mirror of
https://github.com/block/buzz.git
synced 2026-11-12 04:12:21 +00:00
170fe5fa0c
Address the review of the flake check port (99b21e4fb). An integration-only Playwright selection no longer fails flake-select: Playwright exits 1 with a lone "No tests found" error when the file filters match nothing in the smoke project, which listPlaywright now reads as an empty selection, as buzz-app's listBrowser does. Listed journeys are deduplicated, since a looped test() declaration is listed once per iteration, and a selection over 250 journeys skips the Playwright lane with a note: the smoke lane runs ~150 journey-runs per shard in ~11 minutes, so 250 journey-runs per shard across ten shards is what fits the 45-minute timeout after setup and the E2E build. Rust entries now carry the file and the test's in-file module path (`crates/.../auth.rs::tests::name`). repeatRust runs one cargo invocation per package with target flags derived from the path (src/** builds --lib --bins, tests/<stem>.rs builds --test <stem>), so a buzz-relay change no longer compiles every test target of the crate. It lists candidates by in-file path, prefers the test whose full name the file's path predicts, and falls back to every suffix match only when nothing matches (a `#[path]` module), so a same-named test in another module is no longer repeated for a change it never saw. The rust-cache bust knob moves into the shared key (`unit-tests-sherpa-cache-v1`) in both the Unit Tests and flake-rust jobs, since rust-cache ignores `key` once shared-key is set; the contract script checks both and the flake restore's save-if, and the regression harness covers both. flake-select installs desktop dependencies only when a spec changed. Changed .rs files without test functions get a summary row, test.fail() declarations count as tests, a captured nextest 0.9.136 list/JUnit pair pins the classname contract, and CONTRIBUTING lists what the lanes deliberately do not repeat. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Signed-off-by: Matt Toohey <contact@matttoohey.com>
261 lines
11 KiB
YAML
261 lines
11 KiB
YAML
name: CI / Flake Check
|
|
on:
|
|
workflow_call:
|
|
|
|
env:
|
|
CARGO_TERM_COLOR: always
|
|
BUZZ_TEST_POSTGRES_PASSWORD: buzz_dev
|
|
PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/.cache/ms-playwright
|
|
|
|
# Advisory flake check: repeat the tests a pull request changes, ten times
|
|
# each with retries off, so a race inside a test surfaces before it lands on
|
|
# main. The lanes skip when no test changed, so they stay out of the required
|
|
# checks in ci.yml (whose gates reject skipped lanes); their own statuses and
|
|
# step summaries carry the result. Selections reach the lanes through env
|
|
# vars, never interpolated into run scripts; flake-check.mjs admits only plain
|
|
# path and identifier characters.
|
|
jobs:
|
|
flake-select:
|
|
name: Flake check selection
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 10
|
|
permissions:
|
|
contents: read
|
|
outputs:
|
|
rust: ${{ steps.select.outputs.rust }}
|
|
node: ${{ steps.select.outputs.node }}
|
|
playwright: ${{ steps.select.outputs.playwright }}
|
|
playwright_shards: ${{ steps.select.outputs.playwright_shards }}
|
|
playwright_shard_total: ${{ steps.select.outputs.playwright_shard_total }}
|
|
dart: ${{ steps.select.outputs.dart }}
|
|
steps:
|
|
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
|
with:
|
|
# The merge commit's first parent is the base branch tip, so this
|
|
# diff is exactly the pull request's change without a full clone.
|
|
fetch-depth: 2
|
|
- uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1
|
|
# Only a changed spec needs desktop dependencies: Playwright lists its
|
|
# smoke journeys and the desktop app's TypeScript parses it. Pull
|
|
# requests without one skip the install.
|
|
- name: Check for changed Playwright specs
|
|
id: specs
|
|
run: |
|
|
if git diff --quiet --diff-filter=AM HEAD^1 HEAD -- 'desktop/tests/e2e/*.spec.ts'; then
|
|
echo "changed=false" >> "$GITHUB_OUTPUT"
|
|
else
|
|
echo "changed=true" >> "$GITHUB_OUTPUT"
|
|
fi
|
|
- name: Get pnpm store directory
|
|
if: steps.specs.outputs.changed == 'true'
|
|
id: pnpm-cache
|
|
run: echo "STORE_PATH=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT"
|
|
- name: Restore pnpm store cache
|
|
if: steps.specs.outputs.changed == 'true'
|
|
uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5
|
|
with:
|
|
path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
|
|
key: pnpm-${{ runner.os }}-${{ hashFiles('**/pnpm-lock.yaml') }}
|
|
restore-keys: pnpm-${{ runner.os }}-
|
|
- name: Install desktop dependencies
|
|
if: steps.specs.outputs.changed == 'true'
|
|
run: just desktop-install-ci
|
|
- name: Map changed hunks to tests
|
|
id: select
|
|
run: node scripts/flake-check.mjs select HEAD^1 HEAD
|
|
|
|
flake-rust:
|
|
name: Flake check (Rust)
|
|
needs: [flake-select]
|
|
if: needs.flake-select.outputs.rust != ''
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 30
|
|
permissions:
|
|
contents: read
|
|
steps:
|
|
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
|
- uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1
|
|
- uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1
|
|
# Restores the Unit Tests lane's cache (_ci-rust.yml) read-only; this
|
|
# lane never saves one of its own. The restore only hits while the cache
|
|
# key matches: the same shared-key, and rust-cache's environment hash —
|
|
# the CARGO*/RUST*/CC* environment variables, the lockfiles and the rustc
|
|
# version — must stay identical to the Unit Tests job's, so any env this
|
|
# job sets must be set there too (scripts/test-rust-cache-contract.sh
|
|
# checks the key).
|
|
- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32
|
|
with:
|
|
shared-key: unit-tests-sherpa-cache-v1
|
|
save-if: false
|
|
- name: Install cargo-nextest
|
|
uses: taiki-e/install-action@0fd46367812ee04360509b4169d9f659d6892bb2 # v2.79.15
|
|
with:
|
|
tool: cargo-nextest@0.9.136
|
|
# nextest has no repeat flag; the script runs the selection once per
|
|
# iteration under the `flake` profile (.config/nextest.toml).
|
|
- name: Repeat changed Rust tests
|
|
env:
|
|
SELECTED: ${{ needs.flake-select.outputs.rust }}
|
|
run: node scripts/flake-check.mjs rust --iterations=10 $SELECTED
|
|
- name: Rust iteration reports
|
|
if: ${{ !cancelled() }}
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
|
with:
|
|
name: flake-rust
|
|
path: test-results/flake
|
|
if-no-files-found: ignore
|
|
retention-days: 7
|
|
|
|
flake-node:
|
|
name: Flake check (node:test)
|
|
needs: [flake-select]
|
|
if: needs.flake-select.outputs.node != ''
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 15
|
|
permissions:
|
|
contents: read
|
|
steps:
|
|
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
|
- uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1
|
|
- name: Get pnpm store directory
|
|
id: pnpm-cache
|
|
run: echo "STORE_PATH=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT"
|
|
- name: Restore pnpm store cache
|
|
uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5
|
|
with:
|
|
path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
|
|
key: pnpm-${{ runner.os }}-${{ hashFiles('**/pnpm-lock.yaml') }}
|
|
restore-keys: pnpm-${{ runner.os }}-
|
|
- name: Install desktop dependencies
|
|
run: just desktop-install-ci
|
|
- name: Repeat changed node:test files
|
|
env:
|
|
SELECTED: ${{ needs.flake-select.outputs.node }}
|
|
run: node scripts/flake-check.mjs node --iterations=10 $SELECTED
|
|
- name: node:test iteration logs
|
|
if: ${{ !cancelled() }}
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
|
with:
|
|
name: flake-node
|
|
path: test-results/flake
|
|
if-no-files-found: ignore
|
|
retention-days: 7
|
|
|
|
flake-playwright:
|
|
name: Flake check (Playwright smoke, ${{ matrix.shard }}/${{ needs.flake-select.outputs.playwright_shard_total }})
|
|
needs: [flake-select]
|
|
if: needs.flake-select.outputs.playwright != ''
|
|
runs-on: ubuntu-latest
|
|
# Same bound as a smoke shard: Chromium dependency setup is bounded by
|
|
# ci-apt-retry.sh (~25m worst case), and flake-check.mjs sizes the shards
|
|
# so the repetitions fit the rest.
|
|
timeout-minutes: 45
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
shard: ${{ fromJSON(needs.flake-select.outputs.playwright_shards) }}
|
|
permissions:
|
|
contents: read
|
|
steps:
|
|
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
|
- uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1
|
|
- name: Get pnpm store directory
|
|
id: pnpm-cache
|
|
run: echo "STORE_PATH=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT"
|
|
- name: Restore pnpm store cache
|
|
uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5
|
|
with:
|
|
path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
|
|
key: pnpm-${{ runner.os }}-${{ hashFiles('**/pnpm-lock.yaml') }}
|
|
restore-keys: pnpm-${{ runner.os }}-
|
|
- name: Install desktop dependencies
|
|
run: just desktop-install-ci
|
|
- name: Get Playwright version
|
|
id: pw-version
|
|
run: echo "version=$(cd desktop && node -e "console.log(require('@playwright/test/package.json').version)")" >> "$GITHUB_OUTPUT"
|
|
- name: Restore Playwright browser cache
|
|
id: playwright-cache
|
|
uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5
|
|
with:
|
|
path: ${{ env.PLAYWRIGHT_BROWSERS_PATH }}
|
|
key: playwright-${{ runner.os }}-${{ steps.pw-version.outputs.version }}
|
|
- name: Install Playwright Chromium
|
|
if: steps.playwright-cache.outputs.cache-hit != 'true'
|
|
run: cd desktop && pnpm exec playwright install chromium
|
|
- name: Install Playwright system dependencies
|
|
run: cd desktop && ../scripts/ci-apt-retry.sh pnpm exec playwright install-deps chromium
|
|
- name: Desktop E2E build
|
|
run: pnpm -C desktop build:e2e
|
|
# --retries=0 overrides the config's CI retries, so every failed
|
|
# repetition reports as unexpected and fails the step.
|
|
- name: Repeat changed smoke journeys
|
|
env:
|
|
PLAYWRIGHT_JSON_OUTPUT_FILE: playwright-report.json
|
|
SELECTED: ${{ needs.flake-select.outputs.playwright }}
|
|
SHARD: ${{ matrix.shard }}
|
|
SHARD_TOTAL: ${{ needs.flake-select.outputs.playwright_shard_total }}
|
|
run: cd desktop && pnpm exec playwright test --project=smoke --repeat-each=10 --retries=0 --shard="$SHARD/$SHARD_TOTAL" --reporter=list,json $SELECTED
|
|
- name: Summarize repetitions
|
|
if: ${{ !cancelled() }}
|
|
env:
|
|
SHARD: ${{ matrix.shard }}
|
|
SHARD_TOTAL: ${{ needs.flake-select.outputs.playwright_shard_total }}
|
|
run: node scripts/flake-check.mjs playwright-summary desktop/playwright-report.json "Flake check (Playwright smoke, $SHARD/$SHARD_TOTAL)"
|
|
- name: Playwright repetition artifacts
|
|
if: ${{ !cancelled() }}
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
|
with:
|
|
name: flake-playwright-${{ matrix.shard }}
|
|
path: |
|
|
desktop/playwright-report.json
|
|
desktop/test-results
|
|
if-no-files-found: ignore
|
|
retention-days: 7
|
|
|
|
flake-dart:
|
|
name: Flake check (Flutter)
|
|
needs: [flake-select]
|
|
if: needs.flake-select.outputs.dart != ''
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 30
|
|
permissions:
|
|
contents: read
|
|
steps:
|
|
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
|
- uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1
|
|
# Restores the Mobile lane's Hermit and pub caches read-only; that lane
|
|
# saves them.
|
|
- name: Compute Hermit cache key
|
|
id: hermit-bin-hash
|
|
run: |
|
|
hash="$(find ./bin ! -type d | sort | xargs openssl sha256 | openssl sha256 -r | cut -d' ' -f1)"
|
|
echo "hash=$hash" >> "$GITHUB_OUTPUT"
|
|
- name: Restore Hermit package cache
|
|
uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5
|
|
with:
|
|
path: ~/.cache/hermit/pkg
|
|
key: ${{ runner.os }}-hermit-cache-${{ steps.hermit-bin-hash.outputs.hash }}
|
|
restore-keys: ${{ runner.os }}-hermit-cache-
|
|
- name: Prime Flutter SDK
|
|
run: flutter --version
|
|
- name: Restore pub cache
|
|
uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5
|
|
with:
|
|
path: ~/.pub-cache
|
|
key: pub-${{ runner.os }}-${{ hashFiles('mobile/pubspec.lock') }}
|
|
restore-keys: pub-${{ runner.os }}-
|
|
- name: Install dependencies
|
|
run: cd mobile && flutter pub get
|
|
- name: Repeat changed Flutter tests
|
|
env:
|
|
SELECTED: ${{ needs.flake-select.outputs.dart }}
|
|
run: node scripts/flake-check.mjs dart --iterations=10 $SELECTED
|
|
- name: Flutter iteration reports
|
|
if: ${{ !cancelled() }}
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
|
with:
|
|
name: flake-dart
|
|
path: test-results/flake
|
|
if-no-files-found: ignore
|
|
retention-days: 7
|