mirror of
https://github.com/block/buzz.git
synced 2026-10-30 14:01:37 +00:00
eda26b9423
This ports the advisory flake check from block/buzz-app (https://github.com/block/buzz-app/pull/890) to this repository: pull requests repeat the tests they change ten times, with retries off, so a test that passes CI by luck is caught before it merges. ## What it does `flake-select` diffs the PR merge commit against its first parent (`HEAD^1..HEAD`, `fetch-depth: 2`) and maps changed hunks to tests per runner: a hunk inside a test body selects that test; a change anywhere else in a test file, or a new file, selects the whole file. One lane per runner then repeats only that selection 10× with retries disabled and writes a test × iteration table to its step summary. Lanes skip when nothing was selected, run only on `pull_request`, and are not read by any required gate in `ci.yml` — their own statuses carry the result. ## Per runner | Runner | Selection granularity | Repeat mechanism | |---|---|---| | Rust (cargo-nextest) | `crates/<crate>/<path>.rs::<module::test>` from `#[test]`-style attribute spans (comments and literals blanked first). A hunk outside any test body selects the file's tests. `#[ignore]`d and Postgres / `external_infra` tests are excluded. | One `cargo nextest run --profile flake` per package per iteration, with target flags derived from the path (`src/**` → `--lib --bins`, `tests/<stem>.rs` → `--test <stem>`), a filterset of exact `binary_id & test` pairs, and JUnit parsed per iteration. New `[profile.flake]`: `retries = 0`, `fail-fast = false`. | | node:test | Whole changed `desktop/src/**/*.test.mjs`, `*.jsdom-test.mjs`, `desktop/scripts/*.test.mjs` | 10× loop, each file alone, with the flag sets from `desktop/package.json` `test` | | Playwright `smoke` | `file:line` journeys from `test()`/`it()` spans (TypeScript AST) intersected with `playwright test --list --project=smoke`, deduplicated | Native `--repeat-each=10 --retries=0`, sharded `min(10, ceil(journeys / 5))` ways; `playwright-summary` tabulates the JSON report | | Flutter | Whole changed `mobile/test/**/*_test.dart` | 10× `flutter test --file-reporter=json` loop with the mobile lane's `--dart-define` | A runner whose selection exceeds 64 KiB of entries, or more than 250 Playwright journeys, is skipped with a note rather than overrunning its timeout. ## Differences from the buzz-app version - Lanes live in a reusable `_ci-flake.yml` called from `ci.yml`, matching this repo's `_ci-*.yml` convention, instead of inline jobs. - Playwright specs are parsed with the desktop app's own `typescript` package rather than `rolldown/parseAst`. - No Playwright `integration` project lane: it needs the relay artifact chain. Integration-only specs list nothing for `smoke` and the selection stays empty. - Playwright shards are capped at 10, the same bound as the regular smoke lane. - Rust entries carry the file and in-file module path, and the filterset is anchored to exact `(binary, test)` pairs, so a same-named test in another module is not repeated. - nextest target flags are derived from the changed file's path, so a `buzz-relay` change does not compile every test target of the crate. - A journey-count cap (250) skips the Playwright lane for selections ten repetitions could not finish within its 45-minute timeout. ## Heads-up - The Unit Tests `rust-cache` moves from `key: sherpa-cache-v1` to `shared-key: unit-tests-sherpa-cache-v1` so the flake lane can restore it read-only. This changes the cache key once: the first Unit Tests run after merge builds cold. - This PR itself changes no file the selector recognises (`scripts/flake-check.test.mjs` is a root script, run by the Desktop lane), so its own Flake Check lanes are expected to skip. ## Verification - `node --test scripts/flake-check.test.mjs` — all unit tests pass (also wired into the Desktop lane). - `actionlint` on the changed workflows; `biome check` on the two scripts. - `scripts/test-rust-cache-contract.sh` and its regression harness pass with the new shared key. - Synthetic-diff selection runs for each runner, plus `select` against this PR's own merge-base (empty, as expected). - 2-iteration repeats of a real selection on all four runners locally. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Signed-off-by: Matt Toohey <contact@matttoohey.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
278 lines
11 KiB
YAML
278 lines
11 KiB
YAML
name: CI / Desktop
|
|
on:
|
|
workflow_call:
|
|
inputs:
|
|
rust:
|
|
required: true
|
|
type: boolean
|
|
desktop:
|
|
required: true
|
|
type: boolean
|
|
desktop_rust:
|
|
required: true
|
|
type: boolean
|
|
outputs:
|
|
desktop_result:
|
|
value: ${{ jobs.results.outputs.desktop_result }}
|
|
|
|
env:
|
|
CARGO_TERM_COLOR: always
|
|
BUZZ_TEST_POSTGRES_PASSWORD: buzz_dev
|
|
PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/.cache/ms-playwright
|
|
|
|
jobs:
|
|
# Biome, node:test, and the Vite build do not compile Rust, but node:test
|
|
# also pins desktop constants to relay/DB crate sources (for example
|
|
# relayReconnectReplay.test.mjs), so Rust changes select this job too.
|
|
desktop-js:
|
|
name: Desktop JS
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 20
|
|
# Keep in sync with JS_SELECTED in the Desktop gate (pinned by ci-required-gates.test.mjs).
|
|
if: github.event_name == 'push' || inputs.desktop || inputs.rust
|
|
permissions:
|
|
contents: read
|
|
steps:
|
|
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
|
with:
|
|
fetch-depth: 2
|
|
- uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1
|
|
- name: Get pnpm store directory
|
|
id: pnpm-cache
|
|
run: echo "STORE_PATH=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT"
|
|
- name: Restore pnpm store cache
|
|
uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5
|
|
with:
|
|
path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
|
|
key: pnpm-${{ runner.os }}-${{ hashFiles('**/pnpm-lock.yaml') }}
|
|
restore-keys: pnpm-${{ runner.os }}-
|
|
- name: Install desktop dependencies
|
|
run: just desktop-install-ci
|
|
- name: Desktop lint and format
|
|
run: just desktop-check
|
|
- name: Desktop unit tests
|
|
run: just desktop-test
|
|
- name: Flake check selection tests
|
|
run: node --test scripts/flake-check.test.mjs
|
|
- name: Desktop build
|
|
run: just desktop-build
|
|
- name: Save pnpm store cache
|
|
if: github.event_name == 'push'
|
|
uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5
|
|
with:
|
|
path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
|
|
key: pnpm-${{ runner.os }}-${{ hashFiles('**/pnpm-lock.yaml') }}
|
|
|
|
# Tauri Rust lanes run in parallel. The compiled-flags lane only rebuilds
|
|
# buzz_lib under other BUZZ_BUILD_* values, so it restores the core lane's
|
|
# cache read-only; the mesh-llm feature pulls its own dependency graph.
|
|
desktop-tauri:
|
|
name: Desktop Tauri (${{ matrix.lane }})
|
|
runs-on: ubuntu-latest
|
|
# A lane needs up to ~15m. ci-apt-retry.sh bounds the apt step at ~34m worst
|
|
# case (update ~8.5m + install ~25m); 55m fits both, so a degraded mirror
|
|
# fails with the wrapper's error instead of the job timeout.
|
|
timeout-minutes: 55
|
|
# Keep in sync with RUST_SELECTED in the Desktop gate (pinned by ci-required-gates.test.mjs).
|
|
if: github.event_name == 'push' || inputs.desktop_rust || inputs.rust
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- lane: core
|
|
cache: desktop-core
|
|
save-cache: true
|
|
run: just desktop-tauri-clippy && just desktop-tauri-check && just desktop-tauri-test
|
|
- lane: mesh
|
|
cache: desktop-mesh
|
|
save-cache: true
|
|
run: just desktop-tauri-test-mesh
|
|
- lane: compiled-flags
|
|
cache: desktop-core
|
|
save-cache: false
|
|
run: just desktop-tauri-test-compiled-flags
|
|
permissions:
|
|
contents: read
|
|
steps:
|
|
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
|
with:
|
|
fetch-depth: 2
|
|
- uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1
|
|
- uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1
|
|
- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32
|
|
with:
|
|
workspaces: desktop/src-tauri
|
|
shared-key: ${{ matrix.cache }}
|
|
save-if: ${{ matrix.save-cache && github.event_name != 'pull_request' }}
|
|
- name: Install Tauri dependencies (Linux)
|
|
env:
|
|
DEBIAN_FRONTEND: noninteractive
|
|
run: |
|
|
CI_APT_ATTEMPT_SECONDS=90 scripts/ci-apt-retry.sh sudo apt-get update
|
|
scripts/ci-apt-retry.sh sudo apt-get install -y --no-install-recommends \
|
|
build-essential \
|
|
curl \
|
|
file \
|
|
libasound2-dev \
|
|
libayatana-appindicator3-dev \
|
|
libgtk-3-dev \
|
|
librsvg2-dev \
|
|
libssl-dev \
|
|
libwebkit2gtk-4.1-dev \
|
|
libxdo-dev \
|
|
patchelf \
|
|
wget
|
|
- name: Desktop Tauri ${{ matrix.lane }}
|
|
run: ${{ matrix.run }}
|
|
env:
|
|
CMAKE_POLICY_VERSION_MINIMUM: "3.5"
|
|
|
|
desktop-smoke-e2e:
|
|
name: Desktop Smoke E2E (${{ matrix.shard }})
|
|
runs-on: ubuntu-latest
|
|
# A smoke shard needs ~12m, and Chromium dependency setup is bounded by
|
|
# ci-apt-retry.sh (~25m worst case). Leave time for the full suite
|
|
# and its reports to finish.
|
|
timeout-minutes: 45
|
|
# Keep in sync with FRONTEND_SELECTED in the Desktop gate (pinned by ci-required-gates.test.mjs).
|
|
if: github.event_name == 'push' || inputs.desktop
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
# Playwright shards smoke specs by test count, not duration; ten
|
|
# shards keep the slowest one near 11 minutes of test time.
|
|
shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
|
|
permissions:
|
|
contents: read
|
|
steps:
|
|
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
|
- uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1
|
|
- name: Get pnpm store directory
|
|
id: pnpm-cache
|
|
run: echo "STORE_PATH=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT"
|
|
- name: Restore pnpm store cache
|
|
uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5
|
|
with:
|
|
path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
|
|
key: pnpm-${{ runner.os }}-${{ hashFiles('**/pnpm-lock.yaml') }}
|
|
restore-keys: pnpm-${{ runner.os }}-
|
|
- name: Install desktop dependencies
|
|
run: just desktop-install-ci
|
|
- name: Get Playwright version
|
|
id: pw-version
|
|
run: echo "version=$(cd desktop && node -e "console.log(require('@playwright/test/package.json').version)")" >> "$GITHUB_OUTPUT"
|
|
- name: Restore Playwright browser cache
|
|
id: playwright-cache
|
|
uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5
|
|
with:
|
|
path: ${{ env.PLAYWRIGHT_BROWSERS_PATH }}
|
|
key: playwright-${{ runner.os }}-${{ steps.pw-version.outputs.version }}
|
|
- name: Install Playwright Chromium
|
|
if: steps.playwright-cache.outputs.cache-hit != 'true'
|
|
run: cd desktop && pnpm exec playwright install chromium
|
|
- name: Install Playwright system dependencies
|
|
run: cd desktop && ../scripts/ci-apt-retry.sh pnpm exec playwright install-deps chromium
|
|
- name: Save Playwright browser cache
|
|
if: steps.playwright-cache.outputs.cache-hit != 'true' && github.event_name == 'push' && matrix.shard == 1
|
|
uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5
|
|
with:
|
|
path: ${{ env.PLAYWRIGHT_BROWSERS_PATH }}
|
|
key: playwright-${{ runner.os }}-${{ steps.pw-version.outputs.version }}
|
|
- name: Desktop E2E build
|
|
run: pnpm -C desktop build:e2e
|
|
- name: Desktop smoke e2e
|
|
run: cd desktop && pnpm exec playwright test --project=smoke --shard=${{ matrix.shard }}/${{ strategy.job-total }}
|
|
- name: Summarize flaky tests
|
|
if: ${{ !cancelled() }}
|
|
run: node scripts/summarize-flaky-tests.mjs playwright-report.json "Desktop Smoke E2E (${{ matrix.shard }})"
|
|
working-directory: desktop
|
|
- name: Upload desktop smoke e2e artifacts
|
|
if: ${{ !cancelled() }}
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
|
with:
|
|
name: desktop-smoke-e2e-artifacts-${{ matrix.shard }}
|
|
path: |
|
|
desktop/playwright-report
|
|
desktop/playwright-report.json
|
|
desktop/test-results
|
|
if-no-files-found: ignore
|
|
retention-days: 7
|
|
|
|
desktop:
|
|
name: Desktop
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 5
|
|
needs: [desktop-js, desktop-tauri, desktop-smoke-e2e, desktop-windows-build]
|
|
if: always() && (github.event_name == 'push' || inputs.desktop || inputs.desktop_rust || inputs.rust)
|
|
permissions:
|
|
contents: read
|
|
steps:
|
|
- name: Check desktop jobs
|
|
env:
|
|
JS_SELECTED: ${{ github.event_name == 'push' || inputs.desktop || inputs.rust }}
|
|
RUST_SELECTED: ${{ github.event_name == 'push' || inputs.desktop_rust || inputs.rust }}
|
|
FRONTEND_SELECTED: ${{ github.event_name == 'push' || inputs.desktop }}
|
|
JS: ${{ needs.desktop-js.result }}
|
|
TAURI: ${{ needs.desktop-tauri.result }}
|
|
SMOKE: ${{ needs.desktop-smoke-e2e.result }}
|
|
WINDOWS: ${{ needs.desktop-windows-build.result }}
|
|
run: |
|
|
# A selected job must succeed; an unselected job must be skipped.
|
|
status=0
|
|
check() {
|
|
local name="$1" selected="$2" result="$3" want=skipped
|
|
[ "$selected" = true ] && want=success
|
|
if [ "$result" != "$want" ]; then
|
|
echo "$name finished with: $result (expected $want)"
|
|
status=1
|
|
fi
|
|
}
|
|
check "Desktop JS" "$JS_SELECTED" "$JS"
|
|
check "Desktop Tauri lanes" "$RUST_SELECTED" "$TAURI"
|
|
check "Desktop Smoke E2E shards" "$FRONTEND_SELECTED" "$SMOKE"
|
|
check "Desktop Windows Build" "$FRONTEND_SELECTED" "$WINDOWS"
|
|
[ "$status" = 0 ] && echo "Desktop jobs passed"
|
|
exit "$status"
|
|
|
|
desktop-windows-build:
|
|
name: Desktop Windows Build
|
|
runs-on: windows-latest
|
|
timeout-minutes: 20
|
|
# Keep in sync with FRONTEND_SELECTED in the Desktop gate (pinned by ci-required-gates.test.mjs).
|
|
if: github.event_name == 'push' || inputs.desktop
|
|
permissions:
|
|
contents: read
|
|
steps:
|
|
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
|
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
|
with:
|
|
node-version: 24.14.1
|
|
package-manager-cache: false
|
|
- uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4.3.0
|
|
with:
|
|
version: 11.4.0
|
|
- name: Install desktop dependencies
|
|
shell: bash
|
|
run: pnpm install --frozen-lockfile
|
|
- name: Build both protected-feature selections
|
|
shell: pwsh
|
|
run: |
|
|
Remove-Item Env:VITE_BUZZ_BESTIE -ErrorAction SilentlyContinue
|
|
pnpm -C desktop build
|
|
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
|
|
$env:VITE_BUZZ_BESTIE = "1"
|
|
pnpm -C desktop build
|
|
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
|
|
|
|
results:
|
|
name: Results
|
|
if: ${{ always() }}
|
|
needs: [desktop]
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 5
|
|
permissions: {}
|
|
outputs:
|
|
desktop_result: ${{ needs.desktop.result }}
|
|
steps:
|
|
- run: echo "Captured Desktop job results"
|