From 4deb5646e24cb7c9a1fa77f3cedd27b1bacfb742 Mon Sep 17 00:00:00 2001 From: Matt Toohey Date: Tue, 27 Oct 2026 00:09:53 +1100 Subject: [PATCH] ci: repeat changed tests ten times on pull requests Add an advisory flake check that selects the tests a pull request changes and repeats each one ten times with retries off, so a test that passes CI by luck is caught before it merges. This mirrors block/buzz-app#890 (and its sharding follow-up), adapted to the runners this repository actually has. scripts/flake-check.mjs `select` diffs HEAD^1..HEAD and maps hunks to tests per runner: Rust files to `::` entries by parsing `#[test]`-style attributes (shared code changes select the file's tests; `#[ignore]`d and Postgres tests are excluded), desktop Playwright specs to `file:line` smoke journeys via `playwright --list` (non-smoke and unregistered specs are excluded), and desktop node:test and Flutter test files whole. Deleted and renamed-away files drop out of the selection, and any runner whose selection exceeds 64 KiB is dropped with a note so GitHub outputs stay within limits. The reusable _ci-flake.yml adds `flake-select` plus one repeat lane per runner (`flake-rust`, `flake-node`, `flake-playwright` sharded by journey count, `flake-dart`). Each lane skips when its selection is empty, runs only on pull_request, and is not read by any required gate; its status and step summary carry the result. A `flake` nextest profile disables retries and emits JUnit so the Rust lane can report per-test, per-iteration results. The Unit Tests rust-cache gains a `shared-key` so the flake lane can restore it read-only. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Matt Toohey --- .config/nextest.toml | 10 + .github/workflows/_ci-desktop.yml | 2 + .github/workflows/_ci-flake.yml | 244 +++++++++ .github/workflows/_ci-rust.yml | 2 + .github/workflows/ci.yml | 11 + CONTRIBUTING.md | 11 + scripts/flake-check.mjs | 820 ++++++++++++++++++++++++++++++ scripts/flake-check.test.mjs | 454 +++++++++++++++++ 8 files changed, 1554 insertions(+) create mode 100644 .github/workflows/_ci-flake.yml create mode 100644 scripts/flake-check.mjs create mode 100644 scripts/flake-check.test.mjs diff --git a/.config/nextest.toml b/.config/nextest.toml index 1f9f3d156f..8f8d2436b2 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -43,3 +43,13 @@ threads-required = "num-test-threads" filter = "(test(/postgres_tests::/) or binary(/^postgres_/)) and not test(/(^|::)external_infra[^:]*::/)" setup = "postgres-template" run-wrapper = "postgres-isolation" + +[profile.flake] +# The advisory pull-request flake lane (scripts/flake-check.mjs rust) repeats +# a selection and must see every failed repetition, so no retries; the JUnit +# report gives it per-test statuses. +retries = 0 +fail-fast = false + +[profile.flake.junit] +path = "junit.xml" diff --git a/.github/workflows/_ci-desktop.yml b/.github/workflows/_ci-desktop.yml index a0b5e7e628..90eacc59f1 100644 --- a/.github/workflows/_ci-desktop.yml +++ b/.github/workflows/_ci-desktop.yml @@ -52,6 +52,8 @@ jobs: run: just desktop-check - name: Desktop unit tests run: just desktop-test + - name: Flake check selection tests + run: node --test scripts/flake-check.test.mjs - name: Desktop build run: just desktop-build - name: Save pnpm store cache diff --git a/.github/workflows/_ci-flake.yml b/.github/workflows/_ci-flake.yml new file mode 100644 index 0000000000..2cd77f16ed --- /dev/null +++ b/.github/workflows/_ci-flake.yml @@ -0,0 +1,244 @@ +name: CI / Flake Check +on: + workflow_call: + +env: + CARGO_TERM_COLOR: always + BUZZ_TEST_POSTGRES_PASSWORD: buzz_dev + PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/.cache/ms-playwright + +# Advisory flake check: repeat the tests a pull request changes, ten times +# each with retries off, so a race inside a test surfaces before it lands on +# main. The lanes skip when no test changed, so they stay out of the required +# checks in ci.yml (whose gates reject skipped lanes); their own statuses and +# step summaries carry the result. Selections reach the lanes through env +# vars, never interpolated into run scripts; flake-check.mjs admits only plain +# path and identifier characters. +jobs: + flake-select: + name: Flake check selection + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + outputs: + rust: ${{ steps.select.outputs.rust }} + node: ${{ steps.select.outputs.node }} + playwright: ${{ steps.select.outputs.playwright }} + playwright_shards: ${{ steps.select.outputs.playwright_shards }} + playwright_shard_total: ${{ steps.select.outputs.playwright_shard_total }} + dart: ${{ steps.select.outputs.dart }} + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + # The merge commit's first parent is the base branch tip, so this + # diff is exactly the pull request's change without a full clone. + fetch-depth: 2 + - uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1 + - name: Get pnpm store directory + id: pnpm-cache + run: echo "STORE_PATH=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" + - name: Restore pnpm store cache + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5 + with: + path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} + key: pnpm-${{ runner.os }}-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: pnpm-${{ runner.os }}- + # Playwright lists the smoke journeys of a changed spec, and the desktop + # app's TypeScript parses it. + - name: Install desktop dependencies + run: just desktop-install-ci + - name: Map changed hunks to tests + id: select + run: node scripts/flake-check.mjs select HEAD^1 HEAD + + flake-rust: + name: Flake check (Rust) + needs: [flake-select] + if: needs.flake-select.outputs.rust != '' + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1 + - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 + # Restores the Unit Tests lane's cache read-only: this lane only runs on + # pull requests, so it never saves one of its own. + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 + with: + shared-key: unit-tests + key: sherpa-cache-v1 + save-if: ${{ github.event_name != 'pull_request' }} + - name: Install cargo-nextest + uses: taiki-e/install-action@0fd46367812ee04360509b4169d9f659d6892bb2 # v2.79.15 + with: + tool: cargo-nextest@0.9.136 + # nextest has no repeat flag; the script runs the selection once per + # iteration under the `flake` profile (.config/nextest.toml). + - name: Repeat changed Rust tests + env: + SELECTED: ${{ needs.flake-select.outputs.rust }} + run: node scripts/flake-check.mjs rust --iterations=10 $SELECTED + - name: Rust iteration reports + if: ${{ !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: flake-rust + path: test-results/flake + if-no-files-found: ignore + retention-days: 7 + + flake-node: + name: Flake check (node:test) + needs: [flake-select] + if: needs.flake-select.outputs.node != '' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1 + - name: Get pnpm store directory + id: pnpm-cache + run: echo "STORE_PATH=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" + - name: Restore pnpm store cache + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5 + with: + path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} + key: pnpm-${{ runner.os }}-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: pnpm-${{ runner.os }}- + - name: Install desktop dependencies + run: just desktop-install-ci + - name: Repeat changed node:test files + env: + SELECTED: ${{ needs.flake-select.outputs.node }} + run: node scripts/flake-check.mjs node --iterations=10 $SELECTED + - name: node:test iteration logs + if: ${{ !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: flake-node + path: test-results/flake + if-no-files-found: ignore + retention-days: 7 + + flake-playwright: + name: Flake check (Playwright smoke, ${{ matrix.shard }}/${{ needs.flake-select.outputs.playwright_shard_total }}) + needs: [flake-select] + if: needs.flake-select.outputs.playwright != '' + runs-on: ubuntu-latest + # Same bound as a smoke shard: Chromium dependency setup is bounded by + # ci-apt-retry.sh (~25m worst case), and flake-check.mjs sizes the shards + # so the repetitions fit the rest. + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + shard: ${{ fromJSON(needs.flake-select.outputs.playwright_shards) }} + permissions: + contents: read + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1 + - name: Get pnpm store directory + id: pnpm-cache + run: echo "STORE_PATH=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" + - name: Restore pnpm store cache + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5 + with: + path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} + key: pnpm-${{ runner.os }}-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: pnpm-${{ runner.os }}- + - name: Install desktop dependencies + run: just desktop-install-ci + - name: Get Playwright version + id: pw-version + run: echo "version=$(cd desktop && node -e "console.log(require('@playwright/test/package.json').version)")" >> "$GITHUB_OUTPUT" + - name: Restore Playwright browser cache + id: playwright-cache + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5 + with: + path: ${{ env.PLAYWRIGHT_BROWSERS_PATH }} + key: playwright-${{ runner.os }}-${{ steps.pw-version.outputs.version }} + - name: Install Playwright Chromium + if: steps.playwright-cache.outputs.cache-hit != 'true' + run: cd desktop && pnpm exec playwright install chromium + - name: Install Playwright system dependencies + run: cd desktop && ../scripts/ci-apt-retry.sh pnpm exec playwright install-deps chromium + - name: Desktop E2E build + run: pnpm -C desktop build:e2e + # --retries=0 overrides the config's CI retries, so every failed + # repetition reports as unexpected and fails the step. + - name: Repeat changed smoke journeys + env: + PLAYWRIGHT_JSON_OUTPUT_FILE: playwright-report.json + SELECTED: ${{ needs.flake-select.outputs.playwright }} + SHARD: ${{ matrix.shard }} + SHARD_TOTAL: ${{ needs.flake-select.outputs.playwright_shard_total }} + run: cd desktop && pnpm exec playwright test --project=smoke --repeat-each=10 --retries=0 --shard="$SHARD/$SHARD_TOTAL" --reporter=list,json $SELECTED + - name: Summarize repetitions + if: ${{ !cancelled() }} + env: + SHARD: ${{ matrix.shard }} + SHARD_TOTAL: ${{ needs.flake-select.outputs.playwright_shard_total }} + run: node scripts/flake-check.mjs playwright-summary desktop/playwright-report.json "Flake check (Playwright smoke, $SHARD/$SHARD_TOTAL)" + - name: Playwright repetition artifacts + if: ${{ !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: flake-playwright-${{ matrix.shard }} + path: | + desktop/playwright-report.json + desktop/test-results + if-no-files-found: ignore + retention-days: 7 + + flake-dart: + name: Flake check (Flutter) + needs: [flake-select] + if: needs.flake-select.outputs.dart != '' + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1 + # Restores the Mobile lane's Hermit and pub caches read-only; that lane + # saves them. + - name: Compute Hermit cache key + id: hermit-bin-hash + run: | + hash="$(find ./bin ! -type d | sort | xargs openssl sha256 | openssl sha256 -r | cut -d' ' -f1)" + echo "hash=$hash" >> "$GITHUB_OUTPUT" + - name: Restore Hermit package cache + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5 + with: + path: ~/.cache/hermit/pkg + key: ${{ runner.os }}-hermit-cache-${{ steps.hermit-bin-hash.outputs.hash }} + restore-keys: ${{ runner.os }}-hermit-cache- + - name: Prime Flutter SDK + run: flutter --version + - name: Restore pub cache + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5 + with: + path: ~/.pub-cache + key: pub-${{ runner.os }}-${{ hashFiles('mobile/pubspec.lock') }} + restore-keys: pub-${{ runner.os }}- + - name: Install dependencies + run: cd mobile && flutter pub get + - name: Repeat changed Flutter tests + env: + SELECTED: ${{ needs.flake-select.outputs.dart }} + run: node scripts/flake-check.mjs dart --iterations=10 $SELECTED + - name: Flutter iteration reports + if: ${{ !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: flake-dart + path: test-results/flake + if-no-files-found: ignore + retention-days: 7 diff --git a/.github/workflows/_ci-rust.yml b/.github/workflows/_ci-rust.yml index 4f615eb162..fa60042f67 100644 --- a/.github/workflows/_ci-rust.yml +++ b/.github/workflows/_ci-rust.yml @@ -58,6 +58,8 @@ jobs: - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 with: + # Named so the advisory flake lane (_ci-flake.yml) can restore it. + shared-key: unit-tests key: sherpa-cache-v1 save-if: ${{ github.event_name != 'pull_request' }} - name: Install cargo-nextest diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 403c809604..c4cc6590bd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,6 +83,9 @@ jobs: - 'scripts/normative-corpus.json' - 'desktop/**' - 'pnpm-lock.yaml' + - 'scripts/flake-check.mjs' + - 'scripts/flake-check.test.mjs' + - '.github/workflows/_ci-flake.yml' desktop-rust: - 'desktop/src-tauri/**' web: @@ -259,6 +262,14 @@ jobs: mobile: ${{ needs.changes.outputs.mobile == 'true' }} lane: mobile-swift + # Advisory flake check: repeats the tests a pull request changes. Its lanes + # skip when no test changed and none of the required gates below reads them; + # their own statuses and step summaries carry the result. + flake-check: + name: Flake Check + if: github.event_name == 'pull_request' + uses: ./.github/workflows/_ci-flake.yml + security-domain: name: Security Domain needs: [changes] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2beb40bd5a..a5f3aa61f5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -304,6 +304,17 @@ just ci This is the same check that runs in CI. PRs that fail `just ci` will not be merged. If `just ci` fails on formatting, `just fix-all` fixes it in one shot (`rustfmt` + Tauri fmt + desktop, web, and mobile formatters). +Pull requests also get an advisory **flake check**. `scripts/flake-check.mjs +select` maps the diff against the base branch to the Rust tests, desktop +`node --test` files, Playwright smoke journeys and Flutter test files it +changed: a hunk inside a test body selects that test; a change anywhere else +in a test file (imports, helpers, hooks, the code under test beside Rust +tests) or a new file selects the whole file. The `Flake check (…)` lanes then +run that selection ten times with retries off, so a repetition that fails is +a race in the test or the code under it, worth fixing before it lands. The +lanes skip when no test changed and are not required to merge; their step +summaries show the test × iteration table. + --- ## Code Style diff --git a/scripts/flake-check.mjs b/scripts/flake-check.mjs new file mode 100644 index 0000000000..b1c24ed895 --- /dev/null +++ b/scripts/flake-check.mjs @@ -0,0 +1,820 @@ +// Advisory pull-request flake check. `select` maps a diff to the tests it +// changed, per runner; `rust`, `node` and `dart` repeat a selection and report +// every iteration. Playwright repeats natively with --repeat-each, and +// `playwright-summary` tabulates its JSON report afterwards. +import { spawnSync } from "node:child_process"; +import { + appendFileSync, + copyFileSync, + mkdirSync, + readFileSync, + rmSync, +} from "node:fs"; +import { createRequire } from "node:module"; +import { fileURLToPath } from "node:url"; + +const root = fileURLToPath(new URL("../", import.meta.url)); +export const ITERATIONS = 10; +// Repetitions spread across shards, so a whole-file selection stays inside +// the Playwright lane's job timeout. Playwright shards at file × repetition +// granularity (workers: 1, no fullyParallel), and the smoke lane runs ~1,500 +// journeys over ten ~11-minute shards, so five journeys × ten repetitions is a +// few minutes of test time per shard. The regular smoke lane never uses more +// than ten runners, so neither does this one. +const JOURNEYS_PER_SHARD = 5; +const MAX_SHARDS = 10; +export const playwrightShards = (journeys) => + Math.min(MAX_SHARDS, Math.max(1, Math.ceil(journeys / JOURNEYS_PER_SHARD))); +// GitHub caps a job's outputs at 1 MiB in total. A selection this large is a +// sweeping refactor, not a test change worth ten repetitions, so the lane +// skips rather than overrunning its timeout. +export const MAX_OUTPUT_BYTES = 64 * 1024; +export const runners = ["rust", "node", "playwright", "dart"]; +// Mirrors desktop/package.json "test": plain helpers run under the TS loader; +// jsdom tests also preload the DOM shim and force-exit past their timers. +export const nodeTestArgs = (file) => + file.endsWith(".jsdom-test.mjs") + ? [ + "--import", + "./test-jsdom-setup.mjs", + "--import", + "./test-loader.mjs", + "--experimental-strip-types", + "--test-force-exit", + "--test", + ] + : ["--import", "./test-loader.mjs", "--experimental-strip-types", "--test"]; +// Mirrors the mobile test shards in .github/workflows/_ci-clients.yml. +export const dartDefine = + "--dart-define=BUZZ_PUSH_GATEWAY_URL=https://push.example"; +// Mirrors the postgres-ci default-filter in .config/nextest.toml: those tests +// are #[ignore]d and need Postgres, Redis, S3 or MinIO, so a selection that +// reached one would fail every repetition without being flaky. +const infraTests = + "test(/postgres_tests::/) | binary(/^postgres_/) | test(/(^|::)external_infra[^:]*::/)"; + +// Mirrors desktop/package.json "test", desktop/playwright.config.ts testDir, +// the root Cargo workspace's crates and mobile's test directory. Paths are +// later split on whitespace in a shell, so admit only plain names. +export function classify(file) { + if (!/^[\w./@-]+$/.test(file)) return undefined; + if (/^crates\/[^/]+\/(?:src|tests)\/(?!fixtures\/).*\.rs$/.test(file)) + return "rust"; + if ( + /^desktop\/(?:src\/.*\.(?:test|jsdom-test)\.mjs|scripts\/[^/]+\.test\.mjs)$/.test( + file, + ) + ) + return "node"; + if (/^desktop\/tests\/e2e\/[^/]+\.spec\.ts$/.test(file)) return "playwright"; + if (/^mobile\/test\/.*_test\.dart$/.test(file)) return "dart"; + return undefined; +} + +// Line ranges of a -U0 diff: the new-side lines each hunk writes and the +// old-side lines it removes (empty, end < start, for a pure insertion). A pure +// deletion writes no lines and attaches to the line before it (0 at the top of +// the file); `removed` keeps what it took out, since that line cannot tell. +export function hunkRanges(diff) { + return [ + ...diff.matchAll(/^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@/gm), + ].map(([, oldStart, oldCount = "1", start, count = "1"]) => ({ + start: Number(start), + end: Number(start) + Math.max(Number(count), 1) - 1, + removed: { + start: Number(oldStart), + end: Number(oldStart) + Number(oldCount) - 1, + }, + })); +} + +// Members that keep a `test`/`it` chain a test declaration. Anything else on +// the chain is a group (`describe`), a hook (`test.beforeEach`), configuration +// (`test.use`) or a harness object that happens to be named `test`. +const declaring = [ + "only", + "skip", + "fixme", + "fails", + "todo", + "concurrent", + "sequential", + "each", + "for", + "runIf", + "skipIf", +]; + +// The desktop app's TypeScript parses the specs; load it only for a selection +// that needs it, so Rust-only runs work without desktop dependencies. +let typescript; +const ts = () => + (typescript ??= createRequire( + new URL("../desktop/package.json", import.meta.url), + )("typescript")); + +// Line spans of every test declaration in a TypeScript spec, from its first +// line to its closing parenthesis. +export function testCalls(source, file) { + const t = ts(); + const sourceFile = t.createSourceFile( + file, + source, + t.ScriptTarget.Latest, + true, + ); + const lineAt = (position) => + sourceFile.getLineAndCharacterOfPosition(position).line + 1; + const names = (callee) => + t.isIdentifier(callee) + ? [callee.text] + : t.isPropertyAccessExpression(callee) + ? names(callee.expression)?.concat(callee.name.text) + : t.isCallExpression(callee) + ? names(callee.expression) + : undefined; + const calls = []; + const visit = (node) => { + const chain = t.isCallExpression(node) ? names(node.expression) : undefined; + if ( + chain && + ["test", "it"].includes(chain[0]) && + chain.slice(1).every((member) => declaring.includes(member)) + ) { + calls.push({ + start: lineAt(node.getStart(sourceFile)), + end: lineAt(node.end - 1), + }); + return; + } + t.forEachChild(node, visit); + }; + visit(sourceFile); + return calls; +} + +// Rust source with every comment and literal blanked to spaces, newlines kept, +// so attribute and brace scanning never see a `{` or `#[` inside a string, +// char literal or comment. Lifetimes (`'a`) are kept; char literals are not. +export function blankRust(source) { + let out = ""; + let i = 0; + const blank = (end) => { + for (; i < end && i < source.length; i += 1) + out += source[i] === "\n" ? "\n" : " "; + }; + const closingQuote = (from, raw = false, hashes = "") => { + for (let j = from; j < source.length; j += 1) { + if (source[j] === "\\" && !raw) j += 1; + else if (source[j] === '"' && source.startsWith(hashes, j + 1)) + return j + 1 + hashes.length; + } + return source.length; + }; + const rawPrefix = /[bc]?r(#*)"/y; + while (i < source.length) { + const rest = source.slice(i, i + 2); + rawPrefix.lastIndex = i; + const raw = rawPrefix.exec(source); + if (rest.startsWith("//")) { + const end = source.indexOf("\n", i); + blank(end === -1 ? source.length : end); + } else if (rest.startsWith("/*")) { + let depth = 0; + let j = i; + do { + if (source.startsWith("/*", j)) { + depth += 1; + j += 2; + } else if (source.startsWith("*/", j)) { + depth -= 1; + j += 2; + } else j += 1; + } while (depth > 0 && j < source.length); + blank(j); + } else if (raw) { + blank(closingQuote(i + raw[0].length, true, raw[1])); + } else if (/^[bc]?"/.test(rest)) { + blank(closingQuote(i + rest.indexOf('"') + 1)); + } else if (source[i] === "'") { + const width = source.codePointAt(i + 1) > 0xffff ? 2 : 1; + if (source[i + 1] === "\\") blank(source.indexOf("'", i + 2) + 1); + else if (source[i + 1 + width] === "'") blank(i + width + 2); + else { + out += source[i]; + i += 1; + } + } else { + out += source[i]; + i += 1; + } + } + return out; +} + +const matching = (text, open, close, from) => { + let depth = 0; + for (let j = from; j < text.length; j += 1) { + if (text[j] === open) depth += 1; + else if (text[j] === close) { + depth -= 1; + if (depth === 0) return j; + } + } + return text.length - 1; +}; + +// Every test function in a Rust file: a `fn` whose attribute block has an +// attribute path ending in `test` (`#[test]`, `#[tokio::test]`, …), spanning +// the first attribute line to the body's closing brace. `ignored` marks +// `#[ignore]`, which nextest skips by default. +export function rustTests(source) { + const text = blankRust(source); + const lineStarts = [0]; + for (let i = 0; i < text.length; i += 1) + if (text[i] === "\n") lineStarts.push(i + 1); + const lineAt = (offset) => { + let low = 0; + let high = lineStarts.length - 1; + while (low < high) { + const mid = Math.ceil((low + high) / 2); + if (lineStarts[mid] <= offset) low = mid; + else high = mid - 1; + } + return low + 1; + }; + const tests = []; + const attribute = /#\[/g; + for (let match = attribute.exec(text); match; match = attribute.exec(text)) { + const paths = []; + let j = match.index; + while (text.startsWith("#[", j)) { + const close = matching(text, "[", "]", j + 1); + paths.push(/^\s*([\w:]+)/.exec(text.slice(j + 2, close))?.[1] ?? ""); + j = close + 1; + while (/\s/.test(text[j] ?? "")) j += 1; + } + attribute.lastIndex = j; + if (!paths.some((path) => /^(?:\w+::)*test$/.test(path))) continue; + const fn = + /^(?:pub(?:\([\w\s:]*\))?\s+)?(?:async\s+)?(?:unsafe\s+)?fn\s+(\w+)/.exec( + text.slice(j), + ); + if (!fn) continue; + const open = text.indexOf("{", j + fn[0].length); + if (open === -1) continue; + tests.push({ + name: fn[1], + start: lineAt(match.index), + end: lineAt(matching(text, "{", "}", open)), + ignored: paths.includes("ignore"), + }); + } + return tests; +} + +// Bodies of the listed tests: their calls, keyed by the line the runner +// listed. Runners only report a call-site line, which may sit inside the call, +// so each listed line is owned by the enclosing test call; a listed line +// outside every call leaves its lines to the whole-file rule. +export function testBodies(calls, lines) { + const bodies = new Map(); + for (const line of [...lines].sort((a, b) => a - b)) { + const call = calls.find(({ start, end }) => start <= line && line <= end); + if (call && !bodies.has(call)) bodies.set(call, { ...call, line }); + } + return [...bodies.values()].sort((a, b) => a.start - b.start); +} + +// The calls owning every line of a range, or undefined once a line lies +// outside them all. +function owners(calls, { start, end }) { + const found = []; + for (let line = start; line <= end; ) { + const call = calls.find((c) => c.start <= line && line <= c.end); + if (!call) return undefined; + found.push(call); + line = call.end + 1; + } + return found; +} + +// A hunk whose every line lies in a test body selects those tests. A changed +// line outside every body (imports, helpers, hooks or constants before, +// between or after tests, or the code under test in a Rust module) can change +// every test, so it, a new file, or a file with no listed tests selects the +// whole file. Removed lines are judged by the old file's calls (`previous`): a +// deletion attaches to its preceding line, which belongs to the test before a +// removed hook, not to the hook. +export function selectTests( + ranges, + bodies, + { added = false, previous = [] } = {}, +) { + if (added) return { whole: true, reason: "new file" }; + if (bodies.length === 0) return { whole: true, reason: "no tests listed" }; + const selected = new Set(); + for (const { start, end, removed } of ranges) { + if (!owners(previous, removed)) + return { whole: true, reason: "shared code removed" }; + const current = owners(bodies, { start, end }); + if (!current) return { whole: true, reason: "shared code changed" }; + for (const body of current) selected.add(body.line); + } + return { lines: [...selected].sort((a, b) => a - b) }; +} + +// Rust tests a diff selects: hunks inside a test body select that test, any +// other hunk selects every test in the file (the module's code under test +// lives beside them), and `#[ignore]`d tests own their lines but never run. +// A file without test functions is not a test file and selects nothing. +export function selectRust( + diff, + source, + { added = false, previous = "" } = {}, +) { + const tests = rustTests(source); + if (tests.length === 0) return undefined; + const selection = selectTests( + hunkRanges(diff), + tests.map((test) => ({ ...test, line: test.start })), + { added, previous: added ? [] : rustTests(previous) }, + ); + const chosen = selection.whole + ? tests + : tests.filter((test) => selection.lines.includes(test.start)); + return { + ...selection, + names: chosen.filter((test) => !test.ignored).map((test) => test.name), + }; +} + +const run = (command, args, options = {}) => { + const result = spawnSync(command, args, { + cwd: root, + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + ...options, + }); + if (result.error) throw result.error; + return result; +}; +const git = (...args) => { + const result = run("git", args); + if (result.status !== 0) throw new Error(`git ${args[0]}: ${result.stderr}`); + return result.stdout; +}; +const summarize = (text) => { + if (process.env.GITHUB_STEP_SUMMARY) + appendFileSync(process.env.GITHUB_STEP_SUMMARY, text); + else process.stdout.write(text); +}; +const cell = (value) => String(value).replaceAll("|", "\\|"); + +// The `[package]` name of the crate owning a file under crates/. +export function cratePackage(file) { + const manifest = readFileSync( + `${root}crates/${file.split("/")[1]}/Cargo.toml`, + "utf8", + ); + return /^\[package\][\s\S]*?^name\s*=\s*"([^"]+)"/m.exec(manifest)[1]; +} + +// Smoke journeys per spec file as Playwright lists them. Integration-only +// specs list nothing for the smoke project, so a selection of them stays empty. +export function listPlaywright(files) { + const result = run(`${root}bin/pnpm`, [ + "-C", + "desktop", + "--silent", + "exec", + "playwright", + "test", + "--project=smoke", + "--list", + "--reporter=json", + ...files, + ]); + const report = JSON.parse(result.stdout); + if (result.status !== 0 || report.errors.length) + throw new Error( + `playwright --list: ${report.errors.map((error) => error.message).join("\n") || result.stderr}`, + ); + const specs = new Map(files.map((file) => [file, []])); + const collect = (suites) => { + for (const suite of suites) { + for (const spec of suite.specs ?? []) + if (spec.tests?.length) + specs.get(`desktop/tests/e2e/${spec.file}`)?.push(spec.line); + collect(suite.suites ?? []); + } + }; + collect(report.suites); + return specs; +} + +function select(base = "HEAD^1", head = "HEAD") { + const changed = git( + "diff", + "--name-status", + "--no-renames", + "-z", + "--diff-filter=AM", + base, + head, + "--", + ).split("\0"); + const files = Object.fromEntries(runners.map((runner) => [runner, []])); + const added = new Set(); + for (let i = 0; i + 1 < changed.length; i += 2) { + const [status, file] = [changed[i], changed[i + 1]]; + const runner = classify(file); + if (runner) files[runner].push(file); + if (status === "A") added.add(file); + } + const journeys = files.playwright.length + ? listPlaywright(files.playwright) + : new Map(); + const outputs = Object.fromEntries(runners.map((runner) => [runner, []])); + const rows = []; + const previous = (file) => + added.has(file) ? "" : git("show", `${base}:${file}`); + const diff = (file) => + git("diff", "-U0", "--no-color", base, head, "--", file); + for (const runner of runners) { + for (const file of files[runner]) { + let entries = [file]; + let note = "whole file"; + if (runner === "rust") { + const selection = selectRust( + diff(file), + readFileSync(`${root}${file}`, "utf8"), + { + added: added.has(file), + previous: previous(file), + }, + ); + if (!selection) continue; + const crate = cratePackage(file); + entries = selection.names.map((name) => `${crate}::${name}`); + note = + selection.names.length === 0 + ? "excluded (ignored tests only)" + : selection.whole + ? `whole file (${selection.reason}): ${selection.names.length} test(s)` + : `${selection.names.length} changed test(s)`; + } else if (runner === "playwright") { + const listed = journeys.get(file) ?? []; + const source = readFileSync(`${root}${file}`, "utf8"); + const selection = selectTests( + hunkRanges(diff(file)), + testBodies(testCalls(source, file), listed), + { + added: added.has(file), + previous: added.has(file) ? [] : testCalls(previous(file), file), + }, + ); + const lines = selection.whole ? listed : selection.lines; + entries = lines.map((line) => `${file}:${line}`); + note = + lines.length === 0 + ? "excluded (no smoke journeys)" + : selection.whole + ? `whole file (${selection.reason}): ${lines.length} journey(s)` + : `${lines.length} changed journey(s)`; + } + outputs[runner].push(...entries); + rows.push(`| ${cell(file)} | ${runner} | ${cell(note)} |`); + } + if (Buffer.byteLength(outputs[runner].join(" ")) > MAX_OUTPUT_BYTES) { + rows.push( + `| *(${outputs[runner].length} ${runner} entries)* | ${runner} | selection exceeds ${MAX_OUTPUT_BYTES} bytes; lane skipped |`, + ); + outputs[runner] = []; + } + } + summarize( + [ + "## Flake check selection", + "", + ...(rows.length + ? ["| File | Runner | Selection |", "| --- | --- | --- |", ...rows] + : ["No Rust, node:test, Playwright or Flutter test files changed."]), + "", + "", + ].join("\n"), + ); + const shards = playwrightShards(outputs.playwright.length); + const text = [ + ...Object.entries(outputs).map( + ([key, entries]) => `${key}=${entries.join(" ")}\n`, + ), + `playwright_shards=${JSON.stringify(Array.from({ length: shards }, (_, i) => i + 1))}\n`, + `playwright_shard_total=${shards}\n`, + ].join(""); + if (process.env.GITHUB_OUTPUT) + appendFileSync(process.env.GITHUB_OUTPUT, text); + else process.stdout.write(text); +} + +const resultsDir = (runner) => { + const dir = `${root}test-results/flake/${runner}`; + rmSync(dir, { recursive: true, force: true }); + mkdirSync(dir, { recursive: true }); + return dir; +}; + +// Test × iteration table for a repeated selection; a run passes only when its +// process exits 0 and every expected test reports. Tests that never ran in any +// iteration are left out, so the table holds only the repeated ones. +function report(title, runs) { + const columns = runs.map((_, i) => String(i + 1)); + const mark = (status) => + ({ passed: "✅", failed: "❌" })[status] ?? (status ? "⏭" : "—"); + const ran = (status) => status === "passed" || status === "failed"; + const tests = [ + ...new Set(runs.flatMap((run) => [...run.statuses.keys()])), + ].filter((test) => runs.some((run) => ran(run.statuses.get(test)))); + const failures = runs.filter((run) => !run.ok).length; + summarize( + [ + `## ${title}: ${failures} of ${runs.length} iterations failed`, + "", + `| Test | ${columns.join(" | ")} | Failed |`, + `| --- | ${columns.map(() => ":-:").join(" | ")} | ---: |`, + `| **Run** | ${runs.map((run) => (run.ok ? "✅" : "❌")).join(" | ")} | ${failures}/${runs.length} |`, + ...tests.sort().map((test) => { + const statuses = runs.map((run) => run.statuses.get(test)); + return `| ${cell(test)} | ${statuses.map(mark).join(" | ")} | ${statuses.filter((status) => status === "failed").length}/${runs.length} |`; + }), + "", + "", + ].join("\n"), + ); + process.exitCode = failures === 0 ? 0 : 1; +} + +const progress = (i, iterations, ok) => + console.log( + `flake-check: iteration ${i}/${iterations} ${ok ? "passed" : "FAILED"}`, + ); + +// Per-test statuses of a JUnit report, keyed ` `. +export function parseJunit(xml) { + const decode = (text) => + text.replace( + /&(quot|apos|lt|gt|amp);/g, + (_, entity) => + ({ quot: '"', apos: "'", lt: "<", gt: ">", amp: "&" })[entity], + ); + const statuses = new Map(); + for (const chunk of xml.split("")); + const body = chunk.split("")[0]; + const name = decode(/\bname="([^"]*)"/.exec(attributes)[1]); + const classname = decode( + /\bclassname="([^"]*)"/.exec(attributes)?.[1] ?? "", + ); + statuses.set( + `${classname} ${name}`, + /<(?:failure|error)[\s/>]/.test(body) + ? "failed" + : /]/.test(body) + ? "skipped" + : "passed", + ); + } + return statuses; +} + +// nextest has no repeat flag, so run the filterset once per iteration under +// the `flake` profile (.config/nextest.toml: no retries, JUnit per run). The +// selection is listed first: a filterset matching no runnable test is a +// selection of ignored or infrastructure-backed tests, not a failure. +function repeatRust(entries, iterations) { + const byPackage = new Map(); + for (const entry of entries) { + const [, crate, name] = /^(.+)::(\w+)$/.exec(entry) ?? []; + if (!crate) + throw new Error(`rust entry must be ::: ${entry}`); + byPackage.set(crate, (byPackage.get(crate) ?? new Set()).add(name)); + } + const filterset = `(${[...byPackage] + .map( + ([crate, names]) => + `(package(=${crate}) & (${[...names].map((name) => `test(/(^|::)${name}$/)`).join(" | ")}))`, + ) + .join(" | ")}) & !(${infraTests})`; + const packages = [...byPackage.keys()].flatMap((crate) => ["-p", crate]); + const listed = run("cargo", [ + "nextest", + "list", + ...packages, + "-E", + filterset, + "--message-format", + "json", + ]); + if (listed.status !== 0) + throw new Error(`cargo nextest list: ${listed.stderr}`); + const expected = Object.entries( + JSON.parse(listed.stdout)["rust-suites"], + ).flatMap(([binary, suite]) => + Object.entries(suite.testcases) + .filter( + ([, test]) => + !test.ignored && test["filter-match"].status === "matches", + ) + .map(([name]) => `${binary} ${name}`), + ); + if (expected.length === 0) { + summarize( + "## Flake check (Rust): nothing to repeat\n\nEvery selected test is ignored or infrastructure-backed.\n\n", + ); + return; + } + // nextest keeps its store under the workspace root's target/nextest, not + // CARGO_TARGET_DIR; the `flake` profile writes its JUnit report there. + const junit = `${root}target/nextest/flake/junit.xml`; + const dir = resultsDir("rust"); + const runs = []; + for (let i = 1; i <= iterations; i += 1) { + rmSync(junit, { force: true }); + const result = run( + "cargo", + ["nextest", "run", "--profile", "flake", ...packages, "-E", filterset], + { stdio: "inherit" }, + ); + let statuses = new Map(); + try { + statuses = parseJunit(readFileSync(junit, "utf8")); + copyFileSync(junit, `${dir}/iter-${i}.xml`); + } catch { + statuses = new Map(); + } + const ok = + result.status === 0 && + expected.every((test) => statuses.get(test) === "passed"); + runs.push({ ok, statuses }); + progress(i, iterations, ok); + } + report("Flake check (Rust)", runs); +} + +// node:test has no repeat flag either. Each file runs alone, so a failure is +// attributed to its file; the regular lane's cross-file concurrency is not +// reproduced here. +function repeatNode(files, iterations) { + const dir = resultsDir("node"); + const runs = []; + for (let i = 1; i <= iterations; i += 1) { + const statuses = new Map(); + for (const file of files) { + const path = file.replace(/^desktop\//, ""); + const result = run(process.execPath, [...nodeTestArgs(path), path], { + cwd: `${root}desktop`, + }); + process.stdout.write(result.stdout); + process.stderr.write(result.stderr); + appendFileSync( + `${dir}/iter-${i}.log`, + `# ${file}\n${result.stdout}${result.stderr}\n`, + ); + statuses.set(file, result.status === 0 ? "passed" : "failed"); + } + const ok = [...statuses.values()].every((status) => status === "passed"); + runs.push({ ok, statuses }); + progress(i, iterations, ok); + } + report("Flake check (node:test)", runs); +} + +// Per-test statuses of a `dart test` JSON report (one event per line), keyed +// ` › `. Hidden entries are the runner's own file-loading steps. +export function parseDartReport(text, mobileRoot = `${root}mobile/`) { + const started = new Map(); + const statuses = new Map(); + for (const line of text.split("\n")) { + let event; + try { + event = JSON.parse(line); + } catch { + continue; + } + if (event.type === "testStart") started.set(event.test.id, event.test); + if (event.type !== "testDone" || event.hidden) continue; + const test = started.get(event.testID); + if (!test) continue; + const file = fileURLToPath(test.url ?? "file:///").replace(mobileRoot, ""); + statuses.set( + `${file} › ${test.name}`, + event.skipped + ? "skipped" + : event.result === "success" + ? "passed" + : "failed", + ); + } + return statuses; +} + +// `flutter test` has no repeat flag; each iteration recompiles only the +// selected files, as the sharded mobile lane does. +function repeatDart(files, iterations) { + const dir = resultsDir("dart"); + const runs = []; + // The Justfile unsets these so a worktree's git context never reaches flutter. + const { GIT_DIR, GIT_WORK_TREE, ...env } = process.env; + for (let i = 1; i <= iterations; i += 1) { + const reportFile = `${dir}/iter-${i}.json`; + const result = run( + "flutter", + [ + "test", + dartDefine, + `--file-reporter=json:${reportFile}`, + ...files.map((file) => file.replace(/^mobile\//, "")), + ], + { cwd: `${root}mobile`, env, stdio: "inherit" }, + ); + let statuses = new Map(); + try { + statuses = parseDartReport(readFileSync(reportFile, "utf8")); + } catch { + statuses = new Map(); + } + const ok = result.status === 0 && statuses.size > 0; + runs.push({ ok, statuses }); + progress(i, iterations, ok); + } + report("Flake check (Flutter)", runs); +} + +// Playwright's JSON report lists a `--repeat-each` journey once per +// repetition, in order; `unexpected` is a failed repetition since the lane +// runs with --retries=0. +export function playwrightRepetitions(report) { + const repetitions = new Map(); + const collect = (suites) => { + for (const suite of suites) { + for (const spec of suite.specs ?? []) { + const key = `${spec.file}:${spec.line} › ${spec.title}`; + const statuses = repetitions.get(key) ?? []; + for (const test of spec.tests ?? []) + statuses.push( + { expected: "passed", unexpected: "failed", flaky: "failed" }[ + test.status + ] ?? "skipped", + ); + repetitions.set(key, statuses); + } + collect(suite.suites ?? []); + } + }; + collect(report.suites ?? []); + return repetitions; +} + +function playwrightSummary(reportPath, title) { + const repetitions = playwrightRepetitions( + JSON.parse(readFileSync(reportPath, "utf8")), + ); + const iterations = Math.max( + 0, + ...[...repetitions.values()].map((s) => s.length), + ); + const runs = Array.from({ length: iterations }, (_, i) => ({ + ok: [...repetitions.values()].every((statuses) => statuses[i] !== "failed"), + statuses: new Map( + [...repetitions].map(([test, statuses]) => [test, statuses[i]]), + ), + })); + report(title, runs); + // Playwright already failed its own step; the summary only reports. + process.exitCode = 0; +} + +function main() { + const [command, ...args] = process.argv.slice(2); + if (command === "select" && args.length <= 2) return select(...args); + if (command === "playwright-summary" && args.length === 2) + return playwrightSummary(...args); + const repeat = { rust: repeatRust, node: repeatNode, dart: repeatDart }[ + command + ]; + if (repeat) { + const iterations = Number( + args + .find((argument) => argument.startsWith("--iterations=")) + ?.slice("--iterations=".length) ?? ITERATIONS, + ); + const entries = args.filter((argument) => !argument.startsWith("--")); + if (Number.isInteger(iterations) && iterations > 0 && entries.length > 0) + return repeat(entries, iterations); + } + throw new Error( + "usage: flake-check select [base] [head] | rust|node|dart [--iterations=N] | playwright-summary ", + ); +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) main(); diff --git a/scripts/flake-check.test.mjs b/scripts/flake-check.test.mjs new file mode 100644 index 0000000000..876675b973 --- /dev/null +++ b/scripts/flake-check.test.mjs @@ -0,0 +1,454 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; +import { + classify, + cratePackage, + dartDefine, + hunkRanges, + listPlaywright, + nodeTestArgs, + parseDartReport, + parseJunit, + playwrightRepetitions, + playwrightShards, + rustTests, + selectRust, + selectTests, + testBodies, + testCalls, +} from "./flake-check.mjs"; + +const read = (path) => readFileSync(new URL(path, import.meta.url), "utf8"); +// A hunk writing new-side lines start..end; by default it removed nothing. +const hunk = (start, end, removed = { start: start + 1, end: start }) => ({ + start, + end, + removed, +}); +// A -U0 diff of (old start, old count, new start, new count) hunks. +const diff = (...hunks) => + hunks.map(([o, oc, n, nc]) => `@@ -${o},${oc} +${n},${nc} @@`).join("\n"); + +test("classification follows each runner's test file layout", () => { + for (const [file, runner] of [ + ["crates/buzz-core/src/filter.rs", "rust"], + ["crates/buzz-test-client/tests/e2e_relay.rs", "rust"], + ["desktop/src/features/channels/lib/channelDescription.test.mjs", "node"], + ["desktop/src/shared/ui/thing.jsdom-test.mjs", "node"], + ["desktop/scripts/demo-build-config.test.mjs", "node"], + ["desktop/tests/e2e/channels.spec.ts", "playwright"], + ["mobile/test/features/invites/invite_join_provider_test.dart", "dart"], + ]) + assert.equal(classify(file), runner, file); + for (const file of [ + "crates/buzz-core/Cargo.toml", + "crates/buzz-feature-flags/tests/fixtures/relay-feature-selection/src/main.rs", + "desktop/src-tauri/src/lib.rs", + "desktop/src/features/channels/lib/channelDescription.ts", + "desktop/tests/e2e/helpers/animations.ts", + "desktop/tests/e2e/typing-latency.perf.ts", + "desktop/tests/e2e/fixtures/x.spec.ts", + "mobile/lib/main.dart", + "mobile/test/helpers.dart", + "scripts/flake-check.test.mjs", + "web/src/x.test.mjs", + // Selections are shell-split downstream; refuse unusual names. + "desktop/tests/e2e/odd name.spec.ts", + "crates/buzz-core/src/$(id).rs", + ]) + assert.equal(classify(file), undefined, file); +}); + +test("-U0 hunks map to new-side ranges, with deletions attached to the preceding line and their old-side lines kept", () => { + const text = [ + "diff --git a/x.spec.ts b/x.spec.ts", + "--- a/x.spec.ts", + "+++ b/x.spec.ts", + "@@ -1 +1 @@", + '-import { test } from "@playwright/test";', + '+import { expect, test } from "@playwright/test";', + "@@ -12,0 +13,3 @@ test('a', async () => {", + "+ const x = 1;", + "+ const y = 2;", + "+ expect(x).toBe(y - 1);", + "@@ -30,2 +33,0 @@ test('b', async () => {", + "- removed();", + "- alsoRemoved();", + "@@ -1,3 +0,0 @@", + "-gone", + "", + ].join("\n"); + assert.deepEqual(hunkRanges(text), [ + { start: 1, end: 1, removed: { start: 1, end: 1 } }, + { start: 13, end: 15, removed: { start: 12, end: 11 } }, + { start: 33, end: 33, removed: { start: 30, end: 31 } }, + { start: 0, end: 0, removed: { start: 1, end: 3 } }, + ]); + assert.deepEqual(hunkRanges(""), []); +}); + +test("Playwright test calls span the declaration; groups, hooks and configuration are outside", () => { + const spec = [ + 'import { expect, test } from "@playwright/test";', + "test.use({ viewport: { width: 800, height: 600 } });", + "const seed = 1;", + 'test.describe("suite", () => {', + " test.beforeEach(async ({ page }) => {", + ' await page.goto("/");', + " });", + ' test("journey", async ({ page }) => {', + ' test.skip(seed === 2, "not here");', + ' await expect(page.getByText("Go")).toBeVisible();', + " });", + ' for (const mode of ["mouse", "keyboard"]) {', + ' test.fixme(mode + " later", async () => {', + " await Promise.resolve();", + " });", + " }", + "});", + 'test.describe.configure({ mode: "serial" });', + "", + ].join("\n"); + const calls = testCalls(spec, "x.spec.ts"); + assert.deepEqual(calls, [ + { start: 8, end: 11 }, + { start: 13, end: 15 }, + ]); + // Playwright lists a looped declaration once per iteration, at one line. + assert.deepEqual(testBodies(calls, [13, 13, 8]), [ + { start: 8, end: 11, line: 8 }, + { start: 13, end: 15, line: 13 }, + ]); + // A listed line outside every call (a hook) has no body. + assert.deepEqual(testBodies(calls, [5]), []); +}); + +const rustSource = [ + "//! Module docs mentioning #[test] in prose.", + "use std::fmt;", + "", + "pub fn code_under_test(input: &str) -> String {", + " let brace = '{';", + ' let raw = r"\\";', + ' format!("{brace}{raw}{input}")', + "}", + "", + "#[cfg(test)]", + "mod tests {", + " use super::*;", + "", + " fn helper() -> &'static str {", + ' "}"', + " }", + "", + " /// Docs for the first test.", + " #[test]", + " fn plain() {", + ' assert_eq!(code_under_test("x"), "{\\\\x");', + " }", + "", + ' #[tokio::test(flavor = "multi_thread")]', + ' #[ignore = "requires Postgres }"]', + " async fn needs_infra() {", + " /* a block comment with a } brace", + " and #[test] inside */", + " helper();", + " }", + "", + " #[ignore]", + " #[test]", + " pub(crate) fn also_ignored() {}", + "", + " mod postgres_tests {", + " #[sqlx::test]", + " async fn in_nested_module(pool: Pool) -> Result<(), E> {", + " Ok(())", + " }", + " }", + "}", + "", +].join("\n"); + +test("Rust test functions span their attribute block to the closing brace, through strings, chars and comments", () => { + assert.deepEqual(rustTests(rustSource), [ + { name: "plain", start: 19, end: 22, ignored: false }, + { name: "needs_infra", start: 24, end: 30, ignored: true }, + { name: "also_ignored", start: 32, end: 34, ignored: true }, + { name: "in_nested_module", start: 37, end: 40, ignored: false }, + ]); + assert.deepEqual(rustTests("fn main() {}\n"), []); +}); + +test("Rust selection: a hunk inside a test selects it, elsewhere selects every runnable test; ignored tests never run", () => { + const select = (text, options) => + selectRust(text, rustSource, { previous: rustSource, ...options }); + assert.deepEqual(select(diff([21, 1, 21, 1])), { + lines: [19], + names: ["plain"], + }); + // Inside an ignored test: owned, so not shared code, but nothing to run. + assert.deepEqual(select(diff([28, 0, 29, 1])), { lines: [24], names: [] }); + // The code under test, a helper, or a deleted helper (attached to line 13) + // can change every test. + const whole = (reason) => ({ + whole: true, + reason, + names: ["plain", "in_nested_module"], + }); + assert.deepEqual(select(diff([4, 0, 5, 1])), whole("shared code changed")); + assert.deepEqual( + select(diff([21, 1, 21, 1], [14, 0, 15, 1])), + whole("shared code changed"), + ); + assert.deepEqual(select(diff([14, 3, 13, 0])), whole("shared code removed")); + assert.deepEqual( + select(diff([21, 1, 21, 1]), { added: true, previous: "" }), + whole("new file"), + ); + // A file without test functions is not a test file. + assert.equal(selectRust(diff([1, 1, 1, 1]), "fn main() {}\n"), undefined); + assert.equal(cratePackage("crates/buzz-core/src/filter.rs"), "buzz-core"); +}); + +test("hunks inside tests select those tests; shared code, new or unlisted files select the whole file", () => { + const bodies = [ + { start: 10, end: 22, line: 10 }, + { start: 25, end: 37, line: 25 }, + { start: 38, end: 52, line: 40 }, + ]; + // The old file had the same tests, with a hook between the last two. + const previous = [ + { start: 10, end: 22 }, + { start: 25, end: 37 }, + { start: 41, end: 55 }, + ]; + const select = (ranges, options) => + selectTests(ranges, bodies, { previous, ...options }); + // Edit inside one test, including its first and last lines. + assert.deepEqual(select([hunk(12, 14)]), { lines: [10] }); + assert.deepEqual(select([hunk(10, 10)]), { lines: [10] }); + assert.deepEqual(select([hunk(22, 22)]), { lines: [10] }); + // A hunk across two adjacent bodies selects both, by their listed lines. + assert.deepEqual(select([hunk(36, 39)]), { lines: [25, 40] }); + // A hunk that also covers the gap between tests, or lines after the last + // test, touches shared code. + for (const range of [hunk(22, 25), hunk(52, 53), hunk(400, 405)]) + assert.deepEqual(select([range]), { + whole: true, + reason: "shared code changed", + }); + // Several hunks accumulate distinct tests in file order. + assert.deepEqual(select([hunk(41, 41), hunk(11, 11), hunk(12, 12)]), { + lines: [10, 40], + }); + // Replacing lines inside a test, or deleting some of them (attached to the + // preceding line), selects that test when the old lines were its own. + assert.deepEqual(select([hunk(30, 31, { start: 30, end: 32 })]), { + lines: [25], + }); + assert.deepEqual(select([hunk(30, 30, { start: 31, end: 32 })]), { + lines: [25], + }); + // Deleting the hook that followed the second test attaches to that test's + // closing line, but the removed lines were never a test's. + assert.deepEqual(select([hunk(37, 37, { start: 38, end: 40 })]), { + whole: true, + reason: "shared code removed", + }); + // Deletion at the top of the file (line 0) and edits before the first test + // fall back to the whole file. + assert.deepEqual(select([hunk(0, 0, { start: 1, end: 1 })]), { + whole: true, + reason: "shared code removed", + }); + assert.deepEqual(select([hunk(12, 12), hunk(3, 3)]), { + whole: true, + reason: "shared code changed", + }); + assert.deepEqual(select([hunk(12, 12)], { added: true }), { + whole: true, + reason: "new file", + }); + assert.deepEqual(selectTests([hunk(12, 12)], []), { + whole: true, + reason: "no tests listed", + }); + // Without old-side calls, every removal is shared code. + assert.deepEqual( + selectTests([hunk(30, 30, { start: 31, end: 32 })], bodies), + { whole: true, reason: "shared code removed" }, + ); +}); + +test("Playwright lists smoke journeys per spec; integration-only specs list none", () => { + const smoke = "desktop/tests/e2e/channel-sort.spec.ts"; + const integration = "desktop/tests/e2e/agents.spec.ts"; + const listed = listPlaywright([smoke, integration]); + assert.deepEqual(listed.get(integration), []); + const lines = listed.get(smoke); + assert.ok(lines.length > 0, "a smoke spec must list its journeys"); + assert.equal( + testBodies(testCalls(read(`../${smoke}`), smoke), lines).length, + new Set(lines).size, + "every listed journey must be owned by a test call", + ); +}); + +test("Playwright repetitions get one shard per five journeys, at most ten", () => { + assert.deepEqual( + [0, 1, 5, 6, 18, 40, 41, 50, 51, 200].map(playwrightShards), + [1, 1, 1, 2, 4, 8, 9, 10, 10, 10], + ); +}); + +test("reports give per-test statuses per iteration", () => { + assert.deepEqual( + parseJunit( + [ + '<?xml version="1.0" encoding="UTF-8"?>', + '<testsuites name="nextest-run">', + '<testsuite name="buzz-core" tests="3">', + '<testcase name="filter::tests::a" classname="buzz-core" time="0.1"/>', + '<testcase name="filter::tests::b <x>" classname="buzz-core" time="0.1">', + '<failure type="test failure">assertion `left == right` failed</failure>', + "</testcase>", + '<testcase name="filter::tests::c" classname="buzz-core"><skipped/></testcase>', + "</testsuite>", + '<testsuite name="buzz-core::e2e" tests="1">', + '<testcase name="d" classname="buzz-core::e2e" time="0.1"><error/></testcase>', + "</testsuite>", + "</testsuites>", + ].join("\n"), + ), + new Map([ + ["buzz-core filter::tests::a", "passed"], + ["buzz-core filter::tests::b <x>", "failed"], + ["buzz-core filter::tests::c", "skipped"], + ["buzz-core::e2e d", "failed"], + ]), + ); + const event = (value) => JSON.stringify(value); + assert.deepEqual( + parseDartReport( + [ + event({ type: "start", protocolVersion: "0.1.1" }), + event({ + type: "testStart", + test: { id: 1, name: "loading /m/test/a_test.dart", url: null }, + }), + event({ + type: "testDone", + testID: 1, + result: "success", + hidden: true, + skipped: false, + }), + event({ + type: "testStart", + test: { + id: 2, + name: "group renders", + url: "file:///m/test/a_test.dart", + }, + }), + event({ + type: "testStart", + test: { + id: 3, + name: "group fails", + url: "file:///m/test/a_test.dart", + }, + }), + event({ + type: "testStart", + test: { id: 4, name: "later", url: "file:///m/test/b_test.dart" }, + }), + "not json", + event({ + type: "testDone", + testID: 2, + result: "success", + hidden: false, + skipped: false, + }), + event({ + type: "testDone", + testID: 3, + result: "error", + hidden: false, + skipped: false, + }), + event({ + type: "testDone", + testID: 4, + result: "success", + hidden: false, + skipped: true, + }), + event({ type: "done", success: false }), + ].join("\n"), + "/m/", + ), + new Map([ + ["test/a_test.dart › group renders", "passed"], + ["test/a_test.dart › group fails", "failed"], + ["test/b_test.dart › later", "skipped"], + ]), + ); + // Playwright lists a --repeat-each journey once per repetition, in order. + const spec = (title, line, status) => ({ + title, + file: "channels.spec.ts", + line, + tests: [{ status }], + }); + assert.deepEqual( + playwrightRepetitions({ + suites: [ + { + specs: [spec("a", 10, "expected"), spec("a", 10, "unexpected")], + suites: [ + { specs: [spec("b", 30, "skipped"), spec("a", 10, "expected")] }, + ], + }, + ], + }), + new Map([ + ["channels.spec.ts:10 › a", ["passed", "failed", "passed"]], + ["channels.spec.ts:30 › b", ["skipped"]], + ]), + ); +}); + +test("repeat invocations mirror the lanes they stand in for", () => { + // node:test flags come from desktop/package.json "test"; the Flutter define + // from the mobile test shards; nextest's flake profile must keep retries off. + const script = JSON.parse(read("../desktop/package.json")).scripts.test; + for (const file of ["x.test.mjs", "x.jsdom-test.mjs"]) + assert.ok(script.includes(`node ${nodeTestArgs(file).join(" ")} "`), file); + assert.ok( + read("../.github/workflows/_ci-clients.yml").includes( + `scripts/mobile-test-shard.sh \${{ matrix.shard }} 3 ${dartDefine}`, + ), + ); + assert.match( + read("../.config/nextest.toml"), + /^\[profile\.flake\]\n(?:#.*\n)*retries = 0\nfail-fast = false\n\n\[profile\.flake\.junit\]\npath = "junit\.xml"\n/m, + ); + const flake = read("../.github/workflows/_ci-flake.yml"); + for (const runner of ["rust", "node", "dart"]) + assert.ok( + flake.includes(`flake-check.mjs ${runner} --iterations=10 $SELECTED`), + runner, + ); + assert.ok(flake.includes("--repeat-each=10 --retries=0")); + // The lanes are advisory: ci.yml calls the workflow on pull requests only, + // and no required gate depends on it. + const ci = read("../.github/workflows/ci.yml"); + assert.match( + ci, + /^ {2}flake-check:\n(?: {4}.*\n)*? {4}if: github\.event_name == 'pull_request'\n(?: {4}.*\n)*? {4}uses: \.\/\.github\/workflows\/_ci-flake\.yml\n/m, + ); + assert.doesNotMatch(ci, /needs: \[[^\]]*flake/); +});