ci: repeat changed tests ten times on pull requests

Add an advisory flake check that selects the tests a pull request
changes and repeats each one ten times with retries off, so a test
that passes CI by luck is caught before it merges. This mirrors
block/buzz-app#890 (and its sharding follow-up), adapted to the
runners this repository actually has.

scripts/flake-check.mjs `select` diffs HEAD^1..HEAD and maps hunks to
tests per runner: Rust files to `<package>::<fn>` entries by parsing
`#[test]`-style attributes (shared code changes select the file's
tests; `#[ignore]`d and Postgres tests are excluded), desktop
Playwright specs to `file:line` smoke journeys via `playwright --list`
(non-smoke and unregistered specs are excluded), and desktop node:test
and Flutter test files whole. Deleted and renamed-away files drop out
of the selection, and any runner whose selection exceeds 64 KiB is
dropped with a note so GitHub outputs stay within limits.

The reusable _ci-flake.yml adds `flake-select` plus one repeat lane per
runner (`flake-rust`, `flake-node`, `flake-playwright` sharded by
journey count, `flake-dart`). Each lane skips when its selection is
empty, runs only on pull_request, and is not read by any required
gate; its status and step summary carry the result. A `flake` nextest
profile disables retries and emits JUnit so the Rust lane can report
per-test, per-iteration results. The Unit Tests rust-cache gains a
`shared-key` so the flake lane can restore it read-only.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Signed-off-by: Matt Toohey <contact@matttoohey.com>

.github/workflows/_ci-desktop.yml

@@ -52,6 +52,8 @@ jobs:
         run: just desktop-check
       - name: Desktop unit tests
         run: just desktop-test
+      - name: Flake check selection tests
+        run: node --test scripts/flake-check.test.mjs
       - name: Desktop build
         run: just desktop-build
       - name: Save pnpm store cache

.github/workflows/_ci-flake.yml

@@ -0,0 +1,244 @@
+name: CI / Flake Check
+on:
+  workflow_call:
+
+env:
+  CARGO_TERM_COLOR: always
+  BUZZ_TEST_POSTGRES_PASSWORD: buzz_dev
+  PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/.cache/ms-playwright
+
+# Advisory flake check: repeat the tests a pull request changes, ten times
+# each with retries off, so a race inside a test surfaces before it lands on
+# main. The lanes skip when no test changed, so they stay out of the required
+# checks in ci.yml (whose gates reject skipped lanes); their own statuses and
+# step summaries carry the result. Selections reach the lanes through env
+# vars, never interpolated into run scripts; flake-check.mjs admits only plain
+# path and identifier characters.
+jobs:
+  flake-select:
+    name: Flake check selection
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+    outputs:
+      rust: ${{ steps.select.outputs.rust }}
+      node: ${{ steps.select.outputs.node }}
+      playwright: ${{ steps.select.outputs.playwright }}
+      playwright_shards: ${{ steps.select.outputs.playwright_shards }}
+      playwright_shard_total: ${{ steps.select.outputs.playwright_shard_total }}
+      dart: ${{ steps.select.outputs.dart }}
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with:
+          # The merge commit's first parent is the base branch tip, so this
+          # diff is exactly the pull request's change without a full clone.
+          fetch-depth: 2
+      - uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        run: echo "STORE_PATH=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT"
+      - name: Restore pnpm store cache
+        uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5
+        with:
+          path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
+          key: pnpm-${{ runner.os }}-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: pnpm-${{ runner.os }}-
+      # Playwright lists the smoke journeys of a changed spec, and the desktop
+      # app's TypeScript parses it.
+      - name: Install desktop dependencies
+        run: just desktop-install-ci
+      - name: Map changed hunks to tests
+        id: select
+        run: node scripts/flake-check.mjs select HEAD^1 HEAD
+
+  flake-rust:
+    name: Flake check (Rust)
+    needs: [flake-select]
+    if: needs.flake-select.outputs.rust != ''
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+      - uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1
+      - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1
+      # Restores the Unit Tests lane's cache read-only: this lane only runs on
+      # pull requests, so it never saves one of its own.
+      - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32
+        with:
+          shared-key: unit-tests
+          key: sherpa-cache-v1
+          save-if: ${{ github.event_name != 'pull_request' }}
+      - name: Install cargo-nextest
+        uses: taiki-e/install-action@0fd46367812ee04360509b4169d9f659d6892bb2 # v2.79.15
+        with:
+          tool: cargo-nextest@0.9.136
+      # nextest has no repeat flag; the script runs the selection once per
+      # iteration under the `flake` profile (.config/nextest.toml).
+      - name: Repeat changed Rust tests
+        env:
+          SELECTED: ${{ needs.flake-select.outputs.rust }}
+        run: node scripts/flake-check.mjs rust --iterations=10 $SELECTED
+      - name: Rust iteration reports
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: flake-rust
+          path: test-results/flake
+          if-no-files-found: ignore
+          retention-days: 7
+
+  flake-node:
+    name: Flake check (node:test)
+    needs: [flake-select]
+    if: needs.flake-select.outputs.node != ''
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+      - uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        run: echo "STORE_PATH=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT"
+      - name: Restore pnpm store cache
+        uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5
+        with:
+          path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
+          key: pnpm-${{ runner.os }}-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: pnpm-${{ runner.os }}-
+      - name: Install desktop dependencies
+        run: just desktop-install-ci
+      - name: Repeat changed node:test files
+        env:
+          SELECTED: ${{ needs.flake-select.outputs.node }}
+        run: node scripts/flake-check.mjs node --iterations=10 $SELECTED
+      - name: node:test iteration logs
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: flake-node
+          path: test-results/flake
+          if-no-files-found: ignore
+          retention-days: 7
+
+  flake-playwright:
+    name: Flake check (Playwright smoke, ${{ matrix.shard }}/${{ needs.flake-select.outputs.playwright_shard_total }})
+    needs: [flake-select]
+    if: needs.flake-select.outputs.playwright != ''
+    runs-on: ubuntu-latest
+    # Same bound as a smoke shard: Chromium dependency setup is bounded by
+    # ci-apt-retry.sh (~25m worst case), and flake-check.mjs sizes the shards
+    # so the repetitions fit the rest.
+    timeout-minutes: 45
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: ${{ fromJSON(needs.flake-select.outputs.playwright_shards) }}
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+      - uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        run: echo "STORE_PATH=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT"
+      - name: Restore pnpm store cache
+        uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5
+        with:
+          path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
+          key: pnpm-${{ runner.os }}-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: pnpm-${{ runner.os }}-
+      - name: Install desktop dependencies
+        run: just desktop-install-ci
+      - name: Get Playwright version
+        id: pw-version
+        run: echo "version=$(cd desktop && node -e "console.log(require('@playwright/test/package.json').version)")" >> "$GITHUB_OUTPUT"
+      - name: Restore Playwright browser cache
+        id: playwright-cache
+        uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5
+        with:
+          path: ${{ env.PLAYWRIGHT_BROWSERS_PATH }}
+          key: playwright-${{ runner.os }}-${{ steps.pw-version.outputs.version }}
+      - name: Install Playwright Chromium
+        if: steps.playwright-cache.outputs.cache-hit != 'true'
+        run: cd desktop && pnpm exec playwright install chromium
+      - name: Install Playwright system dependencies
+        run: cd desktop && ../scripts/ci-apt-retry.sh pnpm exec playwright install-deps chromium
+      - name: Desktop E2E build
+        run: pnpm -C desktop build:e2e
+      # --retries=0 overrides the config's CI retries, so every failed
+      # repetition reports as unexpected and fails the step.
+      - name: Repeat changed smoke journeys
+        env:
+          PLAYWRIGHT_JSON_OUTPUT_FILE: playwright-report.json
+          SELECTED: ${{ needs.flake-select.outputs.playwright }}
+          SHARD: ${{ matrix.shard }}
+          SHARD_TOTAL: ${{ needs.flake-select.outputs.playwright_shard_total }}
+        run: cd desktop && pnpm exec playwright test --project=smoke --repeat-each=10 --retries=0 --shard="$SHARD/$SHARD_TOTAL" --reporter=list,json $SELECTED
+      - name: Summarize repetitions
+        if: ${{ !cancelled() }}
+        env:
+          SHARD: ${{ matrix.shard }}
+          SHARD_TOTAL: ${{ needs.flake-select.outputs.playwright_shard_total }}
+        run: node scripts/flake-check.mjs playwright-summary desktop/playwright-report.json "Flake check (Playwright smoke, $SHARD/$SHARD_TOTAL)"
+      - name: Playwright repetition artifacts
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: flake-playwright-${{ matrix.shard }}
+          path: |
+            desktop/playwright-report.json
+            desktop/test-results
+          if-no-files-found: ignore
+          retention-days: 7
+
+  flake-dart:
+    name: Flake check (Flutter)
+    needs: [flake-select]
+    if: needs.flake-select.outputs.dart != ''
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+      - uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1
+      # Restores the Mobile lane's Hermit and pub caches read-only; that lane
+      # saves them.
+      - name: Compute Hermit cache key
+        id: hermit-bin-hash
+        run: |
+          hash="$(find ./bin ! -type d | sort | xargs openssl sha256 | openssl sha256 -r | cut -d' ' -f1)"
+          echo "hash=$hash" >> "$GITHUB_OUTPUT"
+      - name: Restore Hermit package cache
+        uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5
+        with:
+          path: ~/.cache/hermit/pkg
+          key: ${{ runner.os }}-hermit-cache-${{ steps.hermit-bin-hash.outputs.hash }}
+          restore-keys: ${{ runner.os }}-hermit-cache-
+      - name: Prime Flutter SDK
+        run: flutter --version
+      - name: Restore pub cache
+        uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5
+        with:
+          path: ~/.pub-cache
+          key: pub-${{ runner.os }}-${{ hashFiles('mobile/pubspec.lock') }}
+          restore-keys: pub-${{ runner.os }}-
+      - name: Install dependencies
+        run: cd mobile && flutter pub get
+      - name: Repeat changed Flutter tests
+        env:
+          SELECTED: ${{ needs.flake-select.outputs.dart }}
+        run: node scripts/flake-check.mjs dart --iterations=10 $SELECTED
+      - name: Flutter iteration reports
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: flake-dart
+          path: test-results/flake
+          if-no-files-found: ignore
+          retention-days: 7

.github/workflows/_ci-rust.yml

@@ -58,6 +58,8 @@ jobs:
       - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1
       - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32
         with:
+          # Named so the advisory flake lane (_ci-flake.yml) can restore it.
+          shared-key: unit-tests
           key: sherpa-cache-v1
           save-if: ${{ github.event_name != 'pull_request' }}
       - name: Install cargo-nextest

.github/workflows/ci.yml

@@ -83,6 +83,9 @@ jobs:
               - 'scripts/normative-corpus.json'
               - 'desktop/**'
               - 'pnpm-lock.yaml'
+              - 'scripts/flake-check.mjs'
+              - 'scripts/flake-check.test.mjs'
+              - '.github/workflows/_ci-flake.yml'
             desktop-rust:
               - 'desktop/src-tauri/**'
             web:
@@ -259,6 +262,14 @@ jobs:
       mobile: ${{ needs.changes.outputs.mobile == 'true' }}
       lane: mobile-swift
 
+  # Advisory flake check: repeats the tests a pull request changes. Its lanes
+  # skip when no test changed and none of the required gates below reads them;
+  # their own statuses and step summaries carry the result.
+  flake-check:
+    name: Flake Check
+    if: github.event_name == 'pull_request'
+    uses: ./.github/workflows/_ci-flake.yml
+
   security-domain:
     name: Security Domain
     needs: [changes]