mirror of
https://github.com/block/buzz.git
synced 2026-10-27 04:30:52 +00:00
f02a4235a9
Address the two High findings of review 5659898932 on #8613: a changed Rust test under desktop/src-tauri, or a changed helper module under a crate's tests/, produced an empty flake selection and every repeat lane skipped, a false "nothing selected" green. Desktop Tauri Rust is now a runner of its own rather than an unclassified path. rustFile places a Rust file in its workspace (the root crates, or desktop/src-tauri and its crates/*), and classify, cratePackage, rustTarget, rustModulePath and rustGroups read that instead of assuming crates/<crate>/. The new flake-tauri lane mirrors the Desktop Tauri core lane's setup (its apt packages, sidecar stubs and desktop-core rust-cache, restored read-only) and runs the same script against desktop/src-tauri/Cargo.toml. nextest reads its configuration from the workspace root, so that workspace gets its own .config/nextest.toml with the flake profile, and the Postgres exclusion applies only in the root workspace: nextest rejects a binary(/^postgres_/) predicate where no such binary exists, which the Tauri workspace's first live run showed. Supporting the surface for real was straightforward, since the workspace is an ordinary cargo workspace that nextest lists and runs by manifest path, so no out-of-scope declaration was needed. A tests/** file without test functions is a helper that integration binaries compile in with `mod common;`. select now finds the package's integration targets that include it (rustIncludes reads `mod <first>;` declarations with comments and strings blanked, `#[path]` attributes, and a tests/<dir>/main.rs owning its directory; a helper no target names falls back to every target) and selects each of those binaries whole, so the ignored and infrastructure exclusions still apply, with a row naming the importers. A helper and its importer changed together select each test once. A src/** file without test functions still selects nothing; a comment beside the row records that this is deliberate, since the regular lanes run what depends on it. Regression tests go through the real select entry point over a synthetic head tree (HEAD with a comment appended, as the reviewer probed): a changed desktop/src-tauri/tests/rodio_mixer_diagnostic.rs yields its three tests under tauri=, and a comment-only change to crates/buzz-agent/tests/common/mod.rs yields rust= entries in exactly anthropic_continuation, fake_llm, permission_boundary and regressions, the four files declaring `mod common;`. Verified live: cargo nextest list with the produced -p/--test flags and filterset matches all 85 entries, and `flake-check.mjs tauri --iterations=2` on the Tauri selection ran green end to end with its JUnit report under desktop/src-tauri/target/nextest. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Signed-off-by: Matt Toohey <contact@matttoohey.com>
327 lines
14 KiB
YAML
327 lines
14 KiB
YAML
name: CI / Flake Check
|
|
on:
|
|
workflow_call:
|
|
|
|
env:
|
|
CARGO_TERM_COLOR: always
|
|
BUZZ_TEST_POSTGRES_PASSWORD: buzz_dev
|
|
PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/.cache/ms-playwright
|
|
|
|
# Advisory flake check: repeat the tests a pull request changes, ten times
|
|
# each with retries off, so a race inside a test surfaces before it lands on
|
|
# main. The lanes skip when no test changed, so they stay out of the required
|
|
# checks in ci.yml (whose gates reject skipped lanes); their own statuses and
|
|
# step summaries carry the result. Selections reach the lanes through env
|
|
# vars, never interpolated into run scripts; flake-check.mjs admits only plain
|
|
# path and identifier characters.
|
|
jobs:
|
|
flake-select:
|
|
name: Flake check selection
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 10
|
|
permissions:
|
|
contents: read
|
|
outputs:
|
|
rust: ${{ steps.select.outputs.rust }}
|
|
tauri: ${{ steps.select.outputs.tauri }}
|
|
node: ${{ steps.select.outputs.node }}
|
|
playwright: ${{ steps.select.outputs.playwright }}
|
|
playwright_shards: ${{ steps.select.outputs.playwright_shards }}
|
|
playwright_shard_total: ${{ steps.select.outputs.playwright_shard_total }}
|
|
dart: ${{ steps.select.outputs.dart }}
|
|
steps:
|
|
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
|
with:
|
|
# The merge commit's first parent is the base branch tip, so this
|
|
# diff is exactly the pull request's change without a full clone.
|
|
fetch-depth: 2
|
|
- uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1
|
|
# Only a changed spec needs desktop dependencies: Playwright lists its
|
|
# smoke journeys and the desktop app's TypeScript parses it. Pull
|
|
# requests without one skip the install.
|
|
- name: Check for changed Playwright specs
|
|
id: specs
|
|
run: |
|
|
if git diff --quiet --diff-filter=AM HEAD^1 HEAD -- 'desktop/tests/e2e/*.spec.ts'; then
|
|
echo "changed=false" >> "$GITHUB_OUTPUT"
|
|
else
|
|
echo "changed=true" >> "$GITHUB_OUTPUT"
|
|
fi
|
|
- name: Get pnpm store directory
|
|
if: steps.specs.outputs.changed == 'true'
|
|
id: pnpm-cache
|
|
run: echo "STORE_PATH=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT"
|
|
- name: Restore pnpm store cache
|
|
if: steps.specs.outputs.changed == 'true'
|
|
uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5
|
|
with:
|
|
path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
|
|
key: pnpm-${{ runner.os }}-${{ hashFiles('**/pnpm-lock.yaml') }}
|
|
restore-keys: pnpm-${{ runner.os }}-
|
|
- name: Install desktop dependencies
|
|
if: steps.specs.outputs.changed == 'true'
|
|
run: just desktop-install-ci
|
|
- name: Map changed hunks to tests
|
|
id: select
|
|
run: node scripts/flake-check.mjs select HEAD^1 HEAD
|
|
|
|
flake-rust:
|
|
name: Flake check (Rust)
|
|
needs: [flake-select]
|
|
if: needs.flake-select.outputs.rust != ''
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 30
|
|
permissions:
|
|
contents: read
|
|
steps:
|
|
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
|
- uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1
|
|
- uses: rui314/setup-mold@339e96b4ff1f3bd865856ba4cbbe8818a7acb038 # v1
|
|
# Restores the Unit Tests lane's cache (_ci-rust.yml) read-only; this
|
|
# lane never saves one of its own. The restore only hits while the cache
|
|
# key matches: the same shared-key, and rust-cache's environment hash —
|
|
# the CARGO*/RUST*/CC* environment variables, the lockfiles and the rustc
|
|
# version — must stay identical to the Unit Tests job's, so any env this
|
|
# job sets must be set there too (scripts/test-rust-cache-contract.sh
|
|
# checks the key).
|
|
- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32
|
|
with:
|
|
shared-key: unit-tests-sherpa-cache-v1
|
|
save-if: false
|
|
- name: Install cargo-nextest
|
|
uses: taiki-e/install-action@0fd46367812ee04360509b4169d9f659d6892bb2 # v2.79.15
|
|
with:
|
|
tool: cargo-nextest@0.9.136
|
|
# nextest has no repeat flag; the script runs the selection once per
|
|
# iteration under the `flake` profile (.config/nextest.toml).
|
|
- name: Repeat changed Rust tests
|
|
env:
|
|
SELECTED: ${{ needs.flake-select.outputs.rust }}
|
|
run: node scripts/flake-check.mjs rust --iterations=10 $SELECTED
|
|
- name: Rust iteration reports
|
|
if: ${{ !cancelled() }}
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: flake-rust
|
|
path: test-results/flake
|
|
if-no-files-found: ignore
|
|
retention-days: 7
|
|
|
|
flake-tauri:
|
|
name: Flake check (Desktop Tauri)
|
|
needs: [flake-select]
|
|
if: needs.flake-select.outputs.tauri != ''
|
|
runs-on: ubuntu-latest
|
|
# Same bound as a Desktop Tauri lane (_ci-desktop.yml): ci-apt-retry.sh
|
|
# bounds the apt step at ~34m worst case and a cold build needs ~15m.
|
|
timeout-minutes: 55
|
|
permissions:
|
|
contents: read
|
|
steps:
|
|
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
|
- uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1
|
|
- uses: rui314/setup-mold@339e96b4ff1f3bd865856ba4cbbe8818a7acb038 # v1
|
|
# Restores the Desktop Tauri core lane's cache (_ci-desktop.yml)
|
|
# read-only, under the same constraints as flake-rust's restore above:
|
|
# the workspace, shared-key and environment hash must stay identical to
|
|
# that job's.
|
|
- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32
|
|
with:
|
|
workspaces: desktop/src-tauri
|
|
shared-key: desktop-core
|
|
save-if: false
|
|
- name: Install Tauri dependencies (Linux)
|
|
env:
|
|
DEBIAN_FRONTEND: noninteractive
|
|
run: |
|
|
CI_APT_ATTEMPT_SECONDS=90 scripts/ci-apt-retry.sh sudo apt-get update
|
|
scripts/ci-apt-retry.sh sudo apt-get install -y --no-install-recommends \
|
|
build-essential \
|
|
curl \
|
|
file \
|
|
libasound2-dev \
|
|
libayatana-appindicator3-dev \
|
|
libgtk-3-dev \
|
|
librsvg2-dev \
|
|
libssl-dev \
|
|
libwebkit2gtk-4.1-dev \
|
|
libxdo-dev \
|
|
patchelf \
|
|
wget
|
|
- name: Install cargo-nextest
|
|
uses: taiki-e/install-action@0fd46367812ee04360509b4169d9f659d6892bb2 # v2.79.15
|
|
with:
|
|
tool: cargo-nextest@0.9.136
|
|
# tauri-build validates the sidecar binaries at compile time; the Desktop
|
|
# Tauri recipes create the same empty stubs.
|
|
- name: Create sidecar stubs
|
|
run: just _ensure-sidecar-stubs
|
|
# Same script as flake-rust, against the Tauri workspace's manifest and
|
|
# its own .config/nextest.toml `flake` profile.
|
|
- name: Repeat changed desktop Tauri tests
|
|
env:
|
|
CMAKE_POLICY_VERSION_MINIMUM: "3.5"
|
|
SELECTED: ${{ needs.flake-select.outputs.tauri }}
|
|
run: node scripts/flake-check.mjs tauri --iterations=10 $SELECTED
|
|
- name: Desktop Tauri iteration reports
|
|
if: ${{ !cancelled() }}
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: flake-tauri
|
|
path: test-results/flake
|
|
if-no-files-found: ignore
|
|
retention-days: 7
|
|
|
|
flake-node:
|
|
name: Flake check (node:test)
|
|
needs: [flake-select]
|
|
if: needs.flake-select.outputs.node != ''
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 15
|
|
permissions:
|
|
contents: read
|
|
steps:
|
|
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
|
- uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1
|
|
- name: Get pnpm store directory
|
|
id: pnpm-cache
|
|
run: echo "STORE_PATH=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT"
|
|
- name: Restore pnpm store cache
|
|
uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5
|
|
with:
|
|
path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
|
|
key: pnpm-${{ runner.os }}-${{ hashFiles('**/pnpm-lock.yaml') }}
|
|
restore-keys: pnpm-${{ runner.os }}-
|
|
- name: Install desktop dependencies
|
|
run: just desktop-install-ci
|
|
- name: Repeat changed node:test files
|
|
env:
|
|
SELECTED: ${{ needs.flake-select.outputs.node }}
|
|
run: node scripts/flake-check.mjs node --iterations=10 $SELECTED
|
|
- name: node:test iteration logs
|
|
if: ${{ !cancelled() }}
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: flake-node
|
|
path: test-results/flake
|
|
if-no-files-found: ignore
|
|
retention-days: 7
|
|
|
|
flake-playwright:
|
|
name: Flake check (Playwright smoke, ${{ matrix.shard }}/${{ needs.flake-select.outputs.playwright_shard_total }})
|
|
needs: [flake-select]
|
|
if: needs.flake-select.outputs.playwright != ''
|
|
runs-on: ubuntu-latest
|
|
# Same bound as a smoke shard: Chromium dependency setup is bounded by
|
|
# ci-apt-retry.sh (~25m worst case), and flake-check.mjs sizes the shards
|
|
# so the repetitions fit the rest.
|
|
timeout-minutes: 45
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
shard: ${{ fromJSON(needs.flake-select.outputs.playwright_shards) }}
|
|
permissions:
|
|
contents: read
|
|
steps:
|
|
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
|
- uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1
|
|
- name: Get pnpm store directory
|
|
id: pnpm-cache
|
|
run: echo "STORE_PATH=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT"
|
|
- name: Restore pnpm store cache
|
|
uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5
|
|
with:
|
|
path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
|
|
key: pnpm-${{ runner.os }}-${{ hashFiles('**/pnpm-lock.yaml') }}
|
|
restore-keys: pnpm-${{ runner.os }}-
|
|
- name: Install desktop dependencies
|
|
run: just desktop-install-ci
|
|
- name: Get Playwright version
|
|
id: pw-version
|
|
run: echo "version=$(cd desktop && node -e "console.log(require('@playwright/test/package.json').version)")" >> "$GITHUB_OUTPUT"
|
|
- name: Restore Playwright browser cache
|
|
id: playwright-cache
|
|
uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5
|
|
with:
|
|
path: ${{ env.PLAYWRIGHT_BROWSERS_PATH }}
|
|
key: playwright-${{ runner.os }}-${{ steps.pw-version.outputs.version }}
|
|
- name: Install Playwright Chromium
|
|
if: steps.playwright-cache.outputs.cache-hit != 'true'
|
|
run: cd desktop && pnpm exec playwright install chromium
|
|
- name: Install Playwright system dependencies
|
|
run: cd desktop && ../scripts/ci-apt-retry.sh pnpm exec playwright install-deps chromium
|
|
- name: Desktop E2E build
|
|
run: pnpm -C desktop build:e2e
|
|
# --retries=0 overrides the config's CI retries, so every failed
|
|
# repetition reports as unexpected and fails the step.
|
|
- name: Repeat changed smoke journeys
|
|
env:
|
|
PLAYWRIGHT_JSON_OUTPUT_FILE: playwright-report.json
|
|
SELECTED: ${{ needs.flake-select.outputs.playwright }}
|
|
SHARD: ${{ matrix.shard }}
|
|
SHARD_TOTAL: ${{ needs.flake-select.outputs.playwright_shard_total }}
|
|
run: cd desktop && pnpm exec playwright test --project=smoke --repeat-each=10 --retries=0 --shard="$SHARD/$SHARD_TOTAL" --reporter=list,json $SELECTED
|
|
- name: Summarize repetitions
|
|
if: ${{ !cancelled() }}
|
|
env:
|
|
SHARD: ${{ matrix.shard }}
|
|
SHARD_TOTAL: ${{ needs.flake-select.outputs.playwright_shard_total }}
|
|
run: node scripts/flake-check.mjs playwright-summary desktop/playwright-report.json "Flake check (Playwright smoke, $SHARD/$SHARD_TOTAL)"
|
|
- name: Playwright repetition artifacts
|
|
if: ${{ !cancelled() }}
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: flake-playwright-${{ matrix.shard }}
|
|
path: |
|
|
desktop/playwright-report.json
|
|
desktop/test-results
|
|
if-no-files-found: ignore
|
|
retention-days: 7
|
|
|
|
flake-dart:
|
|
name: Flake check (Flutter)
|
|
needs: [flake-select]
|
|
if: needs.flake-select.outputs.dart != ''
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 30
|
|
permissions:
|
|
contents: read
|
|
steps:
|
|
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
|
- uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1
|
|
# Restores the Mobile lane's Hermit and pub caches read-only; that lane
|
|
# saves them.
|
|
- name: Compute Hermit cache key
|
|
id: hermit-bin-hash
|
|
run: |
|
|
hash="$(find ./bin ! -type d | sort | xargs openssl sha256 | openssl sha256 -r | cut -d' ' -f1)"
|
|
echo "hash=$hash" >> "$GITHUB_OUTPUT"
|
|
- name: Restore Hermit package cache
|
|
uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5
|
|
with:
|
|
path: ~/.cache/hermit/pkg
|
|
key: ${{ runner.os }}-hermit-cache-${{ steps.hermit-bin-hash.outputs.hash }}
|
|
restore-keys: ${{ runner.os }}-hermit-cache-
|
|
- name: Prime Flutter SDK
|
|
run: flutter --version
|
|
- name: Restore pub cache
|
|
uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5
|
|
with:
|
|
path: ~/.pub-cache
|
|
key: pub-${{ runner.os }}-${{ hashFiles('mobile/pubspec.lock') }}
|
|
restore-keys: pub-${{ runner.os }}-
|
|
- name: Install dependencies
|
|
run: cd mobile && flutter pub get
|
|
- name: Repeat changed Flutter tests
|
|
env:
|
|
SELECTED: ${{ needs.flake-select.outputs.dart }}
|
|
run: node scripts/flake-check.mjs dart --iterations=10 $SELECTED
|
|
- name: Flutter iteration reports
|
|
if: ${{ !cancelled() }}
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: flake-dart
|
|
path: test-results/flake
|
|
if-no-files-found: ignore
|
|
retention-days: 7
|