mirror of
https://github.com/block/buzz.git
synced 2026-11-05 08:51:10 +00:00
280cc23854
The Rust and Desktop Tauri repeats listed only each package's default feature graph and treated an entry nextest did not list as excluded, so a selected `#[cfg(feature = …)]` test (buzz-feature-flags' LaunchDarkly, buzz-push-gateway's App Attest development feature, the desktop crate's mesh-llm) passed as "nothing to repeat" or was silently dropped from a mixed selection. Each package is now listed in the feature graphs the regular lanes test it in, in order, and each entry repeats in the first graph that lists it, so the mesh-llm graph is built only for tests the default graph lacks. Ignored and infrastructure-backed tests are classified from the whole listing (nextest reports an ignored test as a filter mismatch whatever -E says), and an entry no graph lists fails the lane by name. A fake-cargo regression drives the script's entry point through each case, and the graph table is pinned to the Justfile recipes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Matt Toohey <contact@matttoohey.com>
330 lines
14 KiB
YAML
330 lines
14 KiB
YAML
name: CI / Flake Check
|
|
on:
|
|
workflow_call:
|
|
|
|
env:
|
|
CARGO_TERM_COLOR: always
|
|
BUZZ_TEST_POSTGRES_PASSWORD: buzz_dev
|
|
PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/.cache/ms-playwright
|
|
|
|
# Advisory flake check: repeat the tests a pull request changes, ten times
|
|
# each with retries off, so a race inside a test surfaces before it lands on
|
|
# main. The lanes skip when no test changed, so they stay out of the required
|
|
# checks in ci.yml (whose gates reject skipped lanes); their own statuses and
|
|
# step summaries carry the result. Selections reach the lanes through env
|
|
# vars, never interpolated into run scripts; flake-check.mjs admits only plain
|
|
# path and identifier characters.
|
|
jobs:
|
|
flake-select:
|
|
name: Flake check selection
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 10
|
|
permissions:
|
|
contents: read
|
|
outputs:
|
|
rust: ${{ steps.select.outputs.rust }}
|
|
tauri: ${{ steps.select.outputs.tauri }}
|
|
node: ${{ steps.select.outputs.node }}
|
|
playwright: ${{ steps.select.outputs.playwright }}
|
|
playwright_shards: ${{ steps.select.outputs.playwright_shards }}
|
|
playwright_shard_total: ${{ steps.select.outputs.playwright_shard_total }}
|
|
dart: ${{ steps.select.outputs.dart }}
|
|
steps:
|
|
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
|
with:
|
|
# The merge commit's first parent is the base branch tip, so this
|
|
# diff is exactly the pull request's change without a full clone.
|
|
fetch-depth: 2
|
|
- uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1
|
|
# Only a changed spec needs desktop dependencies: Playwright lists its
|
|
# smoke journeys and the desktop app's TypeScript parses it. Pull
|
|
# requests without one skip the install.
|
|
- name: Check for changed Playwright specs
|
|
id: specs
|
|
run: |
|
|
if git diff --quiet --diff-filter=AM HEAD^1 HEAD -- 'desktop/tests/e2e/*.spec.ts'; then
|
|
echo "changed=false" >> "$GITHUB_OUTPUT"
|
|
else
|
|
echo "changed=true" >> "$GITHUB_OUTPUT"
|
|
fi
|
|
- name: Get pnpm store directory
|
|
if: steps.specs.outputs.changed == 'true'
|
|
id: pnpm-cache
|
|
run: echo "STORE_PATH=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT"
|
|
- name: Restore pnpm store cache
|
|
if: steps.specs.outputs.changed == 'true'
|
|
uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5
|
|
with:
|
|
path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
|
|
key: pnpm-${{ runner.os }}-${{ hashFiles('**/pnpm-lock.yaml') }}
|
|
restore-keys: pnpm-${{ runner.os }}-
|
|
- name: Install desktop dependencies
|
|
if: steps.specs.outputs.changed == 'true'
|
|
run: just desktop-install-ci
|
|
- name: Map changed hunks to tests
|
|
id: select
|
|
run: node scripts/flake-check.mjs select HEAD^1 HEAD
|
|
|
|
flake-rust:
|
|
name: Flake check (Rust)
|
|
needs: [flake-select]
|
|
if: needs.flake-select.outputs.rust != ''
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 30
|
|
permissions:
|
|
contents: read
|
|
steps:
|
|
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
|
- uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1
|
|
- uses: rui314/setup-mold@339e96b4ff1f3bd865856ba4cbbe8818a7acb038 # v1
|
|
# Restores the Unit Tests lane's cache (_ci-rust.yml) read-only; this
|
|
# lane never saves one of its own. The restore only hits while the cache
|
|
# key matches: the same shared-key, and rust-cache's environment hash —
|
|
# the CARGO*/RUST*/CC* environment variables, the lockfiles and the rustc
|
|
# version — must stay identical to the Unit Tests job's, so any env this
|
|
# job sets must be set there too (scripts/test-rust-cache-contract.sh
|
|
# checks the key).
|
|
- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32
|
|
with:
|
|
shared-key: unit-tests-sherpa-cache-v1
|
|
save-if: false
|
|
- name: Install cargo-nextest
|
|
uses: taiki-e/install-action@0fd46367812ee04360509b4169d9f659d6892bb2 # v2.79.15
|
|
with:
|
|
tool: cargo-nextest@0.9.136
|
|
# nextest has no repeat flag; the script runs the selection once per
|
|
# iteration under the `flake` profile (.config/nextest.toml).
|
|
- name: Repeat changed Rust tests
|
|
env:
|
|
SELECTED: ${{ needs.flake-select.outputs.rust }}
|
|
run: node scripts/flake-check.mjs rust --iterations=10 $SELECTED
|
|
- name: Rust iteration reports
|
|
if: ${{ !cancelled() }}
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: flake-rust
|
|
path: test-results/flake
|
|
if-no-files-found: ignore
|
|
retention-days: 7
|
|
|
|
flake-tauri:
|
|
name: Flake check (Desktop Tauri)
|
|
needs: [flake-select]
|
|
if: needs.flake-select.outputs.tauri != ''
|
|
runs-on: ubuntu-latest
|
|
# Same bound as a Desktop Tauri lane (_ci-desktop.yml): ci-apt-retry.sh
|
|
# bounds the apt step at ~34m worst case and a cold build needs ~15m.
|
|
timeout-minutes: 55
|
|
permissions:
|
|
contents: read
|
|
steps:
|
|
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
|
- uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1
|
|
- uses: rui314/setup-mold@339e96b4ff1f3bd865856ba4cbbe8818a7acb038 # v1
|
|
# Restores the Desktop Tauri core lane's cache (_ci-desktop.yml)
|
|
# read-only, under the same constraints as flake-rust's restore above:
|
|
# the workspace, shared-key and environment hash must stay identical to
|
|
# that job's.
|
|
- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32
|
|
with:
|
|
workspaces: desktop/src-tauri
|
|
shared-key: desktop-core
|
|
save-if: false
|
|
- name: Install Tauri dependencies (Linux)
|
|
env:
|
|
DEBIAN_FRONTEND: noninteractive
|
|
run: |
|
|
CI_APT_ATTEMPT_SECONDS=90 scripts/ci-apt-retry.sh sudo apt-get update
|
|
scripts/ci-apt-retry.sh sudo apt-get install -y --no-install-recommends \
|
|
build-essential \
|
|
curl \
|
|
file \
|
|
libasound2-dev \
|
|
libayatana-appindicator3-dev \
|
|
libgtk-3-dev \
|
|
librsvg2-dev \
|
|
libssl-dev \
|
|
libwebkit2gtk-4.1-dev \
|
|
libxdo-dev \
|
|
patchelf \
|
|
wget
|
|
- name: Install cargo-nextest
|
|
uses: taiki-e/install-action@0fd46367812ee04360509b4169d9f659d6892bb2 # v2.79.15
|
|
with:
|
|
tool: cargo-nextest@0.9.136
|
|
# tauri-build validates the sidecar binaries at compile time; the Desktop
|
|
# Tauri recipes create the same empty stubs.
|
|
- name: Create sidecar stubs
|
|
run: just _ensure-sidecar-stubs
|
|
# Same script as flake-rust, against the Tauri workspace's manifest and
|
|
# its own .config/nextest.toml `flake` profile. A selected test the
|
|
# default graph does not list is looked for under mesh-llm, whose
|
|
# dependency graph the restored desktop-core cache does not hold, so
|
|
# only such a selection builds it cold.
|
|
- name: Repeat changed desktop Tauri tests
|
|
env:
|
|
CMAKE_POLICY_VERSION_MINIMUM: "3.5"
|
|
SELECTED: ${{ needs.flake-select.outputs.tauri }}
|
|
run: node scripts/flake-check.mjs tauri --iterations=10 $SELECTED
|
|
- name: Desktop Tauri iteration reports
|
|
if: ${{ !cancelled() }}
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: flake-tauri
|
|
path: test-results/flake
|
|
if-no-files-found: ignore
|
|
retention-days: 7
|
|
|
|
flake-node:
|
|
name: Flake check (node:test)
|
|
needs: [flake-select]
|
|
if: needs.flake-select.outputs.node != ''
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 15
|
|
permissions:
|
|
contents: read
|
|
steps:
|
|
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
|
- uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1
|
|
- name: Get pnpm store directory
|
|
id: pnpm-cache
|
|
run: echo "STORE_PATH=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT"
|
|
- name: Restore pnpm store cache
|
|
uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5
|
|
with:
|
|
path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
|
|
key: pnpm-${{ runner.os }}-${{ hashFiles('**/pnpm-lock.yaml') }}
|
|
restore-keys: pnpm-${{ runner.os }}-
|
|
- name: Install desktop dependencies
|
|
run: just desktop-install-ci
|
|
- name: Repeat changed node:test files
|
|
env:
|
|
SELECTED: ${{ needs.flake-select.outputs.node }}
|
|
run: node scripts/flake-check.mjs node --iterations=10 $SELECTED
|
|
- name: node:test iteration logs
|
|
if: ${{ !cancelled() }}
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: flake-node
|
|
path: test-results/flake
|
|
if-no-files-found: ignore
|
|
retention-days: 7
|
|
|
|
flake-playwright:
|
|
name: Flake check (Playwright smoke, ${{ matrix.shard }}/${{ needs.flake-select.outputs.playwright_shard_total }})
|
|
needs: [flake-select]
|
|
if: needs.flake-select.outputs.playwright != ''
|
|
runs-on: ubuntu-latest
|
|
# Same bound as a smoke shard: Chromium dependency setup is bounded by
|
|
# ci-apt-retry.sh (~25m worst case), and flake-check.mjs sizes the shards
|
|
# so the repetitions fit the rest.
|
|
timeout-minutes: 45
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
shard: ${{ fromJSON(needs.flake-select.outputs.playwright_shards) }}
|
|
permissions:
|
|
contents: read
|
|
steps:
|
|
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
|
- uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1
|
|
- name: Get pnpm store directory
|
|
id: pnpm-cache
|
|
run: echo "STORE_PATH=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT"
|
|
- name: Restore pnpm store cache
|
|
uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5
|
|
with:
|
|
path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
|
|
key: pnpm-${{ runner.os }}-${{ hashFiles('**/pnpm-lock.yaml') }}
|
|
restore-keys: pnpm-${{ runner.os }}-
|
|
- name: Install desktop dependencies
|
|
run: just desktop-install-ci
|
|
- name: Get Playwright version
|
|
id: pw-version
|
|
run: echo "version=$(cd desktop && node -e "console.log(require('@playwright/test/package.json').version)")" >> "$GITHUB_OUTPUT"
|
|
- name: Restore Playwright browser cache
|
|
id: playwright-cache
|
|
uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5
|
|
with:
|
|
path: ${{ env.PLAYWRIGHT_BROWSERS_PATH }}
|
|
key: playwright-${{ runner.os }}-${{ steps.pw-version.outputs.version }}
|
|
- name: Install Playwright Chromium
|
|
if: steps.playwright-cache.outputs.cache-hit != 'true'
|
|
run: cd desktop && pnpm exec playwright install chromium
|
|
- name: Install Playwright system dependencies
|
|
run: cd desktop && ../scripts/ci-apt-retry.sh pnpm exec playwright install-deps chromium
|
|
- name: Desktop E2E build
|
|
run: pnpm -C desktop build:e2e
|
|
# --retries=0 overrides the config's CI retries, so every failed
|
|
# repetition reports as unexpected and fails the step.
|
|
- name: Repeat changed smoke journeys
|
|
env:
|
|
PLAYWRIGHT_JSON_OUTPUT_FILE: playwright-report.json
|
|
SELECTED: ${{ needs.flake-select.outputs.playwright }}
|
|
SHARD: ${{ matrix.shard }}
|
|
SHARD_TOTAL: ${{ needs.flake-select.outputs.playwright_shard_total }}
|
|
run: cd desktop && pnpm exec playwright test --project=smoke --repeat-each=10 --retries=0 --shard="$SHARD/$SHARD_TOTAL" --reporter=list,json $SELECTED
|
|
- name: Summarize repetitions
|
|
if: ${{ !cancelled() }}
|
|
env:
|
|
SHARD: ${{ matrix.shard }}
|
|
SHARD_TOTAL: ${{ needs.flake-select.outputs.playwright_shard_total }}
|
|
run: node scripts/flake-check.mjs playwright-summary desktop/playwright-report.json "Flake check (Playwright smoke, $SHARD/$SHARD_TOTAL)"
|
|
- name: Playwright repetition artifacts
|
|
if: ${{ !cancelled() }}
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: flake-playwright-${{ matrix.shard }}
|
|
path: |
|
|
desktop/playwright-report.json
|
|
desktop/test-results
|
|
if-no-files-found: ignore
|
|
retention-days: 7
|
|
|
|
flake-dart:
|
|
name: Flake check (Flutter)
|
|
needs: [flake-select]
|
|
if: needs.flake-select.outputs.dart != ''
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 30
|
|
permissions:
|
|
contents: read
|
|
steps:
|
|
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
|
- uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1
|
|
# Restores the Mobile lane's Hermit and pub caches read-only; that lane
|
|
# saves them.
|
|
- name: Compute Hermit cache key
|
|
id: hermit-bin-hash
|
|
run: |
|
|
hash="$(find ./bin ! -type d | sort | xargs openssl sha256 | openssl sha256 -r | cut -d' ' -f1)"
|
|
echo "hash=$hash" >> "$GITHUB_OUTPUT"
|
|
- name: Restore Hermit package cache
|
|
uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5
|
|
with:
|
|
path: ~/.cache/hermit/pkg
|
|
key: ${{ runner.os }}-hermit-cache-${{ steps.hermit-bin-hash.outputs.hash }}
|
|
restore-keys: ${{ runner.os }}-hermit-cache-
|
|
- name: Prime Flutter SDK
|
|
run: flutter --version
|
|
- name: Restore pub cache
|
|
uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5
|
|
with:
|
|
path: ~/.pub-cache
|
|
key: pub-${{ runner.os }}-${{ hashFiles('mobile/pubspec.lock') }}
|
|
restore-keys: pub-${{ runner.os }}-
|
|
- name: Install dependencies
|
|
run: cd mobile && flutter pub get
|
|
- name: Repeat changed Flutter tests
|
|
env:
|
|
SELECTED: ${{ needs.flake-select.outputs.dart }}
|
|
run: node scripts/flake-check.mjs dart --iterations=10 $SELECTED
|
|
- name: Flutter iteration reports
|
|
if: ${{ !cancelled() }}
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: flake-dart
|
|
path: test-results/flake
|
|
if-no-files-found: ignore
|
|
retention-days: 7
|