diff --git a/.github/workflows/_ci-flake.yml b/.github/workflows/_ci-flake.yml index 2ab3542e4b..94d98c82c9 100644 --- a/.github/workflows/_ci-flake.yml +++ b/.github/workflows/_ci-flake.yml @@ -157,7 +157,10 @@ jobs: - name: Create sidecar stubs run: just _ensure-sidecar-stubs # Same script as flake-rust, against the Tauri workspace's manifest and - # its own .config/nextest.toml `flake` profile. + # its own .config/nextest.toml `flake` profile. A selected test the + # default graph does not list is looked for under mesh-llm, whose + # dependency graph the restored desktop-core cache does not hold, so + # only such a selection builds it cold. - name: Repeat changed desktop Tauri tests env: CMAKE_POLICY_VERSION_MINIMUM: "3.5" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b0c73556f4..9c155a5d17 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -322,6 +322,11 @@ are deliberately not repeated: `#[ignore]`d and Postgres-backed Rust tests, Playwright specs outside the `smoke` project, `*.perf.ts` measurements, and a selection too large for its lane (over 64 KiB of entries, or more Playwright journeys than ten repetitions can finish within the lane's timeout). +A Rust test repeats in the first feature graph the regular lanes test its +package in that lists it (LaunchDarkly, the App Attest development feature, +the desktop crate's `mesh-llm`); a selected test that no such graph lists on +the Linux runner, such as a macOS-only one, fails its lane by name rather +than passing unrepeated. The lanes skip when no test changed and are not required to merge; their step summaries show the selection and the test × iteration table. diff --git a/scripts/flake-check.mjs b/scripts/flake-check.mjs index c9ef8a52cb..aa26b0c4d3 100644 --- a/scripts/flake-check.mjs +++ b/scripts/flake-check.mjs @@ -94,26 +94,26 @@ export const nodeTestArgs = (file) => // Mirrors the mobile test shards in .github/workflows/_ci-clients.yml. export const dartDefine = "--dart-define=BUZZ_PUSH_GATEWAY_URL=https://push.example"; -// Mirrors the postgres-ci default-filter in .config/nextest.toml: those tests -// are #[ignore]d and need Postgres, Redis, S3 or MinIO, so a selection that -// reached one would fail every repetition without being flaky. The convention -// is the root workspace's; nextest rejects a `binary()` predicate matching no -// test binary in the workspace, and the Tauri workspace has none, so it gets -// no exclusion. -const infraTests = { - rust: "test(/postgres_tests::/) | binary(/^postgres_/) | test(/(^|::)external_infra[^:]*::/)", -}; +// Mirrors the postgres-ci default-filter in .config/nextest.toml +// (`test(/postgres_tests::/)`, `binary(/^postgres_/)` and +// `test(/(^|::)external_infra[^:]*::/)`): those tests need Postgres, Redis, +// S3 or MinIO, so a selection that reached one would fail every repetition +// without being flaky. Only the root workspace has such tests. +export const infraTest = (binaryName, name) => + /postgres_tests::/.test(name) || + /^postgres_/.test(binaryName) || + /(^|::)external_infra[^:]*::/.test(name); -// The nextest filterset listing a workspace's candidates for the selected -// in-file test paths, less its infrastructure-backed tests. -export function rustFilterset(runner, paths) { - const suffixes = [...new Set(paths)] - .map((path) => `test(/(^|::)${path}$/)`) - .join(" | "); - return infraTests[runner] - ? `(${suffixes}) & !(${infraTests[runner]})` - : `(${suffixes})`; -} +// The feature graphs the regular lanes test a package in, in order, where +// that is not just its default graph: the Justfile's test-unit runs +// buzz-push-gateway without and with its feature and buzz-feature-flags only +// with LaunchDarkly, and desktop-tauri-test-mesh runs the desktop crate under +// mesh-llm. A test behind `#[cfg(feature = …)]` is listed only in its graph. +export const rustFeatureGraphs = { + "buzz-feature-flags": [["launchdarkly"]], + "buzz-push-gateway": [[], ["personal-dev-app-attest"]], + "buzz-desktop": [[], ["mesh-llm"]], +}; // Mirrors desktop/package.json "test", desktop/playwright.config.ts testDir, // the two cargo workspaces' packages and mobile's test directory. Paths are @@ -875,26 +875,30 @@ export function parseJunit(xml) { // The listed tests an entry stands for: a test the file's module path predicts // exactly when nextest lists one, otherwise every test whose name ends in the // in-file path (a `#[path]` module breaks the prediction; a namesake in another -// module is then repeated too rather than nothing). +// module is then repeated too rather than nothing). `unlisted` keeps the +// entries no candidate stands for. export function chooseRust(tests, candidates) { const chosen = new Map(); - for (const { path, exact } of tests) { + const unlisted = []; + for (const test of tests) { + const { path, exact } = test; const suffix = candidates.filter( ({ name }) => name === path || name.endsWith(`::${path}`), ); const predicted = suffix.filter(({ name }) => name === exact); + if (suffix.length === 0) unlisted.push(test); for (const test of predicted.length ? predicted : suffix) chosen.set(`${test.binary} ${test.name}`, test); } - return [...chosen.values()]; + return { chosen: [...chosen.values()], unlisted }; } // The repeat's cargo invocation per package: `-p ` plus the target // flags of its selected files, each whole flag once (`--test codec --test // connection`; merging tokens instead gave `--test codec connection`, which -// nextest reads as a test-name filter that matches nothing), and the tests -// to list in those targets. Every entry must belong to one workspace, whose -// runner is returned with the groups. +// nextest reads as a test-name filter that matches nothing), the feature +// graphs to list them in, and the tests to list in those targets. Every entry +// must belong to one workspace, whose runner is returned with the groups. export function rustGroups(entries) { const groups = new Map(); const workspaces = new Set(); @@ -910,6 +914,7 @@ export function rustGroups(entries) { if (targets.length === 0) group.every = true; for (const target of targets) group.targets.set(target.join(" "), target); group.tests.push({ + entry, path: test, exact: [...rustModulePath(file), test].join("::"), }); @@ -919,66 +924,104 @@ export function rustGroups(entries) { throw new Error( `rust entries span workspaces: ${[...workspaces].join(", ")}`, ); - for (const [crate, group] of groups) + for (const [crate, group] of groups) { group.args = [ "-p", crate, ...(group.every ? [] : [...group.targets.values()].flat()), ]; + group.graphs = rustFeatureGraphs[crate] ?? [[]]; + } return { groups, runner: [...workspaces][0] }; } // nextest has no repeat flag, so run the selection once per iteration under // the `flake` profile (the workspace's .config/nextest.toml: no retries, JUnit -// per run), one cargo invocation per package so each builds only the targets -// its files can be in. The selection is listed first, by in-file path, and -// narrowed with chooseRust to exact (binary, test) pairs; a selection listing -// no runnable test is one of ignored or infrastructure-backed tests, not a -// failure. The Tauri workspace is addressed by its manifest; the root one is -// cargo's default. +// per run), one cargo invocation per package and feature graph so each builds +// only the targets its files can be in. Each package's targets are listed +// whole in its feature graphs in turn (nextest reports an ignored test as a +// filter mismatch whatever `-E` says, so a filtered listing could not tell an +// ignored entry from an unlisted one), and every entry is narrowed with +// chooseRust to exact (binary, test) pairs in the first graph listing it, so +// a later graph (and its build) is reached only by tests behind its feature. +// An entry standing only for ignored or infrastructure-backed tests is +// excluded; one no graph lists (a test behind a cfg the regular lanes do not +// build here) fails the lane rather than passing unrepeated. The Tauri +// workspace is addressed by its manifest; the root one is cargo's default. function repeatRust(entries, iterations) { const { groups, runner } = rustGroups(entries); + const title = + runner === "rust" ? "Flake check (Rust)" : "Flake check (Desktop Tauri)"; const manifest = rustWorkspaces[runner]; const cargoArgs = runner === "rust" ? [] : ["--manifest-path", manifest]; - const runs = []; - for (const group of groups.values()) { - const listed = run("cargo", [ - "nextest", - "list", - ...cargoArgs, - ...group.args, - "-E", - rustFilterset( - runner, - group.tests.map(({ path }) => path), - ), - "--message-format", - "json", - ]); - if (listed.status !== 0) - throw new Error(`cargo nextest list: ${listed.stderr}`); - const candidates = Object.entries( - JSON.parse(listed.stdout)["rust-suites"], - ).flatMap(([binary, suite]) => - Object.entries(suite.testcases) - .filter( - ([, test]) => - !test.ignored && test["filter-match"].status === "matches", - ) - .map(([name]) => ({ binary, name })), - ); - group.chosen = chooseRust(group.tests, candidates); - group.filterset = group.chosen - .map(({ binary, name }) => `(binary_id(=${binary}) & test(=${name}))`) - .join(" | "); + const invocations = []; + const unlisted = []; + for (const [crate, group] of groups) { + let tests = group.tests; + for (const features of group.graphs) { + if (tests.length === 0) break; + const args = [ + ...group.args, + ...(features.length ? ["--features", features.join(",")] : []), + ]; + const listed = run("cargo", [ + "nextest", + "list", + ...cargoArgs, + ...args, + "--message-format", + "json", + ]); + if (listed.status !== 0) + throw new Error(`cargo nextest list: ${listed.stderr}`); + const candidates = Object.entries( + JSON.parse(listed.stdout)["rust-suites"], + ).flatMap(([binary, suite]) => + Object.entries(suite.testcases).map(([name, test]) => ({ + binary, + name, + runnable: !test.ignored && !infraTest(suite["binary-name"], name), + })), + ); + const choice = chooseRust(tests, candidates); + const chosen = choice.chosen.filter((test) => test.runnable); + if (chosen.length) + invocations.push({ + label: [crate, ...features].join("+"), + args, + chosen, + filterset: chosen + .map( + ({ binary, name }) => `(binary_id(=${binary}) & test(=${name}))`, + ) + .join(" | "), + }); + tests = choice.unlisted; + } + unlisted.push(...tests.map((test) => test.entry)); } - const expected = [...groups.values()].flatMap((group) => - group.chosen.map(({ binary, name }) => `${binary} ${name}`), + if (unlisted.length) { + summarize( + [ + `## ${title}: ${unlisted.length} selected test(s) not listed`, + "", + "No feature graph the regular lanes test these packages in lists them on this runner, so they were not repeated:", + "", + ...unlisted.map((entry) => `- \`${entry}\``), + "", + "", + ].join("\n"), + ); + process.exitCode = 1; + } + const expected = invocations.flatMap((invocation) => + invocation.chosen.map(({ binary, name }) => `${binary} ${name}`), ); if (expected.length === 0) { - summarize( - "## Flake check (Rust): nothing to repeat\n\nEvery selected test is ignored or infrastructure-backed.\n\n", - ); + if (!unlisted.length) + summarize( + `## ${title}: nothing to repeat\n\nEvery selected test is ignored or infrastructure-backed.\n\n`, + ); return; } // nextest keeps its store under the workspace root's target/nextest, not @@ -989,11 +1032,11 @@ function repeatRust(entries, iterations) { "target/nextest/flake/junit.xml", ); const dir = resultsDir(runner); + const runs = []; for (let i = 1; i <= iterations; i += 1) { const statuses = new Map(); let ok = true; - for (const [crate, group] of groups) { - if (group.chosen.length === 0) continue; + for (const { label, args, filterset } of invocations) { rmSync(junit, { force: true }); const result = run( "cargo", @@ -1003,9 +1046,9 @@ function repeatRust(entries, iterations) { ...cargoArgs, "--profile", "flake", - ...group.args, + ...args, "-E", - group.filterset, + filterset, ], { stdio: "inherit" }, ); @@ -1013,7 +1056,7 @@ function repeatRust(entries, iterations) { try { for (const [test, status] of parseJunit(readFileSync(junit, "utf8"))) statuses.set(test, status); - copyFileSync(junit, `${dir}/iter-${i}-${crate}.xml`); + copyFileSync(junit, `${dir}/iter-${i}-${label}.xml`); } catch { ok = false; } @@ -1022,10 +1065,8 @@ function repeatRust(entries, iterations) { runs.push({ ok, statuses }); progress(i, iterations, ok); } - report( - runner === "rust" ? "Flake check (Rust)" : "Flake check (Desktop Tauri)", - runs, - ); + report(title, runs); + if (unlisted.length) process.exitCode = 1; } // node:test has no repeat flag either. Each file runs alone, so a failure is diff --git a/scripts/flake-check.test.mjs b/scripts/flake-check.test.mjs index 4f5f226c32..bbc56cf785 100644 --- a/scripts/flake-check.test.mjs +++ b/scripts/flake-check.test.mjs @@ -1,6 +1,14 @@ import assert from "node:assert/strict"; import { execFileSync, spawnSync } from "node:child_process"; -import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { + copyFileSync, + mkdirSync, + mkdtempSync, + readFileSync, + realpathSync, + rmSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import test from "node:test"; @@ -12,6 +20,7 @@ import { dartDefine, hunkRanges, ITERATIONS, + infraTest, listPlaywright, MAX_PLAYWRIGHT_JOURNEYS, nodeTestArgs, @@ -20,7 +29,7 @@ import { playwrightRepetitions, playwrightShards, runners, - rustFilterset, + rustFeatureGraphs, rustGroups, rustImporters, rustIncludedTests, @@ -335,17 +344,23 @@ test("same-named Rust tests in different modules are distinct, and a hunk select [{ path: "tests::same", exact: "handlers::auth::tests::same" }], candidates, ), - [candidates[1], candidates[3]], + { chosen: [candidates[1], candidates[3]], unlisted: [] }, ); + // An entry no candidate stands for is kept apart, not dropped. + const gated = { path: "tests::gated", exact: "handlers::auth::tests::gated" }; assert.deepEqual( chooseRust( [ { path: "tests::same", exact: "elsewhere::tests::same" }, { path: "tests::other", exact: "handlers::auth::tests::other" }, + gated, ], candidates, ), - [candidates[0], candidates[1], candidates[3], candidates[2]], + { + chosen: [candidates[0], candidates[1], candidates[3], candidates[2]], + unlisted: [gated], + }, ); }); @@ -428,16 +443,28 @@ test("Rust files build only the targets they can be part of, merged per package () => rustGroups(["desktop/src/x.rs::tests::a"]), /rust entry must be/, ); - // The root workspace's listing excludes its Postgres-backed tests by name - // and binary; nextest rejects `binary(/^postgres_/)` in the Tauri - // workspace, where no such binary exists, so that listing carries no - // exclusion. - assert.equal( - rustFilterset("rust", ["tests::a", "b", "tests::a"]), - "(test(/(^|::)tests::a$/) | test(/(^|::)b$/)) & !(test(/postgres_tests::/) | binary(/^postgres_/) | test(/(^|::)external_infra[^:]*::/))", + // Packages the regular lanes test under a feature list in those graphs, in + // order; the rest in their default graph alone. + assert.deepEqual( + [ + "crates/buzz-feature-flags/src/launchdarkly.rs::tests::a", + "crates/buzz-push-gateway/src/lib.rs::tests::b", + "crates/buzz-core/src/filter.rs::tests::c", + ].map((entry) => [...rustGroups([entry]).groups.values()][0].graphs), + [[["launchdarkly"]], [[], ["personal-dev-app-attest"]], [[]]], ); - assert.equal(rustFilterset("tauri", ["a"]), "(test(/(^|::)a$/))"); - assert.doesNotMatch(rustFilterset("tauri", ["a"]), /binary\(/); + assert.deepEqual( + [ + ...rustGroups(["desktop/src-tauri/src/lib.rs::tests::d"]).groups.values(), + ][0].graphs, + [[], ["mesh-llm"]], + ); + // Postgres-backed tests are excluded by name and binary, as the postgres-ci + // default-filter selects them. + assert.ok(infraTest("buzz_db", "event_store::postgres_tests::insert")); + assert.ok(infraTest("postgres_relay", "admits")); + assert.ok(infraTest("buzz_media", "tests::external_infra_s3::upload")); + assert.ok(!infraTest("buzz_db", "tests::postgres_free")); }); test("a helper under tests/ selects the integration binaries that include it", () => { @@ -990,6 +1017,43 @@ test("repeat invocations mirror the lanes they stand in for", () => { /^\[profile\.flake\]\n(?:#.*\n)*retries = 0\nfail-fast = false\n\n\[profile\.flake\.junit\]\npath = "junit\.xml"\n/m, config, ); + // The infrastructure exclusion mirrors the postgres-ci default-filter. + for (const predicate of [ + "test(/postgres_tests::/)", + "binary(/^postgres_/)", + "test(/(^|::)external_infra[^:]*::/)", + ]) + assert.ok(read("../.config/nextest.toml").includes(predicate), predicate); + // Feature graphs mirror the Justfile recipes the regular lanes run: every + // package test-unit runs under a feature, in each graph it runs there, and + // the desktop crate's default and mesh-llm lanes. + const justfile = read("../Justfile"); + const graphs = {}; + for (const [, crate, features] of justfile.matchAll( + /^ +cargo nextest run -p ([\w-]+)(?: --features ([\w,-]+))?$/gm, + )) { + graphs[crate] ??= []; + graphs[crate].push(features?.split(",") ?? []); + } + assert.ok( + justfile.includes( + "\ndesktop-tauri-test: _ensure-sidecar-stubs\n cd desktop/src-tauri && cargo test --workspace\n", + ), + ); + assert.ok( + justfile.includes( + "\ndesktop-tauri-test-mesh: _ensure-sidecar-stubs\n cargo test --manifest-path {{desktop_tauri_manifest}} --features mesh-llm --lib\n", + ), + ); + graphs["buzz-desktop"] = [[], ["mesh-llm"]]; + assert.deepEqual( + rustFeatureGraphs, + Object.fromEntries( + Object.entries(graphs).filter(([, list]) => + list.some((features) => features.length), + ), + ), + ); const flake = read("../.github/workflows/_ci-flake.yml"); for (const runner of ["rust", "tauri", "node", "dart"]) assert.ok( @@ -1022,3 +1086,235 @@ test("repeat invocations mirror the lanes they stand in for", () => { ); assert.doesNotMatch(ci, /needs: \[[^\]]*flake/); }); + +// The repeat, through its entry point, in a scratch tree holding the script +// and the packages its entries name, with a fake `cargo` on PATH listing +// `tests` ({pkg, binary, binaryName, name, feature?, ignored?}; a feature +// test only under that feature) and passing every test a run selects. +function repeatWithFakeCargo(runner, entries, tests) { + const dir = realpathSync(mkdtempSync(join(tmpdir(), "flake-repeat-"))); + try { + mkdirSync(join(dir, "scripts")); + copyFileSync( + join(root, "scripts/flake-check.mjs"), + join(dir, "scripts/flake-check.mjs"), + ); + for (const [crateDir, name] of [ + ["crates/buzz-feature-flags", "buzz-feature-flags"], + ["desktop/src-tauri", "buzz-desktop"], + ]) { + mkdirSync(join(dir, crateDir, "src"), { recursive: true }); + writeFileSync( + join(dir, crateDir, "Cargo.toml"), + `[package]\nname = "${name}"\n`, + ); + writeFileSync(join(dir, crateDir, "src/lib.rs"), ""); + } + mkdirSync(join(dir, "bin")); + writeFileSync( + join(dir, "fake-cargo.mjs"), + `import { appendFileSync, mkdirSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +const args = process.argv.slice(2); +appendFileSync(process.env.FAKE_CARGO_LOG, JSON.stringify(args) + "\\n"); +const value = (flag) => args[args.indexOf(flag) + 1]; +const features = args.includes("--features") ? value("--features").split(",") : []; +const tests = JSON.parse(process.env.FAKE_CARGO_TESTS).filter( + (test) => test.pkg === value("-p") && (!test.feature || features.includes(test.feature)), +); +if (args[1] === "list") { + const suites = {}; + for (const test of tests) { + suites[test.binary] ??= { "binary-name": test.binaryName, testcases: {} }; + suites[test.binary].testcases[test.name] = { + ignored: Boolean(test.ignored), + "filter-match": test.ignored ? { status: "mismatch", reason: "ignored" } : { status: "matches" }, + }; + } + process.stdout.write(JSON.stringify({ "rust-suites": suites })); +} else { + const junit = join( + args.includes("--manifest-path") ? dirname(value("--manifest-path")) : ".", + "target/nextest/flake/junit.xml", + ); + mkdirSync(dirname(junit), { recursive: true }); + const cases = [...value("-E").matchAll(/binary_id\\(=([^)]+)\\) & test\\(=([^)]+)\\)/g)]; + writeFileSync(junit, "" + cases.map(([, binary, name]) => '').join("") + ""); +} +`, + ); + writeFileSync( + join(dir, "bin/cargo"), + `#!/bin/sh\nexec "${process.execPath}" "${join(dir, "fake-cargo.mjs")}" "$@"\n`, + { mode: 0o755 }, + ); + const { GITHUB_STEP_SUMMARY, GITHUB_OUTPUT, ...inherited } = process.env; + const log = join(dir, "cargo.log"); + const summary = join(dir, "summary.md"); + writeFileSync(summary, ""); + const result = spawnSync( + process.execPath, + [ + join(dir, "scripts/flake-check.mjs"), + runner, + "--iterations=2", + ...entries, + ], + { + cwd: dir, + encoding: "utf8", + env: { + ...inherited, + PATH: `${join(dir, "bin")}:${process.env.PATH}`, + GITHUB_STEP_SUMMARY: summary, + FAKE_CARGO_LOG: log, + FAKE_CARGO_TESTS: JSON.stringify(tests), + }, + }, + ); + assert.equal(result.signal, null, result.stderr); + return { + status: result.status, + summary: readFileSync(summary, "utf8"), + calls: readFileSync(log, "utf8") + .trim() + .split("\n") + .map((line) => JSON.parse(line)), + }; + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +test("feature-gated Rust tests repeat in the graph the regular lanes test them in, and unlisted ones fail the lane", () => { + // Reviewer probe on #8613: the repeat listed only the default graph, so a + // selected `#[cfg(feature = …)]` test was dropped and the lane passed as + // "nothing to repeat", or reported only its ungated neighbours. + const features = (call) => + call.includes("--features") ? call[call.indexOf("--features") + 1] : ""; + const flags = { + pkg: "buzz-feature-flags", + binary: "buzz-feature-flags", + binaryName: "buzz_feature_flags", + }; + const root = repeatWithFakeCargo( + "rust", + [ + "crates/buzz-feature-flags/src/lib.rs::tests::plain", + "crates/buzz-feature-flags/src/launchdarkly.rs::tests::null_events", + ], + [ + { ...flags, name: "tests::plain" }, + { + ...flags, + name: "launchdarkly::tests::null_events", + feature: "launchdarkly", + }, + ], + ); + assert.equal(root.status, 0, root.summary); + // buzz-feature-flags is only tested with LaunchDarkly, so it is listed and + // run in that graph alone. + assert.deepEqual( + root.calls.map((call) => [call[1], features(call)]), + [ + ["list", "launchdarkly"], + ["run", "launchdarkly"], + ["run", "launchdarkly"], + ], + ); + assert.match( + root.summary, + /## Flake check \(Rust\): 0 of 2 iterations failed/, + ); + assert.match( + root.summary, + /\| buzz-feature-flags launchdarkly::tests::null_events \| ✅ \| ✅ \| 0\/2 \|/, + ); + assert.match( + root.summary, + /\| buzz-feature-flags tests::plain \| ✅ \| ✅ \| 0\/2 \|/, + ); + + // The desktop crate is tested in its default graph, then under mesh-llm: + // only tests the default graph does not list reach the mesh one, a mesh + // test repeats there, and one no graph lists fails the lane by name while + // the rest still repeat. An ignored test is excluded, not unlisted. + const desktop = { + pkg: "buzz-desktop", + binary: "buzz-desktop", + binaryName: "buzz_lib", + }; + const tauriTests = [ + { ...desktop, name: "tests::plain" }, + { ...desktop, name: "tests::skipped", ignored: true }, + { + ...desktop, + name: "managed_agents::relay_mesh::tests::mesh_only", + feature: "mesh-llm", + }, + ]; + const tauri = repeatWithFakeCargo( + "tauri", + [ + "desktop/src-tauri/src/lib.rs::tests::plain", + "desktop/src-tauri/src/lib.rs::tests::skipped", + "desktop/src-tauri/src/managed_agents/relay_mesh.rs::tests::mesh_only", + "desktop/src-tauri/src/lib.rs::tests::macos_only", + ], + tauriTests, + ); + assert.equal(tauri.status, 1, tauri.summary); + assert.deepEqual( + tauri.calls.map((call) => [call[1], features(call)]), + [ + ["list", ""], + ["list", "mesh-llm"], + ["run", ""], + ["run", "mesh-llm"], + ["run", ""], + ["run", "mesh-llm"], + ], + ); + assert.match( + tauri.summary, + /## Flake check \(Desktop Tauri\): 1 selected test\(s\) not listed\n[^#]*- `desktop\/src-tauri\/src\/lib\.rs::tests::macos_only`\n/, + ); + assert.doesNotMatch(tauri.summary, /skipped`/); + assert.match( + tauri.summary, + /## Flake check \(Desktop Tauri\): 0 of 2 iterations failed/, + ); + assert.match( + tauri.summary, + /\| buzz-desktop managed_agents::relay_mesh::tests::mesh_only \| ✅ \| ✅ \| 0\/2 \|/, + ); + assert.match( + tauri.summary, + /\| buzz-desktop tests::plain \| ✅ \| ✅ \| 0\/2 \|/, + ); + + // A selection of only ignored tests is excluded, and passes as such; one + // of only unlisted tests fails without running anything. + const ignored = repeatWithFakeCargo( + "tauri", + ["desktop/src-tauri/src/lib.rs::tests::skipped"], + tauriTests, + ); + assert.equal(ignored.status, 0, ignored.summary); + assert.match( + ignored.summary, + /## Flake check \(Desktop Tauri\): nothing to repeat/, + ); + const unlisted = repeatWithFakeCargo( + "tauri", + ["desktop/src-tauri/src/lib.rs::tests::macos_only"], + tauriTests, + ); + assert.equal(unlisted.status, 1, unlisted.summary); + assert.deepEqual( + unlisted.calls.map((call) => call[1]), + ["list", "list"], + ); + assert.doesNotMatch(unlisted.summary, /nothing to repeat/); +});