jerry/buzz
1
0
Fork 0
mirror of https://github.com/block/buzz.git synced 2026-10-28 13:01:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ci): repeat feature-gated Rust tests in their CI feature graph and fail on unlisted selections

Browse Source 
The Rust and Desktop Tauri repeats listed only each package's default
feature graph and treated an entry nextest did not list as excluded, so a
selected `#[cfg(feature = …)]` test (buzz-feature-flags' LaunchDarkly,
buzz-push-gateway's App Attest development feature, the desktop crate's
mesh-llm) passed as "nothing to repeat" or was silently dropped from a
mixed selection.

Each package is now listed in the feature graphs the regular lanes test
it in, in order, and each entry repeats in the first graph that lists
it, so the mesh-llm graph is built only for tests the default graph
lacks. Ignored and infrastructure-backed tests are classified from the
whole listing (nextest reports an ignored test as a filter mismatch
whatever -E says), and an entry no graph lists fails the lane by name.
A fake-cargo regression drives the script's entry point through each
case, and the graph table is pinned to the Justfile recipes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Matt Toohey <contact@matttoohey.com>
This commit is contained in:
Matt Toohey
2026-10-28 22:24:57 +11:00
parent 29858b2aff
commit 280cc23854
4 changed files with 436 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
.github/workflows/_ci-flake.yml vendored
View File

@@ -157,7 +157,10 @@ jobs:
- name: Create sidecar stubs
run: just _ensure-sidecar-stubs
# Same script as flake-rust, against the Tauri workspace's manifest and
# its own .config/nextest.toml `flake` profile.
# its own .config/nextest.toml `flake` profile. A selected test the
# default graph does not list is looked for under mesh-llm, whose
# dependency graph the restored desktop-core cache does not hold, so
# only such a selection builds it cold.
- name: Repeat changed desktop Tauri tests
env:
CMAKE_POLICY_VERSION_MINIMUM: "3.5"

5
CONTRIBUTING.md
View File

@@ -322,6 +322,11 @@ are deliberately not repeated: `#[ignore]`d and Postgres-backed Rust tests,
Playwright specs outside the `smoke` project, `*.perf.ts` measurements, and a
selection too large for its lane (over 64 KiB of entries, or more Playwright
journeys than ten repetitions can finish within the lane's timeout).
A Rust test repeats in the first feature graph the regular lanes test its
package in that lists it (LaunchDarkly, the App Attest development feature,
the desktop crate's `mesh-llm`); a selected test that no such graph lists on
the Linux runner, such as a macOS-only one, fails its lane by name rather
than passing unrepeated.
The lanes skip when no test changed and are not required to merge; their
step summaries show the selection and the test × iteration table.

195
scripts/flake-check.mjs
View File

@@ -94,26 +94,26 @@ export const nodeTestArgs = (file) =>
// Mirrors the mobile test shards in .github/workflows/_ci-clients.yml.
export const dartDefine =
"--dart-define=BUZZ_PUSH_GATEWAY_URL=https://push.example";
// Mirrors the postgres-ci default-filter in .config/nextest.toml: those tests
// are #[ignore]d and need Postgres, Redis, S3 or MinIO, so a selection that
// reached one would fail every repetition without being flaky. The convention
// is the root workspace's; nextest rejects a `binary()` predicate matching no
// test binary in the workspace, and the Tauri workspace has none, so it gets
// no exclusion.
const infraTests = {
rust: "test(/postgres_tests::/) | binary(/^postgres_/) | test(/(^|::)external_infra[^:]*::/)",
};
// Mirrors the postgres-ci default-filter in .config/nextest.toml
// (`test(/postgres_tests::/)`, `binary(/^postgres_/)` and
// `test(/(^|::)external_infra[^:]*::/)`): those tests need Postgres, Redis,
// S3 or MinIO, so a selection that reached one would fail every repetition
// without being flaky. Only the root workspace has such tests.
export const infraTest = (binaryName, name) =>
/postgres_tests::/.test(name) ||
/^postgres_/.test(binaryName) ||
/(^|::)external_infra[^:]*::/.test(name);
// The nextest filterset listing a workspace's candidates for the selected
// in-file test paths, less its infrastructure-backed tests.
export function rustFilterset(runner, paths) {
const suffixes = [...new Set(paths)]
.map((path) => `test(/(^|::)${path}$/)`)
.join(" | ");
return infraTests[runner]
? `(${suffixes}) & !(${infraTests[runner]})`
: `(${suffixes})`;
}
// The feature graphs the regular lanes test a package in, in order, where
// that is not just its default graph: the Justfile's test-unit runs
// buzz-push-gateway without and with its feature and buzz-feature-flags only
// with LaunchDarkly, and desktop-tauri-test-mesh runs the desktop crate under
// mesh-llm. A test behind `#[cfg(feature = …)]` is listed only in its graph.
export const rustFeatureGraphs = {
"buzz-feature-flags": [["launchdarkly"]],
"buzz-push-gateway": [[], ["personal-dev-app-attest"]],
"buzz-desktop": [[], ["mesh-llm"]],
};
// Mirrors desktop/package.json "test", desktop/playwright.config.ts testDir,
// the two cargo workspaces' packages and mobile's test directory. Paths are
@@ -875,26 +875,30 @@ export function parseJunit(xml) {
// The listed tests an entry stands for: a test the file's module path predicts
// exactly when nextest lists one, otherwise every test whose name ends in the
// in-file path (a `#[path]` module breaks the prediction; a namesake in another
// module is then repeated too rather than nothing).
// module is then repeated too rather than nothing). `unlisted` keeps the
// entries no candidate stands for.
export function chooseRust(tests, candidates) {
const chosen = new Map();
for (const { path, exact } of tests) {
const unlisted = [];
for (const test of tests) {
const { path, exact } = test;
const suffix = candidates.filter(
({ name }) => name === path || name.endsWith(`::${path}`),
);
const predicted = suffix.filter(({ name }) => name === exact);
if (suffix.length === 0) unlisted.push(test);
for (const test of predicted.length ? predicted : suffix)
chosen.set(`${test.binary} ${test.name}`, test);
}
return [...chosen.values()];
return { chosen: [...chosen.values()], unlisted };
}
// The repeat's cargo invocation per package: `-p <crate>` plus the target
// flags of its selected files, each whole flag once (`--test codec --test
// connection`; merging tokens instead gave `--test codec connection`, which
// nextest reads as a test-name filter that matches nothing), and the tests
// to list in those targets. Every entry must belong to one workspace, whose
// runner is returned with the groups.
// nextest reads as a test-name filter that matches nothing), the feature
// graphs to list them in, and the tests to list in those targets. Every entry
// must belong to one workspace, whose runner is returned with the groups.
export function rustGroups(entries) {
const groups = new Map();
const workspaces = new Set();
@@ -910,6 +914,7 @@ export function rustGroups(entries) {
if (targets.length === 0) group.every = true;
for (const target of targets) group.targets.set(target.join(" "), target);
group.tests.push({
entry,
path: test,
exact: [...rustModulePath(file), test].join("::"),
});
@@ -919,66 +924,104 @@ export function rustGroups(entries) {
throw new Error(
`rust entries span workspaces: ${[...workspaces].join(", ")}`,
);
for (const [crate, group] of groups)
for (const [crate, group] of groups) {
group.args = [
"-p",
crate,
...(group.every ? [] : [...group.targets.values()].flat()),
];
group.graphs = rustFeatureGraphs[crate] ?? [[]];
}
return { groups, runner: [...workspaces][0] };
}
// nextest has no repeat flag, so run the selection once per iteration under
// the `flake` profile (the workspace's .config/nextest.toml: no retries, JUnit
// per run), one cargo invocation per package so each builds only the targets
// its files can be in. The selection is listed first, by in-file path, and
// narrowed with chooseRust to exact (binary, test) pairs; a selection listing
// no runnable test is one of ignored or infrastructure-backed tests, not a
// failure. The Tauri workspace is addressed by its manifest; the root one is
// cargo's default.
// per run), one cargo invocation per package and feature graph so each builds
// only the targets its files can be in. Each package's targets are listed
// whole in its feature graphs in turn (nextest reports an ignored test as a
// filter mismatch whatever `-E` says, so a filtered listing could not tell an
// ignored entry from an unlisted one), and every entry is narrowed with
// chooseRust to exact (binary, test) pairs in the first graph listing it, so
// a later graph (and its build) is reached only by tests behind its feature.
// An entry standing only for ignored or infrastructure-backed tests is
// excluded; one no graph lists (a test behind a cfg the regular lanes do not
// build here) fails the lane rather than passing unrepeated. The Tauri
// workspace is addressed by its manifest; the root one is cargo's default.
function repeatRust(entries, iterations) {
const { groups, runner } = rustGroups(entries);
const title =
runner === "rust" ? "Flake check (Rust)" : "Flake check (Desktop Tauri)";
const manifest = rustWorkspaces[runner];
const cargoArgs = runner === "rust" ? [] : ["--manifest-path", manifest];
const runs = [];
for (const group of groups.values()) {
const listed = run("cargo", [
"nextest",
"list",
...cargoArgs,
...group.args,
"-E",
rustFilterset(
runner,
group.tests.map(({ path }) => path),
),
"--message-format",
"json",
]);
if (listed.status !== 0)
throw new Error(`cargo nextest list: ${listed.stderr}`);
const candidates = Object.entries(
JSON.parse(listed.stdout)["rust-suites"],
).flatMap(([binary, suite]) =>
Object.entries(suite.testcases)
.filter(
([, test]) =>
!test.ignored && test["filter-match"].status === "matches",
)
.map(([name]) => ({ binary, name })),
);
group.chosen = chooseRust(group.tests, candidates);
group.filterset = group.chosen
.map(({ binary, name }) => `(binary_id(=${binary}) & test(=${name}))`)
.join(" | ");
const invocations = [];
const unlisted = [];
for (const [crate, group] of groups) {
let tests = group.tests;
for (const features of group.graphs) {
if (tests.length === 0) break;
const args = [
...group.args,
...(features.length ? ["--features", features.join(",")] : []),
];
const listed = run("cargo", [
"nextest",
"list",
...cargoArgs,
...args,
"--message-format",
"json",
]);
if (listed.status !== 0)
throw new Error(`cargo nextest list: ${listed.stderr}`);
const candidates = Object.entries(
JSON.parse(listed.stdout)["rust-suites"],
).flatMap(([binary, suite]) =>
Object.entries(suite.testcases).map(([name, test]) => ({
binary,
name,
runnable: !test.ignored && !infraTest(suite["binary-name"], name),
})),
);
const choice = chooseRust(tests, candidates);
const chosen = choice.chosen.filter((test) => test.runnable);
if (chosen.length)
invocations.push({
label: [crate, ...features].join("+"),
args,
chosen,
filterset: chosen
.map(
({ binary, name }) => `(binary_id(=${binary}) & test(=${name}))`,
)
.join(" | "),
});
tests = choice.unlisted;
}
unlisted.push(...tests.map((test) => test.entry));
}
const expected = [...groups.values()].flatMap((group) =>
group.chosen.map(({ binary, name }) => `${binary} ${name}`),
if (unlisted.length) {
summarize(
[
`## ${title}: ${unlisted.length} selected test(s) not listed`,
"",
"No feature graph the regular lanes test these packages in lists them on this runner, so they were not repeated:",
"",
...unlisted.map((entry) => `- \`${entry}\``),
"",
"",
].join("\n"),
);
process.exitCode = 1;
}
const expected = invocations.flatMap((invocation) =>
invocation.chosen.map(({ binary, name }) => `${binary} ${name}`),
);
if (expected.length === 0) {
summarize(
"## Flake check (Rust): nothing to repeat\n\nEvery selected test is ignored or infrastructure-backed.\n\n",
);
if (!unlisted.length)
summarize(
`## ${title}: nothing to repeat\n\nEvery selected test is ignored or infrastructure-backed.\n\n`,
);
return;
}
// nextest keeps its store under the workspace root's target/nextest, not
@@ -989,11 +1032,11 @@ function repeatRust(entries, iterations) {
"target/nextest/flake/junit.xml",
);
const dir = resultsDir(runner);
const runs = [];
for (let i = 1; i <= iterations; i += 1) {
const statuses = new Map();
let ok = true;
for (const [crate, group] of groups) {
if (group.chosen.length === 0) continue;
for (const { label, args, filterset } of invocations) {
rmSync(junit, { force: true });
const result = run(
"cargo",
@@ -1003,9 +1046,9 @@ function repeatRust(entries, iterations) {
...cargoArgs,
"--profile",
"flake",
...group.args,
...args,
"-E",
group.filterset,
filterset,
],
{ stdio: "inherit" },
);
@@ -1013,7 +1056,7 @@ function repeatRust(entries, iterations) {
try {
for (const [test, status] of parseJunit(readFileSync(junit, "utf8")))
statuses.set(test, status);
copyFileSync(junit, `${dir}/iter-${i}-${crate}.xml`);
copyFileSync(junit, `${dir}/iter-${i}-${label}.xml`);
} catch {
ok = false;
}
@@ -1022,10 +1065,8 @@ function repeatRust(entries, iterations) {
runs.push({ ok, statuses });
progress(i, iterations, ok);
}
report(
runner === "rust" ? "Flake check (Rust)" : "Flake check (Desktop Tauri)",
runs,
);
report(title, runs);
if (unlisted.length) process.exitCode = 1;
}
// node:test has no repeat flag either. Each file runs alone, so a failure is

322
scripts/flake-check.test.mjs
View File

@@ -1,6 +1,14 @@
import assert from "node:assert/strict";
import { execFileSync, spawnSync } from "node:child_process";
import { mkdtempSync, readFileSync, rmSync } from "node:fs";
import {
copyFileSync,
mkdirSync,
mkdtempSync,
readFileSync,
realpathSync,
rmSync,
writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import test from "node:test";
@@ -12,6 +20,7 @@ import {
dartDefine,
hunkRanges,
ITERATIONS,
infraTest,
listPlaywright,
MAX_PLAYWRIGHT_JOURNEYS,
nodeTestArgs,
@@ -20,7 +29,7 @@ import {
playwrightRepetitions,
playwrightShards,
runners,
rustFilterset,
rustFeatureGraphs,
rustGroups,
rustImporters,
rustIncludedTests,
@@ -335,17 +344,23 @@ test("same-named Rust tests in different modules are distinct, and a hunk select
[{ path: "tests::same", exact: "handlers::auth::tests::same" }],
candidates,
),
[candidates[1], candidates[3]],
{ chosen: [candidates[1], candidates[3]], unlisted: [] },
);
// An entry no candidate stands for is kept apart, not dropped.
const gated = { path: "tests::gated", exact: "handlers::auth::tests::gated" };
assert.deepEqual(
chooseRust(
[
{ path: "tests::same", exact: "elsewhere::tests::same" },
{ path: "tests::other", exact: "handlers::auth::tests::other" },
gated,
],
candidates,
),
[candidates[0], candidates[1], candidates[3], candidates[2]],
{
chosen: [candidates[0], candidates[1], candidates[3], candidates[2]],
unlisted: [gated],
},
);
});
@@ -428,16 +443,28 @@ test("Rust files build only the targets they can be part of, merged per package
() => rustGroups(["desktop/src/x.rs::tests::a"]),
/rust entry must be/,
);
// The root workspace's listing excludes its Postgres-backed tests by name
// and binary; nextest rejects `binary(/^postgres_/)` in the Tauri
// workspace, where no such binary exists, so that listing carries no
// exclusion.
assert.equal(
rustFilterset("rust", ["tests::a", "b", "tests::a"]),
"(test(/(^|::)tests::a$/) | test(/(^|::)b$/)) & !(test(/postgres_tests::/) | binary(/^postgres_/) | test(/(^|::)external_infra[^:]*::/))",
// Packages the regular lanes test under a feature list in those graphs, in
// order; the rest in their default graph alone.
assert.deepEqual(
[
"crates/buzz-feature-flags/src/launchdarkly.rs::tests::a",
"crates/buzz-push-gateway/src/lib.rs::tests::b",
"crates/buzz-core/src/filter.rs::tests::c",
].map((entry) => [...rustGroups([entry]).groups.values()][0].graphs),
[[["launchdarkly"]], [[], ["personal-dev-app-attest"]], [[]]],
);
assert.equal(rustFilterset("tauri", ["a"]), "(test(/(^|::)a$/))");
assert.doesNotMatch(rustFilterset("tauri", ["a"]), /binary\(/);
assert.deepEqual(
[
...rustGroups(["desktop/src-tauri/src/lib.rs::tests::d"]).groups.values(),
][0].graphs,
[[], ["mesh-llm"]],
);
// Postgres-backed tests are excluded by name and binary, as the postgres-ci
// default-filter selects them.
assert.ok(infraTest("buzz_db", "event_store::postgres_tests::insert"));
assert.ok(infraTest("postgres_relay", "admits"));
assert.ok(infraTest("buzz_media", "tests::external_infra_s3::upload"));
assert.ok(!infraTest("buzz_db", "tests::postgres_free"));
});
test("a helper under tests/ selects the integration binaries that include it", () => {
@@ -990,6 +1017,43 @@ test("repeat invocations mirror the lanes they stand in for", () => {
/^\[profile\.flake\]\n(?:#.*\n)*retries = 0\nfail-fast = false\n\n\[profile\.flake\.junit\]\npath = "junit\.xml"\n/m,
config,
);
// The infrastructure exclusion mirrors the postgres-ci default-filter.
for (const predicate of [
"test(/postgres_tests::/)",
"binary(/^postgres_/)",
"test(/(^|::)external_infra[^:]*::/)",
])
assert.ok(read("../.config/nextest.toml").includes(predicate), predicate);
// Feature graphs mirror the Justfile recipes the regular lanes run: every
// package test-unit runs under a feature, in each graph it runs there, and
// the desktop crate's default and mesh-llm lanes.
const justfile = read("../Justfile");
const graphs = {};
for (const [, crate, features] of justfile.matchAll(
/^ +cargo nextest run -p ([\w-]+)(?: --features ([\w,-]+))?$/gm,
)) {
graphs[crate] ??= [];
graphs[crate].push(features?.split(",") ?? []);
}
assert.ok(
justfile.includes(
"\ndesktop-tauri-test: _ensure-sidecar-stubs\n cd desktop/src-tauri && cargo test --workspace\n",
),
);
assert.ok(
justfile.includes(
"\ndesktop-tauri-test-mesh: _ensure-sidecar-stubs\n cargo test --manifest-path {{desktop_tauri_manifest}} --features mesh-llm --lib\n",
),
);
graphs["buzz-desktop"] = [[], ["mesh-llm"]];
assert.deepEqual(
rustFeatureGraphs,
Object.fromEntries(
Object.entries(graphs).filter(([, list]) =>
list.some((features) => features.length),
),
),
);
const flake = read("../.github/workflows/_ci-flake.yml");
for (const runner of ["rust", "tauri", "node", "dart"])
assert.ok(
@@ -1022,3 +1086,235 @@ test("repeat invocations mirror the lanes they stand in for", () => {
);
assert.doesNotMatch(ci, /needs: \[[^\]]*flake/);
});
// The repeat, through its entry point, in a scratch tree holding the script
// and the packages its entries name, with a fake `cargo` on PATH listing
// `tests` ({pkg, binary, binaryName, name, feature?, ignored?}; a feature
// test only under that feature) and passing every test a run selects.
function repeatWithFakeCargo(runner, entries, tests) {
const dir = realpathSync(mkdtempSync(join(tmpdir(), "flake-repeat-")));
try {
mkdirSync(join(dir, "scripts"));
copyFileSync(
join(root, "scripts/flake-check.mjs"),
join(dir, "scripts/flake-check.mjs"),
);
for (const [crateDir, name] of [
["crates/buzz-feature-flags", "buzz-feature-flags"],
["desktop/src-tauri", "buzz-desktop"],
]) {
mkdirSync(join(dir, crateDir, "src"), { recursive: true });
writeFileSync(
join(dir, crateDir, "Cargo.toml"),
`[package]\nname = "${name}"\n`,
);
writeFileSync(join(dir, crateDir, "src/lib.rs"), "");
}
mkdirSync(join(dir, "bin"));
writeFileSync(
join(dir, "fake-cargo.mjs"),
`import { appendFileSync, mkdirSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
const args = process.argv.slice(2);
appendFileSync(process.env.FAKE_CARGO_LOG, JSON.stringify(args) + "\\n");
const value = (flag) => args[args.indexOf(flag) + 1];
const features = args.includes("--features") ? value("--features").split(",") : [];
const tests = JSON.parse(process.env.FAKE_CARGO_TESTS).filter(
(test) => test.pkg === value("-p") && (!test.feature || features.includes(test.feature)),
);
if (args[1] === "list") {
const suites = {};
for (const test of tests) {
suites[test.binary] ??= { "binary-name": test.binaryName, testcases: {} };
suites[test.binary].testcases[test.name] = {
ignored: Boolean(test.ignored),
"filter-match": test.ignored ? { status: "mismatch", reason: "ignored" } : { status: "matches" },
};
}
process.stdout.write(JSON.stringify({ "rust-suites": suites }));
} else {
const junit = join(
args.includes("--manifest-path") ? dirname(value("--manifest-path")) : ".",
"target/nextest/flake/junit.xml",
);
mkdirSync(dirname(junit), { recursive: true });
const cases = [...value("-E").matchAll(/binary_id\\(=([^)]+)\\) & test\\(=([^)]+)\\)/g)];
writeFileSync(junit, "<testsuites>" + cases.map(([, binary, name]) => '<testcase name="' + name + '" classname="' + binary + '"/>').join("") + "</testsuites>");
}
`,
);
writeFileSync(
join(dir, "bin/cargo"),
`#!/bin/sh\nexec "${process.execPath}" "${join(dir, "fake-cargo.mjs")}" "$@"\n`,
{ mode: 0o755 },
);
const { GITHUB_STEP_SUMMARY, GITHUB_OUTPUT, ...inherited } = process.env;
const log = join(dir, "cargo.log");
const summary = join(dir, "summary.md");
writeFileSync(summary, "");
const result = spawnSync(
process.execPath,
[
join(dir, "scripts/flake-check.mjs"),
runner,
"--iterations=2",
...entries,
],
{
cwd: dir,
encoding: "utf8",
env: {
...inherited,
PATH: `${join(dir, "bin")}:${process.env.PATH}`,
GITHUB_STEP_SUMMARY: summary,
FAKE_CARGO_LOG: log,
FAKE_CARGO_TESTS: JSON.stringify(tests),
},
},
);
assert.equal(result.signal, null, result.stderr);
return {
status: result.status,
summary: readFileSync(summary, "utf8"),
calls: readFileSync(log, "utf8")
.trim()
.split("\n")
.map((line) => JSON.parse(line)),
};
} finally {
rmSync(dir, { recursive: true, force: true });
}
}
test("feature-gated Rust tests repeat in the graph the regular lanes test them in, and unlisted ones fail the lane", () => {
// Reviewer probe on #8613: the repeat listed only the default graph, so a
// selected `#[cfg(feature = …)]` test was dropped and the lane passed as
// "nothing to repeat", or reported only its ungated neighbours.
const features = (call) =>
call.includes("--features") ? call[call.indexOf("--features") + 1] : "";
const flags = {
pkg: "buzz-feature-flags",
binary: "buzz-feature-flags",
binaryName: "buzz_feature_flags",
};
const root = repeatWithFakeCargo(
"rust",
[
"crates/buzz-feature-flags/src/lib.rs::tests::plain",
"crates/buzz-feature-flags/src/launchdarkly.rs::tests::null_events",
],
[
{ ...flags, name: "tests::plain" },
{
...flags,
name: "launchdarkly::tests::null_events",
feature: "launchdarkly",
},
],
);
assert.equal(root.status, 0, root.summary);
// buzz-feature-flags is only tested with LaunchDarkly, so it is listed and
// run in that graph alone.
assert.deepEqual(
root.calls.map((call) => [call[1], features(call)]),
[
["list", "launchdarkly"],
["run", "launchdarkly"],
["run", "launchdarkly"],
],
);
assert.match(
root.summary,
/## Flake check \(Rust\): 0 of 2 iterations failed/,
);
assert.match(
root.summary,
/\| buzz-feature-flags launchdarkly::tests::null_events \| ✅ \| ✅ \| 0\/2 \|/,
);
assert.match(
root.summary,
/\| buzz-feature-flags tests::plain \| ✅ \| ✅ \| 0\/2 \|/,
);
// The desktop crate is tested in its default graph, then under mesh-llm:
// only tests the default graph does not list reach the mesh one, a mesh
// test repeats there, and one no graph lists fails the lane by name while
// the rest still repeat. An ignored test is excluded, not unlisted.
const desktop = {
pkg: "buzz-desktop",
binary: "buzz-desktop",
binaryName: "buzz_lib",
};
const tauriTests = [
{ ...desktop, name: "tests::plain" },
{ ...desktop, name: "tests::skipped", ignored: true },
{
...desktop,
name: "managed_agents::relay_mesh::tests::mesh_only",
feature: "mesh-llm",
},
];
const tauri = repeatWithFakeCargo(
"tauri",
[
"desktop/src-tauri/src/lib.rs::tests::plain",
"desktop/src-tauri/src/lib.rs::tests::skipped",
"desktop/src-tauri/src/managed_agents/relay_mesh.rs::tests::mesh_only",
"desktop/src-tauri/src/lib.rs::tests::macos_only",
],
tauriTests,
);
assert.equal(tauri.status, 1, tauri.summary);
assert.deepEqual(
tauri.calls.map((call) => [call[1], features(call)]),
[
["list", ""],
["list", "mesh-llm"],
["run", ""],
["run", "mesh-llm"],
["run", ""],
["run", "mesh-llm"],
],
);
assert.match(
tauri.summary,
/## Flake check \(Desktop Tauri\): 1 selected test\(s\) not listed\n[^#]*- `desktop\/src-tauri\/src\/lib\.rs::tests::macos_only`\n/,
);
assert.doesNotMatch(tauri.summary, /skipped`/);
assert.match(
tauri.summary,
/## Flake check \(Desktop Tauri\): 0 of 2 iterations failed/,
);
assert.match(
tauri.summary,
/\| buzz-desktop managed_agents::relay_mesh::tests::mesh_only \| ✅ \| ✅ \| 0\/2 \|/,
);
assert.match(
tauri.summary,
/\| buzz-desktop tests::plain \| ✅ \| ✅ \| 0\/2 \|/,
);
// A selection of only ignored tests is excluded, and passes as such; one
// of only unlisted tests fails without running anything.
const ignored = repeatWithFakeCargo(
"tauri",
["desktop/src-tauri/src/lib.rs::tests::skipped"],
tauriTests,
);
assert.equal(ignored.status, 0, ignored.summary);
assert.match(
ignored.summary,
/## Flake check \(Desktop Tauri\): nothing to repeat/,
);
const unlisted = repeatWithFakeCargo(
"tauri",
["desktop/src-tauri/src/lib.rs::tests::macos_only"],
tauriTests,
);
assert.equal(unlisted.status, 1, unlisted.summary);
assert.deepEqual(
unlisted.calls.map((call) => call[1]),
["list", "list"],
);
assert.doesNotMatch(unlisted.summary, /nothing to repeat/);
});