mirror of
https://github.com/block/buzz.git
synced 2026-10-27 12:40:30 +00:00
29858b2aff
Address Jude's integrated review of f02a4235a on #8613.
The required Desktop JS gate was red: scripts/flake-check.test.mjs spawns
the real selector, which writes its summary to $GITHUB_STEP_SUMMARY and
its outputs to $GITHUB_OUTPUT when those are set, as they are under
Actions, so the regression test read an empty stdout there. selectAppending
now strips both from the child's environment when it asserts on stdout,
and its `github` mode sets them to temp files and reads those instead; the
Tauri selection is asserted both ways, equal, with nothing on stdout in
the routed mode. The suite passes with and without the two variables set.
A tests/<stem>.rs root with no test function of its own was treated as a
helper: buzz-acp's pool_lifecycle_state compiles src/pool_lifecycle.rs in
by `#[path]`, and a change to it selected git_bootstrap, run_task and
stdio_contract and never pool_lifecycle_state. rustTargetRoot now tells a
binary's root (tests/<stem>.rs, tests/<dir>/main.rs) from a module of one;
rustModules resolves the file modules a file declares by cargo's
conventions (`#[path]` relative to the declaring file, else x.rs or
x/mod.rs under the module's directory, with comments and strings blanked);
rustIncludedTests collects their tests transitively under their module
path, the names nextest lists for the binary. selectRust takes these as
`included`: a whole selection of a root carries them, a hunk inside one of
the root's own tests does not, and a root with none of its own selects
them as shared code changed. A root with no tests at all gets a row and
selects nothing; only a non-root tests/** file without tests takes the
helper branch, and the helper importers' whole-binary expansion uses the
same test set.
Regression tests go through the real select entry point: a comment
appended to pool_lifecycle_state.rs selects exactly the nine
pool_lifecycle::tests::* entries under that file (and fails if the
included tests are dropped), and rustGroups builds `-p buzz-acp --test
pool_lifecycle_state` with those exact names. Verified live: `cargo nextest
list` with the produced filterset matches all nine in
buzz-acp::pool_lifecycle_state, and `flake-check.mjs rust --iterations=2`
over the selection ran green end to end.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Signed-off-by: Matt Toohey <contact@matttoohey.com>
1191 lines
44 KiB
JavaScript
1191 lines
44 KiB
JavaScript
// Advisory pull-request flake check. `select` maps a diff to the tests it
|
||
// changed, per runner; `rust`, `node` and `dart` repeat a selection and report
|
||
// every iteration. Playwright repeats natively with --repeat-each, and
|
||
// `playwright-summary` tabulates its JSON report afterwards.
|
||
import { spawnSync } from "node:child_process";
|
||
import {
|
||
appendFileSync,
|
||
copyFileSync,
|
||
existsSync,
|
||
mkdirSync,
|
||
readdirSync,
|
||
readFileSync,
|
||
rmSync,
|
||
} from "node:fs";
|
||
import { createRequire } from "node:module";
|
||
import { posix as path } from "node:path";
|
||
import { fileURLToPath } from "node:url";
|
||
|
||
const root = fileURLToPath(new URL("../", import.meta.url));
|
||
export const ITERATIONS = 10;
|
||
// Repetitions spread across shards, so a whole-file selection stays inside
|
||
// the Playwright lane's 45-minute job timeout. Playwright shards at file ×
|
||
// repetition granularity (workers: 1, no fullyParallel). The regular smoke
|
||
// lane runs ~150 journeys per shard in ~11 minutes, about 4.5 s each, so five
|
||
// journeys × ten repetitions is a few minutes of test time per shard; it never
|
||
// uses more than ten runners, so neither does this one. Past ten shards each
|
||
// one grows instead, and the timeout minus dependency setup and the E2E build
|
||
// (~10 minutes, more when apt retries) leaves room for about 250 journey-runs
|
||
// (~19 minutes) per shard, so a selection beyond that many journeys skips the
|
||
// lane with a note rather than timing out.
|
||
const JOURNEYS_PER_SHARD = 5;
|
||
const MAX_SHARDS = 10;
|
||
const MAX_JOURNEY_RUNS_PER_SHARD = 250;
|
||
export const playwrightShards = (journeys) =>
|
||
Math.min(MAX_SHARDS, Math.max(1, Math.ceil(journeys / JOURNEYS_PER_SHARD)));
|
||
export const MAX_PLAYWRIGHT_JOURNEYS = Math.floor(
|
||
(MAX_JOURNEY_RUNS_PER_SHARD * MAX_SHARDS) / ITERATIONS,
|
||
);
|
||
// GitHub caps a job's outputs at 1 MiB in total. A selection this large, like
|
||
// one past the journey cap, is a sweeping refactor rather than a test change
|
||
// worth ten repetitions, so the lane skips rather than overrunning its timeout.
|
||
export const MAX_OUTPUT_BYTES = 64 * 1024;
|
||
export const runners = ["rust", "tauri", "node", "playwright", "dart"];
|
||
// The two cargo workspaces, by runner: the root workspace's crates, and the
|
||
// desktop Tauri workspace (desktop/src-tauri and its crates/*), which the
|
||
// Desktop Tauri lane (_ci-desktop.yml) builds with its own manifest, system
|
||
// dependencies and cache. Each is repeated by its own lane with nextest run
|
||
// against that manifest.
|
||
export const rustWorkspaces = {
|
||
rust: "Cargo.toml",
|
||
tauri: "desktop/src-tauri/Cargo.toml",
|
||
};
|
||
const rustLayouts = [
|
||
{
|
||
runner: "rust",
|
||
pattern: /^(crates\/[^/]+)\/(src|tests)\/((?!fixtures\/).*)\.rs$/,
|
||
},
|
||
{
|
||
runner: "tauri",
|
||
pattern:
|
||
/^(desktop\/src-tauri(?:\/crates\/[^/]+)?)\/(src|tests)\/((?!fixtures\/).*)\.rs$/,
|
||
},
|
||
];
|
||
|
||
// A Rust file's place in its workspace: the runner, its package directory,
|
||
// whether it is under src or tests, and its path below that without `.rs`.
|
||
export function rustFile(file) {
|
||
for (const { runner, pattern } of rustLayouts) {
|
||
const match = pattern.exec(file);
|
||
if (match)
|
||
return {
|
||
runner,
|
||
dir: match[1],
|
||
kind: match[2],
|
||
parts: match[3].split("/"),
|
||
};
|
||
}
|
||
return undefined;
|
||
}
|
||
// Mirrors desktop/package.json "test": plain helpers run under the TS loader;
|
||
// jsdom tests also preload the DOM shim and force-exit past their timers.
|
||
export const nodeTestArgs = (file) =>
|
||
file.endsWith(".jsdom-test.mjs")
|
||
? [
|
||
"--import",
|
||
"./test-jsdom-setup.mjs",
|
||
"--import",
|
||
"./test-loader.mjs",
|
||
"--experimental-strip-types",
|
||
"--test-force-exit",
|
||
"--test",
|
||
]
|
||
: ["--import", "./test-loader.mjs", "--experimental-strip-types", "--test"];
|
||
// Mirrors the mobile test shards in .github/workflows/_ci-clients.yml.
|
||
export const dartDefine =
|
||
"--dart-define=BUZZ_PUSH_GATEWAY_URL=https://push.example";
|
||
// Mirrors the postgres-ci default-filter in .config/nextest.toml: those tests
|
||
// are #[ignore]d and need Postgres, Redis, S3 or MinIO, so a selection that
|
||
// reached one would fail every repetition without being flaky. The convention
|
||
// is the root workspace's; nextest rejects a `binary()` predicate matching no
|
||
// test binary in the workspace, and the Tauri workspace has none, so it gets
|
||
// no exclusion.
|
||
const infraTests = {
|
||
rust: "test(/postgres_tests::/) | binary(/^postgres_/) | test(/(^|::)external_infra[^:]*::/)",
|
||
};
|
||
|
||
// The nextest filterset listing a workspace's candidates for the selected
|
||
// in-file test paths, less its infrastructure-backed tests.
|
||
export function rustFilterset(runner, paths) {
|
||
const suffixes = [...new Set(paths)]
|
||
.map((path) => `test(/(^|::)${path}$/)`)
|
||
.join(" | ");
|
||
return infraTests[runner]
|
||
? `(${suffixes}) & !(${infraTests[runner]})`
|
||
: `(${suffixes})`;
|
||
}
|
||
|
||
// Mirrors desktop/package.json "test", desktop/playwright.config.ts testDir,
|
||
// the two cargo workspaces' packages and mobile's test directory. Paths are
|
||
// later split on whitespace in a shell, so admit only plain names. The
|
||
// desktop `*.perf.ts` measurements are deliberately not specs here: they time
|
||
// the app rather than assert on it, so a slow repetition is not a flake.
|
||
export function classify(file) {
|
||
if (!/^[\w./@-]+$/.test(file)) return undefined;
|
||
const rust = rustFile(file);
|
||
if (rust) return rust.runner;
|
||
if (
|
||
/^desktop\/(?:src\/.*\.(?:test|jsdom-test)\.mjs|scripts\/[^/]+\.test\.mjs)$/.test(
|
||
file,
|
||
)
|
||
)
|
||
return "node";
|
||
if (/^desktop\/tests\/e2e\/[^/]+\.spec\.ts$/.test(file)) return "playwright";
|
||
if (/^mobile\/test\/.*_test\.dart$/.test(file)) return "dart";
|
||
return undefined;
|
||
}
|
||
|
||
// Line ranges of a -U0 diff: the new-side lines each hunk writes and the
|
||
// old-side lines it removes (empty, end < start, for a pure insertion). A pure
|
||
// deletion writes no lines and attaches to the line before it (0 at the top of
|
||
// the file); `removed` keeps what it took out, since that line cannot tell.
|
||
export function hunkRanges(diff) {
|
||
return [
|
||
...diff.matchAll(/^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@/gm),
|
||
].map(([, oldStart, oldCount = "1", start, count = "1"]) => ({
|
||
start: Number(start),
|
||
end: Number(start) + Math.max(Number(count), 1) - 1,
|
||
removed: {
|
||
start: Number(oldStart),
|
||
end: Number(oldStart) + Number(oldCount) - 1,
|
||
},
|
||
}));
|
||
}
|
||
|
||
// Members that keep a `test`/`it` chain a test declaration. Anything else on
|
||
// the chain is a group (`describe`), a hook (`test.beforeEach`), configuration
|
||
// (`test.use`) or a harness object that happens to be named `test`.
|
||
const declaring = [
|
||
"only",
|
||
"skip",
|
||
"fixme",
|
||
"fail",
|
||
"fails",
|
||
"todo",
|
||
"concurrent",
|
||
"sequential",
|
||
"each",
|
||
"for",
|
||
"runIf",
|
||
"skipIf",
|
||
];
|
||
|
||
// The desktop app's TypeScript parses the specs; load it only for a selection
|
||
// that needs it, so Rust-only runs work without desktop dependencies.
|
||
let typescript;
|
||
const ts = () =>
|
||
(typescript ??= createRequire(
|
||
new URL("../desktop/package.json", import.meta.url),
|
||
)("typescript"));
|
||
|
||
// Line spans of every test declaration in a TypeScript spec, from its first
|
||
// line to its closing parenthesis.
|
||
export function testCalls(source, file) {
|
||
const t = ts();
|
||
const sourceFile = t.createSourceFile(
|
||
file,
|
||
source,
|
||
t.ScriptTarget.Latest,
|
||
true,
|
||
);
|
||
const lineAt = (position) =>
|
||
sourceFile.getLineAndCharacterOfPosition(position).line + 1;
|
||
const names = (callee) =>
|
||
t.isIdentifier(callee)
|
||
? [callee.text]
|
||
: t.isPropertyAccessExpression(callee)
|
||
? names(callee.expression)?.concat(callee.name.text)
|
||
: t.isCallExpression(callee)
|
||
? names(callee.expression)
|
||
: undefined;
|
||
const calls = [];
|
||
const visit = (node) => {
|
||
const chain = t.isCallExpression(node) ? names(node.expression) : undefined;
|
||
if (
|
||
chain &&
|
||
["test", "it"].includes(chain[0]) &&
|
||
chain.slice(1).every((member) => declaring.includes(member))
|
||
) {
|
||
calls.push({
|
||
start: lineAt(node.getStart(sourceFile)),
|
||
end: lineAt(node.end - 1),
|
||
});
|
||
return;
|
||
}
|
||
t.forEachChild(node, visit);
|
||
};
|
||
visit(sourceFile);
|
||
return calls;
|
||
}
|
||
|
||
// Rust source with every comment and literal blanked to spaces, newlines kept,
|
||
// so attribute and brace scanning never see a `{` or `#[` inside a string,
|
||
// char literal or comment. Lifetimes (`'a`) are kept; char literals are not.
|
||
export function blankRust(source) {
|
||
let out = "";
|
||
let i = 0;
|
||
const blank = (end) => {
|
||
for (; i < end && i < source.length; i += 1)
|
||
out += source[i] === "\n" ? "\n" : " ";
|
||
};
|
||
const closingQuote = (from, raw = false, hashes = "") => {
|
||
for (let j = from; j < source.length; j += 1) {
|
||
if (source[j] === "\\" && !raw) j += 1;
|
||
else if (source[j] === '"' && source.startsWith(hashes, j + 1))
|
||
return j + 1 + hashes.length;
|
||
}
|
||
return source.length;
|
||
};
|
||
const rawPrefix = /[bc]?r(#*)"/y;
|
||
while (i < source.length) {
|
||
const rest = source.slice(i, i + 2);
|
||
rawPrefix.lastIndex = i;
|
||
const raw = rawPrefix.exec(source);
|
||
if (rest.startsWith("//")) {
|
||
const end = source.indexOf("\n", i);
|
||
blank(end === -1 ? source.length : end);
|
||
} else if (rest.startsWith("/*")) {
|
||
let depth = 0;
|
||
let j = i;
|
||
do {
|
||
if (source.startsWith("/*", j)) {
|
||
depth += 1;
|
||
j += 2;
|
||
} else if (source.startsWith("*/", j)) {
|
||
depth -= 1;
|
||
j += 2;
|
||
} else j += 1;
|
||
} while (depth > 0 && j < source.length);
|
||
blank(j);
|
||
} else if (raw) {
|
||
blank(closingQuote(i + raw[0].length, true, raw[1]));
|
||
} else if (/^[bc]?"/.test(rest)) {
|
||
blank(closingQuote(i + rest.indexOf('"') + 1));
|
||
} else if (source[i] === "'") {
|
||
const width = source.codePointAt(i + 1) > 0xffff ? 2 : 1;
|
||
if (source[i + 1] === "\\") blank(source.indexOf("'", i + 2) + 1);
|
||
else if (source[i + 1 + width] === "'") blank(i + width + 2);
|
||
else {
|
||
out += source[i];
|
||
i += 1;
|
||
}
|
||
} else {
|
||
out += source[i];
|
||
i += 1;
|
||
}
|
||
}
|
||
return out;
|
||
}
|
||
|
||
const matching = (text, open, close, from) => {
|
||
let depth = 0;
|
||
for (let j = from; j < text.length; j += 1) {
|
||
if (text[j] === open) depth += 1;
|
||
else if (text[j] === close) {
|
||
depth -= 1;
|
||
if (depth === 0) return j;
|
||
}
|
||
}
|
||
return text.length - 1;
|
||
};
|
||
|
||
// Every test function in a Rust file: a `fn` whose attribute block has an
|
||
// attribute path ending in `test` (`#[test]`, `#[tokio::test]`, …), spanning
|
||
// the first attribute line to the body's closing brace. `name` is the path
|
||
// within the file through its inline modules (`tests::plain`), so a namesake
|
||
// in another module stays a different test. `ignored` marks `#[ignore]`,
|
||
// which nextest skips by default.
|
||
export function rustTests(source) {
|
||
const text = blankRust(source);
|
||
const modules = [];
|
||
const inline = /\bmod\s+(\w+)\s*\{/g;
|
||
for (let m = inline.exec(text); m; m = inline.exec(text)) {
|
||
const open = m.index + m[0].length - 1;
|
||
modules.push({ name: m[1], open, close: matching(text, "{", "}", open) });
|
||
}
|
||
const lineStarts = [0];
|
||
for (let i = 0; i < text.length; i += 1)
|
||
if (text[i] === "\n") lineStarts.push(i + 1);
|
||
const lineAt = (offset) => {
|
||
let low = 0;
|
||
let high = lineStarts.length - 1;
|
||
while (low < high) {
|
||
const mid = Math.ceil((low + high) / 2);
|
||
if (lineStarts[mid] <= offset) low = mid;
|
||
else high = mid - 1;
|
||
}
|
||
return low + 1;
|
||
};
|
||
const tests = [];
|
||
const attribute = /#\[/g;
|
||
for (let match = attribute.exec(text); match; match = attribute.exec(text)) {
|
||
const paths = [];
|
||
let j = match.index;
|
||
while (text.startsWith("#[", j)) {
|
||
const close = matching(text, "[", "]", j + 1);
|
||
paths.push(/^\s*([\w:]+)/.exec(text.slice(j + 2, close))?.[1] ?? "");
|
||
j = close + 1;
|
||
while (/\s/.test(text[j] ?? "")) j += 1;
|
||
}
|
||
attribute.lastIndex = j;
|
||
if (!paths.some((path) => /^(?:\w+::)*test$/.test(path))) continue;
|
||
const fn =
|
||
/^(?:pub(?:\([\w\s:]*\))?\s+)?(?:async\s+)?(?:unsafe\s+)?fn\s+(\w+)/.exec(
|
||
text.slice(j),
|
||
);
|
||
if (!fn) continue;
|
||
const open = text.indexOf("{", j + fn[0].length);
|
||
if (open === -1) continue;
|
||
const path = modules
|
||
.filter((mod) => mod.open < match.index && match.index < mod.close)
|
||
.map((mod) => mod.name);
|
||
tests.push({
|
||
name: [...path, fn[1]].join("::"),
|
||
start: lineAt(match.index),
|
||
end: lineAt(matching(text, "{", "}", open)),
|
||
ignored: paths.includes("ignore"),
|
||
});
|
||
}
|
||
return tests;
|
||
}
|
||
|
||
// Bodies of the listed tests: their calls, keyed by the line the runner
|
||
// listed. Runners only report a call-site line, which may sit inside the call,
|
||
// so each listed line is owned by the enclosing test call; a listed line
|
||
// outside every call leaves its lines to the whole-file rule.
|
||
export function testBodies(calls, lines) {
|
||
const bodies = new Map();
|
||
for (const line of [...lines].sort((a, b) => a - b)) {
|
||
const call = calls.find(({ start, end }) => start <= line && line <= end);
|
||
if (call && !bodies.has(call)) bodies.set(call, { ...call, line });
|
||
}
|
||
return [...bodies.values()].sort((a, b) => a.start - b.start);
|
||
}
|
||
|
||
// The calls owning every line of a range, or undefined once a line lies
|
||
// outside them all.
|
||
function owners(calls, { start, end }) {
|
||
const found = [];
|
||
for (let line = start; line <= end; ) {
|
||
const call = calls.find((c) => c.start <= line && line <= c.end);
|
||
if (!call) return undefined;
|
||
found.push(call);
|
||
line = call.end + 1;
|
||
}
|
||
return found;
|
||
}
|
||
|
||
// A hunk whose every line lies in a test body selects those tests. A changed
|
||
// line outside every body (imports, helpers, hooks or constants before,
|
||
// between or after tests, or the code under test in a Rust module) can change
|
||
// every test, so it, a new file, or a file with no listed tests selects the
|
||
// whole file. Removed lines are judged by the old file's calls (`previous`): a
|
||
// deletion attaches to its preceding line, which belongs to the test before a
|
||
// removed hook, not to the hook.
|
||
export function selectTests(
|
||
ranges,
|
||
bodies,
|
||
{ added = false, previous = [] } = {},
|
||
) {
|
||
if (added) return { whole: true, reason: "new file" };
|
||
if (bodies.length === 0) return { whole: true, reason: "no tests listed" };
|
||
const selected = new Set();
|
||
for (const { start, end, removed } of ranges) {
|
||
if (!owners(previous, removed))
|
||
return { whole: true, reason: "shared code removed" };
|
||
const current = owners(bodies, { start, end });
|
||
if (!current) return { whole: true, reason: "shared code changed" };
|
||
for (const body of current) selected.add(body.line);
|
||
}
|
||
return { lines: [...selected].sort((a, b) => a - b) };
|
||
}
|
||
|
||
// Rust tests a diff selects: hunks inside a test body select that test, any
|
||
// other hunk selects every test in the file (the module's code under test
|
||
// lives beside them) and, for an integration-test root, every test of the
|
||
// file modules it declares (`included`: they are compiled into the same
|
||
// binary, so a root with no test of its own still selects them). `#[ignore]`d
|
||
// tests own their lines but never run. A file with no tests at all selects
|
||
// nothing of its own; `select` decides what that means by where it lives.
|
||
export function selectRust(
|
||
diff,
|
||
source,
|
||
{ added = false, previous = "", included = [] } = {},
|
||
) {
|
||
const tests = rustTests(source);
|
||
if (tests.length === 0 && included.length === 0) return undefined;
|
||
const selection =
|
||
tests.length === 0
|
||
? { whole: true, reason: added ? "new file" : "shared code changed" }
|
||
: selectTests(
|
||
hunkRanges(diff),
|
||
tests.map((test) => ({ ...test, line: test.start })),
|
||
{ added, previous: added ? [] : rustTests(previous) },
|
||
);
|
||
const chosen = selection.whole
|
||
? [...tests, ...included]
|
||
: tests.filter((test) => selection.lines.includes(test.start));
|
||
return {
|
||
...selection,
|
||
names: chosen.filter((test) => !test.ignored).map((test) => test.name),
|
||
};
|
||
}
|
||
|
||
const run = (command, args, options = {}) => {
|
||
const result = spawnSync(command, args, {
|
||
cwd: root,
|
||
encoding: "utf8",
|
||
maxBuffer: 64 * 1024 * 1024,
|
||
...options,
|
||
});
|
||
if (result.error) throw result.error;
|
||
return result;
|
||
};
|
||
const git = (...args) => {
|
||
const result = run("git", args);
|
||
if (result.status !== 0) throw new Error(`git ${args[0]}: ${result.stderr}`);
|
||
return result.stdout;
|
||
};
|
||
const summarize = (text) => {
|
||
if (process.env.GITHUB_STEP_SUMMARY)
|
||
appendFileSync(process.env.GITHUB_STEP_SUMMARY, text);
|
||
else process.stdout.write(text);
|
||
};
|
||
const cell = (value) => String(value).replaceAll("|", "\\|");
|
||
|
||
const crateDir = (file) => `${root}${rustFile(file).dir}`;
|
||
|
||
// The `[package]` name of the package owning a Rust file.
|
||
export function cratePackage(file) {
|
||
const manifest = readFileSync(`${crateDir(file)}/Cargo.toml`, "utf8");
|
||
return /^\[package\][\s\S]*?^name\s*=\s*"([^"]+)"/m.exec(manifest)[1];
|
||
}
|
||
|
||
// Cargo target flags building only the test binaries a file can be part of,
|
||
// one flag per entry so rustGroups can merge files' flags without splitting
|
||
// `--test <name>`: src/** is the lib (when the crate has one) or a bin,
|
||
// tests/<stem>.rs and tests/<dir>/*.rs (with a main.rs) the integration test
|
||
// of that name. A file placed by a custom `[[test]]` path builds every target
|
||
// instead.
|
||
export function rustTarget(file) {
|
||
const { kind, parts } = rustFile(file);
|
||
if (kind === "src") {
|
||
const lib =
|
||
existsSync(`${crateDir(file)}/src/lib.rs`) ||
|
||
/^\[lib\]/m.test(readFileSync(`${crateDir(file)}/Cargo.toml`, "utf8"));
|
||
return lib ? [["--lib"], ["--bins"]] : [["--bins"]];
|
||
}
|
||
if (parts.length === 1) return [["--test", parts[0]]];
|
||
return existsSync(`${crateDir(file)}/tests/${parts[0]}/main.rs`)
|
||
? [["--test", parts[0]]]
|
||
: [];
|
||
}
|
||
|
||
// The module path a file's items carry in their binary under cargo's layout
|
||
// conventions: src/a/b.rs is a::b, a mod.rs/main.rs/lib.rs names its directory
|
||
// or the root, src/bin/x.rs and tests/<target>.rs are roots, tests/<target>/x.rs
|
||
// is x. A `#[path]` attribute can break this; repeatRust then falls back to the
|
||
// in-file path alone.
|
||
export function rustModulePath(file) {
|
||
const { kind, parts: below } = rustFile(file);
|
||
const parts = [...below];
|
||
if (kind === "tests") parts.shift();
|
||
else if (parts[0] === "bin") parts.splice(0, 2);
|
||
if (["mod", "main"].includes(parts.at(-1)) || parts.join() === "lib")
|
||
parts.pop();
|
||
return parts;
|
||
}
|
||
|
||
// Whether an integration-test target (tests/<stem>.rs or tests/<dir>/main.rs)
|
||
// compiles a helper file under tests/ into its binary: a main.rs owns its
|
||
// directory, and any target's source declares `mod <first>;` for the
|
||
// helper's first path component under tests/ (a crate root resolves
|
||
// `mod common;` to tests/common/mod.rs or tests/common.rs), or a `#[path]`
|
||
// attribute names the helper's file or the mod.rs of a directory holding it.
|
||
// Declarations are read with comments and strings blanked; `#[path]` strings
|
||
// are read raw, since blanking would erase them.
|
||
export function rustIncludes(target, source, helper) {
|
||
const targetDir = path.dirname(target);
|
||
const [first] = rustFile(helper).parts;
|
||
if (path.basename(target) === "main.rs" && helper.startsWith(`${targetDir}/`))
|
||
return true;
|
||
if (new RegExp(`\\bmod\\s+${first}\\s*;`).test(blankRust(source)))
|
||
return true;
|
||
for (const [, named] of source.matchAll(/#\[\s*path\s*=\s*"([^"]+)"\s*\]/g)) {
|
||
const resolved = path.normalize(path.join(targetDir, named));
|
||
if (resolved === helper) return true;
|
||
if (
|
||
path.basename(resolved) === "mod.rs" &&
|
||
helper.startsWith(`${path.dirname(resolved)}/`)
|
||
)
|
||
return true;
|
||
}
|
||
return false;
|
||
}
|
||
|
||
// The integration-test targets of a helper's package that include it, as
|
||
// files. A helper no target is found to include (a layout rustIncludes does
|
||
// not read) falls back to every target, so a helper change is never left
|
||
// unrepeated by the scan.
|
||
export function rustImporters(helper) {
|
||
const { dir } = rustFile(helper);
|
||
const tests = `${dir}/tests`;
|
||
const targets = readdirSync(`${root}${tests}`, { withFileTypes: true })
|
||
.flatMap((entry) =>
|
||
entry.isFile() && entry.name.endsWith(".rs")
|
||
? [`${tests}/${entry.name}`]
|
||
: entry.isDirectory() &&
|
||
existsSync(`${root}${tests}/${entry.name}/main.rs`)
|
||
? [`${tests}/${entry.name}/main.rs`]
|
||
: [],
|
||
)
|
||
.filter((target) => target !== helper)
|
||
.sort();
|
||
const importers = targets.filter((target) =>
|
||
rustIncludes(target, readFileSync(`${root}${target}`, "utf8"), helper),
|
||
);
|
||
return {
|
||
importers: importers.length ? importers : targets,
|
||
scanned: importers.length > 0,
|
||
};
|
||
}
|
||
|
||
// Whether a tests/** file is the root of an integration-test binary
|
||
// (tests/<stem>.rs or tests/<dir>/main.rs) rather than a module of one.
|
||
export function rustTargetRoot(file) {
|
||
const place = rustFile(file);
|
||
if (place?.kind !== "tests") return false;
|
||
const { parts } = place;
|
||
return parts.length === 1 || (parts.length === 2 && parts[1] === "main");
|
||
}
|
||
|
||
// The file modules a Rust file declares, as cargo compiles them into its
|
||
// binary: `#[path = "…"] mod x;` names a file relative to the declaring
|
||
// file's directory; a plain `mod x;` is x.rs or x/mod.rs under the module's
|
||
// directory, the file's own for a root (lib.rs, main.rs, mod.rs, a
|
||
// tests/<stem>.rs) and <dir>/<stem>/ otherwise. Declarations are read with
|
||
// comments and strings blanked; `#[path]` strings raw, since blanking erases
|
||
// them. A module no file backs is left out.
|
||
export function rustModules(file, source) {
|
||
const dir = path.dirname(file);
|
||
const stem = path.basename(file, ".rs");
|
||
const owned =
|
||
["lib", "main", "mod"].includes(stem) || rustTargetRoot(file)
|
||
? dir
|
||
: path.join(dir, stem);
|
||
const pathed = new Map();
|
||
for (const [, named, name] of source.matchAll(
|
||
/#\[\s*path\s*=\s*"([^"]+)"\s*\]\s*(?:#\[[^\]]*\]\s*)*(?:pub(?:\([^)]*\))?\s+)?mod\s+(\w+)\s*;/g,
|
||
))
|
||
pathed.set(name, path.normalize(path.join(dir, named)));
|
||
const modules = [];
|
||
for (const [, name] of blankRust(source).matchAll(
|
||
/\b(?:pub(?:\([^)]*\))?\s+)?mod\s+(\w+)\s*;/g,
|
||
)) {
|
||
const found = (
|
||
pathed.has(name)
|
||
? [pathed.get(name)]
|
||
: [`${owned}/${name}.rs`, `${owned}/${name}/mod.rs`]
|
||
).find((candidate) => existsSync(`${root}${candidate}`));
|
||
if (found) modules.push({ name, file: found });
|
||
}
|
||
return modules;
|
||
}
|
||
|
||
// The tests a file compiles in from the file modules it declares,
|
||
// transitively, named by their module path from it
|
||
// (`pool_lifecycle::tests::name` for `#[path = "../src/pool_lifecycle.rs"]
|
||
// mod pool_lifecycle;`), which is how nextest lists them for the binary. An
|
||
// integration-test root with no test function of its own (a wrapper around a
|
||
// source module, as buzz-acp's pool_lifecycle_state) has only these.
|
||
export function rustIncludedTests(file, source, seen = new Set([file])) {
|
||
return rustModules(file, source).flatMap(({ name, file: module }) => {
|
||
if (seen.has(module)) return [];
|
||
seen.add(module);
|
||
const text = readFileSync(`${root}${module}`, "utf8");
|
||
return [...rustTests(text), ...rustIncludedTests(module, text, seen)].map(
|
||
(test) => ({ ...test, name: `${name}::${test.name}` }),
|
||
);
|
||
});
|
||
}
|
||
|
||
// Smoke journeys per spec file as Playwright lists them, each line once (a
|
||
// looped declaration is listed once per iteration). Integration-only specs
|
||
// list nothing for the smoke project, which Playwright reports as its "No
|
||
// tests found" error, so a selection of them stays empty.
|
||
export function listPlaywright(files) {
|
||
const result = run(`${root}bin/pnpm`, [
|
||
"-C",
|
||
"desktop",
|
||
"--silent",
|
||
"exec",
|
||
"playwright",
|
||
"test",
|
||
"--project=smoke",
|
||
"--list",
|
||
"--reporter=json",
|
||
...files,
|
||
]);
|
||
const report = JSON.parse(result.stdout);
|
||
const empty =
|
||
report.errors.length === 1 &&
|
||
/No tests found/.test(report.errors[0].message);
|
||
if ((result.status !== 0 || report.errors.length) && !empty)
|
||
throw new Error(
|
||
`playwright --list: ${report.errors.map((error) => error.message).join("\n") || result.stderr}`,
|
||
);
|
||
const specs = new Map(files.map((file) => [file, new Set()]));
|
||
const collect = (suites) => {
|
||
for (const suite of suites) {
|
||
for (const spec of suite.specs ?? [])
|
||
if (spec.tests?.length)
|
||
specs.get(`desktop/tests/e2e/${spec.file}`)?.add(spec.line);
|
||
collect(suite.suites ?? []);
|
||
}
|
||
};
|
||
collect(empty ? [] : report.suites);
|
||
return new Map([...specs].map(([file, lines]) => [file, [...lines]]));
|
||
}
|
||
|
||
function select(base = "HEAD^1", head = "HEAD") {
|
||
const changed = git(
|
||
"diff",
|
||
"--name-status",
|
||
"--no-renames",
|
||
"-z",
|
||
"--diff-filter=AM",
|
||
base,
|
||
head,
|
||
"--",
|
||
).split("\0");
|
||
const files = Object.fromEntries(runners.map((runner) => [runner, []]));
|
||
const added = new Set();
|
||
for (let i = 0; i + 1 < changed.length; i += 2) {
|
||
const [status, file] = [changed[i], changed[i + 1]];
|
||
const runner = classify(file);
|
||
if (runner) files[runner].push(file);
|
||
if (status === "A") added.add(file);
|
||
}
|
||
const journeys = files.playwright.length
|
||
? listPlaywright(files.playwright)
|
||
: new Map();
|
||
const outputs = Object.fromEntries(runners.map((runner) => [runner, []]));
|
||
const rows = [];
|
||
const previous = (file) =>
|
||
added.has(file) ? "" : git("show", `${base}:${file}`);
|
||
const diff = (file) =>
|
||
git("diff", "-U0", "--no-color", base, head, "--", file);
|
||
// The tests an integration-test root compiles in from its file modules;
|
||
// any other file's tests are its own.
|
||
const includedRust = (file, source) =>
|
||
rustTargetRoot(file) ? rustIncludedTests(file, source) : [];
|
||
// Every runnable test of a Rust file, as entries.
|
||
const wholeRust = (file) => {
|
||
const source = readFileSync(`${root}${file}`, "utf8");
|
||
return [...rustTests(source), ...includedRust(file, source)]
|
||
.filter((test) => !test.ignored)
|
||
.map((test) => `${file}::${test.name}`);
|
||
};
|
||
for (const runner of runners) {
|
||
for (const file of files[runner]) {
|
||
let entries = [file];
|
||
let note = "whole file";
|
||
if (rustWorkspaces[runner]) {
|
||
const source = readFileSync(`${root}${file}`, "utf8");
|
||
const selection = selectRust(diff(file), source, {
|
||
added: added.has(file),
|
||
previous: previous(file),
|
||
included: includedRust(file, source),
|
||
});
|
||
if (selection) {
|
||
entries = selection.names.map((name) => `${file}::${name}`);
|
||
note =
|
||
selection.names.length === 0
|
||
? "excluded (ignored tests only)"
|
||
: selection.whole
|
||
? `whole file (${selection.reason}): ${selection.names.length} test(s)`
|
||
: `${selection.names.length} changed test(s)`;
|
||
} else if (rustTargetRoot(file)) {
|
||
// An integration-test root with no test of its own and none in
|
||
// the modules it declares: a binary with nothing to repeat.
|
||
rows.push(
|
||
`| ${cell(file)} | ${runner} | integration test with no tests |`,
|
||
);
|
||
continue;
|
||
} else if (rustFile(file).kind === "tests") {
|
||
// A tests/** file that is not a binary's root and has no test
|
||
// functions is a helper module that integration binaries compile
|
||
// in (`mod common;`), so a change to it can change every test of
|
||
// each binary that includes it: select those binaries whole. The
|
||
// ignored and infrastructure exclusions still apply downstream, as
|
||
// for any whole-file selection.
|
||
const { importers, scanned } = rustImporters(file);
|
||
entries = importers.flatMap(wholeRust);
|
||
const names = importers.map(
|
||
(importer) => rustFile(importer).parts[0],
|
||
);
|
||
note =
|
||
importers.length === 0
|
||
? "helper with no integration tests to repeat"
|
||
: `helper for${scanned ? "" : " (no importer found; every integration test)"}: ${names.join(", ")}: ${entries.length} test(s)`;
|
||
} else {
|
||
// A src/** file without test functions is the code under test, not
|
||
// a test file. This is deliberate: the regular lanes build and run
|
||
// everything that depends on it, and repeating its crate's whole
|
||
// suite for any source change would not be a changed-test check.
|
||
rows.push(`| ${cell(file)} | ${runner} | no test functions |`);
|
||
continue;
|
||
}
|
||
} else if (runner === "playwright") {
|
||
const listed = journeys.get(file) ?? [];
|
||
const source = readFileSync(`${root}${file}`, "utf8");
|
||
const selection = selectTests(
|
||
hunkRanges(diff(file)),
|
||
testBodies(testCalls(source, file), listed),
|
||
{
|
||
added: added.has(file),
|
||
previous: added.has(file) ? [] : testCalls(previous(file), file),
|
||
},
|
||
);
|
||
const lines = selection.whole ? listed : selection.lines;
|
||
entries = lines.map((line) => `${file}:${line}`);
|
||
note =
|
||
lines.length === 0
|
||
? "excluded (no smoke journeys)"
|
||
: selection.whole
|
||
? `whole file (${selection.reason}): ${lines.length} journey(s)`
|
||
: `${lines.length} changed journey(s)`;
|
||
}
|
||
outputs[runner].push(...entries);
|
||
rows.push(`| ${cell(file)} | ${runner} | ${cell(note)} |`);
|
||
}
|
||
// A helper and a file that includes it, changed together, select the
|
||
// same tests twice.
|
||
outputs[runner] = [...new Set(outputs[runner])];
|
||
const exceeded =
|
||
Buffer.byteLength(outputs[runner].join(" ")) > MAX_OUTPUT_BYTES
|
||
? `selection exceeds ${MAX_OUTPUT_BYTES} bytes`
|
||
: runner === "playwright" &&
|
||
outputs[runner].length > MAX_PLAYWRIGHT_JOURNEYS
|
||
? `selection exceeds ${MAX_PLAYWRIGHT_JOURNEYS} journeys, which ${ITERATIONS} repetitions would not finish in time`
|
||
: undefined;
|
||
if (exceeded) {
|
||
rows.push(
|
||
`| *(${outputs[runner].length} ${runner} entries)* | ${runner} | ${exceeded}; lane skipped |`,
|
||
);
|
||
outputs[runner] = [];
|
||
}
|
||
}
|
||
summarize(
|
||
[
|
||
"## Flake check selection",
|
||
"",
|
||
...(rows.length
|
||
? ["| File | Runner | Selection |", "| --- | --- | --- |", ...rows]
|
||
: [
|
||
"No Rust, desktop Tauri, node:test, Playwright or Flutter test files changed.",
|
||
]),
|
||
"",
|
||
"",
|
||
].join("\n"),
|
||
);
|
||
const shards = playwrightShards(outputs.playwright.length);
|
||
const text = [
|
||
...Object.entries(outputs).map(
|
||
([key, entries]) => `${key}=${entries.join(" ")}\n`,
|
||
),
|
||
`playwright_shards=${JSON.stringify(Array.from({ length: shards }, (_, i) => i + 1))}\n`,
|
||
`playwright_shard_total=${shards}\n`,
|
||
].join("");
|
||
if (process.env.GITHUB_OUTPUT)
|
||
appendFileSync(process.env.GITHUB_OUTPUT, text);
|
||
else process.stdout.write(text);
|
||
}
|
||
|
||
const resultsDir = (runner) => {
|
||
const dir = `${root}test-results/flake/${runner}`;
|
||
rmSync(dir, { recursive: true, force: true });
|
||
mkdirSync(dir, { recursive: true });
|
||
return dir;
|
||
};
|
||
|
||
// Test × iteration table for a repeated selection; a run passes only when its
|
||
// process exits 0 and every expected test reports. Tests that never ran in any
|
||
// iteration are left out, so the table holds only the repeated ones.
|
||
function report(title, runs) {
|
||
const columns = runs.map((_, i) => String(i + 1));
|
||
const mark = (status) =>
|
||
({ passed: "✅", failed: "❌" })[status] ?? (status ? "⏭" : "—");
|
||
const ran = (status) => status === "passed" || status === "failed";
|
||
const tests = [
|
||
...new Set(runs.flatMap((run) => [...run.statuses.keys()])),
|
||
].filter((test) => runs.some((run) => ran(run.statuses.get(test))));
|
||
const failures = runs.filter((run) => !run.ok).length;
|
||
summarize(
|
||
[
|
||
`## ${title}: ${failures} of ${runs.length} iterations failed`,
|
||
"",
|
||
`| Test | ${columns.join(" | ")} | Failed |`,
|
||
`| --- | ${columns.map(() => ":-:").join(" | ")} | ---: |`,
|
||
`| **Run** | ${runs.map((run) => (run.ok ? "✅" : "❌")).join(" | ")} | ${failures}/${runs.length} |`,
|
||
...tests.sort().map((test) => {
|
||
const statuses = runs.map((run) => run.statuses.get(test));
|
||
return `| ${cell(test)} | ${statuses.map(mark).join(" | ")} | ${statuses.filter((status) => status === "failed").length}/${runs.length} |`;
|
||
}),
|
||
"",
|
||
"",
|
||
].join("\n"),
|
||
);
|
||
process.exitCode = failures === 0 ? 0 : 1;
|
||
}
|
||
|
||
const progress = (i, iterations, ok) =>
|
||
console.log(
|
||
`flake-check: iteration ${i}/${iterations} ${ok ? "passed" : "FAILED"}`,
|
||
);
|
||
|
||
// Per-test statuses of a JUnit report, keyed `<classname> <name>`.
|
||
export function parseJunit(xml) {
|
||
const decode = (text) =>
|
||
text.replace(
|
||
/&(quot|apos|lt|gt|amp);/g,
|
||
(_, entity) =>
|
||
({ quot: '"', apos: "'", lt: "<", gt: ">", amp: "&" })[entity],
|
||
);
|
||
const statuses = new Map();
|
||
for (const chunk of xml.split("<testcase").slice(1)) {
|
||
const attributes = chunk.slice(0, chunk.indexOf(">"));
|
||
const body = chunk.split("</testsuite>")[0];
|
||
const name = decode(/\bname="([^"]*)"/.exec(attributes)[1]);
|
||
const classname = decode(
|
||
/\bclassname="([^"]*)"/.exec(attributes)?.[1] ?? "",
|
||
);
|
||
statuses.set(
|
||
`${classname} ${name}`,
|
||
/<(?:failure|error)[\s/>]/.test(body)
|
||
? "failed"
|
||
: /<skipped[\s/>]/.test(body)
|
||
? "skipped"
|
||
: "passed",
|
||
);
|
||
}
|
||
return statuses;
|
||
}
|
||
|
||
// The listed tests an entry stands for: a test the file's module path predicts
|
||
// exactly when nextest lists one, otherwise every test whose name ends in the
|
||
// in-file path (a `#[path]` module breaks the prediction; a namesake in another
|
||
// module is then repeated too rather than nothing).
|
||
export function chooseRust(tests, candidates) {
|
||
const chosen = new Map();
|
||
for (const { path, exact } of tests) {
|
||
const suffix = candidates.filter(
|
||
({ name }) => name === path || name.endsWith(`::${path}`),
|
||
);
|
||
const predicted = suffix.filter(({ name }) => name === exact);
|
||
for (const test of predicted.length ? predicted : suffix)
|
||
chosen.set(`${test.binary} ${test.name}`, test);
|
||
}
|
||
return [...chosen.values()];
|
||
}
|
||
|
||
// The repeat's cargo invocation per package: `-p <crate>` plus the target
|
||
// flags of its selected files, each whole flag once (`--test codec --test
|
||
// connection`; merging tokens instead gave `--test codec connection`, which
|
||
// nextest reads as a test-name filter that matches nothing), and the tests
|
||
// to list in those targets. Every entry must belong to one workspace, whose
|
||
// runner is returned with the groups.
|
||
export function rustGroups(entries) {
|
||
const groups = new Map();
|
||
const workspaces = new Set();
|
||
for (const entry of entries) {
|
||
const [, file, test] = /^([^:]+\.rs)::((?:\w+::)*\w+)$/.exec(entry) ?? [];
|
||
const place = file && rustFile(file);
|
||
if (!place)
|
||
throw new Error(`rust entry must be <file>::<module::test>: ${entry}`);
|
||
workspaces.add(place.runner);
|
||
const crate = cratePackage(file);
|
||
const group = groups.get(crate) ?? { targets: new Map(), tests: [] };
|
||
const targets = rustTarget(file);
|
||
if (targets.length === 0) group.every = true;
|
||
for (const target of targets) group.targets.set(target.join(" "), target);
|
||
group.tests.push({
|
||
path: test,
|
||
exact: [...rustModulePath(file), test].join("::"),
|
||
});
|
||
groups.set(crate, group);
|
||
}
|
||
if (workspaces.size > 1)
|
||
throw new Error(
|
||
`rust entries span workspaces: ${[...workspaces].join(", ")}`,
|
||
);
|
||
for (const [crate, group] of groups)
|
||
group.args = [
|
||
"-p",
|
||
crate,
|
||
...(group.every ? [] : [...group.targets.values()].flat()),
|
||
];
|
||
return { groups, runner: [...workspaces][0] };
|
||
}
|
||
|
||
// nextest has no repeat flag, so run the selection once per iteration under
|
||
// the `flake` profile (the workspace's .config/nextest.toml: no retries, JUnit
|
||
// per run), one cargo invocation per package so each builds only the targets
|
||
// its files can be in. The selection is listed first, by in-file path, and
|
||
// narrowed with chooseRust to exact (binary, test) pairs; a selection listing
|
||
// no runnable test is one of ignored or infrastructure-backed tests, not a
|
||
// failure. The Tauri workspace is addressed by its manifest; the root one is
|
||
// cargo's default.
|
||
function repeatRust(entries, iterations) {
|
||
const { groups, runner } = rustGroups(entries);
|
||
const manifest = rustWorkspaces[runner];
|
||
const cargoArgs = runner === "rust" ? [] : ["--manifest-path", manifest];
|
||
const runs = [];
|
||
for (const group of groups.values()) {
|
||
const listed = run("cargo", [
|
||
"nextest",
|
||
"list",
|
||
...cargoArgs,
|
||
...group.args,
|
||
"-E",
|
||
rustFilterset(
|
||
runner,
|
||
group.tests.map(({ path }) => path),
|
||
),
|
||
"--message-format",
|
||
"json",
|
||
]);
|
||
if (listed.status !== 0)
|
||
throw new Error(`cargo nextest list: ${listed.stderr}`);
|
||
const candidates = Object.entries(
|
||
JSON.parse(listed.stdout)["rust-suites"],
|
||
).flatMap(([binary, suite]) =>
|
||
Object.entries(suite.testcases)
|
||
.filter(
|
||
([, test]) =>
|
||
!test.ignored && test["filter-match"].status === "matches",
|
||
)
|
||
.map(([name]) => ({ binary, name })),
|
||
);
|
||
group.chosen = chooseRust(group.tests, candidates);
|
||
group.filterset = group.chosen
|
||
.map(({ binary, name }) => `(binary_id(=${binary}) & test(=${name}))`)
|
||
.join(" | ");
|
||
}
|
||
const expected = [...groups.values()].flatMap((group) =>
|
||
group.chosen.map(({ binary, name }) => `${binary} ${name}`),
|
||
);
|
||
if (expected.length === 0) {
|
||
summarize(
|
||
"## Flake check (Rust): nothing to repeat\n\nEvery selected test is ignored or infrastructure-backed.\n\n",
|
||
);
|
||
return;
|
||
}
|
||
// nextest keeps its store under the workspace root's target/nextest, not
|
||
// CARGO_TARGET_DIR; the `flake` profile writes its JUnit report there.
|
||
const junit = path.join(
|
||
root,
|
||
path.dirname(manifest),
|
||
"target/nextest/flake/junit.xml",
|
||
);
|
||
const dir = resultsDir(runner);
|
||
for (let i = 1; i <= iterations; i += 1) {
|
||
const statuses = new Map();
|
||
let ok = true;
|
||
for (const [crate, group] of groups) {
|
||
if (group.chosen.length === 0) continue;
|
||
rmSync(junit, { force: true });
|
||
const result = run(
|
||
"cargo",
|
||
[
|
||
"nextest",
|
||
"run",
|
||
...cargoArgs,
|
||
"--profile",
|
||
"flake",
|
||
...group.args,
|
||
"-E",
|
||
group.filterset,
|
||
],
|
||
{ stdio: "inherit" },
|
||
);
|
||
ok &&= result.status === 0;
|
||
try {
|
||
for (const [test, status] of parseJunit(readFileSync(junit, "utf8")))
|
||
statuses.set(test, status);
|
||
copyFileSync(junit, `${dir}/iter-${i}-${crate}.xml`);
|
||
} catch {
|
||
ok = false;
|
||
}
|
||
}
|
||
ok &&= expected.every((test) => statuses.get(test) === "passed");
|
||
runs.push({ ok, statuses });
|
||
progress(i, iterations, ok);
|
||
}
|
||
report(
|
||
runner === "rust" ? "Flake check (Rust)" : "Flake check (Desktop Tauri)",
|
||
runs,
|
||
);
|
||
}
|
||
|
||
// node:test has no repeat flag either. Each file runs alone, so a failure is
|
||
// attributed to its file; the regular lane's cross-file concurrency is not
|
||
// reproduced here.
|
||
function repeatNode(files, iterations) {
|
||
const dir = resultsDir("node");
|
||
const runs = [];
|
||
for (let i = 1; i <= iterations; i += 1) {
|
||
const statuses = new Map();
|
||
for (const file of files) {
|
||
const path = file.replace(/^desktop\//, "");
|
||
const result = run(process.execPath, [...nodeTestArgs(path), path], {
|
||
cwd: `${root}desktop`,
|
||
});
|
||
process.stdout.write(result.stdout);
|
||
process.stderr.write(result.stderr);
|
||
appendFileSync(
|
||
`${dir}/iter-${i}.log`,
|
||
`# ${file}\n${result.stdout}${result.stderr}\n`,
|
||
);
|
||
statuses.set(file, result.status === 0 ? "passed" : "failed");
|
||
}
|
||
const ok = [...statuses.values()].every((status) => status === "passed");
|
||
runs.push({ ok, statuses });
|
||
progress(i, iterations, ok);
|
||
}
|
||
report("Flake check (node:test)", runs);
|
||
}
|
||
|
||
// Per-test statuses of a `dart test` JSON report (one event per line), keyed
|
||
// `<file> › <name>`. Hidden entries are the runner's own file-loading steps.
|
||
export function parseDartReport(text, mobileRoot = `${root}mobile/`) {
|
||
const started = new Map();
|
||
const statuses = new Map();
|
||
for (const line of text.split("\n")) {
|
||
let event;
|
||
try {
|
||
event = JSON.parse(line);
|
||
} catch {
|
||
continue;
|
||
}
|
||
if (event.type === "testStart") started.set(event.test.id, event.test);
|
||
if (event.type !== "testDone" || event.hidden) continue;
|
||
const test = started.get(event.testID);
|
||
if (!test) continue;
|
||
const file = fileURLToPath(test.url ?? "file:///").replace(mobileRoot, "");
|
||
statuses.set(
|
||
`${file} › ${test.name}`,
|
||
event.skipped
|
||
? "skipped"
|
||
: event.result === "success"
|
||
? "passed"
|
||
: "failed",
|
||
);
|
||
}
|
||
return statuses;
|
||
}
|
||
|
||
// `flutter test` has no repeat flag; each iteration recompiles only the
|
||
// selected files, as the sharded mobile lane does.
|
||
function repeatDart(files, iterations) {
|
||
const dir = resultsDir("dart");
|
||
const runs = [];
|
||
// The Justfile unsets these so a worktree's git context never reaches flutter.
|
||
const { GIT_DIR, GIT_WORK_TREE, ...env } = process.env;
|
||
for (let i = 1; i <= iterations; i += 1) {
|
||
const reportFile = `${dir}/iter-${i}.json`;
|
||
const result = run(
|
||
"flutter",
|
||
[
|
||
"test",
|
||
dartDefine,
|
||
`--file-reporter=json:${reportFile}`,
|
||
...files.map((file) => file.replace(/^mobile\//, "")),
|
||
],
|
||
{ cwd: `${root}mobile`, env, stdio: "inherit" },
|
||
);
|
||
let statuses = new Map();
|
||
try {
|
||
statuses = parseDartReport(readFileSync(reportFile, "utf8"));
|
||
} catch {
|
||
statuses = new Map();
|
||
}
|
||
const ok = result.status === 0 && statuses.size > 0;
|
||
runs.push({ ok, statuses });
|
||
progress(i, iterations, ok);
|
||
}
|
||
report("Flake check (Flutter)", runs);
|
||
}
|
||
|
||
// Playwright's JSON report lists a `--repeat-each` journey once per
|
||
// repetition, in order; `unexpected` is a failed repetition since the lane
|
||
// runs with --retries=0.
|
||
export function playwrightRepetitions(report) {
|
||
const repetitions = new Map();
|
||
const collect = (suites) => {
|
||
for (const suite of suites) {
|
||
for (const spec of suite.specs ?? []) {
|
||
const key = `${spec.file}:${spec.line} › ${spec.title}`;
|
||
const statuses = repetitions.get(key) ?? [];
|
||
for (const test of spec.tests ?? [])
|
||
statuses.push(
|
||
{ expected: "passed", unexpected: "failed", flaky: "failed" }[
|
||
test.status
|
||
] ?? "skipped",
|
||
);
|
||
repetitions.set(key, statuses);
|
||
}
|
||
collect(suite.suites ?? []);
|
||
}
|
||
};
|
||
collect(report.suites ?? []);
|
||
return repetitions;
|
||
}
|
||
|
||
function playwrightSummary(reportPath, title) {
|
||
const repetitions = playwrightRepetitions(
|
||
JSON.parse(readFileSync(reportPath, "utf8")),
|
||
);
|
||
const iterations = Math.max(
|
||
0,
|
||
...[...repetitions.values()].map((s) => s.length),
|
||
);
|
||
const runs = Array.from({ length: iterations }, (_, i) => ({
|
||
ok: [...repetitions.values()].every((statuses) => statuses[i] !== "failed"),
|
||
statuses: new Map(
|
||
[...repetitions].map(([test, statuses]) => [test, statuses[i]]),
|
||
),
|
||
}));
|
||
report(title, runs);
|
||
// Playwright already failed its own step; the summary only reports.
|
||
process.exitCode = 0;
|
||
}
|
||
|
||
function main() {
|
||
const [command, ...args] = process.argv.slice(2);
|
||
if (command === "select" && args.length <= 2) return select(...args);
|
||
if (command === "playwright-summary" && args.length === 2)
|
||
return playwrightSummary(...args);
|
||
const repeat = {
|
||
rust: repeatRust,
|
||
tauri: repeatRust,
|
||
node: repeatNode,
|
||
dart: repeatDart,
|
||
}[command];
|
||
if (repeat) {
|
||
const iterations = Number(
|
||
args
|
||
.find((argument) => argument.startsWith("--iterations="))
|
||
?.slice("--iterations=".length) ?? ITERATIONS,
|
||
);
|
||
const entries = args.filter((argument) => !argument.startsWith("--"));
|
||
if (Number.isInteger(iterations) && iterations > 0 && entries.length > 0)
|
||
return repeat(entries, iterations);
|
||
}
|
||
throw new Error(
|
||
"usage: flake-check select [base] [head] | rust|tauri|node|dart [--iterations=N] <entry…> | playwright-summary <report.json> <title>",
|
||
);
|
||
}
|
||
|
||
if (process.argv[1] === fileURLToPath(import.meta.url)) main();
|