// Advisory pull-request flake check. `select` maps a diff to the tests it // changed, per runner; `rust`, `node` and `dart` repeat a selection and report // every iteration. Playwright repeats natively with --repeat-each, and // `playwright-summary` tabulates its JSON report afterwards. import { spawnSync } from "node:child_process"; import { appendFileSync, copyFileSync, existsSync, mkdirSync, readFileSync, rmSync, } from "node:fs"; import { createRequire } from "node:module"; import { fileURLToPath } from "node:url"; const root = fileURLToPath(new URL("../", import.meta.url)); export const ITERATIONS = 10; // Repetitions spread across shards, so a whole-file selection stays inside // the Playwright lane's 45-minute job timeout. Playwright shards at file × // repetition granularity (workers: 1, no fullyParallel). The regular smoke // lane runs ~150 journeys per shard in ~11 minutes, about 4.5 s each, so five // journeys × ten repetitions is a few minutes of test time per shard; it never // uses more than ten runners, so neither does this one. Past ten shards each // one grows instead, and the timeout minus dependency setup and the E2E build // (~10 minutes, more when apt retries) leaves room for about 250 journey-runs // (~19 minutes) per shard, so a selection beyond that many journeys skips the // lane with a note rather than timing out. const JOURNEYS_PER_SHARD = 5; const MAX_SHARDS = 10; const MAX_JOURNEY_RUNS_PER_SHARD = 250; export const playwrightShards = (journeys) => Math.min(MAX_SHARDS, Math.max(1, Math.ceil(journeys / JOURNEYS_PER_SHARD))); export const MAX_PLAYWRIGHT_JOURNEYS = Math.floor( (MAX_JOURNEY_RUNS_PER_SHARD * MAX_SHARDS) / ITERATIONS, ); // GitHub caps a job's outputs at 1 MiB in total. A selection this large, like // one past the journey cap, is a sweeping refactor rather than a test change // worth ten repetitions, so the lane skips rather than overrunning its timeout. export const MAX_OUTPUT_BYTES = 64 * 1024; export const runners = ["rust", "node", "playwright", "dart"]; // Mirrors desktop/package.json "test": plain helpers run under the TS loader; // jsdom tests also preload the DOM shim and force-exit past their timers. export const nodeTestArgs = (file) => file.endsWith(".jsdom-test.mjs") ? [ "--import", "./test-jsdom-setup.mjs", "--import", "./test-loader.mjs", "--experimental-strip-types", "--test-force-exit", "--test", ] : ["--import", "./test-loader.mjs", "--experimental-strip-types", "--test"]; // Mirrors the mobile test shards in .github/workflows/_ci-clients.yml. export const dartDefine = "--dart-define=BUZZ_PUSH_GATEWAY_URL=https://push.example"; // Mirrors the postgres-ci default-filter in .config/nextest.toml: those tests // are #[ignore]d and need Postgres, Redis, S3 or MinIO, so a selection that // reached one would fail every repetition without being flaky. const infraTests = "test(/postgres_tests::/) | binary(/^postgres_/) | test(/(^|::)external_infra[^:]*::/)"; // Mirrors desktop/package.json "test", desktop/playwright.config.ts testDir, // the root Cargo workspace's crates and mobile's test directory. Paths are // later split on whitespace in a shell, so admit only plain names. The // desktop `*.perf.ts` measurements are deliberately not specs here: they time // the app rather than assert on it, so a slow repetition is not a flake. export function classify(file) { if (!/^[\w./@-]+$/.test(file)) return undefined; if (/^crates\/[^/]+\/(?:src|tests)\/(?!fixtures\/).*\.rs$/.test(file)) return "rust"; if ( /^desktop\/(?:src\/.*\.(?:test|jsdom-test)\.mjs|scripts\/[^/]+\.test\.mjs)$/.test( file, ) ) return "node"; if (/^desktop\/tests\/e2e\/[^/]+\.spec\.ts$/.test(file)) return "playwright"; if (/^mobile\/test\/.*_test\.dart$/.test(file)) return "dart"; return undefined; } // Line ranges of a -U0 diff: the new-side lines each hunk writes and the // old-side lines it removes (empty, end < start, for a pure insertion). A pure // deletion writes no lines and attaches to the line before it (0 at the top of // the file); `removed` keeps what it took out, since that line cannot tell. export function hunkRanges(diff) { return [ ...diff.matchAll(/^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@/gm), ].map(([, oldStart, oldCount = "1", start, count = "1"]) => ({ start: Number(start), end: Number(start) + Math.max(Number(count), 1) - 1, removed: { start: Number(oldStart), end: Number(oldStart) + Number(oldCount) - 1, }, })); } // Members that keep a `test`/`it` chain a test declaration. Anything else on // the chain is a group (`describe`), a hook (`test.beforeEach`), configuration // (`test.use`) or a harness object that happens to be named `test`. const declaring = [ "only", "skip", "fixme", "fail", "fails", "todo", "concurrent", "sequential", "each", "for", "runIf", "skipIf", ]; // The desktop app's TypeScript parses the specs; load it only for a selection // that needs it, so Rust-only runs work without desktop dependencies. let typescript; const ts = () => (typescript ??= createRequire( new URL("../desktop/package.json", import.meta.url), )("typescript")); // Line spans of every test declaration in a TypeScript spec, from its first // line to its closing parenthesis. export function testCalls(source, file) { const t = ts(); const sourceFile = t.createSourceFile( file, source, t.ScriptTarget.Latest, true, ); const lineAt = (position) => sourceFile.getLineAndCharacterOfPosition(position).line + 1; const names = (callee) => t.isIdentifier(callee) ? [callee.text] : t.isPropertyAccessExpression(callee) ? names(callee.expression)?.concat(callee.name.text) : t.isCallExpression(callee) ? names(callee.expression) : undefined; const calls = []; const visit = (node) => { const chain = t.isCallExpression(node) ? names(node.expression) : undefined; if ( chain && ["test", "it"].includes(chain[0]) && chain.slice(1).every((member) => declaring.includes(member)) ) { calls.push({ start: lineAt(node.getStart(sourceFile)), end: lineAt(node.end - 1), }); return; } t.forEachChild(node, visit); }; visit(sourceFile); return calls; } // Rust source with every comment and literal blanked to spaces, newlines kept, // so attribute and brace scanning never see a `{` or `#[` inside a string, // char literal or comment. Lifetimes (`'a`) are kept; char literals are not. export function blankRust(source) { let out = ""; let i = 0; const blank = (end) => { for (; i < end && i < source.length; i += 1) out += source[i] === "\n" ? "\n" : " "; }; const closingQuote = (from, raw = false, hashes = "") => { for (let j = from; j < source.length; j += 1) { if (source[j] === "\\" && !raw) j += 1; else if (source[j] === '"' && source.startsWith(hashes, j + 1)) return j + 1 + hashes.length; } return source.length; }; const rawPrefix = /[bc]?r(#*)"/y; while (i < source.length) { const rest = source.slice(i, i + 2); rawPrefix.lastIndex = i; const raw = rawPrefix.exec(source); if (rest.startsWith("//")) { const end = source.indexOf("\n", i); blank(end === -1 ? source.length : end); } else if (rest.startsWith("/*")) { let depth = 0; let j = i; do { if (source.startsWith("/*", j)) { depth += 1; j += 2; } else if (source.startsWith("*/", j)) { depth -= 1; j += 2; } else j += 1; } while (depth > 0 && j < source.length); blank(j); } else if (raw) { blank(closingQuote(i + raw[0].length, true, raw[1])); } else if (/^[bc]?"/.test(rest)) { blank(closingQuote(i + rest.indexOf('"') + 1)); } else if (source[i] === "'") { const width = source.codePointAt(i + 1) > 0xffff ? 2 : 1; if (source[i + 1] === "\\") blank(source.indexOf("'", i + 2) + 1); else if (source[i + 1 + width] === "'") blank(i + width + 2); else { out += source[i]; i += 1; } } else { out += source[i]; i += 1; } } return out; } const matching = (text, open, close, from) => { let depth = 0; for (let j = from; j < text.length; j += 1) { if (text[j] === open) depth += 1; else if (text[j] === close) { depth -= 1; if (depth === 0) return j; } } return text.length - 1; }; // Every test function in a Rust file: a `fn` whose attribute block has an // attribute path ending in `test` (`#[test]`, `#[tokio::test]`, …), spanning // the first attribute line to the body's closing brace. `name` is the path // within the file through its inline modules (`tests::plain`), so a namesake // in another module stays a different test. `ignored` marks `#[ignore]`, // which nextest skips by default. export function rustTests(source) { const text = blankRust(source); const modules = []; const inline = /\bmod\s+(\w+)\s*\{/g; for (let m = inline.exec(text); m; m = inline.exec(text)) { const open = m.index + m[0].length - 1; modules.push({ name: m[1], open, close: matching(text, "{", "}", open) }); } const lineStarts = [0]; for (let i = 0; i < text.length; i += 1) if (text[i] === "\n") lineStarts.push(i + 1); const lineAt = (offset) => { let low = 0; let high = lineStarts.length - 1; while (low < high) { const mid = Math.ceil((low + high) / 2); if (lineStarts[mid] <= offset) low = mid; else high = mid - 1; } return low + 1; }; const tests = []; const attribute = /#\[/g; for (let match = attribute.exec(text); match; match = attribute.exec(text)) { const paths = []; let j = match.index; while (text.startsWith("#[", j)) { const close = matching(text, "[", "]", j + 1); paths.push(/^\s*([\w:]+)/.exec(text.slice(j + 2, close))?.[1] ?? ""); j = close + 1; while (/\s/.test(text[j] ?? "")) j += 1; } attribute.lastIndex = j; if (!paths.some((path) => /^(?:\w+::)*test$/.test(path))) continue; const fn = /^(?:pub(?:\([\w\s:]*\))?\s+)?(?:async\s+)?(?:unsafe\s+)?fn\s+(\w+)/.exec( text.slice(j), ); if (!fn) continue; const open = text.indexOf("{", j + fn[0].length); if (open === -1) continue; const path = modules .filter((mod) => mod.open < match.index && match.index < mod.close) .map((mod) => mod.name); tests.push({ name: [...path, fn[1]].join("::"), start: lineAt(match.index), end: lineAt(matching(text, "{", "}", open)), ignored: paths.includes("ignore"), }); } return tests; } // Bodies of the listed tests: their calls, keyed by the line the runner // listed. Runners only report a call-site line, which may sit inside the call, // so each listed line is owned by the enclosing test call; a listed line // outside every call leaves its lines to the whole-file rule. export function testBodies(calls, lines) { const bodies = new Map(); for (const line of [...lines].sort((a, b) => a - b)) { const call = calls.find(({ start, end }) => start <= line && line <= end); if (call && !bodies.has(call)) bodies.set(call, { ...call, line }); } return [...bodies.values()].sort((a, b) => a.start - b.start); } // The calls owning every line of a range, or undefined once a line lies // outside them all. function owners(calls, { start, end }) { const found = []; for (let line = start; line <= end; ) { const call = calls.find((c) => c.start <= line && line <= c.end); if (!call) return undefined; found.push(call); line = call.end + 1; } return found; } // A hunk whose every line lies in a test body selects those tests. A changed // line outside every body (imports, helpers, hooks or constants before, // between or after tests, or the code under test in a Rust module) can change // every test, so it, a new file, or a file with no listed tests selects the // whole file. Removed lines are judged by the old file's calls (`previous`): a // deletion attaches to its preceding line, which belongs to the test before a // removed hook, not to the hook. export function selectTests( ranges, bodies, { added = false, previous = [] } = {}, ) { if (added) return { whole: true, reason: "new file" }; if (bodies.length === 0) return { whole: true, reason: "no tests listed" }; const selected = new Set(); for (const { start, end, removed } of ranges) { if (!owners(previous, removed)) return { whole: true, reason: "shared code removed" }; const current = owners(bodies, { start, end }); if (!current) return { whole: true, reason: "shared code changed" }; for (const body of current) selected.add(body.line); } return { lines: [...selected].sort((a, b) => a - b) }; } // Rust tests a diff selects: hunks inside a test body select that test, any // other hunk selects every test in the file (the module's code under test // lives beside them), and `#[ignore]`d tests own their lines but never run. // A file without test functions is not a test file and selects nothing. export function selectRust( diff, source, { added = false, previous = "" } = {}, ) { const tests = rustTests(source); if (tests.length === 0) return undefined; const selection = selectTests( hunkRanges(diff), tests.map((test) => ({ ...test, line: test.start })), { added, previous: added ? [] : rustTests(previous) }, ); const chosen = selection.whole ? tests : tests.filter((test) => selection.lines.includes(test.start)); return { ...selection, names: chosen.filter((test) => !test.ignored).map((test) => test.name), }; } const run = (command, args, options = {}) => { const result = spawnSync(command, args, { cwd: root, encoding: "utf8", maxBuffer: 64 * 1024 * 1024, ...options, }); if (result.error) throw result.error; return result; }; const git = (...args) => { const result = run("git", args); if (result.status !== 0) throw new Error(`git ${args[0]}: ${result.stderr}`); return result.stdout; }; const summarize = (text) => { if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, text); else process.stdout.write(text); }; const cell = (value) => String(value).replaceAll("|", "\\|"); const crateDir = (file) => `${root}crates/${file.split("/")[1]}`; // The `[package]` name of the crate owning a file under crates/. export function cratePackage(file) { const manifest = readFileSync(`${crateDir(file)}/Cargo.toml`, "utf8"); return /^\[package\][\s\S]*?^name\s*=\s*"([^"]+)"/m.exec(manifest)[1]; } // Cargo target flags building only the test binaries a file can be part of: // src/** is the lib (when the crate has one) or a bin, tests/.rs and // tests//*.rs (with a main.rs) the integration test of that name. A // file placed by a custom `[[test]]` path builds every target instead. export function rustTarget(file) { const [, , kind, first, second] = file.split("/"); if (kind === "src") { const lib = existsSync(`${crateDir(file)}/src/lib.rs`) || /^\[lib\]/m.test(readFileSync(`${crateDir(file)}/Cargo.toml`, "utf8")); return lib ? ["--lib", "--bins"] : ["--bins"]; } if (second === undefined) return ["--test", first.replace(/\.rs$/, "")]; return existsSync(`${crateDir(file)}/tests/${first}/main.rs`) ? ["--test", first] : []; } // The module path a file's items carry in their binary under cargo's layout // conventions: src/a/b.rs is a::b, a mod.rs/main.rs/lib.rs names its directory // or the root, src/bin/x.rs and tests/.rs are roots, tests//x.rs // is x. A `#[path]` attribute can break this; repeatRust then falls back to the // in-file path alone. export function rustModulePath(file) { const [, , kind, ...parts] = file.replace(/\.rs$/, "").split("/"); if (kind === "tests") parts.shift(); else if (parts[0] === "bin") parts.splice(0, 2); if (["mod", "main"].includes(parts.at(-1)) || parts.join() === "lib") parts.pop(); return parts; } // Smoke journeys per spec file as Playwright lists them, each line once (a // looped declaration is listed once per iteration). Integration-only specs // list nothing for the smoke project, which Playwright reports as its "No // tests found" error, so a selection of them stays empty. export function listPlaywright(files) { const result = run(`${root}bin/pnpm`, [ "-C", "desktop", "--silent", "exec", "playwright", "test", "--project=smoke", "--list", "--reporter=json", ...files, ]); const report = JSON.parse(result.stdout); const empty = report.errors.length === 1 && /No tests found/.test(report.errors[0].message); if ((result.status !== 0 || report.errors.length) && !empty) throw new Error( `playwright --list: ${report.errors.map((error) => error.message).join("\n") || result.stderr}`, ); const specs = new Map(files.map((file) => [file, new Set()])); const collect = (suites) => { for (const suite of suites) { for (const spec of suite.specs ?? []) if (spec.tests?.length) specs.get(`desktop/tests/e2e/${spec.file}`)?.add(spec.line); collect(suite.suites ?? []); } }; collect(empty ? [] : report.suites); return new Map([...specs].map(([file, lines]) => [file, [...lines]])); } function select(base = "HEAD^1", head = "HEAD") { const changed = git( "diff", "--name-status", "--no-renames", "-z", "--diff-filter=AM", base, head, "--", ).split("\0"); const files = Object.fromEntries(runners.map((runner) => [runner, []])); const added = new Set(); for (let i = 0; i + 1 < changed.length; i += 2) { const [status, file] = [changed[i], changed[i + 1]]; const runner = classify(file); if (runner) files[runner].push(file); if (status === "A") added.add(file); } const journeys = files.playwright.length ? listPlaywright(files.playwright) : new Map(); const outputs = Object.fromEntries(runners.map((runner) => [runner, []])); const rows = []; const previous = (file) => added.has(file) ? "" : git("show", `${base}:${file}`); const diff = (file) => git("diff", "-U0", "--no-color", base, head, "--", file); for (const runner of runners) { for (const file of files[runner]) { let entries = [file]; let note = "whole file"; if (runner === "rust") { const selection = selectRust( diff(file), readFileSync(`${root}${file}`, "utf8"), { added: added.has(file), previous: previous(file), }, ); if (!selection) { rows.push(`| ${cell(file)} | ${runner} | no test functions |`); continue; } entries = selection.names.map((name) => `${file}::${name}`); note = selection.names.length === 0 ? "excluded (ignored tests only)" : selection.whole ? `whole file (${selection.reason}): ${selection.names.length} test(s)` : `${selection.names.length} changed test(s)`; } else if (runner === "playwright") { const listed = journeys.get(file) ?? []; const source = readFileSync(`${root}${file}`, "utf8"); const selection = selectTests( hunkRanges(diff(file)), testBodies(testCalls(source, file), listed), { added: added.has(file), previous: added.has(file) ? [] : testCalls(previous(file), file), }, ); const lines = selection.whole ? listed : selection.lines; entries = lines.map((line) => `${file}:${line}`); note = lines.length === 0 ? "excluded (no smoke journeys)" : selection.whole ? `whole file (${selection.reason}): ${lines.length} journey(s)` : `${lines.length} changed journey(s)`; } outputs[runner].push(...entries); rows.push(`| ${cell(file)} | ${runner} | ${cell(note)} |`); } const exceeded = Buffer.byteLength(outputs[runner].join(" ")) > MAX_OUTPUT_BYTES ? `selection exceeds ${MAX_OUTPUT_BYTES} bytes` : runner === "playwright" && outputs[runner].length > MAX_PLAYWRIGHT_JOURNEYS ? `selection exceeds ${MAX_PLAYWRIGHT_JOURNEYS} journeys, which ${ITERATIONS} repetitions would not finish in time` : undefined; if (exceeded) { rows.push( `| *(${outputs[runner].length} ${runner} entries)* | ${runner} | ${exceeded}; lane skipped |`, ); outputs[runner] = []; } } summarize( [ "## Flake check selection", "", ...(rows.length ? ["| File | Runner | Selection |", "| --- | --- | --- |", ...rows] : ["No Rust, node:test, Playwright or Flutter test files changed."]), "", "", ].join("\n"), ); const shards = playwrightShards(outputs.playwright.length); const text = [ ...Object.entries(outputs).map( ([key, entries]) => `${key}=${entries.join(" ")}\n`, ), `playwright_shards=${JSON.stringify(Array.from({ length: shards }, (_, i) => i + 1))}\n`, `playwright_shard_total=${shards}\n`, ].join(""); if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, text); else process.stdout.write(text); } const resultsDir = (runner) => { const dir = `${root}test-results/flake/${runner}`; rmSync(dir, { recursive: true, force: true }); mkdirSync(dir, { recursive: true }); return dir; }; // Test × iteration table for a repeated selection; a run passes only when its // process exits 0 and every expected test reports. Tests that never ran in any // iteration are left out, so the table holds only the repeated ones. function report(title, runs) { const columns = runs.map((_, i) => String(i + 1)); const mark = (status) => ({ passed: "✅", failed: "❌" })[status] ?? (status ? "⏭" : "—"); const ran = (status) => status === "passed" || status === "failed"; const tests = [ ...new Set(runs.flatMap((run) => [...run.statuses.keys()])), ].filter((test) => runs.some((run) => ran(run.statuses.get(test)))); const failures = runs.filter((run) => !run.ok).length; summarize( [ `## ${title}: ${failures} of ${runs.length} iterations failed`, "", `| Test | ${columns.join(" | ")} | Failed |`, `| --- | ${columns.map(() => ":-:").join(" | ")} | ---: |`, `| **Run** | ${runs.map((run) => (run.ok ? "✅" : "❌")).join(" | ")} | ${failures}/${runs.length} |`, ...tests.sort().map((test) => { const statuses = runs.map((run) => run.statuses.get(test)); return `| ${cell(test)} | ${statuses.map(mark).join(" | ")} | ${statuses.filter((status) => status === "failed").length}/${runs.length} |`; }), "", "", ].join("\n"), ); process.exitCode = failures === 0 ? 0 : 1; } const progress = (i, iterations, ok) => console.log( `flake-check: iteration ${i}/${iterations} ${ok ? "passed" : "FAILED"}`, ); // Per-test statuses of a JUnit report, keyed ` `. export function parseJunit(xml) { const decode = (text) => text.replace( /&(quot|apos|lt|gt|amp);/g, (_, entity) => ({ quot: '"', apos: "'", lt: "<", gt: ">", amp: "&" })[entity], ); const statuses = new Map(); for (const chunk of xml.split("")); const body = chunk.split("")[0]; const name = decode(/\bname="([^"]*)"/.exec(attributes)[1]); const classname = decode( /\bclassname="([^"]*)"/.exec(attributes)?.[1] ?? "", ); statuses.set( `${classname} ${name}`, /<(?:failure|error)[\s/>]/.test(body) ? "failed" : /]/.test(body) ? "skipped" : "passed", ); } return statuses; } // The listed tests an entry stands for: a test the file's module path predicts // exactly when nextest lists one, otherwise every test whose name ends in the // in-file path (a `#[path]` module breaks the prediction; a namesake in another // module is then repeated too rather than nothing). export function chooseRust(tests, candidates) { const chosen = new Map(); for (const { path, exact } of tests) { const suffix = candidates.filter( ({ name }) => name === path || name.endsWith(`::${path}`), ); const predicted = suffix.filter(({ name }) => name === exact); for (const test of predicted.length ? predicted : suffix) chosen.set(`${test.binary} ${test.name}`, test); } return [...chosen.values()]; } // nextest has no repeat flag, so run the selection once per iteration under // the `flake` profile (.config/nextest.toml: no retries, JUnit per run), one // cargo invocation per package so each builds only the targets its files can // be in. The selection is listed first, by in-file path, and narrowed with // chooseRust to exact (binary, test) pairs; a selection listing no runnable // test is one of ignored or infrastructure-backed tests, not a failure. function repeatRust(entries, iterations) { const groups = new Map(); for (const entry of entries) { const [, file, path] = /^(crates\/[^:]+\.rs)::((?:\w+::)*\w+)$/.exec(entry) ?? []; if (!file) throw new Error(`rust entry must be ::: ${entry}`); const crate = cratePackage(file); const group = groups.get(crate) ?? { targets: new Set(), tests: [] }; const targets = rustTarget(file); if (targets.length === 0) group.every = true; for (const target of targets) group.targets.add(target); group.tests.push({ path, exact: [...rustModulePath(file), path].join("::"), }); groups.set(crate, group); } const runs = []; for (const [crate, group] of groups) { group.args = ["-p", crate, ...(group.every ? [] : group.targets)]; const suffixes = [...new Set(group.tests.map(({ path }) => path))] .map((path) => `test(/(^|::)${path}$/)`) .join(" | "); const listed = run("cargo", [ "nextest", "list", ...group.args, "-E", `(${suffixes}) & !(${infraTests})`, "--message-format", "json", ]); if (listed.status !== 0) throw new Error(`cargo nextest list: ${listed.stderr}`); const candidates = Object.entries( JSON.parse(listed.stdout)["rust-suites"], ).flatMap(([binary, suite]) => Object.entries(suite.testcases) .filter( ([, test]) => !test.ignored && test["filter-match"].status === "matches", ) .map(([name]) => ({ binary, name })), ); group.chosen = chooseRust(group.tests, candidates); group.filterset = group.chosen .map(({ binary, name }) => `(binary_id(=${binary}) & test(=${name}))`) .join(" | "); } const expected = [...groups.values()].flatMap((group) => group.chosen.map(({ binary, name }) => `${binary} ${name}`), ); if (expected.length === 0) { summarize( "## Flake check (Rust): nothing to repeat\n\nEvery selected test is ignored or infrastructure-backed.\n\n", ); return; } // nextest keeps its store under the workspace root's target/nextest, not // CARGO_TARGET_DIR; the `flake` profile writes its JUnit report there. const junit = `${root}target/nextest/flake/junit.xml`; const dir = resultsDir("rust"); for (let i = 1; i <= iterations; i += 1) { const statuses = new Map(); let ok = true; for (const [crate, group] of groups) { if (group.chosen.length === 0) continue; rmSync(junit, { force: true }); const result = run( "cargo", [ "nextest", "run", "--profile", "flake", ...group.args, "-E", group.filterset, ], { stdio: "inherit" }, ); ok &&= result.status === 0; try { for (const [test, status] of parseJunit(readFileSync(junit, "utf8"))) statuses.set(test, status); copyFileSync(junit, `${dir}/iter-${i}-${crate}.xml`); } catch { ok = false; } } ok &&= expected.every((test) => statuses.get(test) === "passed"); runs.push({ ok, statuses }); progress(i, iterations, ok); } report("Flake check (Rust)", runs); } // node:test has no repeat flag either. Each file runs alone, so a failure is // attributed to its file; the regular lane's cross-file concurrency is not // reproduced here. function repeatNode(files, iterations) { const dir = resultsDir("node"); const runs = []; for (let i = 1; i <= iterations; i += 1) { const statuses = new Map(); for (const file of files) { const path = file.replace(/^desktop\//, ""); const result = run(process.execPath, [...nodeTestArgs(path), path], { cwd: `${root}desktop`, }); process.stdout.write(result.stdout); process.stderr.write(result.stderr); appendFileSync( `${dir}/iter-${i}.log`, `# ${file}\n${result.stdout}${result.stderr}\n`, ); statuses.set(file, result.status === 0 ? "passed" : "failed"); } const ok = [...statuses.values()].every((status) => status === "passed"); runs.push({ ok, statuses }); progress(i, iterations, ok); } report("Flake check (node:test)", runs); } // Per-test statuses of a `dart test` JSON report (one event per line), keyed // ` › `. Hidden entries are the runner's own file-loading steps. export function parseDartReport(text, mobileRoot = `${root}mobile/`) { const started = new Map(); const statuses = new Map(); for (const line of text.split("\n")) { let event; try { event = JSON.parse(line); } catch { continue; } if (event.type === "testStart") started.set(event.test.id, event.test); if (event.type !== "testDone" || event.hidden) continue; const test = started.get(event.testID); if (!test) continue; const file = fileURLToPath(test.url ?? "file:///").replace(mobileRoot, ""); statuses.set( `${file} › ${test.name}`, event.skipped ? "skipped" : event.result === "success" ? "passed" : "failed", ); } return statuses; } // `flutter test` has no repeat flag; each iteration recompiles only the // selected files, as the sharded mobile lane does. function repeatDart(files, iterations) { const dir = resultsDir("dart"); const runs = []; // The Justfile unsets these so a worktree's git context never reaches flutter. const { GIT_DIR, GIT_WORK_TREE, ...env } = process.env; for (let i = 1; i <= iterations; i += 1) { const reportFile = `${dir}/iter-${i}.json`; const result = run( "flutter", [ "test", dartDefine, `--file-reporter=json:${reportFile}`, ...files.map((file) => file.replace(/^mobile\//, "")), ], { cwd: `${root}mobile`, env, stdio: "inherit" }, ); let statuses = new Map(); try { statuses = parseDartReport(readFileSync(reportFile, "utf8")); } catch { statuses = new Map(); } const ok = result.status === 0 && statuses.size > 0; runs.push({ ok, statuses }); progress(i, iterations, ok); } report("Flake check (Flutter)", runs); } // Playwright's JSON report lists a `--repeat-each` journey once per // repetition, in order; `unexpected` is a failed repetition since the lane // runs with --retries=0. export function playwrightRepetitions(report) { const repetitions = new Map(); const collect = (suites) => { for (const suite of suites) { for (const spec of suite.specs ?? []) { const key = `${spec.file}:${spec.line} › ${spec.title}`; const statuses = repetitions.get(key) ?? []; for (const test of spec.tests ?? []) statuses.push( { expected: "passed", unexpected: "failed", flaky: "failed" }[ test.status ] ?? "skipped", ); repetitions.set(key, statuses); } collect(suite.suites ?? []); } }; collect(report.suites ?? []); return repetitions; } function playwrightSummary(reportPath, title) { const repetitions = playwrightRepetitions( JSON.parse(readFileSync(reportPath, "utf8")), ); const iterations = Math.max( 0, ...[...repetitions.values()].map((s) => s.length), ); const runs = Array.from({ length: iterations }, (_, i) => ({ ok: [...repetitions.values()].every((statuses) => statuses[i] !== "failed"), statuses: new Map( [...repetitions].map(([test, statuses]) => [test, statuses[i]]), ), })); report(title, runs); // Playwright already failed its own step; the summary only reports. process.exitCode = 0; } function main() { const [command, ...args] = process.argv.slice(2); if (command === "select" && args.length <= 2) return select(...args); if (command === "playwright-summary" && args.length === 2) return playwrightSummary(...args); const repeat = { rust: repeatRust, node: repeatNode, dart: repeatDart }[ command ]; if (repeat) { const iterations = Number( args .find((argument) => argument.startsWith("--iterations=")) ?.slice("--iterations=".length) ?? ITERATIONS, ); const entries = args.filter((argument) => !argument.startsWith("--")); if (Number.isInteger(iterations) && iterations > 0 && entries.length > 0) return repeat(entries, iterations); } throw new Error( "usage: flake-check select [base] [head] | rust|node|dart [--iterations=N] | playwright-summary ", ); } if (process.argv[1] === fileURLToPath(import.meta.url)) main();