mirror of
https://github.com/block/buzz.git
synced 2026-10-28 21:11:48 +00:00
f961cade65
Add an advisory flake check that selects the tests a pull request changes and repeats each one ten times with retries off, so a test that passes CI by luck is caught before it merges. This mirrors block/buzz-app#890 (and its sharding follow-up), adapted to the runners this repository actually has. scripts/flake-check.mjs `select` diffs HEAD^1..HEAD and maps hunks to tests per runner: Rust files to `<package>::<fn>` entries by parsing `#[test]`-style attributes (shared code changes select the file's tests; `#[ignore]`d and Postgres tests are excluded), desktop Playwright specs to `file:line` smoke journeys via `playwright --list` (non-smoke and unregistered specs are excluded), and desktop node:test and Flutter test files whole. Deleted and renamed-away files drop out of the selection, and any runner whose selection exceeds 64 KiB is dropped with a note so GitHub outputs stay within limits. The reusable _ci-flake.yml adds `flake-select` plus one repeat lane per runner (`flake-rust`, `flake-node`, `flake-playwright` sharded by journey count, `flake-dart`). Each lane skips when its selection is empty, runs only on pull_request, and is not read by any required gate; its status and step summary carry the result. A `flake` nextest profile disables retries and emits JUnit so the Rust lane can report per-test, per-iteration results. The Unit Tests rust-cache gains a `shared-key` so the flake lane can restore it read-only. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Signed-off-by: Matt Toohey <contact@matttoohey.com>
455 lines
15 KiB
JavaScript
455 lines
15 KiB
JavaScript
import assert from "node:assert/strict";
|
||
import { readFileSync } from "node:fs";
|
||
import test from "node:test";
|
||
import {
|
||
classify,
|
||
cratePackage,
|
||
dartDefine,
|
||
hunkRanges,
|
||
listPlaywright,
|
||
nodeTestArgs,
|
||
parseDartReport,
|
||
parseJunit,
|
||
playwrightRepetitions,
|
||
playwrightShards,
|
||
rustTests,
|
||
selectRust,
|
||
selectTests,
|
||
testBodies,
|
||
testCalls,
|
||
} from "./flake-check.mjs";
|
||
|
||
const read = (path) => readFileSync(new URL(path, import.meta.url), "utf8");
|
||
// A hunk writing new-side lines start..end; by default it removed nothing.
|
||
const hunk = (start, end, removed = { start: start + 1, end: start }) => ({
|
||
start,
|
||
end,
|
||
removed,
|
||
});
|
||
// A -U0 diff of (old start, old count, new start, new count) hunks.
|
||
const diff = (...hunks) =>
|
||
hunks.map(([o, oc, n, nc]) => `@@ -${o},${oc} +${n},${nc} @@`).join("\n");
|
||
|
||
test("classification follows each runner's test file layout", () => {
|
||
for (const [file, runner] of [
|
||
["crates/buzz-core/src/filter.rs", "rust"],
|
||
["crates/buzz-test-client/tests/e2e_relay.rs", "rust"],
|
||
["desktop/src/features/channels/lib/channelDescription.test.mjs", "node"],
|
||
["desktop/src/shared/ui/thing.jsdom-test.mjs", "node"],
|
||
["desktop/scripts/demo-build-config.test.mjs", "node"],
|
||
["desktop/tests/e2e/channels.spec.ts", "playwright"],
|
||
["mobile/test/features/invites/invite_join_provider_test.dart", "dart"],
|
||
])
|
||
assert.equal(classify(file), runner, file);
|
||
for (const file of [
|
||
"crates/buzz-core/Cargo.toml",
|
||
"crates/buzz-feature-flags/tests/fixtures/relay-feature-selection/src/main.rs",
|
||
"desktop/src-tauri/src/lib.rs",
|
||
"desktop/src/features/channels/lib/channelDescription.ts",
|
||
"desktop/tests/e2e/helpers/animations.ts",
|
||
"desktop/tests/e2e/typing-latency.perf.ts",
|
||
"desktop/tests/e2e/fixtures/x.spec.ts",
|
||
"mobile/lib/main.dart",
|
||
"mobile/test/helpers.dart",
|
||
"scripts/flake-check.test.mjs",
|
||
"web/src/x.test.mjs",
|
||
// Selections are shell-split downstream; refuse unusual names.
|
||
"desktop/tests/e2e/odd name.spec.ts",
|
||
"crates/buzz-core/src/$(id).rs",
|
||
])
|
||
assert.equal(classify(file), undefined, file);
|
||
});
|
||
|
||
test("-U0 hunks map to new-side ranges, with deletions attached to the preceding line and their old-side lines kept", () => {
|
||
const text = [
|
||
"diff --git a/x.spec.ts b/x.spec.ts",
|
||
"--- a/x.spec.ts",
|
||
"+++ b/x.spec.ts",
|
||
"@@ -1 +1 @@",
|
||
'-import { test } from "@playwright/test";',
|
||
'+import { expect, test } from "@playwright/test";',
|
||
"@@ -12,0 +13,3 @@ test('a', async () => {",
|
||
"+ const x = 1;",
|
||
"+ const y = 2;",
|
||
"+ expect(x).toBe(y - 1);",
|
||
"@@ -30,2 +33,0 @@ test('b', async () => {",
|
||
"- removed();",
|
||
"- alsoRemoved();",
|
||
"@@ -1,3 +0,0 @@",
|
||
"-gone",
|
||
"",
|
||
].join("\n");
|
||
assert.deepEqual(hunkRanges(text), [
|
||
{ start: 1, end: 1, removed: { start: 1, end: 1 } },
|
||
{ start: 13, end: 15, removed: { start: 12, end: 11 } },
|
||
{ start: 33, end: 33, removed: { start: 30, end: 31 } },
|
||
{ start: 0, end: 0, removed: { start: 1, end: 3 } },
|
||
]);
|
||
assert.deepEqual(hunkRanges(""), []);
|
||
});
|
||
|
||
test("Playwright test calls span the declaration; groups, hooks and configuration are outside", () => {
|
||
const spec = [
|
||
'import { expect, test } from "@playwright/test";',
|
||
"test.use({ viewport: { width: 800, height: 600 } });",
|
||
"const seed = 1;",
|
||
'test.describe("suite", () => {',
|
||
" test.beforeEach(async ({ page }) => {",
|
||
' await page.goto("/");',
|
||
" });",
|
||
' test("journey", async ({ page }) => {',
|
||
' test.skip(seed === 2, "not here");',
|
||
' await expect(page.getByText("Go")).toBeVisible();',
|
||
" });",
|
||
' for (const mode of ["mouse", "keyboard"]) {',
|
||
' test.fixme(mode + " later", async () => {',
|
||
" await Promise.resolve();",
|
||
" });",
|
||
" }",
|
||
"});",
|
||
'test.describe.configure({ mode: "serial" });',
|
||
"",
|
||
].join("\n");
|
||
const calls = testCalls(spec, "x.spec.ts");
|
||
assert.deepEqual(calls, [
|
||
{ start: 8, end: 11 },
|
||
{ start: 13, end: 15 },
|
||
]);
|
||
// Playwright lists a looped declaration once per iteration, at one line.
|
||
assert.deepEqual(testBodies(calls, [13, 13, 8]), [
|
||
{ start: 8, end: 11, line: 8 },
|
||
{ start: 13, end: 15, line: 13 },
|
||
]);
|
||
// A listed line outside every call (a hook) has no body.
|
||
assert.deepEqual(testBodies(calls, [5]), []);
|
||
});
|
||
|
||
const rustSource = [
|
||
"//! Module docs mentioning #[test] in prose.",
|
||
"use std::fmt;",
|
||
"",
|
||
"pub fn code_under_test(input: &str) -> String {",
|
||
" let brace = '{';",
|
||
' let raw = r"\\";',
|
||
' format!("{brace}{raw}{input}")',
|
||
"}",
|
||
"",
|
||
"#[cfg(test)]",
|
||
"mod tests {",
|
||
" use super::*;",
|
||
"",
|
||
" fn helper() -> &'static str {",
|
||
' "}"',
|
||
" }",
|
||
"",
|
||
" /// Docs for the first test.",
|
||
" #[test]",
|
||
" fn plain() {",
|
||
' assert_eq!(code_under_test("x"), "{\\\\x");',
|
||
" }",
|
||
"",
|
||
' #[tokio::test(flavor = "multi_thread")]',
|
||
' #[ignore = "requires Postgres }"]',
|
||
" async fn needs_infra() {",
|
||
" /* a block comment with a } brace",
|
||
" and #[test] inside */",
|
||
" helper();",
|
||
" }",
|
||
"",
|
||
" #[ignore]",
|
||
" #[test]",
|
||
" pub(crate) fn also_ignored() {}",
|
||
"",
|
||
" mod postgres_tests {",
|
||
" #[sqlx::test]",
|
||
" async fn in_nested_module(pool: Pool) -> Result<(), E> {",
|
||
" Ok(())",
|
||
" }",
|
||
" }",
|
||
"}",
|
||
"",
|
||
].join("\n");
|
||
|
||
test("Rust test functions span their attribute block to the closing brace, through strings, chars and comments", () => {
|
||
assert.deepEqual(rustTests(rustSource), [
|
||
{ name: "plain", start: 19, end: 22, ignored: false },
|
||
{ name: "needs_infra", start: 24, end: 30, ignored: true },
|
||
{ name: "also_ignored", start: 32, end: 34, ignored: true },
|
||
{ name: "in_nested_module", start: 37, end: 40, ignored: false },
|
||
]);
|
||
assert.deepEqual(rustTests("fn main() {}\n"), []);
|
||
});
|
||
|
||
test("Rust selection: a hunk inside a test selects it, elsewhere selects every runnable test; ignored tests never run", () => {
|
||
const select = (text, options) =>
|
||
selectRust(text, rustSource, { previous: rustSource, ...options });
|
||
assert.deepEqual(select(diff([21, 1, 21, 1])), {
|
||
lines: [19],
|
||
names: ["plain"],
|
||
});
|
||
// Inside an ignored test: owned, so not shared code, but nothing to run.
|
||
assert.deepEqual(select(diff([28, 0, 29, 1])), { lines: [24], names: [] });
|
||
// The code under test, a helper, or a deleted helper (attached to line 13)
|
||
// can change every test.
|
||
const whole = (reason) => ({
|
||
whole: true,
|
||
reason,
|
||
names: ["plain", "in_nested_module"],
|
||
});
|
||
assert.deepEqual(select(diff([4, 0, 5, 1])), whole("shared code changed"));
|
||
assert.deepEqual(
|
||
select(diff([21, 1, 21, 1], [14, 0, 15, 1])),
|
||
whole("shared code changed"),
|
||
);
|
||
assert.deepEqual(select(diff([14, 3, 13, 0])), whole("shared code removed"));
|
||
assert.deepEqual(
|
||
select(diff([21, 1, 21, 1]), { added: true, previous: "" }),
|
||
whole("new file"),
|
||
);
|
||
// A file without test functions is not a test file.
|
||
assert.equal(selectRust(diff([1, 1, 1, 1]), "fn main() {}\n"), undefined);
|
||
assert.equal(cratePackage("crates/buzz-core/src/filter.rs"), "buzz-core");
|
||
});
|
||
|
||
test("hunks inside tests select those tests; shared code, new or unlisted files select the whole file", () => {
|
||
const bodies = [
|
||
{ start: 10, end: 22, line: 10 },
|
||
{ start: 25, end: 37, line: 25 },
|
||
{ start: 38, end: 52, line: 40 },
|
||
];
|
||
// The old file had the same tests, with a hook between the last two.
|
||
const previous = [
|
||
{ start: 10, end: 22 },
|
||
{ start: 25, end: 37 },
|
||
{ start: 41, end: 55 },
|
||
];
|
||
const select = (ranges, options) =>
|
||
selectTests(ranges, bodies, { previous, ...options });
|
||
// Edit inside one test, including its first and last lines.
|
||
assert.deepEqual(select([hunk(12, 14)]), { lines: [10] });
|
||
assert.deepEqual(select([hunk(10, 10)]), { lines: [10] });
|
||
assert.deepEqual(select([hunk(22, 22)]), { lines: [10] });
|
||
// A hunk across two adjacent bodies selects both, by their listed lines.
|
||
assert.deepEqual(select([hunk(36, 39)]), { lines: [25, 40] });
|
||
// A hunk that also covers the gap between tests, or lines after the last
|
||
// test, touches shared code.
|
||
for (const range of [hunk(22, 25), hunk(52, 53), hunk(400, 405)])
|
||
assert.deepEqual(select([range]), {
|
||
whole: true,
|
||
reason: "shared code changed",
|
||
});
|
||
// Several hunks accumulate distinct tests in file order.
|
||
assert.deepEqual(select([hunk(41, 41), hunk(11, 11), hunk(12, 12)]), {
|
||
lines: [10, 40],
|
||
});
|
||
// Replacing lines inside a test, or deleting some of them (attached to the
|
||
// preceding line), selects that test when the old lines were its own.
|
||
assert.deepEqual(select([hunk(30, 31, { start: 30, end: 32 })]), {
|
||
lines: [25],
|
||
});
|
||
assert.deepEqual(select([hunk(30, 30, { start: 31, end: 32 })]), {
|
||
lines: [25],
|
||
});
|
||
// Deleting the hook that followed the second test attaches to that test's
|
||
// closing line, but the removed lines were never a test's.
|
||
assert.deepEqual(select([hunk(37, 37, { start: 38, end: 40 })]), {
|
||
whole: true,
|
||
reason: "shared code removed",
|
||
});
|
||
// Deletion at the top of the file (line 0) and edits before the first test
|
||
// fall back to the whole file.
|
||
assert.deepEqual(select([hunk(0, 0, { start: 1, end: 1 })]), {
|
||
whole: true,
|
||
reason: "shared code removed",
|
||
});
|
||
assert.deepEqual(select([hunk(12, 12), hunk(3, 3)]), {
|
||
whole: true,
|
||
reason: "shared code changed",
|
||
});
|
||
assert.deepEqual(select([hunk(12, 12)], { added: true }), {
|
||
whole: true,
|
||
reason: "new file",
|
||
});
|
||
assert.deepEqual(selectTests([hunk(12, 12)], []), {
|
||
whole: true,
|
||
reason: "no tests listed",
|
||
});
|
||
// Without old-side calls, every removal is shared code.
|
||
assert.deepEqual(
|
||
selectTests([hunk(30, 30, { start: 31, end: 32 })], bodies),
|
||
{ whole: true, reason: "shared code removed" },
|
||
);
|
||
});
|
||
|
||
test("Playwright lists smoke journeys per spec; integration-only specs list none", () => {
|
||
const smoke = "desktop/tests/e2e/channel-sort.spec.ts";
|
||
const integration = "desktop/tests/e2e/agents.spec.ts";
|
||
const listed = listPlaywright([smoke, integration]);
|
||
assert.deepEqual(listed.get(integration), []);
|
||
const lines = listed.get(smoke);
|
||
assert.ok(lines.length > 0, "a smoke spec must list its journeys");
|
||
assert.equal(
|
||
testBodies(testCalls(read(`../${smoke}`), smoke), lines).length,
|
||
new Set(lines).size,
|
||
"every listed journey must be owned by a test call",
|
||
);
|
||
});
|
||
|
||
test("Playwright repetitions get one shard per five journeys, at most ten", () => {
|
||
assert.deepEqual(
|
||
[0, 1, 5, 6, 18, 40, 41, 50, 51, 200].map(playwrightShards),
|
||
[1, 1, 1, 2, 4, 8, 9, 10, 10, 10],
|
||
);
|
||
});
|
||
|
||
test("reports give per-test statuses per iteration", () => {
|
||
assert.deepEqual(
|
||
parseJunit(
|
||
[
|
||
'<?xml version="1.0" encoding="UTF-8"?>',
|
||
'<testsuites name="nextest-run">',
|
||
'<testsuite name="buzz-core" tests="3">',
|
||
'<testcase name="filter::tests::a" classname="buzz-core" time="0.1"/>',
|
||
'<testcase name="filter::tests::b <x>" classname="buzz-core" time="0.1">',
|
||
'<failure type="test failure">assertion `left == right` failed</failure>',
|
||
"</testcase>",
|
||
'<testcase name="filter::tests::c" classname="buzz-core"><skipped/></testcase>',
|
||
"</testsuite>",
|
||
'<testsuite name="buzz-core::e2e" tests="1">',
|
||
'<testcase name="d" classname="buzz-core::e2e" time="0.1"><error/></testcase>',
|
||
"</testsuite>",
|
||
"</testsuites>",
|
||
].join("\n"),
|
||
),
|
||
new Map([
|
||
["buzz-core filter::tests::a", "passed"],
|
||
["buzz-core filter::tests::b <x>", "failed"],
|
||
["buzz-core filter::tests::c", "skipped"],
|
||
["buzz-core::e2e d", "failed"],
|
||
]),
|
||
);
|
||
const event = (value) => JSON.stringify(value);
|
||
assert.deepEqual(
|
||
parseDartReport(
|
||
[
|
||
event({ type: "start", protocolVersion: "0.1.1" }),
|
||
event({
|
||
type: "testStart",
|
||
test: { id: 1, name: "loading /m/test/a_test.dart", url: null },
|
||
}),
|
||
event({
|
||
type: "testDone",
|
||
testID: 1,
|
||
result: "success",
|
||
hidden: true,
|
||
skipped: false,
|
||
}),
|
||
event({
|
||
type: "testStart",
|
||
test: {
|
||
id: 2,
|
||
name: "group renders",
|
||
url: "file:///m/test/a_test.dart",
|
||
},
|
||
}),
|
||
event({
|
||
type: "testStart",
|
||
test: {
|
||
id: 3,
|
||
name: "group fails",
|
||
url: "file:///m/test/a_test.dart",
|
||
},
|
||
}),
|
||
event({
|
||
type: "testStart",
|
||
test: { id: 4, name: "later", url: "file:///m/test/b_test.dart" },
|
||
}),
|
||
"not json",
|
||
event({
|
||
type: "testDone",
|
||
testID: 2,
|
||
result: "success",
|
||
hidden: false,
|
||
skipped: false,
|
||
}),
|
||
event({
|
||
type: "testDone",
|
||
testID: 3,
|
||
result: "error",
|
||
hidden: false,
|
||
skipped: false,
|
||
}),
|
||
event({
|
||
type: "testDone",
|
||
testID: 4,
|
||
result: "success",
|
||
hidden: false,
|
||
skipped: true,
|
||
}),
|
||
event({ type: "done", success: false }),
|
||
].join("\n"),
|
||
"/m/",
|
||
),
|
||
new Map([
|
||
["test/a_test.dart › group renders", "passed"],
|
||
["test/a_test.dart › group fails", "failed"],
|
||
["test/b_test.dart › later", "skipped"],
|
||
]),
|
||
);
|
||
// Playwright lists a --repeat-each journey once per repetition, in order.
|
||
const spec = (title, line, status) => ({
|
||
title,
|
||
file: "channels.spec.ts",
|
||
line,
|
||
tests: [{ status }],
|
||
});
|
||
assert.deepEqual(
|
||
playwrightRepetitions({
|
||
suites: [
|
||
{
|
||
specs: [spec("a", 10, "expected"), spec("a", 10, "unexpected")],
|
||
suites: [
|
||
{ specs: [spec("b", 30, "skipped"), spec("a", 10, "expected")] },
|
||
],
|
||
},
|
||
],
|
||
}),
|
||
new Map([
|
||
["channels.spec.ts:10 › a", ["passed", "failed", "passed"]],
|
||
["channels.spec.ts:30 › b", ["skipped"]],
|
||
]),
|
||
);
|
||
});
|
||
|
||
test("repeat invocations mirror the lanes they stand in for", () => {
|
||
// node:test flags come from desktop/package.json "test"; the Flutter define
|
||
// from the mobile test shards; nextest's flake profile must keep retries off.
|
||
const script = JSON.parse(read("../desktop/package.json")).scripts.test;
|
||
for (const file of ["x.test.mjs", "x.jsdom-test.mjs"])
|
||
assert.ok(script.includes(`node ${nodeTestArgs(file).join(" ")} "`), file);
|
||
assert.ok(
|
||
read("../.github/workflows/_ci-clients.yml").includes(
|
||
`scripts/mobile-test-shard.sh \${{ matrix.shard }} 3 ${dartDefine}`,
|
||
),
|
||
);
|
||
assert.match(
|
||
read("../.config/nextest.toml"),
|
||
/^\[profile\.flake\]\n(?:#.*\n)*retries = 0\nfail-fast = false\n\n\[profile\.flake\.junit\]\npath = "junit\.xml"\n/m,
|
||
);
|
||
const flake = read("../.github/workflows/_ci-flake.yml");
|
||
for (const runner of ["rust", "node", "dart"])
|
||
assert.ok(
|
||
flake.includes(`flake-check.mjs ${runner} --iterations=10 $SELECTED`),
|
||
runner,
|
||
);
|
||
assert.ok(flake.includes("--repeat-each=10 --retries=0"));
|
||
// The lanes are advisory: ci.yml calls the workflow on pull requests only,
|
||
// and no required gate depends on it.
|
||
const ci = read("../.github/workflows/ci.yml");
|
||
assert.match(
|
||
ci,
|
||
/^ {2}flake-check:\n(?: {4}.*\n)*? {4}if: github\.event_name == 'pull_request'\n(?: {4}.*\n)*? {4}uses: \.\/\.github\/workflows\/_ci-flake\.yml\n/m,
|
||
);
|
||
assert.doesNotMatch(ci, /needs: \[[^\]]*flake/);
|
||
});
|