"use strict"; const assert = require("node:assert/strict"); const { readFileSync } = require("node:fs"); const path = require("node:path"); const test = require("node:test"); const { invalidate, invalidatePullRequestUpdate, post, prepare, prepareBaseReconciliation, withGithubRetry, } = require("./codex-security-review.js"); const BASE_SHA = "a".repeat(40); const OLD_BASE_SHA = "b".repeat(40); const HEAD_SHA = "c".repeat(40); const OTHER_HEAD_SHA = "d".repeat(40); const NEW_BASE_SHA = "e".repeat(40); const MARKER = ""; const CURRENT_REVIEW_LABEL = "codex-security-review-current"; function pullRequest({ author = "outside-contributor", baseSha = OLD_BASE_SHA, headSha = HEAD_SHA, labels = [], } = {}) { return { author_association: "CONTRIBUTOR", user: { login: author }, state: "open", base: { ref: "main", sha: baseSha, repo: { full_name: "block/buzz" }, }, head: { sha: headSha, repo: { full_name: "outside/buzz" }, }, changed_files: 1, labels, }; } function harness({ pull = pullRequest(), comments = [], files = [], labeledIssues = [], liveMainShas = [BASE_SHA], permissions = { "block-member": "write", "outside-contributor": "read", "renovate[bot]": "none", }, } = {}) { const storedComments = [...comments]; const created = []; const updated = []; const addedLabels = []; const removedLabels = []; const removeLabelCalls = []; const outputs = new Map(); const failures = []; const notices = []; const info = []; const warnings = []; const listComments = async () => storedComments; const listFiles = async () => files; let labelExists = false; let mainRefRead = 0; const github = { paginate: async (method, args) => method(args), rest: { git: { getRef: async () => { const sha = liveMainShas[Math.min(mainRefRead, liveMainShas.length - 1)]; mainRefRead += 1; return { data: { object: { type: "commit", sha } } }; }, }, issues: { listComments, listForRepo: async () => labeledIssues, getLabel: async () => { if (!labelExists) { throw Object.assign(new Error("not found"), { status: 404 }); } return { data: { name: CURRENT_REVIEW_LABEL } }; }, createLabel: async () => { labelExists = true; }, addLabels: async (input) => { labelExists = true; addedLabels.push(input); }, removeLabel: async (input) => { removeLabelCalls.push(input); if (!labelExists) { throw Object.assign(new Error("not found"), { status: 404 }); } labelExists = false; removedLabels.push(input); }, createComment: async (input) => { created.push(input); storedComments.push( reviewComment(input.body, { id: 100 + storedComments.length }), ); }, updateComment: async (input) => { updated.push(input); const comment = storedComments.find( (candidate) => candidate.id === input.comment_id, ); if (comment) { comment.body = input.body; } }, }, repos: { getCollaboratorPermissionLevel: async ({ username }) => { const permission = permissions[username]; if (permission === undefined) { throw Object.assign(new Error("not found"), { status: 404 }); } if (permission instanceof Error) { throw permission; } if (Array.isArray(permission)) { const next = permission.shift(); if (next instanceof Error) { throw next; } return { data: { permission: next } }; } return { data: { permission } }; }, }, pulls: { get: async () => ({ data: pull }), listFiles, }, }, }; const core = { info: (message) => info.push(message), notice: (message) => notices.push(message), warning: (message) => warnings.push(message), setFailed: (message) => failures.push(message), setOutput: (name, value) => outputs.set(name, value), }; const context = { actor: "block-member", eventName: "issue_comment", payload: { comment: { body: `@buzz-security-review ${HEAD_SHA}`, user: { login: "block-member" }, }, issue: { number: 6816 }, }, repo: { owner: "block", repo: "buzz" }, }; return { context, core, addedLabels, created, failures, github, info, notices, outputs, removeLabelCalls, removedLabels, storedComments, updated, warnings, }; } function reviewComment(body, { id = 42 } = {}) { return { id, body, user: { login: "github-actions[bot]", type: "Bot" }, }; } const NO_FINDINGS_REVIEW = { overall_risk: "NONE", summary: "No findings.", findings: [], notes: [], }; async function postReview(state, review = NO_FINDINGS_REVIEW) { const environment = { CODEX_MODEL: "gpt-6.1-sol", GITHUB_REPOSITORY: "block/buzz", GITHUB_RUN_ID: "1234", GITHUB_SERVER_URL: "https://github.com", REVIEW_BASE_SHA: BASE_SHA, REVIEW_COMMIT_RANGE: `${BASE_SHA}...${HEAD_SHA}`, REVIEW_HEAD_REPO: "outside/buzz", REVIEW_HEAD_SHA: HEAD_SHA, REVIEW_JSON: JSON.stringify(review), REVIEW_PR_NUMBER: "6816", REVIEW_TRIGGER_ACTOR: "block-member", }; const previous = Object.fromEntries( Object.keys(environment).map((key) => [key, process.env[key]]), ); Object.assign(process.env, environment); try { await post(state); } finally { for (const [key, value] of Object.entries(previous)) { if (value === undefined) { delete process.env[key]; } else { process.env[key] = value; } } } } test("prepare binds a member command to the named head SHA", async () => { const current = harness(); await prepare(current); assert.deepEqual(current.failures, []); assert.equal(current.outputs.get("authorized"), "true"); assert.equal(current.outputs.get("head_sha"), HEAD_SHA); assert.equal(current.outputs.get("base_sha"), BASE_SHA); assert.notEqual(current.outputs.get("base_sha"), OLD_BASE_SHA); assert.equal( current.outputs.get("commit_range"), `${BASE_SHA}...${HEAD_SHA}`, ); const moved = harness({ pull: pullRequest({ headSha: OTHER_HEAD_SHA }) }); await prepare(moved); assert.equal(moved.outputs.size, 0); assert.equal(moved.failures.length, 1); assert.match(moved.failures[0], new RegExp(OTHER_HEAD_SHA)); assert.match( moved.failures[0], new RegExp(`@buzz-security-review ${OTHER_HEAD_SHA}`), ); }); function pullRequestTrigger(options) { const state = harness(options); state.context.eventName = "pull_request_target"; state.context.payload.pull_request = { number: 6816, head: { sha: HEAD_SHA }, author_association: "MEMBER", }; return state; } test("pull request authorization uses the live author's write access", async () => { // Private org members report CONTRIBUTOR; write access still authorizes. const member = pullRequestTrigger({ pull: pullRequest({ author: "block-member" }) }); await prepare(member); assert.equal(member.outputs.get("authorized"), "true"); assert.deepEqual(member.failures, []); for (const author of [ "outside-contributor", "nonexistent-user", "renovate[bot]", ]) { const untrusted = pullRequestTrigger({ pull: pullRequest({ author }) }); await prepare(untrusted); assert.equal(untrusted.outputs.get("authorized"), undefined); assert.deepEqual(untrusted.failures, []); assert.match(untrusted.info.at(-1), /requires authorization/); } }); test("permission lookup errors fail closed", async () => { const failure = Object.assign(new Error("forbidden"), { status: 403 }); const author = pullRequestTrigger({ pull: pullRequest({ author: "block-member" }), permissions: { "block-member": failure }, }); await assert.rejects(prepare(author), /forbidden/); assert.equal(author.outputs.size, 0); const commenter = harness({ permissions: { "block-member": failure } }); await assert.rejects(prepare(commenter), /forbidden/); assert.equal(commenter.outputs.size, 0); }); test("transient permission lookup errors are retried", async () => { const unavailable = Object.assign(new Error("unavailable"), { status: 503, response: { headers: { "retry-after": "0" } }, }); const state = harness({ permissions: { "block-member": [unavailable, "write"] }, }); await prepare(state); assert.equal(state.outputs.get("authorized"), "true"); assert.equal(state.warnings.length, 1); }); test("stale-marking proceeds when the permission lookup fails", async () => { const state = harness({ pull: pullRequest({ author: "block-member", headSha: OTHER_HEAD_SHA }), permissions: { "block-member": Object.assign(new Error("forbidden"), { status: 403 }), }, comments: [ reviewComment( `${MARKER}\n\nold review`, ), ], }); state.context.eventName = "pull_request_target"; state.context.payload.pull_request = { number: 6816 }; await state.github.rest.issues.addLabels({ issue_number: 6816, labels: [CURRENT_REVIEW_LABEL], }); await invalidatePullRequestUpdate(state); assert.equal(state.updated.length, 1); assert.match(state.updated[0].body, /review required for the current range/); assert.equal(state.removedLabels.length, 1); const unreviewed = harness({ pull: pullRequest({ author: "block-member" }), permissions: { "block-member": Object.assign(new Error("forbidden"), { status: 403 }), }, }); unreviewed.context.eventName = "pull_request_target"; unreviewed.context.payload.pull_request = { number: 6816 }; await invalidatePullRequestUpdate(unreviewed); assert.equal(unreviewed.created.length, 1); assert.equal(unreviewed.warnings.length, 1); }); test("review commands require a commenter with write access", async () => { for (const login of ["outside-contributor", "nonexistent-user"]) { const state = harness(); state.context.payload.comment.user.login = login; await prepare(state); assert.equal(state.outputs.size, 0); assert.deepEqual(state.failures, []); assert.match(state.info.at(-1), /require write access/); } const malformed = harness(); malformed.context.payload.comment = { body: "@buzz-security-review foo", user: { login: "outside-contributor" }, }; await prepare(malformed); assert.deepEqual(malformed.failures, []); assert.equal(malformed.outputs.size, 0); const trusted = harness({ permissions: { "block-member": "admin" } }); await prepare(trusted); assert.equal(trusted.outputs.get("authorized"), "true"); }); test("workflow leaves comment authorization to the trusted prepare step", () => { const workflow = readFileSync( path.join(__dirname, "../workflows/codex-security-review.yml"), "utf8", ); assert.doesNotMatch(workflow, /comment\.author_association/); }); test("PR mutation jobs use pull request write permission", () => { const workflow = readFileSync( path.join(__dirname, "../workflows/codex-security-review.yml"), "utf8", ); for (const jobName of [ "reconcile-base-reviews", "invalidate-previous-review", "post-review", ]) { const start = workflow.indexOf(` ${jobName}:\n`); assert.notEqual(start, -1, `missing workflow job ${jobName}`); const remainder = workflow.slice(start + 2); const nextJob = remainder.search(/^ [a-z][a-z0-9-]*:\n/m); const job = nextJob === -1 ? workflow.slice(start) : workflow.slice(start, start + 2 + nextJob); assert.match(job, /^ pull-requests: write$/m); assert.doesNotMatch(job, /^ issues: write$/m); } }); test("prepare rejects review commands without a full exact SHA", async () => { const state = harness(); state.context.payload.comment.body = "@buzz-security-review"; await prepare(state); assert.equal(state.outputs.size, 0); assert.equal(state.failures.length, 1); assert.match(state.failures[0], //); }); test("invalidation compares the complete base and head range", async () => { const stale = harness({ comments: [ reviewComment( `${MARKER}\n\nold review`, ), ], }); await stale.github.rest.issues.addLabels({ issue_number: 6816, labels: [CURRENT_REVIEW_LABEL], }); await invalidate({ github: stale.github, context: stale.context, core: stale.core, prNumber: 6816, existingOnly: true, }); assert.equal(stale.updated.length, 1); assert.match(stale.updated[0].body, /review required for the current range/); assert.ok(stale.updated[0].body.includes(`${BASE_SHA}...${HEAD_SHA}`)); assert.match( stale.updated[0].body, new RegExp(`@buzz-security-review ${HEAD_SHA}`), ); assert.equal(stale.removedLabels.length, 1); await invalidate({ github: stale.github, context: stale.context, core: stale.core, prNumber: 6816, existingOnly: true, }); assert.equal(stale.updated.length, 1); assert.match(stale.info.at(-1), /already has the current stale-review notice/); const current = harness({ comments: [ reviewComment( `${MARKER}\n\ncurrent review`, ), ], }); await invalidate({ github: current.github, context: current.context, core: current.core, prNumber: 6816, existingOnly: true, }); assert.equal(current.updated.length, 0); assert.match(current.info.at(-1), /current range/); }); test("base reconciliation batches every labeled review with a durable cursor", async () => { const labeledIssues = Array.from({ length: 34 }, (_, index) => ({ number: index + 1, pull_request: {}, })); const first = harness({ labeledIssues }); first.context.eventName = "repository_dispatch"; first.context.payload.client_payload = { after_pr: "1", main_sha: BASE_SHA, }; await prepareBaseReconciliation(first); const firstBatch = JSON.parse(first.outputs.get("pr_numbers")); assert.equal(firstBatch.length, 32); assert.equal(firstBatch[0], 2); assert.equal(firstBatch.at(-1), 33); assert.equal(first.outputs.get("main_sha"), BASE_SHA); assert.equal(first.outputs.get("should_continue"), "true"); assert.equal(first.outputs.get("next_after"), "33"); assert.equal(first.outputs.get("next_pass"), "1"); const second = harness({ labeledIssues }); second.context.eventName = "repository_dispatch"; second.context.payload.client_payload = { after_pr: "33", main_sha: BASE_SHA, }; await prepareBaseReconciliation(second); assert.deepEqual(JSON.parse(second.outputs.get("pr_numbers")), [34]); assert.equal(second.outputs.get("should_continue"), "true"); assert.equal(second.outputs.get("next_after"), "0"); assert.equal(second.outputs.get("next_pass"), "2"); const disappearedTail = harness(); disappearedTail.context.eventName = "repository_dispatch"; disappearedTail.context.payload.client_payload = { after_pr: "33", main_sha: BASE_SHA, pass: "1", }; await prepareBaseReconciliation(disappearedTail); assert.deepEqual( JSON.parse(disappearedTail.outputs.get("pr_numbers")), [0], ); assert.equal(disappearedTail.outputs.get("should_continue"), "true"); assert.equal(disappearedTail.outputs.get("next_after"), "0"); assert.equal(disappearedTail.outputs.get("next_pass"), "2"); const retry = harness({ labeledIssues: [labeledIssues.at(-1)] }); retry.context.eventName = "repository_dispatch"; retry.context.payload.client_payload = { after_pr: "0", main_sha: BASE_SHA, pass: "2", }; await prepareBaseReconciliation(retry); assert.deepEqual(JSON.parse(retry.outputs.get("pr_numbers")), [34]); assert.equal(retry.outputs.get("should_continue"), "false"); assert.equal(retry.outputs.get("next_after"), "0"); assert.equal(retry.outputs.get("next_pass"), "2"); }); test("base reconciliation stops a continuation from an older main commit", async () => { const state = harness({ labeledIssues: [{ number: 6816, pull_request: {} }], liveMainShas: [NEW_BASE_SHA], }); state.context.eventName = "repository_dispatch"; state.context.payload.client_payload = { after_pr: "256", main_sha: BASE_SHA, }; await prepareBaseReconciliation(state); assert.deepEqual(JSON.parse(state.outputs.get("pr_numbers")), [0]); assert.equal(state.outputs.get("main_sha"), BASE_SHA); assert.equal(state.outputs.get("should_continue"), "false"); assert.equal(state.outputs.get("next_after"), "0"); assert.equal(state.outputs.get("next_pass"), "1"); assert.match(state.info.at(-1), /superseded main commit/); }); test("GitHub rate limits use bounded retry delays", async () => { const waits = []; const warnings = []; let attempts = 0; const rateLimitError = Object.assign(new Error("secondary rate limit"), { status: 403, response: { status: 403, headers: { "retry-after": "0" }, data: { message: "secondary rate limit" }, }, }); const result = await withGithubRetry( async () => { attempts += 1; if (attempts < 3) { throw rateLimitError; } return "completed"; }, { core: { warning: (message) => warnings.push(message) }, sleep: async (milliseconds) => waits.push(milliseconds), }, ); assert.equal(result, "completed"); assert.equal(attempts, 3); assert.deepEqual(waits, [1000, 1000]); assert.equal(warnings.length, 2); }); test("base reconciliation does not create comments on unreviewed PRs", async () => { const state = harness(); await invalidate({ github: state.github, context: state.context, core: state.core, prNumber: 6816, existingOnly: true, }); assert.equal(state.created.length, 0); assert.equal(state.updated.length, 0); }); test("pull request updates invalidate member reviews without adding placeholders", async () => { const reviewed = harness({ pull: pullRequest({ author: "block-member", headSha: OTHER_HEAD_SHA, }), comments: [ reviewComment( `${MARKER}\n\nold review`, ), ], }); reviewed.context.eventName = "pull_request_target"; reviewed.context.payload.pull_request = { number: 6816, author_association: "CONTRIBUTOR", }; await reviewed.github.rest.issues.addLabels({ issue_number: 6816, labels: [CURRENT_REVIEW_LABEL], }); await invalidatePullRequestUpdate(reviewed); assert.equal(reviewed.updated.length, 1); assert.ok( reviewed.updated[0].body.includes(`${BASE_SHA}...${OTHER_HEAD_SHA}`), ); assert.equal(reviewed.removedLabels.length, 1); const unreviewed = harness({ pull: pullRequest({ author: "block-member" }), }); unreviewed.context.eventName = "pull_request_target"; unreviewed.context.payload.pull_request = { number: 6816, author_association: "CONTRIBUTOR", }; await invalidatePullRequestUpdate(unreviewed); assert.equal(unreviewed.created.length, 0); assert.equal(unreviewed.updated.length, 0); assert.equal(unreviewed.removeLabelCalls.length, 0); const external = harness({ pull: pullRequest({ author: "outside-contributor" }), }); external.context.eventName = "pull_request_target"; external.context.payload.pull_request = { number: 6816, author_association: "MEMBER", }; await invalidatePullRequestUpdate(external); assert.equal(external.created.length, 1); assert.match(external.created[0].body, /review required for the current range/); }); test("post preserves finding text while rendering it as inert code", async () => { const findingPath = "src/x)www.example.com/review.js"; const state = harness({ files: [{ filename: findingPath }] }); const summary = "Keep