buzz/scripts/test-manifest-validator.mjs

#!/usr/bin/env node
/**
 * Schema-negative validator tests for the manifest generator.
 *
 * Every validator rule in generate-model-capabilities.mjs must have a
 * failing-input test here — if validation is missing, these tests would
 * not catch the defect.
 *
 * Uses Node.js built-in test runner (node --test).
 */

import { test } from "node:test";
import assert from "node:assert/strict";
import { readFileSync, writeFileSync, unlinkSync, mkdirSync, mkdtempSync } from "node:fs";
import { join, dirname } from "node:path";
import { fileURLToPath } from "node:url";
import { tmpdir } from "node:os";
import { execFileSync, spawnSync } from "node:child_process";

const __dirname = dirname(fileURLToPath(import.meta.url));
const repoRoot = join(__dirname, "..");
const manifestPath = join(repoRoot, "scripts", "model-capabilities.json");
const generatorPath = join(repoRoot, "scripts", "generate-model-capabilities.mjs");

/** Load the real manifest so we can mutate copies. */
const BASE_MANIFEST = JSON.parse(readFileSync(manifestPath, "utf8"));

/**
 * Run the generator with a mutated manifest, returning { exitCode, stderr, stdout }.
 * Writes the mutated manifest to a temp file and overrides the manifest path via env.
 */
function runGeneratorWithManifest(manifestOverride) {
  // Write mutated manifest to a temp path
  const tmpDir = mkdtempSync(join(tmpdir(), "test-validator-"));
  const tmpManifest = join(tmpDir, "model-capabilities.json");
  const tmpOutputDir = join(tmpDir, "out");
  mkdirSync(tmpOutputDir, { recursive: true });

  writeFileSync(tmpManifest, JSON.stringify(manifestOverride));

  // Run the generator via node, pointing MANIFEST_PATH env at the temp file
  // The generator reads from process.env.MANIFEST_PATH if set (we add this support)
  const result = spawnSync(
    process.execPath,
    [generatorPath, "--manifest-path", tmpManifest, "--output-dir", tmpOutputDir],
    {
      encoding: "utf8",
      env: { ...process.env },
    },
  );

  // Cleanup
  try { unlinkSync(tmpManifest); } catch {}

  return { exitCode: result.status ?? 1, stderr: result.stderr, stdout: result.stdout };
}

/**
 * Assert that the generator REJECTS the given manifest (exits non-zero).
 * The optional `expectedMessage` is checked in stderr if provided.
 */
function assertRejects(label, manifest, expectedMessage) {
  const { exitCode, stderr, stdout } = runGeneratorWithManifest(manifest);
  assert.notEqual(exitCode, 0, `${label}: expected generator to fail but it succeeded.\nstdout: ${stdout}\nstderr: ${stderr}`);
  if (expectedMessage) {
    const combined = stderr + stdout;
    assert.ok(
      combined.includes(expectedMessage),
      `${label}: expected error message "${expectedMessage}" not found.\nstdout: ${stdout}\nstderr: ${stderr}`,
    );
  }
}

/** Deep clone the base manifest and apply a mutator function. */
function mutate(fn) {
  const clone = JSON.parse(JSON.stringify(BASE_MANIFEST));
  fn(clone);
  return clone;
}

// ---------------------------------------------------------------------------
// Rule: invalid enum value in family_rule.thinking_mode
// ---------------------------------------------------------------------------
test("schema-negative: invalid thinking_mode in family rule is rejected", () => {
  assertRejects(
    "invalid thinking_mode",
    mutate((m) => {
      m.family_rules[0].thinking_mode = "invalid-mode";
    }),
    "thinking_mode",
  );
});

// ---------------------------------------------------------------------------
// Rule: invalid enum value in family_rule.databricks_v2_wire_route
// ---------------------------------------------------------------------------
test("schema-negative: invalid databricks_v2_wire_route in family rule is rejected", () => {
  assertRejects(
    "invalid databricks_v2_wire_route",
    mutate((m) => {
      m.family_rules[0].databricks_v2_wire_route = "chat-completions";
    }),
    "databricks_v2_wire_route",
  );
});

// ---------------------------------------------------------------------------
// Rule: invalid enum value in family_rule.supported_efforts[]
// ---------------------------------------------------------------------------
test("schema-negative: invalid effort value in family rule supported_efforts is rejected", () => {
  assertRejects(
    "invalid supported_efforts value",
    mutate((m) => {
      m.family_rules[0].supported_efforts = ["low", "ultra-high"];
    }),
    "supported_efforts",
  );
});

// ---------------------------------------------------------------------------
// Rule: empty supported_efforts array in family rule
// ---------------------------------------------------------------------------
test("schema-negative: empty supported_efforts in family rule is rejected", () => {
  assertRejects(
    "empty supported_efforts",
    mutate((m) => {
      m.family_rules[0].supported_efforts = [];
    }),
    "supported_efforts",
  );
});

// ---------------------------------------------------------------------------
// Rule: default_effort not in supported_efforts (non-null)
// ---------------------------------------------------------------------------
test("schema-negative: default_effort not in supported_efforts is rejected", () => {
  assertRejects(
    "default_effort not in supported_efforts",
    mutate((m) => {
      m.family_rules[0].supported_efforts = ["low", "medium"];
      m.family_rules[0].default_effort = "high"; // not in list
    }),
    "default_effort",
  );
});

// ---------------------------------------------------------------------------
// Rule: invalid normalization_policy in family rule
// ---------------------------------------------------------------------------
test("schema-negative: invalid normalization_policy in family rule is rejected", () => {
  assertRejects(
    "invalid normalization_policy",
    mutate((m) => {
      m.family_rules[0].normalization_policy = "pass-through-all";
    }),
    "normalization_policy",
  );
});

// ---------------------------------------------------------------------------
// Rule: invalid match_kind in family rule
// ---------------------------------------------------------------------------
test("schema-negative: invalid match_kind in family rule is rejected", () => {
  assertRejects(
    "invalid match_kind",
    mutate((m) => {
      m.family_rules[0].match_kind = "regex";
    }),
    "match_kind",
  );
});

// ---------------------------------------------------------------------------
// Rule: duplicate family rule id
// ---------------------------------------------------------------------------
test("schema-negative: duplicate family rule id is rejected", () => {
  assertRejects(
    "duplicate family rule id",
    mutate((m) => {
      m.family_rules.push({ ...m.family_rules[0] }); // duplicate id
    }),
    "duplicate",
  );
});

// ---------------------------------------------------------------------------
// Rule: duplicate exact_record (provider, raw_model_id) key
// ---------------------------------------------------------------------------
test("schema-negative: duplicate exact_record key is rejected", () => {
  assertRejects(
    "duplicate exact_record key",
    mutate((m) => {
      m.exact_records.push({ ...m.exact_records[0] }); // duplicate
    }),
    "duplicate",
  );
});

// ---------------------------------------------------------------------------
// Rule: exact_record missing provider
// ---------------------------------------------------------------------------
test("schema-negative: exact_record missing provider is rejected", () => {
  assertRejects(
    "exact_record missing provider",
    mutate((m) => {
      m.exact_records.push({ raw_model_id: "some-model" });
    }),
    "provider",
  );
});

// ---------------------------------------------------------------------------
// Rule: exact_record missing raw_model_id
// ---------------------------------------------------------------------------
test("schema-negative: exact_record missing raw_model_id is rejected", () => {
  assertRejects(
    "exact_record missing raw_model_id",
    mutate((m) => {
      m.exact_records.push({ provider: "databricks_v2" });
    }),
    "raw_model_id",
  );
});

// ---------------------------------------------------------------------------
// Rule: provider fallback record missing blank state
// ---------------------------------------------------------------------------
test("schema-negative: provider fallback missing blank state is rejected", () => {
  assertRejects(
    "provider fallback missing blank",
    mutate((m) => {
      delete m.provider_fallbacks.anthropic.blank;
    }),
    "blank",
  );
});

// ---------------------------------------------------------------------------
// Rule: provider fallback record missing concrete_unknown state
// ---------------------------------------------------------------------------
test("schema-negative: provider fallback missing concrete_unknown state is rejected", () => {
  assertRejects(
    "provider fallback missing concrete_unknown",
    mutate((m) => {
      delete m.provider_fallbacks.anthropic.concrete_unknown;
    }),
    "concrete_unknown",
  );
});

// ---------------------------------------------------------------------------
// Rule: invalid thinking_mode in provider fallback
// ---------------------------------------------------------------------------
test("schema-negative: invalid thinking_mode in provider fallback is rejected", () => {
  assertRejects(
    "invalid thinking_mode in fallback",
    mutate((m) => {
      m.provider_fallbacks.anthropic.blank.thinking_mode = "always-on";
    }),
    "thinking_mode",
  );
});

// ---------------------------------------------------------------------------
// Rule: invalid databricks_v2_wire_route in provider fallback
// ---------------------------------------------------------------------------
test("schema-negative: invalid wire_route in provider fallback is rejected", () => {
  assertRejects(
    "invalid wire_route in fallback",
    mutate((m) => {
      m.provider_fallbacks.anthropic.blank.databricks_v2_wire_route = "http-sse";
    }),
    "databricks_v2_wire_route",
  );
});

// ---------------------------------------------------------------------------
// Rule: invalid default_effort in provider fallback (not in supported_efforts)
// ---------------------------------------------------------------------------
test("schema-negative: default_effort not in supported_efforts in fallback is rejected", () => {
  assertRejects(
    "default_effort not in supported_efforts in fallback",
    mutate((m) => {
      m.provider_fallbacks.openai.blank.supported_efforts = ["low", "medium"];
      m.provider_fallbacks.openai.blank.default_effort = "high"; // not in list
    }),
    "default_effort",
  );
});

// ---------------------------------------------------------------------------
// Rule: family rule missing id
// ---------------------------------------------------------------------------
test("schema-negative: family rule missing id is rejected", () => {
  assertRejects(
    "family rule missing id",
    mutate((m) => {
      m.family_rules.push({
        match_kind: "prefix",
        match_value: "test-",
        providers: ["anthropic"],
        match_priority: 1,
        thinking_mode: "none",
        supported_efforts: ["low"],
        default_effort: null,
        databricks_v2_wire_route: "not-applicable",
        normalization_policy: "none",
        // id deliberately omitted
      });
    }),
    "id",
  );
});

// ---------------------------------------------------------------------------
// Rule: duplicate registry_label IDs
// ---------------------------------------------------------------------------
test("schema-negative: duplicate registry_label ID is rejected", () => {
  assertRejects(
    "duplicate registry_label ID",
    mutate((m) => {
      // Array format — duplicate id is structurally detectable
      m.registry_labels = [
        { id: "databricks-gpt-5-5", label: "GPT-5.5" },
        { id: "databricks-gpt-5-5", label: "GPT-5.5 duplicate" },
      ];
    }),
    "duplicate",
  );
});

// ---------------------------------------------------------------------------
// Rule: registry_label entry missing id (empty string)
// ---------------------------------------------------------------------------
test("schema-negative: registry_label entry with empty id is rejected", () => {
  assertRejects(
    "registry_label empty id",
    mutate((m) => {
      m.registry_labels = [{ id: "", label: "Some Label" }];
    }),
    "id",
  );
});

// ---------------------------------------------------------------------------
// Rule: registry_label entry with unsafe characters in id
// ---------------------------------------------------------------------------
test("schema-negative: registry_label entry with unsafe id chars is rejected", () => {
  assertRejects(
    "registry_label unsafe id",
    mutate((m) => {
      m.registry_labels = [{ id: 'bad"id', label: "Some Label" }];
    }),
    "unsafe",
  );
});

// ---------------------------------------------------------------------------
// Rule: duplicate databricks_v2_known_models IDs
// ---------------------------------------------------------------------------
test("schema-negative: duplicate databricks_v2_known_models ID is rejected", () => {
  assertRejects(
    "duplicate known model ID",
    mutate((m) => {
      m.databricks_v2_known_models = ["databricks-gpt-5-5", "databricks-gpt-5-5"];
    }),
    "duplicate",
  );
});

// ---------------------------------------------------------------------------
// Rule: unsafe characters in match_value (family rule)
// ---------------------------------------------------------------------------
test("schema-negative: family rule match_value with unsafe chars is rejected", () => {
  assertRejects(
    "family rule match_value with backslash",
    mutate((m) => {
      // Inject a backslash into an existing rule's match_value — would break Rust string literal
      const rule = m.family_rules.find((r) => r.id === "anthropic-manual-budget-claude3");
      rule.match_value = "claude-3\\evil";
    }),
    "unsafe",
  );
});

// ---------------------------------------------------------------------------
// Rule: unsafe characters in known-model ID
// ---------------------------------------------------------------------------
test("schema-negative: databricks_v2_known_models ID with unsafe chars is rejected", () => {
  assertRejects(
    "known-model ID with double-quote",
    mutate((m) => {
      m.databricks_v2_known_models = ['databricks-gpt-5-5', 'bad"id'];
    }),
    "unsafe",
  );
});

// ---------------------------------------------------------------------------
// Rule: unsafe characters in exact_record registry_label
// ---------------------------------------------------------------------------
test("schema-negative: exact_record registry_label with unsafe chars is rejected", () => {
  assertRejects(
    "exact_record registry_label with backslash",
    mutate((m) => {
      const rec = m.exact_records.find((r) => r.raw_model_id === "databricks-gpt-5-4-mini");
      rec.registry_label = "GPT-5.4 Mini\\injected";
    }),
    "unsafe",
  );
});

// ===========================================================================
// Pricing record validation rules
// ===========================================================================

// Helper: insert a bad pricing record (replaces the first real record)
function mutatePricing(fn) {
  return mutate((m) => {
    const rec = JSON.parse(JSON.stringify(m.pricing_records[0]));
    fn(rec, m);
    m.pricing_records[0] = rec;
  });
}

// ---------------------------------------------------------------------------
// Rule: pricing record missing authority
// ---------------------------------------------------------------------------
test("schema-negative: pricing record missing authority is rejected", () => {
  assertRejects(
    "pricing record missing authority",
    mutate((m) => {
      m.pricing_records.push({
        model: "some-model",
        usd_per_mtok: { input: 1, output: 5, cache_read: null, cache_write: null },
        _source: "test",
      });
    }),
    "authority",
  );
});

// ---------------------------------------------------------------------------
// Rule: pricing record with unregistered authority is rejected
// ---------------------------------------------------------------------------
test("schema-negative: pricing record with path-bearing authority is rejected", () => {
  assertRejects(
    "path-bearing authority",
    mutatePricing((rec) => {
      rec.authority = "api.openai.com/v1";
    }),
    "unregistered authority",
  );
});

test("schema-negative: pricing record with scheme in authority is rejected", () => {
  assertRejects(
    "scheme in authority",
    mutatePricing((rec) => {
      rec.authority = "https://api.anthropic.com";
    }),
    "unregistered authority",
  );
});

test("schema-negative: pricing record with custom/unknown authority is rejected", () => {
  assertRejects(
    "unknown authority",
    mutatePricing((rec) => {
      rec.authority = "custom.openai-compat.example.com";
    }),
    "unregistered authority",
  );
});

// ---------------------------------------------------------------------------
// Rule: pricing record with empty model is rejected
// ---------------------------------------------------------------------------
test("schema-negative: pricing record with empty model is rejected", () => {
  assertRejects(
    "empty model",
    mutatePricing((rec) => {
      rec.model = "";
    }),
    "model",
  );
});

// ---------------------------------------------------------------------------
// Rule: unsafe characters in key fields are rejected
// ---------------------------------------------------------------------------
test("schema-negative: pricing record model with double-quote is rejected", () => {
  assertRejects(
    "model with double-quote",
    mutatePricing((rec) => {
      rec.model = 'bad"model';
    }),
    "unsafe",
  );
});

test("schema-negative: pricing record model with backslash is rejected", () => {
  assertRejects(
    "model with backslash",
    mutatePricing((rec) => {
      rec.model = "bad\\model";
    }),
    "unsafe",
  );
});

test("schema-negative: pricing record model with NUL character is rejected", () => {
  assertRejects(
    "model with NUL",
    mutatePricing((rec) => {
      rec.model = "bad\x00model";
    }),
    "unsafe",
  );
});

test("schema-negative: pricing record model with control character is rejected", () => {
  assertRejects(
    "model with control char",
    mutatePricing((rec) => {
      rec.model = "bad\x01model";
    }),
    "unsafe",
  );
});

// ---------------------------------------------------------------------------
// Rule: malformed rates are rejected
// ---------------------------------------------------------------------------
test("schema-negative: pricing record missing output rate is rejected", () => {
  assertRejects(
    "missing output",
    mutatePricing((rec) => {
      delete rec.usd_per_mtok.output;
    }),
    "output",
  );
});

test("schema-negative: pricing record with negative input rate is rejected", () => {
  assertRejects(
    "negative input",
    mutatePricing((rec) => {
      rec.usd_per_mtok.input = -1;
    }),
    "input",
  );
});

test("schema-negative: pricing record with non-finite input rate is rejected", () => {
  assertRejects(
    "Infinity input",
    mutatePricing((rec) => {
      rec.usd_per_mtok.input = Infinity;
    }),
    "input",
  );
});

test("schema-negative: pricing record with NaN output rate is rejected", () => {
  assertRejects(
    "NaN output",
    mutatePricing((rec) => {
      rec.usd_per_mtok.output = NaN;
    }),
    "output",
  );
});

test("schema-negative: pricing record with negative cache_read rate is rejected", () => {
  assertRejects(
    "negative cache_read",
    mutatePricing((rec) => {
      rec.usd_per_mtok.cache_read = -0.5;
    }),
    "cache_read",
  );
});

test("schema-negative: pricing record with string cache_write rate is rejected", () => {
  // Note: Infinity/NaN cannot be tested via JSON mutation (they serialize to null,
  // which is valid for cache fields meaning "not published"). Use a string value,
  // which is also invalid and does survive JSON serialization.
  assertRejects(
    "string cache_write",
    mutatePricing((rec) => {
      rec.usd_per_mtok.cache_write = "not-a-number";
    }),
    "cache_write",
  );
});

test("schema-negative: pricing record with deleted cache_read is rejected", () => {
  assertRejects(
    "deleted cache_read",
    mutatePricing((rec) => {
      delete rec.usd_per_mtok.cache_read;
    }),
    "cache_read",
  );
});

test("schema-negative: pricing record with deleted cache_write is rejected", () => {
  assertRejects(
    "deleted cache_write",
    mutatePricing((rec) => {
      delete rec.usd_per_mtok.cache_write;
    }),
    "cache_write",
  );
});

// ---------------------------------------------------------------------------
// Rule: missing provenance is rejected
// ---------------------------------------------------------------------------
test("schema-negative: pricing record missing _source provenance is rejected", () => {
  assertRejects(
    "missing _source",
    mutatePricing((rec) => {
      delete rec._source;
    }),
    "_source",
  );
});

test("schema-negative: pricing record with empty _source provenance is rejected", () => {
  assertRejects(
    "empty _source",
    mutatePricing((rec) => {
      rec._source = "";
    }),
    "_source",
  );
});

// ---------------------------------------------------------------------------
// Rule: duplicate (authority, model) pairs are rejected
// ---------------------------------------------------------------------------
test("schema-negative: duplicate pricing record (authority, model) is rejected", () => {
  assertRejects(
    "duplicate pricing record",
    mutate((m) => {
      // Add a record identical to the first one
      m.pricing_records.push({ ...m.pricing_records[0] });
    }),
    "duplicate",
  );
});

// ---------------------------------------------------------------------------
// Rule: case-colliding pairs must not silently generate (they are rejected
// at validation because all authorities are already lowercase registered
// tokens; a record with uppercase authority is caught before emit)
// ---------------------------------------------------------------------------
test("schema-negative: pricing record with uppercase authority is rejected before emit", () => {
  assertRejects(
    "uppercase authority",
    mutatePricing((rec) => {
      rec.authority = "API.ANTHROPIC.COM";
    }),
    "unregistered authority",
  );
});

console.log("\nSchema-negative validator tests complete.");