Files
fusion/scripts/pg-setup-boundary-probe.mjs
gsxdsm 6d7b4a3ac3 FN-9140: make Vitest timeout ownership surveys deterministic
Replace ambiguous setup-boundary observations with calibrated, repeatable timeout ownership evidence.

- record fixture lifecycle events in an append-only JSONL ledger with process-safe ordering
- classify four timeout-budget arms across repeated isolate-mode cells and fail closed on incomplete evidence
- expand connectionless unit coverage and document the terminal insufficient-data survey result

Files changed:
 .../test-failures/postgres-ddl-admission-bound.md  |   2 +
 .../vitest-setup-boundary-timeout-ownership.md     |  37 +++
 docs/testing.md                                    |   2 +-
 scripts/__tests__/pg-setup-boundary-probe.test.mjs | 195 ++++++++----
 scripts/pg-setup-boundary-probe.mjs                | 341 ++++++++++++---------
 5 files changed, 371 insertions(+), 206 deletions(-)

Fusion-Task-Id: FN-9140

Fusion-Task-Lineage: 9c6970b8-af80-400a-b2b2-49718d4fe87f

Co-authored-by: Fusion (runfusion.ai) <noreply@runfusion.ai>
2026-08-16 22:47:53 -07:00

248 lines
18 KiB
JavaScript

#!/usr/bin/env node
import { spawn } from "node:child_process";
import { appendFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { createRequire } from "node:module";
import { tmpdir } from "node:os";
import { dirname, join, resolve } from "node:path";
import { pathToFileURL } from "node:url";
const require = createRequire(import.meta.url);
export const BOUNDARIES = ["global-setup", "setup-top-level-await", "setup-before-all", "per-file-before-all"];
export const ARMS = ["A", "B", "C", "R"];
const DIAGNOSTIC_LIMIT = 4000;
/*
FNXC:PgSetupBoundaryProbe 2026-08-17-23:40:
FN-9140 replaces stdout scraping and one-shot, non-falsifying observations with an append-only
ledger, repeated fixture-local budget arms, and calibration. Parent and fork clocks have different
monotonic origins, so only same-pid ordering uses monotonicNs; cross-pid ordering requires an epoch
separation margin. SMALL and LARGE are measurement parameters written only to the temp fixture.
*/
export function deriveBudgets(delayMs) {
return { D: delayMs, SMALL: Math.floor(delayMs / 4), LARGE: delayMs * 5 };
}
export function parseProbeArgs(argv) {
const result = { workers: 2, files: 6, repeats: 3, isolate: "both", delayMs: 4000, orderingMarginMs: 50, json: undefined, runTimeoutMs: undefined };
const names = new Map([["--workers", "workers"], ["--files", "files"], ["--repeats", "repeats"], ["--delay-ms", "delayMs"], ["--ordering-margin-ms", "orderingMarginMs"], ["--run-timeout-ms", "runTimeoutMs"]]);
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === "--help") return { help: true };
if (arg === "--json") { result.json = argv[++index]; if (!result.json) throw new Error("--json requires a path"); continue; }
if (arg === "--isolate") { result.isolate = argv[++index]; if (!["both", "true", "false"].includes(result.isolate)) throw new Error("--isolate must be both, true, or false"); continue; }
const name = names.get(arg);
if (!name) throw new Error(`Unknown argument: ${arg}`);
const value = Number(argv[++index]);
if (!Number.isSafeInteger(value) || value <= 0) throw new Error(`${arg} must be a positive integer`);
result[name] = value;
}
if (result.files <= result.workers) throw new Error("indeterminate configuration: --files must be strictly greater than --workers");
const budgets = deriveBudgets(result.delayMs);
if (budgets.SMALL >= budgets.D || budgets.LARGE <= budgets.D) throw new Error("delay does not produce discriminating budgets");
result.runTimeoutMs ??= Math.max(60_000, budgets.LARGE + result.delayMs * result.files + 30_000);
return result;
}
/** Parse append-only JSONL, intentionally ignoring an interrupted final write. */
export function parseLedger(text) {
const lines = text.split(/\r?\n/);
return lines.flatMap((line, index) => {
if (!line.trim()) return [];
try {
const event = JSON.parse(line);
if (!event || typeof event !== "object" || !Number.isFinite(event.pid) || !Number.isFinite(event.epochMs) || typeof event.monotonicNs !== "string") return [];
return [{ ...event, ledgerIndex: index }];
} catch { return []; }
});
}
function eventDuration(events, boundary) {
const start = events.find((event) => event.type === "boundary" && event.boundary === boundary && event.phase === "start");
const end = events.find((event) => event.type === "boundary" && event.boundary === boundary && event.phase === "end" && event.pid === start?.pid && event.file === start?.file);
return start && end ? end.epochMs - start.epochMs : null;
}
export function summarizeLedger(boundary, events, { files, orderingMarginMs }) {
const starts = events.filter((event) => event.type === "boundary" && event.boundary === boundary && event.phase === "start");
const pairs = new Set(starts.map((event) => `${event.pid}:${event.file ?? ""}`));
const pidCount = new Set(starts.map((event) => event.pid)).size;
const fileCount = new Set(starts.map((event) => event.file).filter(Boolean)).size;
/*
FNXC:PgSetupBoundaryProbe 2026-08-17-05:34:
A setup-file callback can be evaluated repeatedly in one worker. Without a test-file identity,
pid/file pairs cannot distinguish that from worker-scoped setup, even when PIDs are recycled.
*/
const hasFileIdentity = starts.every((event) => typeof event.file === "string" && event.file.length > 0);
const granularity = starts.length === 1 ? "per invocation"
: !hasFileIdentity ? "indeterminate"
: pairs.size === files && fileCount === files ? "per file"
: pairs.size === pidCount ? "per worker"
: "indeterminate";
const tests = events.filter((event) => event.type === "test" && event.phase === "start");
let ordering = "not-observed";
if (starts.length && tests.length) {
const last = starts.toSorted((a, b) => a.pid === b.pid ? Number(BigInt(a.monotonicNs) - BigInt(b.monotonicNs)) : a.epochMs - b.epochMs).at(-1);
const first = tests.toSorted((a, b) => a.pid === b.pid ? Number(BigInt(a.monotonicNs) - BigInt(b.monotonicNs)) : a.epochMs - b.epochMs)[0];
if (last.pid === first.pid) ordering = BigInt(last.monotonicNs) < BigInt(first.monotonicNs) ? "before-first-test" : "after-first-test";
else if (Math.abs(first.epochMs - last.epochMs) < orderingMarginMs) ordering = "indeterminate";
else ordering = last.epochMs < first.epochMs ? "before-first-test" : "after-first-test";
}
return { boundary, executions: starts.length, distinctPidFiles: pairs.size, workersObserved: pidCount, filesObserved: fileCount, granularity, ordering, durationMs: eventDuration(events, boundary) };
}
function excerpt(output) { return output.slice(-DIAGNOSTIC_LIMIT); }
export function classifyArm({ exitCode, output = "", events = [], boundary, delayMs, timedOut = false }) {
if (timedOut) return { outcome: "run-timeout" };
const summary = summarizeLedger(boundary, events, { files: Number.MAX_SAFE_INTEGER, orderingMarginMs: 1 });
if (exitCode === 0) {
if (summary.durationMs == null || summary.durationMs < delayMs * 0.9) return { outcome: "failed-unclassified", diagnostic: "boundary end missing or duration below tolerance" };
return { outcome: "passed", durationMs: summary.durationMs };
}
if (/Test timed out in \d+ms\./i.test(output)) return { outcome: "timed-out-test" };
if (/Hook timed out in \d+ms\./i.test(output)) return { outcome: "timed-out-hook" };
return { outcome: "failed-unclassified", diagnostic: excerpt(output) };
}
export function resolveOwnership(arms) {
const A = arms.A?.outcome; const B = arms.B?.outcome; const C = arms.C?.outcome; const R = arms.R?.outcome;
if ([A, B, C, R].some((outcome) => outcome === "run-timeout")) return "run-timeout";
if ([A, B, C, R].some((outcome) => outcome === "failed-unclassified")) return "failed-unclassified";
if (A === "passed" && R === "passed" && B === "passed" && C === "passed") return "off-budget";
if (A === "timed-out-test" && B === "timed-out-test" && C === "passed" && R === "passed") return "test-timeout";
if (A === "timed-out-hook" && C === "timed-out-hook" && B === "passed" && R === "passed") return "hook-timeout";
if (A !== "passed" && B === "timed-out-test" && C === "timed-out-hook" && R === "passed") return "both-budgets";
// Arm R alone is an intentionally inconclusive calibration observation, never a survey fact.
if (A === undefined && B === undefined && C === undefined && R === "passed") return "completed-within-budget";
return "failed-unclassified";
}
/*
FNXC:PgSetupBoundaryProbe 2026-08-17-05:12:
FN-9140's qualifying off-budget evidence belongs to arm A, not the last arm executed. Preserve every
arm summary and require it to agree across repeats so an after-test A observation or arm-level flap
cannot be masked by a stable reference arm.
*/
export function summarizeRepeats(boundary, repeats) {
const values = repeats.map((repeat) => ({
ownership: repeat.ownership,
arms: Object.fromEntries(ARMS.map((arm) => [arm, repeat.arms[arm]?.outcome])),
summaries: Object.fromEntries(ARMS.map((arm) => [arm, {
ordering: repeat.summaries[arm]?.ordering,
granularity: repeat.summaries[arm]?.granularity,
}])),
}));
const deterministic = values.every((value) => JSON.stringify(value) === JSON.stringify(values[0]));
const evidence = values[0]?.summaries.A;
return {
boundary,
repeats,
deterministic,
ownership: deterministic ? values[0]?.ownership : "failed-unclassified",
ordering: deterministic ? evidence?.ordering : "indeterminate",
granularity: deterministic ? evidence?.granularity : "indeterminate",
};
}
export function decideSurveyVerdict({ calibration, cells }) {
if (calibration !== "passed") return "insufficient-data";
if (cells.some((cell) => !cell.deterministic || ["failed-unclassified", "run-timeout"].includes(cell.ownership) || cell.ordering === "indeterminate")) return "insufficient-data";
return cells.some((cell) => cell.ownership === "off-budget" && cell.ordering === "before-first-test") ? "prerequisite-established" : "prerequisite-not-established";
}
export function fixtureFiles({ boundary, arm, options, ledgerPath, vitestApiUrl, control }) {
const budgets = deriveBudgets(options.delayMs);
const budget = arm === "A" ? [budgets.SMALL, budgets.SMALL] : arm === "B" ? [budgets.SMALL, budgets.LARGE] : arm === "C" ? [budgets.LARGE, budgets.SMALL] : [budgets.LARGE, budgets.LARGE];
const emit = (type, name, phase, fileExpression = '""') => `appendFileSync(process.env.FUSION_BOUNDARY_LEDGER, JSON.stringify({type:${JSON.stringify(type)},boundary:${JSON.stringify(name)},phase:${JSON.stringify(phase)},pid:process.pid,file:${fileExpression},isolate:${JSON.stringify(String(options.isolate))},arm:${JSON.stringify(arm)},epochMs:Date.now(),monotonicNs:process.hrtime.bigint().toString()})+'\\n');`;
const sleep = `await new Promise(resolve => setTimeout(resolve, ${options.delayMs}));`;
const boundaryBody = (fileExpression) => `${emit("boundary", boundary, "start", fileExpression)} ${sleep} ${emit("boundary", boundary, "end", fileExpression)}`;
const testBody = control === "test" ? `${emit("boundary", "control-test", "start")} ${sleep} ${emit("boundary", "control-test", "end")}` : "";
const config = `export default { test: { include:["tests/**/*.test.mjs"], pool:"forks", isolate:${options.isolate}, maxWorkers:${options.workers}, minWorkers:${options.workers}, fileParallelism:true, testTimeout:${budget[0]}, hookTimeout:${budget[1]}, setupFiles:["./setup.mjs"]${boundary === "global-setup" ? ',globalSetup:["./global-setup.mjs"]' : ""} } };\n`;
/*
FNXC:PgSetupBoundaryProbe 2026-08-17-05:34:
FN-9140 must not infer per-file setup execution from worker PIDs. beforeAll can read Vitest's
active test path; top-level setup has no supported file context and is deliberately indeterminate.
*/
const setup = `import { appendFileSync } from "node:fs"; import { beforeAll, expect } from ${JSON.stringify(vitestApiUrl)}; ${boundary === "setup-top-level-await" ? boundaryBody() : ""} ${boundary === "setup-before-all" || control === "hook" ? `beforeAll(async()=>{${control === "hook" ? `${emit("boundary", "control-hook", "start")} ${sleep} ${emit("boundary", "control-hook", "end")}` : boundaryBody('expect.getState().testPath ?? ""')}});` : ""}`;
const global = `import { appendFileSync } from "node:fs"; export default async function(){ ${boundary === "global-setup" ? boundaryBody() : ""} }`;
const files = { "vitest.config.mjs": config, "setup.mjs": setup, "global-setup.mjs": global };
for (let index = 0; index < options.files; index += 1) files[`tests/${index}.test.mjs`] = `import { appendFileSync } from "node:fs"; import { beforeAll,test } from ${JSON.stringify(vitestApiUrl)}; ${boundary === "per-file-before-all" ? `beforeAll(async()=>{${boundaryBody(JSON.stringify(String(index)))}});` : ""} test(${JSON.stringify(String(index))}, async()=>{${testBody} ${emit("test", "test", "start", JSON.stringify(String(index)))} ${emit("test", "test", "end", JSON.stringify(String(index)))}});`;
return files;
}
function writeFixture(directory, files) { for (const [file, content] of Object.entries(files)) { const target = join(directory, file); mkdirSync(dirname(target), { recursive: true }); writeFileSync(target, content); } }
function vitestEntry() { return join(dirname(require.resolve("vitest", { paths: [resolve(process.cwd(), "packages/core")] })), "vitest.mjs"); }
export function resolveVitestRunner(resolveEntry = vitestEntry) {
try {
const entry = resolveEntry();
return { entry, vitestVersion: JSON.parse(readFileSync(join(dirname(entry), "package.json"), "utf8")).version };
} catch (error) {
return { entry: undefined, vitestVersion: "unresolvable", resolutionError: error instanceof Error ? error.message : String(error) };
}
}
export async function runFixtureCell({ boundary, arm, options, control, runner = resolveVitestRunner() }) {
const directory = mkdtempSync(join(tmpdir(), "fusion-pg-setup-boundary-")); const ledgerPath = join(directory, "ledger.jsonl");
try {
if (!runner.entry) {
const run = { exitCode: 1, output: `Vitest resolution failed: ${runner.resolutionError}`, timedOut: false, events: [] };
return { ...run, arm: classifyArm({ ...run, boundary: control ? `control-${control}` : boundary, delayMs: options.delayMs }) };
}
try {
const api = pathToFileURL(join(dirname(runner.entry), "dist", "index.js")).href;
writeFixture(directory, fixtureFiles({ boundary, arm, options, ledgerPath, vitestApiUrl: api, control }));
const run = await new Promise((done) => {
const child = spawn(process.execPath, [runner.entry, "run", "--config", "vitest.config.mjs", "--reporter=verbose"], { cwd: directory, env: { ...process.env, FUSION_BOUNDARY_LEDGER: ledgerPath }, stdio: ["ignore", "pipe", "pipe"] }); let output = ""; let timedOut = false;
child.stdout.on("data", (chunk) => { output += chunk; }); child.stderr.on("data", (chunk) => { output += chunk; });
child.once("error", (error) => { output += `Vitest spawn failed: ${error.message}`; });
const timer = setTimeout(() => { timedOut = true; child.kill("SIGKILL"); }, options.runTimeoutMs);
child.on("close", (exitCode) => { clearTimeout(timer); done({ exitCode: exitCode ?? 1, output, timedOut }); });
});
let events = [];
try { events = parseLedger(readFileSync(ledgerPath, "utf8")); } catch { /* A setup failure may occur before the first ledger append. */ }
return { ...run, events, arm: classifyArm({ ...run, events, boundary: control ? `control-${control}` : boundary, delayMs: options.delayMs }) };
} catch (error) {
const run = { exitCode: 1, output: `Vitest fixture failed: ${error instanceof Error ? error.message : String(error)}`, timedOut: false, events: [] };
return { ...run, arm: classifyArm({ ...run, boundary: control ? `control-${control}` : boundary, delayMs: options.delayMs }) };
}
} finally { rmSync(directory, { recursive: true, force: true }); }
}
export async function runSurvey(options, runner = resolveVitestRunner()) {
const budgets = deriveBudgets(options.delayMs); const isolateModes = options.isolate === "both" ? [true, false] : [options.isolate === "true"];
const calibrationResults = [];
for (const [control, arm, expected] of [["hook", "C", "hook-timeout"], ["test", "B", "test-timeout"], ["test", "R", "completed-within-budget"]]) {
const run = await runFixtureCell({ boundary: "setup-before-all", arm, options: { ...options, isolate: true }, control, runner });
const outcome = control === "test" && arm === "R" && run.arm.outcome === "passed" ? "completed-within-budget" : run.arm.outcome === "timed-out-hook" ? "hook-timeout" : run.arm.outcome === "timed-out-test" ? "test-timeout" : run.arm.outcome;
calibrationResults.push({ control, arm, expected, actual: outcome });
}
const calibration = calibrationResults.every((row) => row.actual === row.expected) ? "passed" : "failed";
const cells = [];
for (const isolate of isolateModes) for (const boundary of BOUNDARIES) {
const repeats = [];
for (let repeat = 0; repeat < options.repeats; repeat += 1) {
const arms = {}; const summaries = {};
for (const arm of ARMS) {
const run = await runFixtureCell({ boundary, arm, options: { ...options, isolate }, runner });
arms[arm] = run.arm;
summaries[arm] = summarizeLedger(boundary, run.events, options);
}
repeats.push({ repeat, arms, summaries, ownership: resolveOwnership(arms) });
}
cells.push({ isolate, ...summarizeRepeats(boundary, repeats) });
}
return { vitestVersion: runner.vitestVersion, nodeVersion: process.version, pool: "forks", isolateModes, workers: options.workers, files: options.files, repeats: options.repeats, budgetMatrix: { ...budgets, orderingMarginMs: options.orderingMarginMs, runTimeoutMs: options.runTimeoutMs }, calibration, calibrationFailureReason: calibration === "failed" ? "classifier-uncalibrated" : undefined, runnerResolutionError: runner.resolutionError, calibrationResults, cells, verdict: decideSurveyVerdict({ calibration, cells }) };
}
async function main() {
const options = parseProbeArgs(process.argv.slice(2));
if (options.help) return console.log("Usage: node scripts/pg-setup-boundary-probe.mjs --workers 2 --files 6 --repeats 3 --delay-ms 4000 [--json path]");
const isolateModes = options.isolate === "both" ? [true, false] : [options.isolate === "true"];
console.log(`[pg-setup-boundary-probe] per-cell bound ${options.runTimeoutMs}ms; total bounded cells ${(3 + BOUNDARIES.length * isolateModes.length * ARMS.length) * options.repeats}`);
const report = await runSurvey(options);
if (options.json) writeFileSync(options.json, `${JSON.stringify(report, null, 2)}\n`);
console.log(JSON.stringify(report, null, 2));
}
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) main().catch((error) => { console.error(`[pg-setup-boundary-probe] ${error.message}`); process.exitCode = 1; });