root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

FN-9149: Add PostgreSQL timeout-boundary diagnostics

Browse Source 
Instrument the opt-in PostgreSQL test harness to attribute loaded-lane timeout failures without changing default behavior.

- Add bounded setup, body, and teardown watchdog probes with host, cluster, and template evidence.
- Wire observer records into the harness and loaded-failure census with explicit suppression and attribution handling.
- Cover observer inertness, boundary behavior, and census joins while documenting the 27-worker campaign findings.

Files changed:
 ...res-loaded-lane-unrelated-failure-population.md |  31 +-
 docs/testing.md                                    |  27 ++
 .../core/src/__test-utils__/pg-test-harness.ts     | 113 +++++-
 .../__test-utils__/pg-timeout-boundary-observer.ts | 451 +++++++++++++++++++++
 .../pg-test-harness-observer-inertness.test.ts     |  31 ++
 .../__tests__/pg-timeout-boundary-observer.test.ts | 177 ++++++++
 .../__tests__/pg-loaded-failure-census.test.mjs    |  31 ++
 scripts/pg-loaded-failure-census.mjs               | 101 ++++-
 8 files changed, 934 insertions(+), 28 deletions(-)

Fusion-Task-Id: FN-9149

Fusion-Task-Lineage: 4df4ee28-5369-41ae-bb0c-e7e9ae78d873

Co-authored-by: Fusion (runfusion.ai) <noreply@runfusion.ai>
This commit is contained in:
gsxdsm
2026-08-19 08:41:14 -07:00
parent 161edaa694
commit a1977e052b
8 changed files with 934 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
scripts/__tests__/pg-loaded-failure-census.test.mjs
View File

@@ -8,6 +8,8 @@ import {
classifyLifecyclePosition,
extractFailingFiles,
parseDiagnosticsJsonl,
parseBoundaryObserverJsonl,
classifyBoundaryAttribution,
stripAnsi,
} from "../pg-loaded-failure-census.mjs";
@@ -49,6 +51,35 @@ test("censuses every high-failure file and joins snapshot diagnostics", () => {
assert.equal(extractFailingFiles(fixture("high-run.txt")).length, 25);
});
test("joins out-of-order watchdog payloads by file and boundary, not line order", () => {
const failure = { file: "src/__tests__/postgres/body-case.test.ts", lifecyclePosition: "test body" };
const parsed = parseBoundaryObserverJsonl(`${JSON.stringify({ testFile: failure.file, boundary: "body", trigger: "boundary-complete", timestamp: "2026-01-01T00:00:02Z", host: { loadavg1: 0, cpuCount: 8, eventLoopLagMs: 0 } })}\n${JSON.stringify({ testFile: failure.file, boundary: "body", trigger: "boundary-watchdog", timestamp: "2026-01-01T00:00:01Z", settledDuringProbe: true, host: { loadavg1: 0, cpuCount: 8, eventLoopLagMs: 0 }, cluster: { activity: [{ state: "active", blockingPids: [44] }], locks: [] }, template: { markerPresent: true } })}\nmalformed`);
assert.equal(parsed.malformedLines, 1);
assert.equal(classifyBoundaryAttribution(failure, parsed.rows).classification, "cluster-implicated");
});
test("keeps explicit unobservable sets and suppressed watchdog failures distinct from joined attribution", () => {
const body = { file: "src/__tests__/postgres/direct.test.ts", lifecyclePosition: "test body" };
assert.equal(classifyBoundaryAttribution(body, [], [body.file]).classification, "body-unobservable");
const suppressed = [{ testFile: body.file, boundary: "body", trigger: "boundary-watchdog", probeSuppressed: "single-flight", host: { loadavg1: 0, cpuCount: 8, eventLoopLagMs: 0 } }];
assert.equal(classifyBoundaryAttribution(body, suppressed).classification, "unjoined");
const fully = { file: "src/__tests__/postgres/no-harness.test.ts", lifecyclePosition: "afterEach" };
const census = buildCensus({
log: ` FAIL ${fully.file} > leaves no harness boundary\nError: afterEach hook timed out in 15000ms.\n\n Test Files 1 failed (1)\n`,
fullyUnobservableFiles: [fully.file],
});
assert.equal(census.fullyUnobservableFailingFileCount, 1);
assert.deepEqual(census.fullyUnobservableFailingFiles, [fully.file]);
assert.equal(census.attributions[0].boundaryAttribution.classification, "unjoined");
});
test("requires a golden advisory waiter, not a holder, for template convoy attribution", () => {
const failure = { file: "src/__tests__/postgres/template.test.ts", lifecyclePosition: "beforeAll hook" };
const base = { testFile: failure.file, boundary: "setup", trigger: "boundary-watchdog", host: { loadavg1: 0, cpuCount: 8, eventLoopLagMs: 0 }, cluster: { activity: [], locks: [] } };
assert.notEqual(classifyBoundaryAttribution(failure, [{ ...base, template: { advisoryHolders: [10], advisoryWaiters: [], isOwner: false } }]).classification, "template-convoy");
assert.equal(classifyBoundaryAttribution(failure, [{ ...base, template: { advisoryHolders: [10], advisoryWaiters: [11], isOwner: false } }]).classification, "template-convoy");
});
test("reports a complete healthy run as measured zero rather than insufficient data", () => {
const census = buildCensus({ log: fixture("low-run.txt"), diagnostics: [], ordinarySlotCeiling: 97 });
assert.equal(census.status, "measured");

101
scripts/pg-loaded-failure-census.mjs
View File

@@ -34,7 +34,12 @@ export function parseDiagnosticsJsonl(text) {
}
function normalizeFile(value) {
const match = String(value).replaceAll("\\", "/").match(/(?:[\w@.-]+\/)*[\w@.-]+(?:\.pg)?\.test\.[cm]?[jt]sx?/i);
const normalized = String(value).replaceAll("\\", "/");
// Vitest diagnostics use absolute paths while runner failures use repo paths.
// Canonicalize at the test-root segment before any key-based observer join.
const testRoot = normalized.indexOf("src/__tests__/");
if (testRoot >= 0) return normalized.slice(testRoot);
const match = normalized.match(/(?:[\w@.-]+\/)*[\w@.-]+(?:\.pg)?\.test\.[cm]?[jt]sx?/i);
return match?.[0] ?? null;
}
@@ -93,6 +98,86 @@ export function parseFileSummary(log) {
return { complete: true, totalFiles: failed + passed + skipped, reportedFailedFiles: failed };
}
export function parseBoundaryObserverJsonl(text) {
return parseDiagnosticsJsonl(text);
}
function boundaryForLifecycle(lifecyclePosition) {
if (lifecyclePosition === "beforeAll hook" || lifecyclePosition === "in-test setup") return "setup";
if (lifecyclePosition === "afterEach" || lifecyclePosition === "afterAll hook" || lifecyclePosition === "global setup-teardown") return "teardown";
return "body";
}
function observerFile(record) {
return normalizeFile(record?.testFile);
}
/**
* FNXC:PgTimeoutBoundaryObserver 2026-08-19-13:51:
* Watchdog payloads win because they are the only records carrying a cluster
* snapshot. Completion-only and suppressed records may support host evidence,
* but must never be promoted to cluster causation by inference.
*/
export function classifyBoundaryAttribution(failure, observerRecords, bodyUnobservableFiles = [], fullyUnobservableFiles = []) {
const boundary = boundaryForLifecycle(failure.lifecyclePosition);
const sameFile = observerRecords.filter((record) => observerFile(record) === failure.file && record.boundary === boundary);
if (fullyUnobservableFiles.includes(failure.file)) {
// These files never enter a harness-owned boundary, so their missing join
// is an explicit coverage limit rather than a failed observer correlation.
return { classification: "unjoined", boundary, record: null, hostOnly: false, fullyUnobservable: true };
}
if (boundary === "body" && bodyUnobservableFiles.includes(failure.file)) {
return { classification: "body-unobservable", boundary, record: null, hostOnly: false, fullyUnobservable: false };
}
const watchdog = sameFile.filter((record) => record.trigger === "boundary-watchdog");
const record = watchdog.find((candidate) => !candidate.probeSuppressed && candidate.cluster && candidate.template)
?? watchdog[0]
?? sameFile[0]
?? null;
if (!record) return { classification: "unjoined", boundary, record: null, hostOnly: false };
const hostOnly = record.trigger !== "boundary-watchdog" || Boolean(record.probeSuppressed) || !record.cluster;
if (hostOnly) {
const load = Number(record?.host?.loadavg1);
const cpus = Number(record?.host?.cpuCount);
const lag = Number(record?.host?.eventLoopLagMs);
return { classification: (Number.isFinite(load) && Number.isFinite(cpus) && load >= cpus) || lag >= 100 ? "host-implicated" : "unjoined", boundary, record, hostOnly: true };
}
const template = record.template ?? {};
// A holder alone is not a convoy: only a non-owner waiter proves the
// timed-out boundary was queued behind the golden template advisory lock.
if (Array.isArray(template.advisoryWaiters) && template.advisoryWaiters.length > 0 && template.isOwner === false) {
return { classification: "template-convoy", boundary, record, hostOnly: false, fullyUnobservable: false };
}
const cluster = record.cluster ?? {};
const active = Array.isArray(cluster.activity) && cluster.activity.some((row) => row?.state === "active" || row?.wait_event || row?.blockingPids?.length);
const blocked = Array.isArray(cluster.locks) && cluster.locks.some((lock) => lock?.granted === false || lock?.blockingPids?.length);
if (active || blocked) return { classification: "cluster-implicated", boundary, record, hostOnly: false };
const load = Number(record?.host?.loadavg1);
const cpus = Number(record?.host?.cpuCount);
const lag = Number(record?.host?.eventLoopLagMs);
if ((Number.isFinite(load) && Number.isFinite(cpus) && load >= cpus) || lag >= 100) return { classification: "host-implicated", boundary, record, hostOnly: false };
return { classification: "unjoined", boundary, record, hostOnly: false };
}
export function summarizeBoundaryObserver(records, failures, bodyUnobservableFiles = [], fullyUnobservableFiles = []) {
const rows = Array.isArray(records) ? records : [];
const attributions = failures.map((failure) => ({ ...failure, boundaryAttribution: classifyBoundaryAttribution(failure, rows, bodyUnobservableFiles, fullyUnobservableFiles) }));
const suppression = {};
for (const row of rows) {
const reason = row?.probeSuppressed === "single-flight" ? "concurrency" : row?.probeSuppressed;
if (reason) suppression[reason] = (suppression[reason] ?? 0) + 1;
}
return {
boundaryObserver: rows.length ? "present" : "absent",
boundaryAttributionHistogram: Object.fromEntries(Object.entries(Object.groupBy(attributions, (row) => row.boundaryAttribution.classification)).map(([key, values]) => [key, values.length])),
observerProbeSuppression: suppression,
settledDuringProbeCount: rows.filter((row) => row?.settledDuringProbe === true).length,
fullyUnobservableFailingFiles: attributions.filter((row) => row.boundaryAttribution.fullyUnobservable).map((row) => row.file),
fullyUnobservableFailingFileCount: attributions.filter((row) => row.boundaryAttribution.fullyUnobservable).length,
attributions,
};
}
export function summarizeDiagnostics(diagnostics) {
const input = Array.isArray(diagnostics) ? diagnostics : [];
const waits = new Map();
@@ -126,7 +211,7 @@ export function summarizeDiagnostics(diagnostics) {
};
}
export function buildCensus({ log, diagnostics = [], ordinarySlotCeiling = null, subjects = [] }) {
export function buildCensus({ log, diagnostics = [], boundaryObserver = [], bodyUnobservableFiles = [], fullyUnobservableFiles = [], ordinarySlotCeiling = null, subjects = [] }) {
const summary = parseFileSummary(log);
if (!summary.complete) {
return { status: "insufficient-data", reason: "missing Test Files summary", totalFiles: null, failingFiles: [], failingFileCount: null };
@@ -136,6 +221,7 @@ export function buildCensus({ log, diagnostics = [], ordinarySlotCeiling = null,
return { status: "insufficient-data", reason: `summary reports ${summary.reportedFailedFiles} failed files but ${failingFiles.length} failure blocks were parsed`, totalFiles: summary.totalFiles, failingFiles, failingFileCount: null };
}
const diagnosticSummary = summarizeDiagnostics(diagnostics);
const observerSummary = summarizeBoundaryObserver(boundaryObserver, failingFiles, bodyUnobservableFiles, fullyUnobservableFiles);
const ceiling = Number.isFinite(ordinarySlotCeiling) && ordinarySlotCeiling >= 0 ? ordinarySlotCeiling : null;
return {
status: "measured",
@@ -148,15 +234,19 @@ export function buildCensus({ log, diagnostics = [], ordinarySlotCeiling = null,
ordinarySlotCeiling: ceiling,
backendHeadroom: ceiling != null && diagnosticSummary.peakBackends != null ? ceiling - diagnosticSummary.peakBackends : null,
...diagnosticSummary,
...observerSummary,
};
}
function parseArgs(args) {
const result = { log: undefined, diagnostics: undefined, ordinarySlotCeiling: null, subjects: [] };
const result = { log: undefined, diagnostics: undefined, boundaryObserver: undefined, bodyUnobservableFiles: undefined, fullyUnobservableFiles: undefined, ordinarySlotCeiling: null, subjects: [] };
for (let index = 0; index < args.length; index += 1) {
const argument = args[index];
if (argument === "--log") result.log = args[++index];
else if (argument === "--diagnostics") result.diagnostics = args[++index];
else if (argument === "--boundary-observer") result.boundaryObserver = args[++index];
else if (argument === "--body-unobservable-files") result.bodyUnobservableFiles = args[++index];
else if (argument === "--fully-unobservable-files") result.fullyUnobservableFiles = args[++index];
else if (argument === "--ordinary-slot-ceiling") result.ordinarySlotCeiling = Number(args[++index]);
else if (argument === "--subject") result.subjects.push(args[++index]);
else throw new Error(`Unknown argument: ${argument}`);
@@ -168,5 +258,8 @@ function parseArgs(args) {
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
const args = parseArgs(process.argv.slice(2));
const parsed = args.diagnostics ? parseDiagnosticsJsonl(readFileSync(args.diagnostics, "utf8")) : { rows: [], malformedLines: 0 };
console.log(JSON.stringify({ ...buildCensus({ log: readFileSync(args.log, "utf8"), diagnostics: parsed.rows, ordinarySlotCeiling: args.ordinarySlotCeiling, subjects: args.subjects }), malformedDiagnosticLines: parsed.malformedLines }, null, 2));
const observer = args.boundaryObserver ? parseBoundaryObserverJsonl(readFileSync(args.boundaryObserver, "utf8")) : { rows: [], malformedLines: 0 };
const bodyUnobservableFiles = args.bodyUnobservableFiles ? readFileSync(args.bodyUnobservableFiles, "utf8").split(/\\r?\\n/).map(normalizeFile).filter(Boolean) : [];
const fullyUnobservableFiles = args.fullyUnobservableFiles ? readFileSync(args.fullyUnobservableFiles, "utf8").split(/\\r?\\n/).map(normalizeFile).filter(Boolean) : [];
console.log(JSON.stringify({ ...buildCensus({ log: readFileSync(args.log, "utf8"), diagnostics: parsed.rows, boundaryObserver: observer.rows, bodyUnobservableFiles, fullyUnobservableFiles, ordinarySlotCeiling: args.ordinarySlotCeiling, subjects: args.subjects }), malformedDiagnosticLines: parsed.malformedLines, malformedBoundaryObserverLines: observer.malformedLines }, null, 2));
}