FN-9149: Add PostgreSQL timeout-boundary diagnostics

Instrument the opt-in PostgreSQL test harness to attribute loaded-lane timeout failures without changing default behavior.

- Add bounded setup, body, and teardown watchdog probes with host, cluster, and template evidence.
- Wire observer records into the harness and loaded-failure census with explicit suppression and attribution handling.
- Cover observer inertness, boundary behavior, and census joins while documenting the 27-worker campaign findings.

Files changed:
 ...res-loaded-lane-unrelated-failure-population.md |  31 +-
 docs/testing.md                                    |  27 ++
 .../core/src/__test-utils__/pg-test-harness.ts     | 113 +++++-
 .../__test-utils__/pg-timeout-boundary-observer.ts | 451 +++++++++++++++++++++
 .../pg-test-harness-observer-inertness.test.ts     |  31 ++
 .../__tests__/pg-timeout-boundary-observer.test.ts | 177 ++++++++
 .../__tests__/pg-loaded-failure-census.test.mjs    |  31 ++
 scripts/pg-loaded-failure-census.mjs               | 101 ++++-
 8 files changed, 934 insertions(+), 28 deletions(-)

Fusion-Task-Id: FN-9149

Fusion-Task-Lineage: 4df4ee28-5369-41ae-bb0c-e7e9ae78d873

Co-authored-by: Fusion (runfusion.ai) <noreply@runfusion.ai>
This commit is contained in:
gsxdsm
2026-08-19 08:41:14 -07:00
parent 161edaa694
commit a1977e052b
8 changed files with 934 additions and 28 deletions

View File

@@ -8,6 +8,8 @@ import {
classifyLifecyclePosition,
extractFailingFiles,
parseDiagnosticsJsonl,
parseBoundaryObserverJsonl,
classifyBoundaryAttribution,
stripAnsi,
} from "../pg-loaded-failure-census.mjs";
@@ -49,6 +51,35 @@ test("censuses every high-failure file and joins snapshot diagnostics", () => {
assert.equal(extractFailingFiles(fixture("high-run.txt")).length, 25);
});
test("joins out-of-order watchdog payloads by file and boundary, not line order", () => {
const failure = { file: "src/__tests__/postgres/body-case.test.ts", lifecyclePosition: "test body" };
const parsed = parseBoundaryObserverJsonl(`${JSON.stringify({ testFile: failure.file, boundary: "body", trigger: "boundary-complete", timestamp: "2026-01-01T00:00:02Z", host: { loadavg1: 0, cpuCount: 8, eventLoopLagMs: 0 } })}\n${JSON.stringify({ testFile: failure.file, boundary: "body", trigger: "boundary-watchdog", timestamp: "2026-01-01T00:00:01Z", settledDuringProbe: true, host: { loadavg1: 0, cpuCount: 8, eventLoopLagMs: 0 }, cluster: { activity: [{ state: "active", blockingPids: [44] }], locks: [] }, template: { markerPresent: true } })}\nmalformed`);
assert.equal(parsed.malformedLines, 1);
assert.equal(classifyBoundaryAttribution(failure, parsed.rows).classification, "cluster-implicated");
});
test("keeps explicit unobservable sets and suppressed watchdog failures distinct from joined attribution", () => {
const body = { file: "src/__tests__/postgres/direct.test.ts", lifecyclePosition: "test body" };
assert.equal(classifyBoundaryAttribution(body, [], [body.file]).classification, "body-unobservable");
const suppressed = [{ testFile: body.file, boundary: "body", trigger: "boundary-watchdog", probeSuppressed: "single-flight", host: { loadavg1: 0, cpuCount: 8, eventLoopLagMs: 0 } }];
assert.equal(classifyBoundaryAttribution(body, suppressed).classification, "unjoined");
const fully = { file: "src/__tests__/postgres/no-harness.test.ts", lifecyclePosition: "afterEach" };
const census = buildCensus({
log: ` FAIL ${fully.file} > leaves no harness boundary\nError: afterEach hook timed out in 15000ms.\n\n Test Files 1 failed (1)\n`,
fullyUnobservableFiles: [fully.file],
});
assert.equal(census.fullyUnobservableFailingFileCount, 1);
assert.deepEqual(census.fullyUnobservableFailingFiles, [fully.file]);
assert.equal(census.attributions[0].boundaryAttribution.classification, "unjoined");
});
test("requires a golden advisory waiter, not a holder, for template convoy attribution", () => {
const failure = { file: "src/__tests__/postgres/template.test.ts", lifecyclePosition: "beforeAll hook" };
const base = { testFile: failure.file, boundary: "setup", trigger: "boundary-watchdog", host: { loadavg1: 0, cpuCount: 8, eventLoopLagMs: 0 }, cluster: { activity: [], locks: [] } };
assert.notEqual(classifyBoundaryAttribution(failure, [{ ...base, template: { advisoryHolders: [10], advisoryWaiters: [], isOwner: false } }]).classification, "template-convoy");
assert.equal(classifyBoundaryAttribution(failure, [{ ...base, template: { advisoryHolders: [10], advisoryWaiters: [11], isOwner: false } }]).classification, "template-convoy");
});
test("reports a complete healthy run as measured zero rather than insufficient data", () => {
const census = buildCensus({ log: fixture("low-run.txt"), diagnostics: [], ordinarySlotCeiling: 97 });
assert.equal(census.status, "measured");