FN-9149: Add PostgreSQL timeout-boundary diagnostics
Instrument the opt-in PostgreSQL test harness to attribute loaded-lane timeout failures without changing default behavior. - Add bounded setup, body, and teardown watchdog probes with host, cluster, and template evidence. - Wire observer records into the harness and loaded-failure census with explicit suppression and attribution handling. - Cover observer inertness, boundary behavior, and census joins while documenting the 27-worker campaign findings. Files changed: ...res-loaded-lane-unrelated-failure-population.md | 31 +- docs/testing.md | 27 ++ .../core/src/__test-utils__/pg-test-harness.ts | 113 +++++- .../__test-utils__/pg-timeout-boundary-observer.ts | 451 +++++++++++++++++++++ .../pg-test-harness-observer-inertness.test.ts | 31 ++ .../__tests__/pg-timeout-boundary-observer.test.ts | 177 ++++++++ .../__tests__/pg-loaded-failure-census.test.mjs | 31 ++ scripts/pg-loaded-failure-census.mjs | 101 ++++- 8 files changed, 934 insertions(+), 28 deletions(-) Fusion-Task-Id: FN-9149 Fusion-Task-Lineage: 4df4ee28-5369-41ae-bb0c-e7e9ae78d873 Co-authored-by: Fusion (runfusion.ai) <noreply@runfusion.ai>
This commit is contained in:
@@ -8,6 +8,8 @@ import {
|
||||
classifyLifecyclePosition,
|
||||
extractFailingFiles,
|
||||
parseDiagnosticsJsonl,
|
||||
parseBoundaryObserverJsonl,
|
||||
classifyBoundaryAttribution,
|
||||
stripAnsi,
|
||||
} from "../pg-loaded-failure-census.mjs";
|
||||
|
||||
@@ -49,6 +51,35 @@ test("censuses every high-failure file and joins snapshot diagnostics", () => {
|
||||
assert.equal(extractFailingFiles(fixture("high-run.txt")).length, 25);
|
||||
});
|
||||
|
||||
test("joins out-of-order watchdog payloads by file and boundary, not line order", () => {
|
||||
const failure = { file: "src/__tests__/postgres/body-case.test.ts", lifecyclePosition: "test body" };
|
||||
const parsed = parseBoundaryObserverJsonl(`${JSON.stringify({ testFile: failure.file, boundary: "body", trigger: "boundary-complete", timestamp: "2026-01-01T00:00:02Z", host: { loadavg1: 0, cpuCount: 8, eventLoopLagMs: 0 } })}\n${JSON.stringify({ testFile: failure.file, boundary: "body", trigger: "boundary-watchdog", timestamp: "2026-01-01T00:00:01Z", settledDuringProbe: true, host: { loadavg1: 0, cpuCount: 8, eventLoopLagMs: 0 }, cluster: { activity: [{ state: "active", blockingPids: [44] }], locks: [] }, template: { markerPresent: true } })}\nmalformed`);
|
||||
assert.equal(parsed.malformedLines, 1);
|
||||
assert.equal(classifyBoundaryAttribution(failure, parsed.rows).classification, "cluster-implicated");
|
||||
});
|
||||
|
||||
test("keeps explicit unobservable sets and suppressed watchdog failures distinct from joined attribution", () => {
|
||||
const body = { file: "src/__tests__/postgres/direct.test.ts", lifecyclePosition: "test body" };
|
||||
assert.equal(classifyBoundaryAttribution(body, [], [body.file]).classification, "body-unobservable");
|
||||
const suppressed = [{ testFile: body.file, boundary: "body", trigger: "boundary-watchdog", probeSuppressed: "single-flight", host: { loadavg1: 0, cpuCount: 8, eventLoopLagMs: 0 } }];
|
||||
assert.equal(classifyBoundaryAttribution(body, suppressed).classification, "unjoined");
|
||||
const fully = { file: "src/__tests__/postgres/no-harness.test.ts", lifecyclePosition: "afterEach" };
|
||||
const census = buildCensus({
|
||||
log: ` FAIL ${fully.file} > leaves no harness boundary\nError: afterEach hook timed out in 15000ms.\n\n Test Files 1 failed (1)\n`,
|
||||
fullyUnobservableFiles: [fully.file],
|
||||
});
|
||||
assert.equal(census.fullyUnobservableFailingFileCount, 1);
|
||||
assert.deepEqual(census.fullyUnobservableFailingFiles, [fully.file]);
|
||||
assert.equal(census.attributions[0].boundaryAttribution.classification, "unjoined");
|
||||
});
|
||||
|
||||
test("requires a golden advisory waiter, not a holder, for template convoy attribution", () => {
|
||||
const failure = { file: "src/__tests__/postgres/template.test.ts", lifecyclePosition: "beforeAll hook" };
|
||||
const base = { testFile: failure.file, boundary: "setup", trigger: "boundary-watchdog", host: { loadavg1: 0, cpuCount: 8, eventLoopLagMs: 0 }, cluster: { activity: [], locks: [] } };
|
||||
assert.notEqual(classifyBoundaryAttribution(failure, [{ ...base, template: { advisoryHolders: [10], advisoryWaiters: [], isOwner: false } }]).classification, "template-convoy");
|
||||
assert.equal(classifyBoundaryAttribution(failure, [{ ...base, template: { advisoryHolders: [10], advisoryWaiters: [11], isOwner: false } }]).classification, "template-convoy");
|
||||
});
|
||||
|
||||
test("reports a complete healthy run as measured zero rather than insufficient data", () => {
|
||||
const census = buildCensus({ log: fixture("low-run.txt"), diagnostics: [], ordinarySlotCeiling: 97 });
|
||||
assert.equal(census.status, "measured");
|
||||
|
||||
Reference in New Issue
Block a user