Files
fusion/packages/dashboard/src/monitor-trait.ts
gsxdsm 0d854263ee Address PR review feedback round 2 (#1683)
- monitor-store: add releaseIncidentFixTaskClaim (guarded UPDATE that only
  clears an in-flight sentinel, never a real attached task id) and make
  countRecentAutoFixTasks ignore sentinel placeholders, so a claim stranded by
  a failed createTask can't permanently absorb/suppress future regressions
- monitor-trait: release the claim if createTask throws after a successful
  claim, returning an error outcome instead of stranding the sentinel
- tests: release-vs-real-id, sentinel-excluded count, createTask-failure-then-reopen
- fix a type-unsound narrowing in the concurrency test (cast to the full union
  exposed the error variant's missing incidentId); use a discriminated guard
- add FNXC annotations on the new release/count paths and the concurrency harness

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-16 15:11:09 -07:00

239 lines
9.2 KiB
TypeScript

import type {
Task,
TaskCreateInput,
TaskStore,
TraitDefinition,
} from "@fusion/core";
import { getTraitRegistry, registerTraitHookImpl } from "@fusion/core";
import { createSessionDiagnostics } from "./ai-session-diagnostics.js";
import {
attachFixTask,
claimIncidentForFixTask,
countRecentAutoFixTasks,
decideStormGuard,
ingestIncidentSignal,
releaseIncidentFixTaskClaim,
type IncidentSignalInput,
type StormGuardConfig,
} from "./monitor-store.js";
/**
* U13 — Monitor stage trait.
*
* A column carrying the `monitor` trait watches post-ship work. When a card
* enters it, the trait records that the shipped change is now being monitored.
* Separately, a regression signal (an inbound U11 error signal arriving after a
* ship) is fed through {@link runMonitorOnRegression}, which opens — through the
* storm guard — at most ONE linked fix task in `triage`, closing the loop back to
* Triage (U12).
*
* Mirrors `triage-trait.ts`: the trait DEFINITION is registered as a built-in so
* plugins cannot override it; the IMPLEMENTATION lives in dashboard because it
* reuses the monitor store + the task store, wired through the core→dashboard DI
* seam (`registerTraitHookImpl`). The trait never re-triggers on a fix task it
* itself opened (no self-loop), mirroring U12.
*/
const diagnostics = createSessionDiagnostics("monitor-trait");
/** Registry id of the monitor trait. */
export const MONITOR_TRAIT_ID = "monitor";
/** Column an auto-opened fix task lands in (back to the start of the loop). */
export const MONITOR_FIX_ROUTE_COLUMN = "triage";
/** Metadata marking a task as a Fusion-opened monitor fix task (self-loop guard). */
export const MONITOR_FIX_TASK_META_KEY = "monitorFixForIncidentId";
/** Metadata carrying the grouping key the fix task addresses. */
export const MONITOR_FIX_GROUPING_META_KEY = "monitorFixGroupingKey";
export const MONITOR_TRAIT_DEFINITION: TraitDefinition = {
id: MONITOR_TRAIT_ID,
name: "Monitor",
description:
"Watch post-ship work; on a regression signal, open a single linked fix task (storm-guarded) back in triage.",
builtin: true,
flags: { notify: true },
hooks: { onEnter: true },
configSchema: {
fields: [
{ key: "threshold", type: "number", description: "Firings before a fix task opens" },
{ key: "sustainedMs", type: "number", description: "Sustained open-duration that satisfies the gate (ms)" },
{ key: "maxTasksPerWindow", type: "number", description: "Circuit breaker: max auto-fix tasks per window" },
],
},
};
/**
* True if a task is a Fusion-opened monitor fix task (never re-triage / never
* re-trigger the guard on these — no self-loop).
*/
export function isMonitorFixTask(task: Task): boolean {
const meta = (task.source?.sourceMetadata ?? {}) as Record<string, unknown>;
return typeof meta[MONITOR_FIX_TASK_META_KEY] === "string";
}
function buildFixTaskInput(
signal: IncidentSignalInput,
incidentId: string,
): TaskCreateInput {
const title = `Fix regression: ${signal.title}`;
const lines = [title];
if (signal.link) lines.push(`\nSource: ${signal.link}`);
lines.push(`\nGrouping key: ${signal.groupingKey}`);
lines.push(`Incident: ${incidentId}`);
return {
title,
description: lines.join("\n"),
column: MONITOR_FIX_ROUTE_COLUMN as TaskCreateInput["column"],
priority: signal.severity === "critical" ? "urgent" : "high",
source: {
sourceType: "automation",
sourceMetadata: {
[MONITOR_FIX_TASK_META_KEY]: incidentId,
[MONITOR_FIX_GROUPING_META_KEY]: signal.groupingKey,
signalSource: signal.source,
signalSeverity: signal.severity,
},
},
};
}
export interface MonitorDeps {
store: TaskStore;
config?: StormGuardConfig;
/** Injectable clock for deterministic tests. */
nowMs?: number;
}
export type MonitorRegressionOutcome =
| { kind: "fix-task-opened"; taskId: string; incidentId: string }
| { kind: "absorbed"; incidentId: string; existingFixTaskId: string | null; reason: string }
| { kind: "suppressed"; incidentId: string; reason: string }
| { kind: "error"; reason: string };
/**
* Handle a post-ship regression signal. Ingests it into the incidents table
* (opening or absorbing into an open incident by groupingKey), then runs the
* storm guard:
*
* - absorb → an open incident already has a fix task; bump occurrence, no new task.
* - suppress → flapping (gate not met) or circuit-breaker tripped; no new task.
* - open → create exactly one fix task in triage and link it to the incident.
*
* Idempotent across a burst sharing one groupingKey: the FIRST firing past the
* gate opens the task and links it; every subsequent firing finds the linked
* incident and absorbs. A Fusion-opened fix task never re-enters this path.
*
* FNXC:Monitor 2026-06-16-14:05: only one fix task may be opened per open
* incident window; concurrent regression ingests must not duplicate. The
* create-then-link step is guarded by an atomic incident-level claim
* (claimIncidentForFixTask) so the await on task creation cannot interleave two
* winners for the same open incident.
*/
export async function runMonitorOnRegression(
signal: IncidentSignalInput,
deps: MonitorDeps,
): Promise<MonitorRegressionOutcome> {
const { store, config, nowMs } = deps;
const db = store.getDatabase();
let incidentId: string;
try {
const { incident } = ingestIncidentSignal(db, signal);
incidentId = incident.incidentId;
const recent = countRecentAutoFixTasks(db, config, nowMs);
const decision = decideStormGuard(incident, recent, config, nowMs);
if (decision.action === "absorb") {
return {
kind: "absorbed",
incidentId,
existingFixTaskId: decision.existingFixTaskId,
reason: decision.reason,
};
}
if (decision.action === "suppress") {
return { kind: "suppressed", incidentId, reason: decision.reason };
}
// open-fix-task: claim the incident BEFORE the await on task creation. The
// claim is an atomic conditional UPDATE (set fixTaskId WHERE fixTaskId IS
// NULL), so under concurrent regression ingests for the same open incident
// exactly one caller wins. Losers absorb instead of opening a duplicate task
// — without this, two callers could both pass decideStormGuard (fixTaskId
// still null), both await store.createTask, and both attach, opening two
// tasks where only the last link wins.
if (!claimIncidentForFixTask(db, incidentId)) {
const linked = decision.incident.fixTaskId ?? null;
return {
kind: "absorbed",
incidentId,
existingFixTaskId: linked,
reason: "fix-task-claimed-concurrently",
};
}
// FNXC:Monitor 2026-06-16-15:40: a fix-task claim must be released if task
// creation fails so a stranded sentinel can't permanently absorb/suppress
// future regressions. The claim wrote a non-null sentinel to fixTaskId; if
// createTask throws here, attachFixTask never overwrites it, leaving the
// incident pseudo-linked forever. Release the claim (back to NULL, only when
// still the sentinel) before surfacing an error outcome so a later regression
// can open a fix task again.
let task: Task;
try {
task = await store.createTask(buildFixTaskInput(signal, incidentId));
} catch (createErr) {
releaseIncidentFixTaskClaim(db, incidentId);
diagnostics.errorFromException("Monitor fix-task creation failed; released claim", createErr, {
groupingKey: signal.groupingKey,
incidentId,
});
return {
kind: "error",
reason: createErr instanceof Error ? createErr.message : String(createErr),
};
}
attachFixTask(db, incidentId, task.id);
return { kind: "fix-task-opened", taskId: task.id, incidentId };
} catch (err) {
diagnostics.errorFromException("Monitor regression handling failed", err, {
groupingKey: signal.groupingKey,
});
return { kind: "error", reason: err instanceof Error ? err.message : String(err) };
}
}
// ── Registration (DI seam) ──────────────────────────────────────────────────
let registered = false;
/**
* Register the monitor trait definition + onEnter hook implementation. The
* onEnter hook records that a shipped task is now monitored; regression-driven
* fix-task creation runs through {@link runMonitorOnRegression} from the signal
* ingestion path, not from onEnter. Idempotent.
*/
export function registerMonitorTrait(): void {
if (registered) return;
const registry = getTraitRegistry();
if (!registry.has(MONITOR_TRAIT_ID)) {
registry.register(MONITOR_TRAIT_DEFINITION);
}
registerTraitHookImpl(MONITOR_TRAIT_ID, "onEnter", (...args: unknown[]) => {
const ctx = args[0] as { task?: Task } | undefined;
if (!ctx?.task) return undefined;
// Post-ship watch is currently a no-op marker hook; the loop-closing work is
// signal-driven (runMonitorOnRegression). Returning undefined keeps the
// card in place — monitoring is observational, not a routing action.
return undefined;
});
registered = true;
}
/** Test-only: reset the registration latch. */
export function __resetMonitorTraitForTests(): void {
registered = false;
}