FN-5658: add goal injection visibility diagnostics

Improve anchoring visibility by classifying and emitting goal-injection diagnostics across engine lanes.

- add goal-injection diagnostics module with result/failure classification and audit emission helpers
- wire heartbeat and executor prompt construction to resolve classified goal context and emit diagnostic events
- expose diagnostic APIs from engine index and add new run-audit event type for prompt goal injection
- add unit and wiring tests plus diagnostics docs updates for the new observability path

Files changed:
 docs/diagnostics.md                                |   8 +
 .../goal-injection-diagnostics-wiring.test.ts      |  87 +++++++++
 .../__tests__/goal-injection-diagnostics.test.ts   | 150 +++++++++++++++
 packages/engine/src/agent-heartbeat.ts             |  34 +++-
 packages/engine/src/executor.ts                    |  35 +++-
 packages/engine/src/goal-injection-diagnostics.ts  | 209 +++++++++++++++++++++
 packages/engine/src/index.ts                       |  10 +
 packages/engine/src/run-audit.ts                   |   7 +-
 8 files changed, 521 insertions(+), 19 deletions(-)

Fusion-Task-Id: FN-5658

Fusion-Task-Lineage: f4feaa37-d9b1-4bb6-b8c9-35a9b7a02f57
This commit is contained in:
gsxdsm
2026-05-29 20:50:08 -07:00
parent ab1f6a7ffa
commit ee0b1cd9c8
8 changed files with 521 additions and 19 deletions

View File

@@ -0,0 +1,87 @@
import { describe, expect, it, vi } from "vitest";
import type { Goal } from "@fusion/core";
import { emitGoalInjectionDiagnostic, resolveGoalContextForDiagnostics } from "../goal-injection-diagnostics.js";
function goal(id: string, title: string, createdAt: string): Goal {
return {
id,
title,
description: undefined,
status: "active",
createdAt,
updatedAt: createdAt,
};
}
describe("goal injection diagnostics wiring seam", () => {
it("emits applied audit metadata for positive injection", async () => {
const goals = [goal("G-1", "one", "2026-01-01T00:00:00.000Z"), goal("G-2", "two", "2026-01-02T00:00:00.000Z")];
const recordRunAuditEvent = vi.fn().mockResolvedValue(undefined);
const store = { logEntry: vi.fn().mockResolvedValue(undefined), recordRunAuditEvent } as any;
const resolution = resolveGoalContextForDiagnostics({ listActiveGoals: () => goals });
await emitGoalInjectionDiagnostic({
lane: "executor",
...resolution.classification,
runId: "exec-run",
agentId: "agent-1",
taskId: "FN-1",
store,
runContext: { runId: "exec-run", agentId: "agent-1", taskId: "FN-1", phase: "execute" },
});
expect(recordRunAuditEvent).toHaveBeenCalledTimes(1);
const event = recordRunAuditEvent.mock.calls[0][0];
expect(event.mutationType).toBe("prompt:goal-injection");
expect(event.metadata).toMatchObject({ outcome: "applied", goalCount: 2, goalIds: ["G-1", "G-2"] });
});
it("emits no-goals audit metadata when active goals are empty", async () => {
const recordRunAuditEvent = vi.fn().mockResolvedValue(undefined);
const store = { logEntry: vi.fn().mockResolvedValue(undefined), recordRunAuditEvent } as any;
const resolution = resolveGoalContextForDiagnostics({ listActiveGoals: () => [] });
await emitGoalInjectionDiagnostic({
lane: "executor",
...resolution.classification,
runId: "exec-run",
agentId: "agent-1",
taskId: "FN-1",
store,
runContext: { runId: "exec-run", agentId: "agent-1", taskId: "FN-1", phase: "execute" },
});
expect(recordRunAuditEvent).toHaveBeenCalledTimes(1);
expect(recordRunAuditEvent.mock.calls[0][0].metadata).toMatchObject({ outcome: "no-goals", goalCount: 0, goalIds: [] });
});
it("classifies list failure and keeps prompt construction alive", async () => {
const recordRunAuditEvent = vi.fn().mockResolvedValue(undefined);
const store = { logEntry: vi.fn().mockResolvedValue(undefined), recordRunAuditEvent } as any;
const resolution = resolveGoalContextForDiagnostics({
listActiveGoals: () => {
throw new TypeError("boom");
},
});
await expect(
emitGoalInjectionDiagnostic({
lane: "executor",
...resolution.classification,
runId: "exec-run",
agentId: "agent-1",
taskId: "FN-1",
store,
runContext: { runId: "exec-run", agentId: "agent-1", taskId: "FN-1", phase: "execute" },
}),
).resolves.toBeTruthy();
expect(recordRunAuditEvent).toHaveBeenCalledTimes(1);
expect(recordRunAuditEvent.mock.calls[0][0].metadata).toMatchObject({
outcome: "disabled-or-failed",
reason: "list-failed",
errorClass: "TypeError",
});
expect(resolution.goalContext).toBe("");
});
});

View File

@@ -0,0 +1,150 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
const { warnSpy } = vi.hoisted(() => ({
warnSpy: vi.fn(),
}));
vi.mock("../logger.js", () => ({
createLogger: () => ({ warn: warnSpy, log: vi.fn(), error: vi.fn() }),
}));
import {
emitGoalInjectionDiagnostic,
type GoalInjectionDiagnosticInput,
} from "../goal-injection-diagnostics.js";
function buildInput(overrides: Partial<GoalInjectionDiagnosticInput> = {}): GoalInjectionDiagnosticInput {
return {
lane: "executor",
outcome: "applied",
goalCount: 2,
goalIds: ["G-1", "G-2"],
truncated: false,
runId: "run-1",
agentId: "agent-1",
taskId: "FN-1",
...overrides,
};
}
describe("emitGoalInjectionDiagnostic", () => {
beforeEach(() => {
warnSpy.mockReset();
});
it("emits applied outcome to task log and run audit", async () => {
const logEntry = vi.fn().mockResolvedValue(undefined);
const recordRunAuditEvent = vi.fn().mockResolvedValue(undefined);
const store = { logEntry, recordRunAuditEvent } as any;
const result = await emitGoalInjectionDiagnostic(buildInput({ store, runContext: { runId: "run-1", agentId: "agent-1", taskId: "FN-1" } }));
expect(result.outcome).toBe("applied");
expect(result.goalCount).toBe(2);
expect(result.goalIds).toEqual(["G-1", "G-2"]);
expect(result.truncated).toBe(false);
expect(result.reason).toBeUndefined();
expect(logEntry).toHaveBeenCalledTimes(1);
expect(logEntry.mock.calls[0][1]).toContain("[goal-injection] applied");
expect(logEntry.mock.calls[0][1]).toContain('ids=["G-1","G-2"]');
expect(recordRunAuditEvent).toHaveBeenCalledTimes(1);
const auditInput = recordRunAuditEvent.mock.calls[0][0];
expect(auditInput.mutationType).toBe("prompt:goal-injection");
expect(auditInput.metadata).toMatchObject({ outcome: "applied", goalCount: 2, goalIds: ["G-1", "G-2"], truncated: false });
});
it("reflects truncated applied outcome", async () => {
const recordRunAuditEvent = vi.fn().mockResolvedValue(undefined);
const store = { logEntry: vi.fn().mockResolvedValue(undefined), recordRunAuditEvent } as any;
const result = await emitGoalInjectionDiagnostic(buildInput({ store, runContext: { runId: "run-1", agentId: "agent-1" }, truncated: true }));
expect(result.truncated).toBe(true);
expect(recordRunAuditEvent.mock.calls[0][0].metadata.truncated).toBe(true);
});
it("emits no-goals outcome", async () => {
const logEntry = vi.fn().mockResolvedValue(undefined);
const recordRunAuditEvent = vi.fn().mockResolvedValue(undefined);
const store = { logEntry, recordRunAuditEvent } as any;
const result = await emitGoalInjectionDiagnostic(
buildInput({
store,
runContext: { runId: "run-1", agentId: "agent-1", taskId: "FN-1" },
outcome: "no-goals",
goalCount: 0,
goalIds: [],
}),
);
expect(result.goalCount).toBe(0);
expect(result.goalIds).toEqual([]);
expect(result.truncated).toBe(false);
expect(logEntry.mock.calls[0][1]).toContain("no-goals count=0 ids=[]");
expect(recordRunAuditEvent).toHaveBeenCalledTimes(1);
});
it("supports disabled-or-failed reasons and errorClass", async () => {
for (const reason of ["config-disabled", "store-unavailable", "list-failed", "injector-threw"] as const) {
const result = await emitGoalInjectionDiagnostic(
buildInput({
outcome: "disabled-or-failed",
goalCount: 0,
goalIds: [],
truncated: false,
reason,
errorClass: reason === "list-failed" || reason === "injector-threw" ? "Error" : undefined,
}),
);
expect(result.reason).toBe(reason);
if (reason === "list-failed" || reason === "injector-threw") {
expect(result.errorClass).toBe("Error");
}
}
});
it("does not include forbidden payload keys", async () => {
const recordRunAuditEvent = vi.fn().mockResolvedValue(undefined);
const store = { logEntry: vi.fn().mockResolvedValue(undefined), recordRunAuditEvent } as any;
const result = await emitGoalInjectionDiagnostic(buildInput({ store, runContext: { runId: "run-1", agentId: "agent-1" } }));
const forbidden = ["title", "description", "body", "prompt", "text"];
const keys = Object.keys(result);
const metadataKeys = Object.keys(recordRunAuditEvent.mock.calls[0][0].metadata);
for (const key of forbidden) {
expect(keys).not.toContain(key);
expect(metadataKeys).not.toContain(key);
}
});
it("returns record without side effects when store missing", async () => {
const result = await emitGoalInjectionDiagnostic(buildInput({ store: undefined, runContext: null }));
expect(result.outcome).toBe("applied");
});
it("skips audit when runContext missing and warns", async () => {
const store = { logEntry: vi.fn().mockResolvedValue(undefined), recordRunAuditEvent: vi.fn().mockResolvedValue(undefined) } as any;
await emitGoalInjectionDiagnostic(buildInput({ store, runContext: null }));
expect(store.recordRunAuditEvent).not.toHaveBeenCalled();
expect(warnSpy).toHaveBeenCalled();
});
it("isolates side-effect failures", async () => {
const storeLogFails = {
logEntry: vi.fn().mockRejectedValue(new Error("log failed")),
recordRunAuditEvent: vi.fn().mockResolvedValue(undefined),
} as any;
const first = await emitGoalInjectionDiagnostic(buildInput({ store: storeLogFails, runContext: { runId: "run-1", agentId: "agent-1" } }));
expect(first.outcome).toBe("applied");
expect(storeLogFails.recordRunAuditEvent).toHaveBeenCalledTimes(1);
const storeAuditFails = {
logEntry: vi.fn().mockResolvedValue(undefined),
recordRunAuditEvent: vi.fn().mockRejectedValue(new Error("audit failed")),
} as any;
const second = await emitGoalInjectionDiagnostic(buildInput({ store: storeAuditFails, runContext: { runId: "run-1", agentId: "agent-1", taskId: "FN-1" } }));
expect(second.outcome).toBe("applied");
expect(storeAuditFails.logEntry).toHaveBeenCalledTimes(1);
});
});

View File

@@ -32,7 +32,10 @@ import {
} from "./agent-instructions.js";
import { resolveHeartbeatPromptTemplate, resolveHeartbeatScopeDisciplineMode, selectHeartbeatProcedure } from "./heartbeat-procedure-resolver.js";
import { buildPromptLayers, collapsePromptLayers } from "./prompt-layers.js";
import { buildGoalContextSection } from "./goal-context-injector.js";
import {
emitGoalInjectionDiagnostic,
resolveGoalContextForDiagnostics,
} from "./goal-injection-diagnostics.js";
import { emitGoalAnchoringAudit } from "./goal-anchoring-audit.js";
import { createLogger, heartbeatLog, formatError } from "./logger.js";
import { acquireTaskWorktree } from "./worktree-acquisition.js";
@@ -2384,18 +2387,31 @@ export class HeartbeatMonitor {
heartbeatLog.log(`applied plugin prompt contributions for heartbeat surface`);
}
const activeGoals = typeof taskStore.getGoalStore === "function"
? taskStore.getGoalStore().listGoals({ status: "active" })
: [];
const heartbeatGoalInjection = buildGoalContextSection({ activeGoals });
const heartbeatGoalResolution = resolveGoalContextForDiagnostics({
listActiveGoals:
typeof taskStore.getGoalStore === "function"
? () => taskStore.getGoalStore().listGoals({ status: "active" })
: undefined,
});
const heartbeatGoalContext = heartbeatGoalResolution.goalContext;
const heartbeatGoalClassification = heartbeatGoalResolution.classification;
await emitGoalAnchoringAudit(audit, {
lane: "heartbeat",
taskId,
goalsInjected: heartbeatGoalInjection.emittedGoalIds.length,
truncated: !!heartbeatGoalInjection.truncated,
reason: heartbeatGoalInjection.emittedGoalIds.length === 0 ? "no-active-goals" : undefined,
goalsInjected: heartbeatGoalClassification.goalCount,
truncated: heartbeatGoalClassification.truncated,
reason: heartbeatGoalClassification.outcome === "no-goals" ? "no-active-goals" : undefined,
});
await emitGoalInjectionDiagnostic({
lane: "heartbeat",
...heartbeatGoalClassification,
runId: run.id,
agentId,
store: taskStore,
runContext: engineRunContext,
});
const heartbeatGoalContext = heartbeatGoalInjection.text;
const heartbeatLayers = buildPromptLayers({
basePrompt: baseHeartbeatSystemPrompt,

View File

@@ -99,7 +99,10 @@ import {
buildPluginPromptSection,
} from "./agent-instructions.js";
import { buildPromptLayers, collapsePromptLayers } from "./prompt-layers.js";
import { buildGoalContextSection } from "./goal-context-injector.js";
import {
emitGoalInjectionDiagnostic,
resolveGoalContextForDiagnostics,
} from "./goal-injection-diagnostics.js";
import { emitGoalAnchoringAudit } from "./goal-anchoring-audit.js";
import type { AgentReflectionService } from "./agent-reflection.js";
import { createRunAuditor, generateSyntheticRunId, type EngineRunContext, type RunAuditor } from "./run-audit.js";
@@ -4154,18 +4157,32 @@ export class TaskExecutor {
executorLog.log(`${task.id}: applied plugin prompt contributions for executor-system surface`);
}
const activeGoals = typeof this.store.getGoalStore === "function"
? this.store.getGoalStore().listGoals({ status: "active" })
: [];
const executorGoalInjection = buildGoalContextSection({ activeGoals });
const executorGoalResolution = resolveGoalContextForDiagnostics({
listActiveGoals:
typeof this.store.getGoalStore === "function"
? () => this.store.getGoalStore().listGoals({ status: "active" })
: undefined,
});
const executorGoalContext = executorGoalResolution.goalContext;
const executorGoalClassification = executorGoalResolution.classification;
await emitGoalAnchoringAudit(audit, {
lane: "executor",
taskId: task.id,
goalsInjected: executorGoalInjection.emittedGoalIds.length,
truncated: !!executorGoalInjection.truncated,
reason: executorGoalInjection.emittedGoalIds.length === 0 ? "no-active-goals" : undefined,
goalsInjected: executorGoalClassification.goalCount,
truncated: executorGoalClassification.truncated,
reason: executorGoalClassification.outcome === "no-goals" ? "no-active-goals" : undefined,
});
await emitGoalInjectionDiagnostic({
lane: "executor",
...executorGoalClassification,
runId: engineRunContext.runId,
agentId: engineRunContext.agentId,
taskId: task.id,
store: this.store,
runContext: engineRunContext,
});
const executorGoalContext = executorGoalInjection.text;
const executorLayers = buildPromptLayers({
basePrompt: getExecutorSystemPrompt(settings),

View File

@@ -0,0 +1,209 @@
import type { Goal } from "@fusion/core";
import { buildGoalContextSection, type GoalInjectionResult } from "./goal-context-injector.js";
import type { TaskStore } from "@fusion/core";
import type { EngineRunContext } from "./run-audit.js";
import { createLogger } from "./logger.js";
const diagnosticsLog = createLogger("goal-injection-diagnostics");
export type GoalInjectionOutcome = "applied" | "no-goals" | "disabled-or-failed";
export type GoalInjectionDisabledReason =
| "config-disabled"
| "store-unavailable"
| "list-failed"
| "injector-threw";
export interface GoalInjectionDiagnostic {
lane: "heartbeat" | "executor";
outcome: GoalInjectionOutcome;
goalCount: number;
goalIds: string[];
truncated: boolean;
reason?: GoalInjectionDisabledReason;
errorClass?: string;
runId?: string;
agentId?: string;
taskId?: string;
timestamp: string;
}
export interface GoalInjectionDiagnosticInput extends Omit<GoalInjectionDiagnostic, "timestamp"> {
store?: TaskStore;
runContext?: EngineRunContext | null;
}
export interface GoalInjectionClassification {
outcome: GoalInjectionOutcome;
goalCount: number;
goalIds: string[];
truncated: boolean;
reason?: GoalInjectionDisabledReason;
errorClass?: string;
}
export function classifyGoalInjectionResult(result: GoalInjectionResult): GoalInjectionClassification {
if (result.emittedGoalIds.length === 0) {
return {
outcome: "no-goals",
goalCount: 0,
goalIds: [],
truncated: false,
};
}
return {
outcome: "applied",
goalCount: result.emittedGoalIds.length,
goalIds: result.emittedGoalIds,
truncated: result.truncated !== null,
};
}
export function classifyGoalInjectionFailure(
reason: GoalInjectionDisabledReason,
error?: unknown,
): GoalInjectionClassification {
const resolvedErrorClass =
error && typeof error === "object" && "constructor" in error && typeof (error as { constructor?: { name?: unknown } }).constructor?.name === "string"
? (error as { constructor: { name: string } }).constructor.name
: undefined;
return {
outcome: "disabled-or-failed",
goalCount: 0,
goalIds: [],
truncated: false,
reason,
...(resolvedErrorClass ? { errorClass: resolvedErrorClass } : {}),
};
}
export interface GoalContextResolution {
goalContext: string;
classification: GoalInjectionClassification;
}
export interface ResolveGoalContextInput {
listActiveGoals?: () => Goal[];
injector?: (activeGoals: Goal[]) => GoalInjectionResult;
disabledReason?: GoalInjectionDisabledReason;
}
export function resolveGoalContextForDiagnostics(input: ResolveGoalContextInput): GoalContextResolution {
if (!input.listActiveGoals) {
return {
goalContext: "",
classification: classifyGoalInjectionFailure(input.disabledReason ?? "store-unavailable"),
};
}
try {
const activeGoals = input.listActiveGoals();
const injector = input.injector ?? ((goals: Goal[]) => buildGoalContextSection({ activeGoals: goals }));
try {
const injectionResult = injector(activeGoals);
return {
goalContext: injectionResult.text,
classification: classifyGoalInjectionResult(injectionResult),
};
} catch (injectorError) {
return {
goalContext: "",
classification: classifyGoalInjectionFailure("injector-threw", injectorError),
};
}
} catch (listError) {
return {
goalContext: "",
classification: classifyGoalInjectionFailure("list-failed", listError),
};
}
}
function formatAgentLogLine(input: GoalInjectionDiagnostic): string {
const ids = JSON.stringify(input.goalIds);
const reason = input.reason ? ` reason=${input.reason}` : "";
const errorClass = input.errorClass ? ` err=${input.errorClass}` : "";
return `[goal-injection] ${input.outcome} count=${input.goalCount} ids=${ids} truncated=${String(input.truncated)}${reason}${errorClass}`;
}
/**
* Emit per-run goal-context injection diagnostics for executor/heartbeat lanes.
*
* Outcomes:
* - `applied`: at least one active goal ID was injected.
* - `no-goals`: injector executed successfully but active goal set was empty.
* - `disabled-or-failed`: injection was disabled or list/injector execution failed.
*
* Guardrail: this emitter stores goal IDs/counts only and must never persist
* prompt body text, goal titles, or goal descriptions to avoid prompt/PII leakage.
*
* This helper is called from FN-5653 wiring sites (executor/heartbeat), not from
* inside the pure goal-context injector module.
*/
export async function emitGoalInjectionDiagnostic(
input: GoalInjectionDiagnosticInput,
): Promise<GoalInjectionDiagnostic> {
const record: GoalInjectionDiagnostic = {
lane: input.lane,
outcome: input.outcome,
goalCount: input.goalCount,
goalIds: [...input.goalIds],
truncated: input.truncated,
...(input.reason ? { reason: input.reason } : {}),
...(input.errorClass ? { errorClass: input.errorClass } : {}),
...(input.runId ? { runId: input.runId } : {}),
...(input.agentId ? { agentId: input.agentId } : {}),
...(input.taskId ? { taskId: input.taskId } : {}),
timestamp: new Date().toISOString(),
};
if (input.store && record.taskId) {
try {
await input.store.logEntry(record.taskId, formatAgentLogLine(record), undefined, input.runContext ?? undefined);
} catch (error) {
diagnosticsLog.warn(
`failed to append goal-injection task log for ${record.taskId}: ${error instanceof Error ? error.message : String(error)}`,
);
}
}
const auditStore = input.store;
const hasAuditWriter = Boolean(auditStore && typeof auditStore.recordRunAuditEvent === "function");
if (!input.runContext || !hasAuditWriter || !auditStore) {
diagnosticsLog.warn(
`goal-injection diagnostic emitted without run-audit side effect (lane=${record.lane}, hasContext=${Boolean(input.runContext)}, hasWriter=${Boolean(hasAuditWriter)})`,
);
return record;
}
try {
await auditStore.recordRunAuditEvent({
taskId: input.runContext.taskId,
agentId: input.runContext.agentId,
runId: input.runContext.runId,
domain: "database",
mutationType: "prompt:goal-injection",
target: record.lane,
metadata: {
lane: record.lane,
outcome: record.outcome,
goalCount: record.goalCount,
goalIds: record.goalIds,
truncated: record.truncated,
...(record.reason ? { reason: record.reason } : {}),
...(record.errorClass ? { errorClass: record.errorClass } : {}),
...(record.runId ? { runId: record.runId } : {}),
...(record.agentId ? { agentId: record.agentId } : {}),
...(record.taskId ? { taskId: record.taskId } : {}),
},
});
} catch (error) {
diagnosticsLog.warn(
`failed to append goal-injection run-audit event for lane=${record.lane}: ${error instanceof Error ? error.message : String(error)}`,
);
}
return record;
}

View File

@@ -182,6 +182,16 @@ export {
type GoalInjectionResult,
type GoalInjectionTruncationEvent,
} from "./goal-context-injector.js";
export {
classifyGoalInjectionFailure,
classifyGoalInjectionResult,
emitGoalInjectionDiagnostic,
type GoalInjectionClassification,
type GoalInjectionDiagnostic,
type GoalInjectionDiagnosticInput,
type GoalInjectionDisabledReason,
type GoalInjectionOutcome,
} from "./goal-injection-diagnostics.js";
export {
emitGoalAnchoringAudit,
emitGoalRetrievalAudit,

View File

@@ -593,7 +593,12 @@ export type DatabaseMutationType =
/** Goal anchoring observability events (FN-5655). */
| "goal:injection-applied"
| "goal:injection-skipped"
| "goal:retrieval-invoked";
| "goal:retrieval-invoked"
/**
* Goal injection diagnostic event (FN-5658).
* Metadata: { lane, outcome, goalCount, goalIds, truncated, reason?, errorClass?, runId?, agentId?, taskId? }
*/
| "prompt:goal-injection";
// ── Filesystem mutation types ─────────────────────────────────────────────────