FN-5655: add goal anchoring audit events across lanes
Add audit guardrails that track goal anchoring injection across executor, heartbeat, and CLI/dashboard surfaces. - add a dedicated engine goal-anchoring audit emitter and export it - emit goal anchoring audit events from executor and heartbeat goal-context injection paths - extend run-audit wiring and CLI extension output to surface goal anchoring events - add engine, CLI, and dashboard regression tests for goal-tool and audit event behavior - document the new audit coverage and include a published changeset entry Files changed: .changeset/fn-5655-goal-anchoring-audit.md | 7 ++ docs/architecture.md | 1 + docs/dashboard-guide.md | 1 + .../__tests__/extension-goal-tools-audit.test.ts | 65 +++++++++++++++ packages/cli/src/extension.ts | 28 +++++++ .../__tests__/routes-run-audit-goal-events.test.ts | 72 ++++++++++++++++ .../src/__tests__/goal-anchoring-audit.test.ts | 95 ++++++++++++++++++++++ packages/engine/src/agent-heartbeat.ts | 20 +++-- packages/engine/src/executor.ts | 20 +++-- packages/engine/src/goal-anchoring-audit.ts | 89 ++++++++++++++++++++ packages/engine/src/index.ts | 10 +++ packages/engine/src/run-audit.ts | 6 +- 12 files changed, 399 insertions(+), 15 deletions(-) Fusion-Task-Id: FN-5655 Fusion-Task-Lineage: f3c10568-4050-42c4-9fe2-b84bf749b89d
This commit is contained in:
gsxdsm
95
packages/engine/src/__tests__/goal-anchoring-audit.test.ts
Normal file
95
packages/engine/src/__tests__/goal-anchoring-audit.test.ts
Normal file
@@ -0,0 +1,95 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { RunAuditEventInput, TaskStore } from "@fusion/core";
|
||||
import { createRunAuditor } from "../run-audit.js";
|
||||
import {
|
||||
emitGoalAnchoringAudit,
|
||||
emitGoalRetrievalAudit,
|
||||
GOAL_INJECTION_APPLIED,
|
||||
GOAL_INJECTION_SKIPPED,
|
||||
GOAL_RETRIEVAL_INVOKED,
|
||||
} from "../goal-anchoring-audit.js";
|
||||
|
||||
describe("goal anchoring audit helpers", () => {
|
||||
it("emits applied injection audit", async () => {
|
||||
const database = vi.fn(async () => {});
|
||||
await emitGoalAnchoringAudit({ database } as any, {
|
||||
lane: "heartbeat",
|
||||
taskId: "FN-1",
|
||||
goalsInjected: 3,
|
||||
});
|
||||
expect(database).toHaveBeenCalledWith(expect.objectContaining({
|
||||
type: GOAL_INJECTION_APPLIED,
|
||||
target: "FN-1",
|
||||
metadata: expect.objectContaining({ lane: "heartbeat", count: 3 }),
|
||||
}));
|
||||
});
|
||||
|
||||
it("emits skipped injection audit with reason and default target", async () => {
|
||||
const database = vi.fn(async () => {});
|
||||
await emitGoalAnchoringAudit({ database } as any, {
|
||||
lane: "executor",
|
||||
goalsInjected: 0,
|
||||
reason: "no-active-goals",
|
||||
});
|
||||
expect(database).toHaveBeenCalledWith(expect.objectContaining({
|
||||
type: GOAL_INJECTION_SKIPPED,
|
||||
target: "goals",
|
||||
metadata: expect.objectContaining({ reason: "no-active-goals", count: 0 }),
|
||||
}));
|
||||
});
|
||||
|
||||
it("includes truncated metadata when present", async () => {
|
||||
const database = vi.fn(async () => {});
|
||||
await emitGoalAnchoringAudit({ database } as any, {
|
||||
lane: "heartbeat",
|
||||
goalsInjected: 1,
|
||||
truncated: true,
|
||||
});
|
||||
expect(database).toHaveBeenCalledWith(expect.objectContaining({
|
||||
metadata: expect.objectContaining({ truncated: true }),
|
||||
}));
|
||||
});
|
||||
|
||||
it("emits retrieval audit when run context exists", () => {
|
||||
const recordRunAuditEvent = vi.fn();
|
||||
const store = { recordRunAuditEvent } as unknown as TaskStore;
|
||||
emitGoalRetrievalAudit(store, { runId: "r1", agentId: "a1", taskId: "FN-1" }, { toolName: "fn_goal_list", resultCount: 2 });
|
||||
expect(recordRunAuditEvent).toHaveBeenCalledTimes(1);
|
||||
expect(recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({
|
||||
domain: "database",
|
||||
mutationType: GOAL_RETRIEVAL_INVOKED,
|
||||
target: "goals",
|
||||
metadata: expect.objectContaining({ toolName: "fn_goal_list", count: 2, notFound: false }),
|
||||
}));
|
||||
});
|
||||
|
||||
it("skips retrieval audit when runId or agentId is missing", () => {
|
||||
const recordRunAuditEvent = vi.fn();
|
||||
const store = { recordRunAuditEvent } as unknown as TaskStore;
|
||||
emitGoalRetrievalAudit(store, { agentId: "a1" }, { toolName: "fn_goal_list", resultCount: 2 });
|
||||
emitGoalRetrievalAudit(store, { runId: "r1" }, { toolName: "fn_goal_list", resultCount: 2 });
|
||||
expect(recordRunAuditEvent).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("swallows retrieval audit failures and warns once", () => {
|
||||
const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
const store = { recordRunAuditEvent: vi.fn(() => { throw new Error("boom"); }) } as unknown as TaskStore;
|
||||
expect(() => emitGoalRetrievalAudit(store, { runId: "r1", agentId: "a1" }, { toolName: "fn_goal_show", resultCount: 0, goalId: "G-1", notFound: true })).not.toThrow();
|
||||
expect(warn).toHaveBeenCalledTimes(1);
|
||||
warn.mockRestore();
|
||||
});
|
||||
|
||||
it("persists heartbeat-style events through createRunAuditor", async () => {
|
||||
const events: RunAuditEventInput[] = [];
|
||||
const store = { recordRunAuditEvent: vi.fn((input: RunAuditEventInput) => events.push(input)) } as unknown as TaskStore;
|
||||
const auditor = createRunAuditor(store, { runId: "run-1", agentId: "agent-1", taskId: "FN-9", phase: "heartbeat" });
|
||||
|
||||
await emitGoalAnchoringAudit(auditor, { lane: "heartbeat", taskId: "FN-9", goalsInjected: 2 });
|
||||
await emitGoalAnchoringAudit(auditor, { lane: "heartbeat", taskId: "FN-9", goalsInjected: 0, reason: "no-active-goals" });
|
||||
|
||||
const goalEvents = events.filter((event) => String(event.mutationType).startsWith("goal:"));
|
||||
expect(goalEvents).toHaveLength(2);
|
||||
expect(goalEvents[0]).toMatchObject({ mutationType: GOAL_INJECTION_APPLIED, metadata: expect.objectContaining({ count: 2, lane: "heartbeat" }) });
|
||||
expect(goalEvents[1]).toMatchObject({ mutationType: GOAL_INJECTION_SKIPPED, metadata: expect.objectContaining({ count: 0, reason: "no-active-goals" }) });
|
||||
});
|
||||
});
|
||||
@@ -17,7 +17,7 @@
|
||||
* - onTerminated: Called when a heartbeat run is terminated
|
||||
*/
|
||||
|
||||
import type { AgentStore, AgentHeartbeatRun, HeartbeatInvocationSource, AgentHeartbeatConfig, AgentBudgetStatus, Message, MessageStore, TaskStore, TaskDetail, AgentRole, Agent, InboxTask, RunMutationContext, Settings, AgentConfigRevision, ReflectionStore, ChatStore, ChatRoom, ChatRoomMessage, AgentMemoryInclusionMode, Goal } from "@fusion/core";
|
||||
import type { AgentStore, AgentHeartbeatRun, HeartbeatInvocationSource, AgentHeartbeatConfig, AgentBudgetStatus, Message, MessageStore, TaskStore, TaskDetail, AgentRole, Agent, InboxTask, RunMutationContext, Settings, AgentConfigRevision, ReflectionStore, ChatStore, ChatRoom, ChatRoomMessage, AgentMemoryInclusionMode } from "@fusion/core";
|
||||
import { AutoClaimSnapshotManager, type AutoClaimCandidate } from "./auto-claim-snapshot.js";
|
||||
import { ApprovalRequestStore, buildExecutionMemoryInstructions, isEphemeralAgent, hasAgentIdentity, resolveEffectiveAgentPermissionPolicy, canAgentTakeImplementationTask, canAgentTakeImplementationTaskForExplicitRouting, resolvePersistAgentThinkingLog, resolveAgentMemoryInclusionMode } from "@fusion/core";
|
||||
import type { ToolDefinition } from "@earendil-works/pi-coding-agent";
|
||||
@@ -33,6 +33,7 @@ import {
|
||||
import { resolveHeartbeatPromptTemplate, resolveHeartbeatScopeDisciplineMode, selectHeartbeatProcedure } from "./heartbeat-procedure-resolver.js";
|
||||
import { buildPromptLayers, collapsePromptLayers } from "./prompt-layers.js";
|
||||
import { buildGoalContextSection } from "./goal-context-injector.js";
|
||||
import { emitGoalAnchoringAudit } from "./goal-anchoring-audit.js";
|
||||
import { createLogger, heartbeatLog, formatError } from "./logger.js";
|
||||
import { acquireTaskWorktree } from "./worktree-acquisition.js";
|
||||
import { createRunAuditor, type EngineRunContext } from "./run-audit.js";
|
||||
@@ -2383,13 +2384,18 @@ export class HeartbeatMonitor {
|
||||
heartbeatLog.log(`applied plugin prompt contributions for heartbeat surface`);
|
||||
}
|
||||
|
||||
const goalStore = this.taskStore && typeof (this.taskStore as { getGoalStore?: unknown }).getGoalStore === "function"
|
||||
? (this.taskStore as { getGoalStore: () => { listGoals?: (input: { status: "active" }) => Goal[] } }).getGoalStore()
|
||||
: undefined;
|
||||
const activeGoals = typeof goalStore?.listGoals === "function"
|
||||
? goalStore.listGoals({ status: "active" })
|
||||
const activeGoals = typeof taskStore.getGoalStore === "function"
|
||||
? taskStore.getGoalStore().listGoals({ status: "active" })
|
||||
: [];
|
||||
const heartbeatGoalContext = buildGoalContextSection({ activeGoals }).text;
|
||||
const heartbeatGoalInjection = buildGoalContextSection({ activeGoals });
|
||||
await emitGoalAnchoringAudit(audit, {
|
||||
lane: "heartbeat",
|
||||
taskId,
|
||||
goalsInjected: heartbeatGoalInjection.emittedGoalIds.length,
|
||||
truncated: !!heartbeatGoalInjection.truncated,
|
||||
reason: heartbeatGoalInjection.emittedGoalIds.length === 0 ? "no-active-goals" : undefined,
|
||||
});
|
||||
const heartbeatGoalContext = heartbeatGoalInjection.text;
|
||||
|
||||
const heartbeatLayers = buildPromptLayers({
|
||||
basePrompt: baseHeartbeatSystemPrompt,
|
||||
|
||||
@@ -8,7 +8,7 @@ const execAsync = promisify(exec);
|
||||
import { delimiter, isAbsolute, join, relative, resolve as resolvePath } from "node:path";
|
||||
import { existsSync, realpathSync } from "node:fs";
|
||||
import { readFile, rm, writeFile } from "node:fs/promises";
|
||||
import type { TaskStore, Task, TaskDetail, TaskTokenUsage, StepStatus, Settings, WorkflowStep, MissionStore, Slice, AgentState, AgentCapability, RunMutationContext, AgentHeartbeatConfig, Agent, AgentMemoryInclusionMode, ProjectSettings, Goal } from "@fusion/core";
|
||||
import type { TaskStore, Task, TaskDetail, TaskTokenUsage, StepStatus, Settings, WorkflowStep, MissionStore, Slice, AgentState, AgentCapability, RunMutationContext, AgentHeartbeatConfig, Agent, AgentMemoryInclusionMode, ProjectSettings } from "@fusion/core";
|
||||
import { RetryStormError, TaskDeletedError, serializeRetryStormError } from "@fusion/core";
|
||||
import {
|
||||
ApprovalRequestStore,
|
||||
@@ -100,6 +100,7 @@ import {
|
||||
} from "./agent-instructions.js";
|
||||
import { buildPromptLayers, collapsePromptLayers } from "./prompt-layers.js";
|
||||
import { buildGoalContextSection } from "./goal-context-injector.js";
|
||||
import { emitGoalAnchoringAudit } from "./goal-anchoring-audit.js";
|
||||
import type { AgentReflectionService } from "./agent-reflection.js";
|
||||
import { createRunAuditor, generateSyntheticRunId, type EngineRunContext, type RunAuditor } from "./run-audit.js";
|
||||
import { AutoRecoveryDispatcher } from "./auto-recovery.js";
|
||||
@@ -4153,13 +4154,18 @@ export class TaskExecutor {
|
||||
executorLog.log(`${task.id}: applied plugin prompt contributions for executor-system surface`);
|
||||
}
|
||||
|
||||
const goalStore = typeof (this.store as { getGoalStore?: unknown }).getGoalStore === "function"
|
||||
? (this.store as { getGoalStore: () => { listGoals?: (input: { status: "active" }) => Goal[] } }).getGoalStore()
|
||||
: undefined;
|
||||
const activeGoals = typeof goalStore?.listGoals === "function"
|
||||
? goalStore.listGoals({ status: "active" })
|
||||
const activeGoals = typeof this.store.getGoalStore === "function"
|
||||
? this.store.getGoalStore().listGoals({ status: "active" })
|
||||
: [];
|
||||
const executorGoalContext = buildGoalContextSection({ activeGoals }).text;
|
||||
const executorGoalInjection = buildGoalContextSection({ activeGoals });
|
||||
await emitGoalAnchoringAudit(audit, {
|
||||
lane: "executor",
|
||||
taskId: task.id,
|
||||
goalsInjected: executorGoalInjection.emittedGoalIds.length,
|
||||
truncated: !!executorGoalInjection.truncated,
|
||||
reason: executorGoalInjection.emittedGoalIds.length === 0 ? "no-active-goals" : undefined,
|
||||
});
|
||||
const executorGoalContext = executorGoalInjection.text;
|
||||
|
||||
const executorLayers = buildPromptLayers({
|
||||
basePrompt: getExecutorSystemPrompt(settings),
|
||||
|
||||
89
packages/engine/src/goal-anchoring-audit.ts
Normal file
89
packages/engine/src/goal-anchoring-audit.ts
Normal file
@@ -0,0 +1,89 @@
|
||||
import type { TaskStore } from "@fusion/core";
|
||||
|
||||
import type { RunAuditor } from "./run-audit.js";
|
||||
|
||||
/** Goal context was injected into heartbeat/executor prompts for the Slice 2 cite-rate experiment. */
|
||||
export const GOAL_INJECTION_APPLIED = "goal:injection-applied";
|
||||
/** Goal injector ran but produced no prompt context for the Slice 2 cite-rate experiment. */
|
||||
export const GOAL_INJECTION_SKIPPED = "goal:injection-skipped";
|
||||
/** Goal retrieval tools were invoked at runtime for the Slice 2 cite-rate experiment. */
|
||||
export const GOAL_RETRIEVAL_INVOKED = "goal:retrieval-invoked";
|
||||
|
||||
const GOAL_AUDIT_TYPES = [GOAL_INJECTION_APPLIED, GOAL_INJECTION_SKIPPED, GOAL_RETRIEVAL_INVOKED] as const;
|
||||
if (new Set(GOAL_AUDIT_TYPES).size !== GOAL_AUDIT_TYPES.length || GOAL_AUDIT_TYPES.some((value) => typeof value !== "string" || !value.startsWith("goal:"))) {
|
||||
throw new Error("Goal anchoring audit mutation types must be unique goal:* strings.");
|
||||
}
|
||||
|
||||
/** Engine lane that attempted goal anchoring; powers dashboard filtering for cite-rate observability. */
|
||||
export type GoalAnchoringLane = "heartbeat" | "executor";
|
||||
|
||||
/**
|
||||
* Structured goal-injection audit payload.
|
||||
* Counts/IDs only: never include prompt bodies, goal titles, or goal descriptions.
|
||||
*/
|
||||
export type GoalInjectionAuditInput = {
|
||||
lane: GoalAnchoringLane;
|
||||
taskId?: string;
|
||||
goalsInjected: number;
|
||||
truncated?: boolean;
|
||||
reason?: "no-active-goals" | "injector-empty";
|
||||
};
|
||||
|
||||
/**
|
||||
* Structured goal-retrieval audit payload.
|
||||
* Counts/IDs only: never include goal titles/descriptions or any prompt text.
|
||||
*/
|
||||
export type GoalRetrievalAuditInput = {
|
||||
toolName: "fn_goal_list" | "fn_goal_show";
|
||||
resultCount: number;
|
||||
goalId?: string;
|
||||
notFound?: boolean;
|
||||
};
|
||||
|
||||
/**
|
||||
* Emit a database-domain run-audit event for goal injection observability.
|
||||
* These events are consumed from the existing run-audit dashboard timeline with start/end time filters.
|
||||
*/
|
||||
export async function emitGoalAnchoringAudit(auditor: RunAuditor, input: GoalInjectionAuditInput): Promise<void> {
|
||||
const isApplied = input.goalsInjected > 0;
|
||||
await auditor.database({
|
||||
type: isApplied ? GOAL_INJECTION_APPLIED : GOAL_INJECTION_SKIPPED,
|
||||
target: input.taskId ?? "goals",
|
||||
metadata: {
|
||||
lane: input.lane,
|
||||
count: input.goalsInjected,
|
||||
...(typeof input.truncated === "boolean" ? { truncated: input.truncated } : {}),
|
||||
...(input.reason ? { reason: input.reason } : {}),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Emit a database-domain run-audit event when goal retrieval tools are invoked.
|
||||
* Uses only identifiers/counts so cite-rate monitoring can query the timeline without sensitive payloads.
|
||||
*/
|
||||
export function emitGoalRetrievalAudit(
|
||||
store: TaskStore,
|
||||
ctx: { runId?: string; agentId?: string; taskId?: string },
|
||||
input: GoalRetrievalAuditInput,
|
||||
): void {
|
||||
if (!ctx.runId || !ctx.agentId) return;
|
||||
|
||||
try {
|
||||
store.recordRunAuditEvent({
|
||||
runId: ctx.runId,
|
||||
agentId: ctx.agentId,
|
||||
taskId: ctx.taskId,
|
||||
domain: "database",
|
||||
mutationType: GOAL_RETRIEVAL_INVOKED,
|
||||
target: input.goalId ?? "goals",
|
||||
metadata: {
|
||||
toolName: input.toolName,
|
||||
count: input.resultCount,
|
||||
notFound: input.notFound ?? false,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.warn("[fusion-extension] goal retrieval audit emission skipped", error);
|
||||
}
|
||||
}
|
||||
@@ -182,6 +182,16 @@ export {
|
||||
type GoalInjectionResult,
|
||||
type GoalInjectionTruncationEvent,
|
||||
} from "./goal-context-injector.js";
|
||||
export {
|
||||
emitGoalAnchoringAudit,
|
||||
emitGoalRetrievalAudit,
|
||||
GOAL_INJECTION_APPLIED,
|
||||
GOAL_INJECTION_SKIPPED,
|
||||
GOAL_RETRIEVAL_INVOKED,
|
||||
type GoalAnchoringLane,
|
||||
type GoalInjectionAuditInput,
|
||||
type GoalRetrievalAuditInput,
|
||||
} from "./goal-anchoring-audit.js";
|
||||
export {
|
||||
resolveWorktrunkBinary,
|
||||
installWorktrunk,
|
||||
|
||||
@@ -589,7 +589,11 @@ export type DatabaseMutationType =
|
||||
*/
|
||||
| "merger:transient-failure-auto-recovered"
|
||||
/** Metadata: { taskId, transientClass, recoveryCount, maxRecoveries, errorSnippet } */
|
||||
| "merger:transient-failure-budget-exhausted";
|
||||
| "merger:transient-failure-budget-exhausted"
|
||||
/** Goal anchoring observability events (FN-5655). */
|
||||
| "goal:injection-applied"
|
||||
| "goal:injection-skipped"
|
||||
| "goal:retrieval-invoked";
|
||||
|
||||
// ── Filesystem mutation types ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
Reference in New Issue
Block a user