FN-5655: add goal anchoring audit events across lanes
Add audit guardrails that track goal anchoring injection across executor, heartbeat, and CLI/dashboard surfaces. - add a dedicated engine goal-anchoring audit emitter and export it - emit goal anchoring audit events from executor and heartbeat goal-context injection paths - extend run-audit wiring and CLI extension output to surface goal anchoring events - add engine, CLI, and dashboard regression tests for goal-tool and audit event behavior - document the new audit coverage and include a published changeset entry Files changed: .changeset/fn-5655-goal-anchoring-audit.md | 7 ++ docs/architecture.md | 1 + docs/dashboard-guide.md | 1 + .../__tests__/extension-goal-tools-audit.test.ts | 65 +++++++++++++++ packages/cli/src/extension.ts | 28 +++++++ .../__tests__/routes-run-audit-goal-events.test.ts | 72 ++++++++++++++++ .../src/__tests__/goal-anchoring-audit.test.ts | 95 ++++++++++++++++++++++ packages/engine/src/agent-heartbeat.ts | 20 +++-- packages/engine/src/executor.ts | 20 +++-- packages/engine/src/goal-anchoring-audit.ts | 89 ++++++++++++++++++++ packages/engine/src/index.ts | 10 +++ packages/engine/src/run-audit.ts | 6 +- 12 files changed, 399 insertions(+), 15 deletions(-) Fusion-Task-Id: FN-5655 Fusion-Task-Lineage: f3c10568-4050-42c4-9fe2-b84bf749b89d
This commit is contained in:
@@ -0,0 +1,65 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { mkdtemp, mkdir, rm } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { TaskStore } from "@fusion/core";
|
||||
import kbExtension from "../extension.js";
|
||||
import { GOAL_RETRIEVAL_INVOKED } from "@fusion/engine";
|
||||
|
||||
interface RegisteredTool {
|
||||
name: string;
|
||||
execute: (toolCallId: string, params: any, signal: AbortSignal | undefined, onUpdate: ((update: any) => void) | undefined, ctx: any) => Promise<any>;
|
||||
}
|
||||
|
||||
function createMockAPI() {
|
||||
const tools = new Map<string, RegisteredTool>();
|
||||
return {
|
||||
registerTool(def: RegisteredTool) { tools.set(def.name, def); },
|
||||
registerCommand() {},
|
||||
registerShortcut() {},
|
||||
registerFlag() {},
|
||||
on() {},
|
||||
tools,
|
||||
} as any;
|
||||
}
|
||||
|
||||
describe("extension goal tools retrieval audit", () => {
|
||||
let tmpDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tmpDir = await mkdtemp(join(tmpdir(), "kb-goal-audit-"));
|
||||
await mkdir(join(tmpDir, ".fusion"), { recursive: true });
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(tmpDir, { recursive: true, force: true });
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("emits retrieval audit for fn_goal_list and fn_goal_show branches", async () => {
|
||||
const recordSpy = vi.spyOn(TaskStore.prototype, "recordRunAuditEvent");
|
||||
const api = createMockAPI();
|
||||
kbExtension(api);
|
||||
|
||||
const createTool = api.tools.get("fn_goal_create");
|
||||
const listTool = api.tools.get("fn_goal_list");
|
||||
const showTool = api.tools.get("fn_goal_show");
|
||||
const ctx = { cwd: tmpDir, runId: "run-1", agentId: "agent-1", taskId: "FN-1" };
|
||||
|
||||
await createTool.execute("c1", { title: "Goal one" }, undefined, undefined, ctx);
|
||||
const listResult = await listTool.execute("l1", { status: "active" }, undefined, undefined, ctx);
|
||||
const goalId = listResult.details.goals[0].id as string;
|
||||
|
||||
await showTool.execute("s1", { id: goalId }, undefined, undefined, ctx);
|
||||
await showTool.execute("s2", { id: "G-404" }, undefined, undefined, ctx);
|
||||
|
||||
const goalAuditCalls = recordSpy.mock.calls
|
||||
.map((call) => call[0])
|
||||
.filter((event) => event.mutationType === GOAL_RETRIEVAL_INVOKED);
|
||||
|
||||
expect(goalAuditCalls).toHaveLength(3);
|
||||
expect(goalAuditCalls[0]).toMatchObject({ metadata: expect.objectContaining({ toolName: "fn_goal_list", count: 1 }) });
|
||||
expect(goalAuditCalls[1]).toMatchObject({ target: goalId, metadata: expect.objectContaining({ toolName: "fn_goal_show", count: 1, notFound: false }) });
|
||||
expect(goalAuditCalls[2]).toMatchObject({ target: "G-404", metadata: expect.objectContaining({ toolName: "fn_goal_show", count: 0, notFound: true }) });
|
||||
});
|
||||
});
|
||||
@@ -46,6 +46,7 @@ import {
|
||||
type FinalizePlanOverride,
|
||||
fetchWebContent,
|
||||
assertNoSecretPlaintext,
|
||||
emitGoalRetrievalAudit,
|
||||
} from "@fusion/engine";
|
||||
import * as dashboard from "@fusion/dashboard";
|
||||
import { resolve, basename, extname, join } from "node:path";
|
||||
@@ -2402,6 +2403,11 @@ export default function kbExtension(pi: ExtensionAPI) {
|
||||
}),
|
||||
|
||||
async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
|
||||
const fnCtx = ctx as typeof ctx & {
|
||||
agentId?: string;
|
||||
runId?: string;
|
||||
taskId?: string;
|
||||
};
|
||||
const store = await getStore(ctx.cwd);
|
||||
const goalStore = store.getGoalStore();
|
||||
const status = params.status ?? "active";
|
||||
@@ -2409,6 +2415,11 @@ export default function kbExtension(pi: ExtensionAPI) {
|
||||
const activeCount = goalStore.listGoals({ status: "active" }).length;
|
||||
const softWarning = activeCount >= 3;
|
||||
|
||||
emitGoalRetrievalAudit(store, fnCtx, {
|
||||
toolName: "fn_goal_list",
|
||||
resultCount: goals.length,
|
||||
});
|
||||
|
||||
const lines: string[] = [];
|
||||
lines.push(`Goals (${goals.length}) [filter: ${status}]`);
|
||||
lines.push(`Active: ${activeCount}/5`);
|
||||
@@ -2537,11 +2548,22 @@ export default function kbExtension(pi: ExtensionAPI) {
|
||||
}),
|
||||
|
||||
async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
|
||||
const fnCtx = ctx as typeof ctx & {
|
||||
agentId?: string;
|
||||
runId?: string;
|
||||
taskId?: string;
|
||||
};
|
||||
const store = await getStore(ctx.cwd);
|
||||
const goalStore = store.getGoalStore();
|
||||
const goal = goalStore.getGoal(params.id);
|
||||
|
||||
if (!goal) {
|
||||
emitGoalRetrievalAudit(store, fnCtx, {
|
||||
toolName: "fn_goal_show",
|
||||
resultCount: 0,
|
||||
goalId: params.id,
|
||||
notFound: true,
|
||||
});
|
||||
return {
|
||||
isError: true,
|
||||
content: [{ type: "text", text: `Goal ${params.id} not found` }],
|
||||
@@ -2558,6 +2580,12 @@ export default function kbExtension(pi: ExtensionAPI) {
|
||||
lines.push(`Description: ${goal.description}`);
|
||||
}
|
||||
|
||||
emitGoalRetrievalAudit(store, fnCtx, {
|
||||
toolName: "fn_goal_show",
|
||||
resultCount: 1,
|
||||
goalId: params.id,
|
||||
});
|
||||
|
||||
return {
|
||||
content: [{ type: "text", text: lines.join("\n") }],
|
||||
details: { goal },
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { request } from "../test-request.js";
|
||||
|
||||
const mockGetRunDetail = vi.fn();
|
||||
const mockGetRunAuditEvents = vi.fn();
|
||||
|
||||
vi.mock("@fusion/core", () => ({
|
||||
AgentStore: class MockAgentStore {
|
||||
init = vi.fn().mockResolvedValue(undefined);
|
||||
getRunDetail = mockGetRunDetail;
|
||||
},
|
||||
ChatStore: class MockChatStore {
|
||||
init = vi.fn().mockResolvedValue(undefined);
|
||||
},
|
||||
deterministicGuardLocks: new Map(),
|
||||
}));
|
||||
|
||||
class MockStore {
|
||||
getRunAuditEvents = mockGetRunAuditEvents;
|
||||
getAgentLogsByTimeRange = vi.fn().mockResolvedValue([]);
|
||||
getMutationsForRun = vi.fn().mockResolvedValue([]);
|
||||
getRootDir() { return "/tmp/fn-5655-test"; }
|
||||
getFusionDir() { return "/tmp/fn-5655-test/.fusion"; }
|
||||
getDatabase() { return { exec: vi.fn(), prepare: vi.fn().mockReturnValue({ run: vi.fn(), get: vi.fn(), all: vi.fn().mockReturnValue([]) }) }; }
|
||||
}
|
||||
|
||||
describe("run-audit goal event route filtering", () => {
|
||||
let app: ReturnType<typeof import("../server.js").createServer>;
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.clearAllMocks();
|
||||
const { createServer } = await import("../server.js");
|
||||
app = createServer(new MockStore() as any);
|
||||
mockGetRunDetail.mockResolvedValue({ id: "run-1", agentId: "agent-1", startedAt: "2026-01-01T00:00:00.000Z", endedAt: null, status: "active", contextSnapshot: { taskId: "FN-1" } });
|
||||
|
||||
const allEvents = [
|
||||
{ id: "e1", timestamp: "2026-01-01T00:00:00.000Z", runId: "run-1", agentId: "agent-1", domain: "database", mutationType: "goal:injection-applied", target: "FN-1", metadata: { count: 2, lane: "heartbeat" } },
|
||||
{ id: "e2", timestamp: "2026-01-01T00:05:00.000Z", runId: "run-1", agentId: "agent-1", domain: "database", mutationType: "goal:injection-skipped", target: "goals", metadata: { count: 0, lane: "executor" } },
|
||||
{ id: "e3", timestamp: "2026-01-01T00:10:00.000Z", runId: "run-1", agentId: "agent-1", domain: "database", mutationType: "goal:retrieval-invoked", target: "goals", metadata: { count: 3, toolName: "fn_goal_list" } },
|
||||
];
|
||||
|
||||
mockGetRunAuditEvents.mockImplementation((filter: { startTime?: string; endTime?: string; domain?: string }) => {
|
||||
let events = allEvents;
|
||||
if (filter.domain) events = events.filter((event) => event.domain === filter.domain);
|
||||
if (filter.startTime) events = events.filter((event) => event.timestamp >= filter.startTime!);
|
||||
if (filter.endTime) events = events.filter((event) => event.timestamp <= filter.endTime!);
|
||||
return events;
|
||||
});
|
||||
});
|
||||
|
||||
it("returns only goal events in requested database time window", async () => {
|
||||
const response = await request(
|
||||
app,
|
||||
"GET",
|
||||
"/api/agents/agent-1/runs/run-1/audit?domain=database&startTime=2026-01-01T00:04:00.000Z&endTime=2026-01-01T00:06:00.000Z",
|
||||
);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.events).toHaveLength(1);
|
||||
expect(response.body.events[0].mutationType).toBe("goal:injection-skipped");
|
||||
});
|
||||
|
||||
it("returns all goal events with mutationType strings preserved", async () => {
|
||||
const response = await request(app, "GET", "/api/agents/agent-1/runs/run-1/audit?domain=database");
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.events.map((event: { mutationType: string }) => event.mutationType)).toEqual([
|
||||
"goal:injection-applied",
|
||||
"goal:injection-skipped",
|
||||
"goal:retrieval-invoked",
|
||||
]);
|
||||
});
|
||||
});
|
||||
95
packages/engine/src/__tests__/goal-anchoring-audit.test.ts
Normal file
95
packages/engine/src/__tests__/goal-anchoring-audit.test.ts
Normal file
@@ -0,0 +1,95 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { RunAuditEventInput, TaskStore } from "@fusion/core";
|
||||
import { createRunAuditor } from "../run-audit.js";
|
||||
import {
|
||||
emitGoalAnchoringAudit,
|
||||
emitGoalRetrievalAudit,
|
||||
GOAL_INJECTION_APPLIED,
|
||||
GOAL_INJECTION_SKIPPED,
|
||||
GOAL_RETRIEVAL_INVOKED,
|
||||
} from "../goal-anchoring-audit.js";
|
||||
|
||||
describe("goal anchoring audit helpers", () => {
|
||||
it("emits applied injection audit", async () => {
|
||||
const database = vi.fn(async () => {});
|
||||
await emitGoalAnchoringAudit({ database } as any, {
|
||||
lane: "heartbeat",
|
||||
taskId: "FN-1",
|
||||
goalsInjected: 3,
|
||||
});
|
||||
expect(database).toHaveBeenCalledWith(expect.objectContaining({
|
||||
type: GOAL_INJECTION_APPLIED,
|
||||
target: "FN-1",
|
||||
metadata: expect.objectContaining({ lane: "heartbeat", count: 3 }),
|
||||
}));
|
||||
});
|
||||
|
||||
it("emits skipped injection audit with reason and default target", async () => {
|
||||
const database = vi.fn(async () => {});
|
||||
await emitGoalAnchoringAudit({ database } as any, {
|
||||
lane: "executor",
|
||||
goalsInjected: 0,
|
||||
reason: "no-active-goals",
|
||||
});
|
||||
expect(database).toHaveBeenCalledWith(expect.objectContaining({
|
||||
type: GOAL_INJECTION_SKIPPED,
|
||||
target: "goals",
|
||||
metadata: expect.objectContaining({ reason: "no-active-goals", count: 0 }),
|
||||
}));
|
||||
});
|
||||
|
||||
it("includes truncated metadata when present", async () => {
|
||||
const database = vi.fn(async () => {});
|
||||
await emitGoalAnchoringAudit({ database } as any, {
|
||||
lane: "heartbeat",
|
||||
goalsInjected: 1,
|
||||
truncated: true,
|
||||
});
|
||||
expect(database).toHaveBeenCalledWith(expect.objectContaining({
|
||||
metadata: expect.objectContaining({ truncated: true }),
|
||||
}));
|
||||
});
|
||||
|
||||
it("emits retrieval audit when run context exists", () => {
|
||||
const recordRunAuditEvent = vi.fn();
|
||||
const store = { recordRunAuditEvent } as unknown as TaskStore;
|
||||
emitGoalRetrievalAudit(store, { runId: "r1", agentId: "a1", taskId: "FN-1" }, { toolName: "fn_goal_list", resultCount: 2 });
|
||||
expect(recordRunAuditEvent).toHaveBeenCalledTimes(1);
|
||||
expect(recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({
|
||||
domain: "database",
|
||||
mutationType: GOAL_RETRIEVAL_INVOKED,
|
||||
target: "goals",
|
||||
metadata: expect.objectContaining({ toolName: "fn_goal_list", count: 2, notFound: false }),
|
||||
}));
|
||||
});
|
||||
|
||||
it("skips retrieval audit when runId or agentId is missing", () => {
|
||||
const recordRunAuditEvent = vi.fn();
|
||||
const store = { recordRunAuditEvent } as unknown as TaskStore;
|
||||
emitGoalRetrievalAudit(store, { agentId: "a1" }, { toolName: "fn_goal_list", resultCount: 2 });
|
||||
emitGoalRetrievalAudit(store, { runId: "r1" }, { toolName: "fn_goal_list", resultCount: 2 });
|
||||
expect(recordRunAuditEvent).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("swallows retrieval audit failures and warns once", () => {
|
||||
const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
const store = { recordRunAuditEvent: vi.fn(() => { throw new Error("boom"); }) } as unknown as TaskStore;
|
||||
expect(() => emitGoalRetrievalAudit(store, { runId: "r1", agentId: "a1" }, { toolName: "fn_goal_show", resultCount: 0, goalId: "G-1", notFound: true })).not.toThrow();
|
||||
expect(warn).toHaveBeenCalledTimes(1);
|
||||
warn.mockRestore();
|
||||
});
|
||||
|
||||
it("persists heartbeat-style events through createRunAuditor", async () => {
|
||||
const events: RunAuditEventInput[] = [];
|
||||
const store = { recordRunAuditEvent: vi.fn((input: RunAuditEventInput) => events.push(input)) } as unknown as TaskStore;
|
||||
const auditor = createRunAuditor(store, { runId: "run-1", agentId: "agent-1", taskId: "FN-9", phase: "heartbeat" });
|
||||
|
||||
await emitGoalAnchoringAudit(auditor, { lane: "heartbeat", taskId: "FN-9", goalsInjected: 2 });
|
||||
await emitGoalAnchoringAudit(auditor, { lane: "heartbeat", taskId: "FN-9", goalsInjected: 0, reason: "no-active-goals" });
|
||||
|
||||
const goalEvents = events.filter((event) => String(event.mutationType).startsWith("goal:"));
|
||||
expect(goalEvents).toHaveLength(2);
|
||||
expect(goalEvents[0]).toMatchObject({ mutationType: GOAL_INJECTION_APPLIED, metadata: expect.objectContaining({ count: 2, lane: "heartbeat" }) });
|
||||
expect(goalEvents[1]).toMatchObject({ mutationType: GOAL_INJECTION_SKIPPED, metadata: expect.objectContaining({ count: 0, reason: "no-active-goals" }) });
|
||||
});
|
||||
});
|
||||
@@ -17,7 +17,7 @@
|
||||
* - onTerminated: Called when a heartbeat run is terminated
|
||||
*/
|
||||
|
||||
import type { AgentStore, AgentHeartbeatRun, HeartbeatInvocationSource, AgentHeartbeatConfig, AgentBudgetStatus, Message, MessageStore, TaskStore, TaskDetail, AgentRole, Agent, InboxTask, RunMutationContext, Settings, AgentConfigRevision, ReflectionStore, ChatStore, ChatRoom, ChatRoomMessage, AgentMemoryInclusionMode, Goal } from "@fusion/core";
|
||||
import type { AgentStore, AgentHeartbeatRun, HeartbeatInvocationSource, AgentHeartbeatConfig, AgentBudgetStatus, Message, MessageStore, TaskStore, TaskDetail, AgentRole, Agent, InboxTask, RunMutationContext, Settings, AgentConfigRevision, ReflectionStore, ChatStore, ChatRoom, ChatRoomMessage, AgentMemoryInclusionMode } from "@fusion/core";
|
||||
import { AutoClaimSnapshotManager, type AutoClaimCandidate } from "./auto-claim-snapshot.js";
|
||||
import { ApprovalRequestStore, buildExecutionMemoryInstructions, isEphemeralAgent, hasAgentIdentity, resolveEffectiveAgentPermissionPolicy, canAgentTakeImplementationTask, canAgentTakeImplementationTaskForExplicitRouting, resolvePersistAgentThinkingLog, resolveAgentMemoryInclusionMode } from "@fusion/core";
|
||||
import type { ToolDefinition } from "@earendil-works/pi-coding-agent";
|
||||
@@ -33,6 +33,7 @@ import {
|
||||
import { resolveHeartbeatPromptTemplate, resolveHeartbeatScopeDisciplineMode, selectHeartbeatProcedure } from "./heartbeat-procedure-resolver.js";
|
||||
import { buildPromptLayers, collapsePromptLayers } from "./prompt-layers.js";
|
||||
import { buildGoalContextSection } from "./goal-context-injector.js";
|
||||
import { emitGoalAnchoringAudit } from "./goal-anchoring-audit.js";
|
||||
import { createLogger, heartbeatLog, formatError } from "./logger.js";
|
||||
import { acquireTaskWorktree } from "./worktree-acquisition.js";
|
||||
import { createRunAuditor, type EngineRunContext } from "./run-audit.js";
|
||||
@@ -2383,13 +2384,18 @@ export class HeartbeatMonitor {
|
||||
heartbeatLog.log(`applied plugin prompt contributions for heartbeat surface`);
|
||||
}
|
||||
|
||||
const goalStore = this.taskStore && typeof (this.taskStore as { getGoalStore?: unknown }).getGoalStore === "function"
|
||||
? (this.taskStore as { getGoalStore: () => { listGoals?: (input: { status: "active" }) => Goal[] } }).getGoalStore()
|
||||
: undefined;
|
||||
const activeGoals = typeof goalStore?.listGoals === "function"
|
||||
? goalStore.listGoals({ status: "active" })
|
||||
const activeGoals = typeof taskStore.getGoalStore === "function"
|
||||
? taskStore.getGoalStore().listGoals({ status: "active" })
|
||||
: [];
|
||||
const heartbeatGoalContext = buildGoalContextSection({ activeGoals }).text;
|
||||
const heartbeatGoalInjection = buildGoalContextSection({ activeGoals });
|
||||
await emitGoalAnchoringAudit(audit, {
|
||||
lane: "heartbeat",
|
||||
taskId,
|
||||
goalsInjected: heartbeatGoalInjection.emittedGoalIds.length,
|
||||
truncated: !!heartbeatGoalInjection.truncated,
|
||||
reason: heartbeatGoalInjection.emittedGoalIds.length === 0 ? "no-active-goals" : undefined,
|
||||
});
|
||||
const heartbeatGoalContext = heartbeatGoalInjection.text;
|
||||
|
||||
const heartbeatLayers = buildPromptLayers({
|
||||
basePrompt: baseHeartbeatSystemPrompt,
|
||||
|
||||
@@ -8,7 +8,7 @@ const execAsync = promisify(exec);
|
||||
import { delimiter, isAbsolute, join, relative, resolve as resolvePath } from "node:path";
|
||||
import { existsSync, realpathSync } from "node:fs";
|
||||
import { readFile, rm, writeFile } from "node:fs/promises";
|
||||
import type { TaskStore, Task, TaskDetail, TaskTokenUsage, StepStatus, Settings, WorkflowStep, MissionStore, Slice, AgentState, AgentCapability, RunMutationContext, AgentHeartbeatConfig, Agent, AgentMemoryInclusionMode, ProjectSettings, Goal } from "@fusion/core";
|
||||
import type { TaskStore, Task, TaskDetail, TaskTokenUsage, StepStatus, Settings, WorkflowStep, MissionStore, Slice, AgentState, AgentCapability, RunMutationContext, AgentHeartbeatConfig, Agent, AgentMemoryInclusionMode, ProjectSettings } from "@fusion/core";
|
||||
import { RetryStormError, TaskDeletedError, serializeRetryStormError } from "@fusion/core";
|
||||
import {
|
||||
ApprovalRequestStore,
|
||||
@@ -100,6 +100,7 @@ import {
|
||||
} from "./agent-instructions.js";
|
||||
import { buildPromptLayers, collapsePromptLayers } from "./prompt-layers.js";
|
||||
import { buildGoalContextSection } from "./goal-context-injector.js";
|
||||
import { emitGoalAnchoringAudit } from "./goal-anchoring-audit.js";
|
||||
import type { AgentReflectionService } from "./agent-reflection.js";
|
||||
import { createRunAuditor, generateSyntheticRunId, type EngineRunContext, type RunAuditor } from "./run-audit.js";
|
||||
import { AutoRecoveryDispatcher } from "./auto-recovery.js";
|
||||
@@ -4153,13 +4154,18 @@ export class TaskExecutor {
|
||||
executorLog.log(`${task.id}: applied plugin prompt contributions for executor-system surface`);
|
||||
}
|
||||
|
||||
const goalStore = typeof (this.store as { getGoalStore?: unknown }).getGoalStore === "function"
|
||||
? (this.store as { getGoalStore: () => { listGoals?: (input: { status: "active" }) => Goal[] } }).getGoalStore()
|
||||
: undefined;
|
||||
const activeGoals = typeof goalStore?.listGoals === "function"
|
||||
? goalStore.listGoals({ status: "active" })
|
||||
const activeGoals = typeof this.store.getGoalStore === "function"
|
||||
? this.store.getGoalStore().listGoals({ status: "active" })
|
||||
: [];
|
||||
const executorGoalContext = buildGoalContextSection({ activeGoals }).text;
|
||||
const executorGoalInjection = buildGoalContextSection({ activeGoals });
|
||||
await emitGoalAnchoringAudit(audit, {
|
||||
lane: "executor",
|
||||
taskId: task.id,
|
||||
goalsInjected: executorGoalInjection.emittedGoalIds.length,
|
||||
truncated: !!executorGoalInjection.truncated,
|
||||
reason: executorGoalInjection.emittedGoalIds.length === 0 ? "no-active-goals" : undefined,
|
||||
});
|
||||
const executorGoalContext = executorGoalInjection.text;
|
||||
|
||||
const executorLayers = buildPromptLayers({
|
||||
basePrompt: getExecutorSystemPrompt(settings),
|
||||
|
||||
89
packages/engine/src/goal-anchoring-audit.ts
Normal file
89
packages/engine/src/goal-anchoring-audit.ts
Normal file
@@ -0,0 +1,89 @@
|
||||
import type { TaskStore } from "@fusion/core";
|
||||
|
||||
import type { RunAuditor } from "./run-audit.js";
|
||||
|
||||
/** Goal context was injected into heartbeat/executor prompts for the Slice 2 cite-rate experiment. */
|
||||
export const GOAL_INJECTION_APPLIED = "goal:injection-applied";
|
||||
/** Goal injector ran but produced no prompt context for the Slice 2 cite-rate experiment. */
|
||||
export const GOAL_INJECTION_SKIPPED = "goal:injection-skipped";
|
||||
/** Goal retrieval tools were invoked at runtime for the Slice 2 cite-rate experiment. */
|
||||
export const GOAL_RETRIEVAL_INVOKED = "goal:retrieval-invoked";
|
||||
|
||||
const GOAL_AUDIT_TYPES = [GOAL_INJECTION_APPLIED, GOAL_INJECTION_SKIPPED, GOAL_RETRIEVAL_INVOKED] as const;
|
||||
if (new Set(GOAL_AUDIT_TYPES).size !== GOAL_AUDIT_TYPES.length || GOAL_AUDIT_TYPES.some((value) => typeof value !== "string" || !value.startsWith("goal:"))) {
|
||||
throw new Error("Goal anchoring audit mutation types must be unique goal:* strings.");
|
||||
}
|
||||
|
||||
/** Engine lane that attempted goal anchoring; powers dashboard filtering for cite-rate observability. */
|
||||
export type GoalAnchoringLane = "heartbeat" | "executor";
|
||||
|
||||
/**
|
||||
* Structured goal-injection audit payload.
|
||||
* Counts/IDs only: never include prompt bodies, goal titles, or goal descriptions.
|
||||
*/
|
||||
export type GoalInjectionAuditInput = {
|
||||
lane: GoalAnchoringLane;
|
||||
taskId?: string;
|
||||
goalsInjected: number;
|
||||
truncated?: boolean;
|
||||
reason?: "no-active-goals" | "injector-empty";
|
||||
};
|
||||
|
||||
/**
|
||||
* Structured goal-retrieval audit payload.
|
||||
* Counts/IDs only: never include goal titles/descriptions or any prompt text.
|
||||
*/
|
||||
export type GoalRetrievalAuditInput = {
|
||||
toolName: "fn_goal_list" | "fn_goal_show";
|
||||
resultCount: number;
|
||||
goalId?: string;
|
||||
notFound?: boolean;
|
||||
};
|
||||
|
||||
/**
|
||||
* Emit a database-domain run-audit event for goal injection observability.
|
||||
* These events are consumed from the existing run-audit dashboard timeline with start/end time filters.
|
||||
*/
|
||||
export async function emitGoalAnchoringAudit(auditor: RunAuditor, input: GoalInjectionAuditInput): Promise<void> {
|
||||
const isApplied = input.goalsInjected > 0;
|
||||
await auditor.database({
|
||||
type: isApplied ? GOAL_INJECTION_APPLIED : GOAL_INJECTION_SKIPPED,
|
||||
target: input.taskId ?? "goals",
|
||||
metadata: {
|
||||
lane: input.lane,
|
||||
count: input.goalsInjected,
|
||||
...(typeof input.truncated === "boolean" ? { truncated: input.truncated } : {}),
|
||||
...(input.reason ? { reason: input.reason } : {}),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Emit a database-domain run-audit event when goal retrieval tools are invoked.
|
||||
* Uses only identifiers/counts so cite-rate monitoring can query the timeline without sensitive payloads.
|
||||
*/
|
||||
export function emitGoalRetrievalAudit(
|
||||
store: TaskStore,
|
||||
ctx: { runId?: string; agentId?: string; taskId?: string },
|
||||
input: GoalRetrievalAuditInput,
|
||||
): void {
|
||||
if (!ctx.runId || !ctx.agentId) return;
|
||||
|
||||
try {
|
||||
store.recordRunAuditEvent({
|
||||
runId: ctx.runId,
|
||||
agentId: ctx.agentId,
|
||||
taskId: ctx.taskId,
|
||||
domain: "database",
|
||||
mutationType: GOAL_RETRIEVAL_INVOKED,
|
||||
target: input.goalId ?? "goals",
|
||||
metadata: {
|
||||
toolName: input.toolName,
|
||||
count: input.resultCount,
|
||||
notFound: input.notFound ?? false,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.warn("[fusion-extension] goal retrieval audit emission skipped", error);
|
||||
}
|
||||
}
|
||||
@@ -182,6 +182,16 @@ export {
|
||||
type GoalInjectionResult,
|
||||
type GoalInjectionTruncationEvent,
|
||||
} from "./goal-context-injector.js";
|
||||
export {
|
||||
emitGoalAnchoringAudit,
|
||||
emitGoalRetrievalAudit,
|
||||
GOAL_INJECTION_APPLIED,
|
||||
GOAL_INJECTION_SKIPPED,
|
||||
GOAL_RETRIEVAL_INVOKED,
|
||||
type GoalAnchoringLane,
|
||||
type GoalInjectionAuditInput,
|
||||
type GoalRetrievalAuditInput,
|
||||
} from "./goal-anchoring-audit.js";
|
||||
export {
|
||||
resolveWorktrunkBinary,
|
||||
installWorktrunk,
|
||||
|
||||
@@ -589,7 +589,11 @@ export type DatabaseMutationType =
|
||||
*/
|
||||
| "merger:transient-failure-auto-recovered"
|
||||
/** Metadata: { taskId, transientClass, recoveryCount, maxRecoveries, errorSnippet } */
|
||||
| "merger:transient-failure-budget-exhausted";
|
||||
| "merger:transient-failure-budget-exhausted"
|
||||
/** Goal anchoring observability events (FN-5655). */
|
||||
| "goal:injection-applied"
|
||||
| "goal:injection-skipped"
|
||||
| "goal:retrieval-invoked";
|
||||
|
||||
// ── Filesystem mutation types ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
Reference in New Issue
Block a user