FN-5655: add goal anchoring audit events across lanes
Add audit guardrails that track goal anchoring injection across executor, heartbeat, and CLI/dashboard surfaces. - add a dedicated engine goal-anchoring audit emitter and export it - emit goal anchoring audit events from executor and heartbeat goal-context injection paths - extend run-audit wiring and CLI extension output to surface goal anchoring events - add engine, CLI, and dashboard regression tests for goal-tool and audit event behavior - document the new audit coverage and include a published changeset entry Files changed: .changeset/fn-5655-goal-anchoring-audit.md | 7 ++ docs/architecture.md | 1 + docs/dashboard-guide.md | 1 + .../__tests__/extension-goal-tools-audit.test.ts | 65 +++++++++++++++ packages/cli/src/extension.ts | 28 +++++++ .../__tests__/routes-run-audit-goal-events.test.ts | 72 ++++++++++++++++ .../src/__tests__/goal-anchoring-audit.test.ts | 95 ++++++++++++++++++++++ packages/engine/src/agent-heartbeat.ts | 20 +++-- packages/engine/src/executor.ts | 20 +++-- packages/engine/src/goal-anchoring-audit.ts | 89 ++++++++++++++++++++ packages/engine/src/index.ts | 10 +++ packages/engine/src/run-audit.ts | 6 +- 12 files changed, 399 insertions(+), 15 deletions(-) Fusion-Task-Id: FN-5655 Fusion-Task-Lineage: f3c10568-4050-42c4-9fe2-b84bf749b89d
This commit is contained in:
@@ -0,0 +1,65 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { mkdtemp, mkdir, rm } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { TaskStore } from "@fusion/core";
|
||||
import kbExtension from "../extension.js";
|
||||
import { GOAL_RETRIEVAL_INVOKED } from "@fusion/engine";
|
||||
|
||||
interface RegisteredTool {
|
||||
name: string;
|
||||
execute: (toolCallId: string, params: any, signal: AbortSignal | undefined, onUpdate: ((update: any) => void) | undefined, ctx: any) => Promise<any>;
|
||||
}
|
||||
|
||||
function createMockAPI() {
|
||||
const tools = new Map<string, RegisteredTool>();
|
||||
return {
|
||||
registerTool(def: RegisteredTool) { tools.set(def.name, def); },
|
||||
registerCommand() {},
|
||||
registerShortcut() {},
|
||||
registerFlag() {},
|
||||
on() {},
|
||||
tools,
|
||||
} as any;
|
||||
}
|
||||
|
||||
describe("extension goal tools retrieval audit", () => {
|
||||
let tmpDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tmpDir = await mkdtemp(join(tmpdir(), "kb-goal-audit-"));
|
||||
await mkdir(join(tmpDir, ".fusion"), { recursive: true });
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(tmpDir, { recursive: true, force: true });
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("emits retrieval audit for fn_goal_list and fn_goal_show branches", async () => {
|
||||
const recordSpy = vi.spyOn(TaskStore.prototype, "recordRunAuditEvent");
|
||||
const api = createMockAPI();
|
||||
kbExtension(api);
|
||||
|
||||
const createTool = api.tools.get("fn_goal_create");
|
||||
const listTool = api.tools.get("fn_goal_list");
|
||||
const showTool = api.tools.get("fn_goal_show");
|
||||
const ctx = { cwd: tmpDir, runId: "run-1", agentId: "agent-1", taskId: "FN-1" };
|
||||
|
||||
await createTool.execute("c1", { title: "Goal one" }, undefined, undefined, ctx);
|
||||
const listResult = await listTool.execute("l1", { status: "active" }, undefined, undefined, ctx);
|
||||
const goalId = listResult.details.goals[0].id as string;
|
||||
|
||||
await showTool.execute("s1", { id: goalId }, undefined, undefined, ctx);
|
||||
await showTool.execute("s2", { id: "G-404" }, undefined, undefined, ctx);
|
||||
|
||||
const goalAuditCalls = recordSpy.mock.calls
|
||||
.map((call) => call[0])
|
||||
.filter((event) => event.mutationType === GOAL_RETRIEVAL_INVOKED);
|
||||
|
||||
expect(goalAuditCalls).toHaveLength(3);
|
||||
expect(goalAuditCalls[0]).toMatchObject({ metadata: expect.objectContaining({ toolName: "fn_goal_list", count: 1 }) });
|
||||
expect(goalAuditCalls[1]).toMatchObject({ target: goalId, metadata: expect.objectContaining({ toolName: "fn_goal_show", count: 1, notFound: false }) });
|
||||
expect(goalAuditCalls[2]).toMatchObject({ target: "G-404", metadata: expect.objectContaining({ toolName: "fn_goal_show", count: 0, notFound: true }) });
|
||||
});
|
||||
});
|
||||
@@ -46,6 +46,7 @@ import {
|
||||
type FinalizePlanOverride,
|
||||
fetchWebContent,
|
||||
assertNoSecretPlaintext,
|
||||
emitGoalRetrievalAudit,
|
||||
} from "@fusion/engine";
|
||||
import * as dashboard from "@fusion/dashboard";
|
||||
import { resolve, basename, extname, join } from "node:path";
|
||||
@@ -2402,6 +2403,11 @@ export default function kbExtension(pi: ExtensionAPI) {
|
||||
}),
|
||||
|
||||
async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
|
||||
const fnCtx = ctx as typeof ctx & {
|
||||
agentId?: string;
|
||||
runId?: string;
|
||||
taskId?: string;
|
||||
};
|
||||
const store = await getStore(ctx.cwd);
|
||||
const goalStore = store.getGoalStore();
|
||||
const status = params.status ?? "active";
|
||||
@@ -2409,6 +2415,11 @@ export default function kbExtension(pi: ExtensionAPI) {
|
||||
const activeCount = goalStore.listGoals({ status: "active" }).length;
|
||||
const softWarning = activeCount >= 3;
|
||||
|
||||
emitGoalRetrievalAudit(store, fnCtx, {
|
||||
toolName: "fn_goal_list",
|
||||
resultCount: goals.length,
|
||||
});
|
||||
|
||||
const lines: string[] = [];
|
||||
lines.push(`Goals (${goals.length}) [filter: ${status}]`);
|
||||
lines.push(`Active: ${activeCount}/5`);
|
||||
@@ -2537,11 +2548,22 @@ export default function kbExtension(pi: ExtensionAPI) {
|
||||
}),
|
||||
|
||||
async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
|
||||
const fnCtx = ctx as typeof ctx & {
|
||||
agentId?: string;
|
||||
runId?: string;
|
||||
taskId?: string;
|
||||
};
|
||||
const store = await getStore(ctx.cwd);
|
||||
const goalStore = store.getGoalStore();
|
||||
const goal = goalStore.getGoal(params.id);
|
||||
|
||||
if (!goal) {
|
||||
emitGoalRetrievalAudit(store, fnCtx, {
|
||||
toolName: "fn_goal_show",
|
||||
resultCount: 0,
|
||||
goalId: params.id,
|
||||
notFound: true,
|
||||
});
|
||||
return {
|
||||
isError: true,
|
||||
content: [{ type: "text", text: `Goal ${params.id} not found` }],
|
||||
@@ -2558,6 +2580,12 @@ export default function kbExtension(pi: ExtensionAPI) {
|
||||
lines.push(`Description: ${goal.description}`);
|
||||
}
|
||||
|
||||
emitGoalRetrievalAudit(store, fnCtx, {
|
||||
toolName: "fn_goal_show",
|
||||
resultCount: 1,
|
||||
goalId: params.id,
|
||||
});
|
||||
|
||||
return {
|
||||
content: [{ type: "text", text: lines.join("\n") }],
|
||||
details: { goal },
|
||||
|
||||
Reference in New Issue
Block a user