root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

FN-5758: add cited-goal audit aggregation and runtime route

Browse Source 
Add end-to-end goal-citation audit trail plumbing across core, engine, CLI, and dashboard.

- extend goal citation extraction with aggregation logic and new core coverage
- expose cited-goal runtime data through dashboard routes with dedicated API tests
- wire goal-citation audit details into engine diagnostics and CLI extension audit expectations
- document the new audit behavior and publish a patch changeset for @runfusion/fusion

Files changed:
 .changeset/fn-5758-goal-citation-audit.md          |  9 +++
 docs/dashboard-guide.md                            |  2 +
 docs/diagnostics.md                                |  4 ++
 .../__tests__/extension-goal-tools-audit.test.ts   |  6 +-
 packages/cli/src/extension.ts                      |  3 +
 .../goal-citation-audit-aggregation.test.ts        | 63 ++++++++++++++++++
 packages/core/src/goal-citation-extractor.ts       | 54 ++++++++++++++-
 packages/core/src/index.ts                         |  1 +
 .../src/__tests__/routes-run-cited-goals.test.ts   | 77 ++++++++++++++++++++++
 packages/dashboard/src/routes.ts                   | 11 ++++
 .../src/routes/register-agent-runtime-routes.ts    | 49 ++++++++++++++
 .../src/__tests__/goal-anchoring-audit.test.ts     | 40 +++++++----
 packages/engine/src/goal-anchoring-audit.ts        |  4 ++
 packages/engine/src/goal-injection-diagnostics.ts  |  1 +
 14 files changed, 308 insertions(+), 16 deletions(-)

Fusion-Task-Id: FN-5758

Fusion-Task-Lineage: b077c0b7-6ca4-489d-8c36-73d80a2afdb2
This commit is contained in:
gsxdsm
2026-05-30 22:53:24 -07:00
parent d98ff8780c
commit 194dfa9387
14 changed files with 308 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
packages/cli/src/__tests__/extension-goal-tools-audit.test.ts
View File

@@ -58,8 +58,8 @@ describe("extension goal tools retrieval audit", () => {
.filter((event) => event.mutationType === GOAL_RETRIEVAL_INVOKED);
expect(goalAuditCalls).toHaveLength(3);
expect(goalAuditCalls[0]).toMatchObject({ metadata: expect.objectContaining({ toolName: "fn_goal_list", count: 1 }) });
expect(goalAuditCalls[1]).toMatchObject({ target: goalId, metadata: expect.objectContaining({ toolName: "fn_goal_show", count: 1, notFound: false }) });
expect(goalAuditCalls[2]).toMatchObject({ target: "G-404", metadata: expect.objectContaining({ toolName: "fn_goal_show", count: 0, notFound: true }) });
expect(goalAuditCalls[0]).toMatchObject({ metadata: expect.objectContaining({ toolName: "fn_goal_list", count: 1, goalIds: [goalId] }) });
expect(goalAuditCalls[1]).toMatchObject({ target: goalId, metadata: expect.objectContaining({ toolName: "fn_goal_show", count: 1, goalIds: [goalId], notFound: false }) });
expect(goalAuditCalls[2]).toMatchObject({ target: "G-404", metadata: expect.objectContaining({ toolName: "fn_goal_show", count: 0, goalIds: [], notFound: true }) });
});
});

3
packages/cli/src/extension.ts
View File

@@ -2418,6 +2418,7 @@ export default function kbExtension(pi: ExtensionAPI) {
emitGoalRetrievalAudit(store, fnCtx, {
toolName: "fn_goal_list",
resultCount: goals.length,
goalIds: goals.map((goal) => goal.id),
});
const lines: string[] = [];
@@ -2562,6 +2563,7 @@ export default function kbExtension(pi: ExtensionAPI) {
toolName: "fn_goal_show",
resultCount: 0,
goalId: params.id,
goalIds: [],
notFound: true,
});
return {
@@ -2584,6 +2586,7 @@ export default function kbExtension(pi: ExtensionAPI) {
toolName: "fn_goal_show",
resultCount: 1,
goalId: params.id,
goalIds: [params.id],
});
return {

63
packages/core/src/__tests__/goal-citation-audit-aggregation.test.ts Normal file
View File

@@ -0,0 +1,63 @@
import { describe, expect, it } from "vitest";
import type { RunAuditEvent } from "../types.js";
import { collectCitedGoalIdsFromAudit } from "../goal-citation-extractor.js";
function event(partial: Partial<RunAuditEvent>): RunAuditEvent {
return {
id: "e-1",
timestamp: new Date().toISOString(),
agentId: "agent-1",
runId: "run-1",
domain: "database",
mutationType: "task:update",
target: "task",
...partial,
};
}
describe("collectCitedGoalIdsFromAudit", () => {
it("returns empty collections for empty events", () => {
expect(collectCitedGoalIdsFromAudit([])).toEqual({
injectedGoalIds: [],
retrievedGoalIds: [],
citedGoalIds: [],
});
});
it("collects injection goal ids", () => {
const result = collectCitedGoalIdsFromAudit([
event({ mutationType: "goal:injection-applied", metadata: { goalIds: ["G-A", "G-B"] } }),
event({ mutationType: "prompt:goal-injection", metadata: { goalIds: ["G-B", "G-C"] } }),
]);
expect(result).toEqual({
injectedGoalIds: ["G-A", "G-B", "G-C"],
retrievedGoalIds: [],
citedGoalIds: ["G-A", "G-B", "G-C"],
});
});
it("collects retrieval goal ids from metadata, target, and goalId", () => {
const result = collectCitedGoalIdsFromAudit([
event({ mutationType: "goal:retrieval-invoked", target: "G-A", metadata: { goalIds: ["G-B"], goalId: "G-C" } }),
event({ mutationType: "goal:retrieval-invoked", target: "goals", metadata: { goalId: "G-D" } }),
]);
expect(result).toEqual({
injectedGoalIds: [],
retrievedGoalIds: ["G-B", "G-A", "G-C", "G-D"],
citedGoalIds: ["G-B", "G-A", "G-C", "G-D"],
});
});
it("dedupes and ignores malformed/non-goal ids", () => {
const result = collectCitedGoalIdsFromAudit([
event({ mutationType: "goal:injection-skipped", metadata: { goalIds: ["G-1", "FN-1", 42, "G-1"] } }),
event({ mutationType: "goal:retrieval-invoked", target: "goals", metadata: { goalIds: ["G-1", "G-2", "task-1"], goalId: "G-2" } }),
event({ mutationType: "goal:retrieval-invoked", target: "FN-9", metadata: { goalId: "not-a-goal" } }),
]);
expect(result).toEqual({
injectedGoalIds: ["G-1"],
retrievedGoalIds: ["G-1", "G-2"],
citedGoalIds: ["G-1", "G-2"],
});
});
});

54
packages/core/src/goal-citation-extractor.ts
View File

@@ -1,4 +1,4 @@
import type { GoalCitationMatch } from "./types.js";
import type { GoalCitationMatch, RunAuditEvent } from "./types.js";
export const GOAL_ID_PATTERN = /\bG-[0-9A-Z]+(?:-[0-9A-Z]+)*\b/g;
@@ -27,6 +27,58 @@ export function extractGoalCitations(text: string): GoalCitationMatch[] {
return matches;
}
const INJECTION_AUDIT_TYPES = new Set(["goal:injection-applied", "goal:injection-skipped", "prompt:goal-injection"]);
const RETRIEVAL_AUDIT_TYPE = "goal:retrieval-invoked";
const GOAL_ID_EXACT_PATTERN = new RegExp(`^${GOAL_ID_PATTERN.source.replace(/\\b/g, "")}$`);
function isGoalId(value: string): boolean {
return GOAL_ID_EXACT_PATTERN.test(value);
}
function collectGoalIds(ids: Set<string>, values: unknown): void {
if (!Array.isArray(values)) return;
for (const value of values) {
if (typeof value !== "string") continue;
if (!isGoalId(value)) continue;
ids.add(value);
}
}
function collectGoalId(ids: Set<string>, value: unknown): void {
if (typeof value !== "string") return;
if (!isGoalId(value)) return;
ids.add(value);
}
export function collectCitedGoalIdsFromAudit(events: RunAuditEvent[]): {
injectedGoalIds: string[];
retrievedGoalIds: string[];
citedGoalIds: string[];
} {
const injectedGoalIds = new Set<string>();
const retrievedGoalIds = new Set<string>();
for (const event of events) {
const metadata = event.metadata ?? {};
if (INJECTION_AUDIT_TYPES.has(event.mutationType)) {
collectGoalIds(injectedGoalIds, metadata.goalIds);
continue;
}
if (event.mutationType === RETRIEVAL_AUDIT_TYPE) {
collectGoalIds(retrievedGoalIds, metadata.goalIds);
collectGoalId(retrievedGoalIds, event.target);
collectGoalId(retrievedGoalIds, metadata.goalId);
}
}
return {
injectedGoalIds: [...injectedGoalIds],
retrievedGoalIds: [...retrievedGoalIds],
citedGoalIds: [...new Set([...injectedGoalIds, ...retrievedGoalIds])],
};
}
export function buildSnippet(text: string, index: number, max = GOAL_CITATION_SNIPPET_MAX): string {
const normalized = String(text ?? "");
if (normalized.length === 0 || max <= 0) {

1
packages/core/src/index.ts
View File

@@ -736,6 +736,7 @@ export type {
export {
extractGoalCitations,
buildSnippet,
collectCitedGoalIdsFromAudit,
GOAL_ID_PATTERN,
GOAL_CITATION_SNIPPET_MAX,
} from "./goal-citation-extractor.js";

77
packages/dashboard/src/__tests__/routes-run-cited-goals.test.ts Normal file
View File

@@ -0,0 +1,77 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { request } from "../test-request.js";
const mockGetRunDetail = vi.fn();
const mockGetRunAuditEvents = vi.fn();
vi.mock("@fusion/core", async () => {
const actual = await vi.importActual<typeof import("@fusion/core")>("@fusion/core");
return {
...actual,
AgentStore: class MockAgentStore {
init = vi.fn().mockResolvedValue(undefined);
getRunDetail = mockGetRunDetail;
},
ChatStore: class MockChatStore {
init = vi.fn().mockResolvedValue(undefined);
},
deterministicGuardLocks: new Map(),
};
});
class MockStore {
getRunAuditEvents = mockGetRunAuditEvents;
getAgentLogsByTimeRange = vi.fn().mockResolvedValue([]);
getMutationsForRun = vi.fn().mockResolvedValue([]);
getRootDir() { return "/tmp/fn-5758-test"; }
getFusionDir() { return "/tmp/fn-5758-test/.fusion"; }
getDatabase() { return { exec: vi.fn(), prepare: vi.fn().mockReturnValue({ run: vi.fn(), get: vi.fn(), all: vi.fn().mockReturnValue([]) }) }; }
}
describe("run cited goals route", () => {
let app: ReturnType<typeof import("../server.js").createServer>;
beforeEach(async () => {
vi.clearAllMocks();
const { createServer } = await import("../server.js");
app = createServer(new MockStore() as any);
});
it("returns aggregated cited goal ids for a run", async () => {
mockGetRunDetail.mockResolvedValue({ id: "run-1", agentId: "agent-1", startedAt: "2026-01-01T00:00:00.000Z", status: "done", contextSnapshot: { taskId: "FN-1" } });
mockGetRunAuditEvents.mockReturnValue([
{ id: "e1", timestamp: "2026-01-01T00:00:00.000Z", runId: "run-1", agentId: "agent-1", domain: "database", mutationType: "goal:injection-applied", target: "FN-1", metadata: { goalIds: ["G-A", "G-B"] } },
{ id: "e2", timestamp: "2026-01-01T00:00:01.000Z", runId: "run-1", agentId: "agent-1", domain: "database", mutationType: "goal:retrieval-invoked", target: "G-C", metadata: { goalIds: ["G-B"] } },
]);
const response = await request(app, "GET", "/api/agents/agent-1/runs/run-1/cited-goals");
expect(response.status).toBe(200);
expect(response.body).toEqual({
runId: "run-1",
taskId: "FN-1",
injectedGoalIds: ["G-A", "G-B"],
retrievedGoalIds: ["G-B", "G-C"],
citedGoalIds: ["G-A", "G-B", "G-C"],
});
});
it("returns empty arrays when no goal events exist", async () => {
mockGetRunDetail.mockResolvedValue({ id: "run-1", agentId: "agent-1", startedAt: "2026-01-01T00:00:00.000Z", status: "done", contextSnapshot: {} });
mockGetRunAuditEvents.mockReturnValue([]);
const response = await request(app, "GET", "/api/agents/agent-1/runs/run-1/cited-goals");
expect(response.status).toBe(200);
expect(response.body).toEqual({
runId: "run-1",
injectedGoalIds: [],
retrievedGoalIds: [],
citedGoalIds: [],
});
});
it("returns 404 for unknown run", async () => {
mockGetRunDetail.mockResolvedValue(null);
const response = await request(app, "GET", "/api/agents/agent-1/runs/run-missing/cited-goals");
expect(response.status).toBe(404);
});
});

11
packages/dashboard/src/routes.ts
View File

@@ -629,6 +629,17 @@ export interface RunAuditResponse {
hasMore: boolean;
}
/**
* Response shape for GET /api/agents/:id/runs/:runId/cited-goals
*/
export interface RunCitedGoalsResponse {
runId: string;
taskId?: string;
injectedGoalIds: string[];
retrievedGoalIds: string[];
citedGoalIds: string[];
}
/**
* Response shape for GET /api/agents/:id/runs/:runId/timeline
*/

49
packages/dashboard/src/routes/register-agent-runtime-routes.ts
View File

@@ -4,6 +4,7 @@ import type { ApiRoutesContext } from "./types.js";
type NormalizedAuditEvent = import("../routes.js").NormalizedRunAuditEvent;
type TimelineEntry = import("../routes.js").TimelineEntry;
type RunAuditResponse = import("../routes.js").RunAuditResponse;
type RunCitedGoalsResponse = import("../routes.js").RunCitedGoalsResponse;
type RunTimelineResponse = import("../routes.js").RunTimelineResponse;
interface AgentRuntimeRouteDeps {
@@ -1519,6 +1520,54 @@ export function registerAgentRuntimeRoutes(ctx: ApiRoutesContext, deps: AgentRun
}
});
/**
* GET /api/agents/:id/runs/:runId/cited-goals
* Get cited goal IDs aggregated from goal-related run-audit events for a specific run.
*/
router.get("/agents/:id/runs/:runId/cited-goals", async (req, res) => {
try {
const { store: scopedStore } = await getProjectContext(req);
const { AgentStore, collectCitedGoalIdsFromAudit } = await import("@fusion/core");
const agentStore = new AgentStore({ rootDir: scopedStore.getFusionDir() });
await agentStore.init();
const runId = req.params.runId;
if (!runId || runId.trim().length === 0) {
throw badRequest("runId is required");
}
const run = await agentStore.getRunDetail(req.params.id, req.params.runId);
if (!run) {
throw notFound("Run not found");
}
const goalEvents = scopedStore.getRunAuditEvents({
runId: req.params.runId,
domain: "database",
});
const { injectedGoalIds, retrievedGoalIds, citedGoalIds } = collectCitedGoalIdsFromAudit(goalEvents);
const taskId = run.contextSnapshot?.taskId as string | undefined;
const response: RunCitedGoalsResponse = {
runId: req.params.runId,
taskId,
injectedGoalIds,
retrievedGoalIds,
citedGoalIds,
};
res.json(response);
} catch (err: unknown) {
if (err instanceof ApiError) {
throw err;
}
if ((err instanceof Error ? err.message : String(err)).includes("not found")) {
throw notFound(err instanceof Error ? err.message : String(err));
} else {
rethrowAsApiError(err);
}
}
});
/**
* GET /api/agents/:id/runs/:runId/timeline
* Get a correlated timeline combining run-audit events and agent logs for a specific run.

40
packages/engine/src/__tests__/goal-anchoring-audit.test.ts
View File

@@ -1,5 +1,5 @@
import { describe, expect, it, vi } from "vitest";
import type { RunAuditEventInput, TaskStore } from "@fusion/core";
import { collectCitedGoalIdsFromAudit, type RunAuditEventInput, type TaskStore } from "@fusion/core";
import { createRunAuditor } from "../run-audit.js";
import {
emitGoalAnchoringAudit,
@@ -20,7 +20,7 @@ describe("goal anchoring audit helpers", () => {
expect(database).toHaveBeenCalledWith(expect.objectContaining({
type: GOAL_INJECTION_APPLIED,
target: "FN-1",
metadata: expect.objectContaining({ lane: "heartbeat", count: 3 }),
metadata: expect.objectContaining({ lane: "heartbeat", count: 3, goalIds: [] }),
}));
});
@@ -34,7 +34,7 @@ describe("goal anchoring audit helpers", () => {
expect(database).toHaveBeenCalledWith(expect.objectContaining({
type: GOAL_INJECTION_SKIPPED,
target: "goals",
metadata: expect.objectContaining({ reason: "no-active-goals", count: 0 }),
metadata: expect.objectContaining({ reason: "no-active-goals", count: 0, goalIds: [] }),
}));
});
@@ -43,23 +43,24 @@ describe("goal anchoring audit helpers", () => {
await emitGoalAnchoringAudit({ database } as any, {
lane: "heartbeat",
goalsInjected: 1,
goalIds: ["G-ALPHA"],
truncated: true,
});
expect(database).toHaveBeenCalledWith(expect.objectContaining({
metadata: expect.objectContaining({ truncated: true }),
metadata: expect.objectContaining({ truncated: true, goalIds: ["G-ALPHA"] }),
}));
});
it("emits retrieval audit when run context exists", () => {
const recordRunAuditEvent = vi.fn();
const store = { recordRunAuditEvent } as unknown as TaskStore;
emitGoalRetrievalAudit(store, { runId: "r1", agentId: "a1", taskId: "FN-1" }, { toolName: "fn_goal_list", resultCount: 2 });
emitGoalRetrievalAudit(store, { runId: "r1", agentId: "a1", taskId: "FN-1" }, { toolName: "fn_goal_list", resultCount: 2, goalIds: ["G-1", "G-2"] });
expect(recordRunAuditEvent).toHaveBeenCalledTimes(1);
expect(recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({
domain: "database",
mutationType: GOAL_RETRIEVAL_INVOKED,
target: "goals",
metadata: expect.objectContaining({ toolName: "fn_goal_list", count: 2, notFound: false }),
metadata: expect.objectContaining({ toolName: "fn_goal_list", count: 2, goalIds: ["G-1", "G-2"], notFound: false }),
}));
});
@@ -79,17 +80,32 @@ describe("goal anchoring audit helpers", () => {
warn.mockRestore();
});
it("persists heartbeat-style events through createRunAuditor", async () => {
it("persists goal IDs across injection/retrieval and aggregates cited IDs", async () => {
const events: RunAuditEventInput[] = [];
const store = { recordRunAuditEvent: vi.fn((input: RunAuditEventInput) => events.push(input)) } as unknown as TaskStore;
const auditor = createRunAuditor(store, { runId: "run-1", agentId: "agent-1", taskId: "FN-9", phase: "heartbeat" });
await emitGoalAnchoringAudit(auditor, { lane: "heartbeat", taskId: "FN-9", goalsInjected: 2 });
await emitGoalAnchoringAudit(auditor, { lane: "heartbeat", taskId: "FN-9", goalsInjected: 0, reason: "no-active-goals" });
await emitGoalAnchoringAudit(auditor, { lane: "heartbeat", taskId: "FN-9", goalsInjected: 2, goalIds: ["G-A", "G-B"] });
await emitGoalAnchoringAudit(auditor, { lane: "heartbeat", taskId: "FN-9", goalsInjected: 0, goalIds: [], reason: "no-active-goals" });
emitGoalRetrievalAudit(store, { runId: "run-1", agentId: "agent-1", taskId: "FN-9" }, { toolName: "fn_goal_list", resultCount: 2, goalIds: ["G-A", "G-C"] });
emitGoalRetrievalAudit(store, { runId: "run-1", agentId: "agent-1", taskId: "FN-9" }, { toolName: "fn_goal_show", resultCount: 1, goalId: "G-B", goalIds: ["G-B"] });
const goalEvents = events.filter((event) => String(event.mutationType).startsWith("goal:"));
expect(goalEvents).toHaveLength(2);
expect(goalEvents[0]).toMatchObject({ mutationType: GOAL_INJECTION_APPLIED, metadata: expect.objectContaining({ count: 2, lane: "heartbeat" }) });
expect(goalEvents[1]).toMatchObject({ mutationType: GOAL_INJECTION_SKIPPED, metadata: expect.objectContaining({ count: 0, reason: "no-active-goals" }) });
expect(goalEvents).toHaveLength(4);
expect(goalEvents[0]).toMatchObject({ mutationType: GOAL_INJECTION_APPLIED, metadata: expect.objectContaining({ count: 2, lane: "heartbeat", goalIds: ["G-A", "G-B"] }) });
expect(goalEvents[1]).toMatchObject({ mutationType: GOAL_INJECTION_SKIPPED, metadata: expect.objectContaining({ count: 0, reason: "no-active-goals", goalIds: [] }) });
const aggregate = collectCitedGoalIdsFromAudit(goalEvents as any);
expect(aggregate).toEqual({
injectedGoalIds: ["G-A", "G-B"],
retrievedGoalIds: ["G-A", "G-C", "G-B"],
citedGoalIds: ["G-A", "G-B", "G-C"],
});
for (const event of goalEvents) {
expect(JSON.stringify(event.metadata ?? {})).not.toContain("Description:");
expect(JSON.stringify(event.metadata ?? {})).not.toContain("title");
expect(JSON.stringify(event.metadata ?? {})).not.toContain("goalContext");
}
});
});

4
packages/engine/src/goal-anchoring-audit.ts
View File

@@ -25,6 +25,7 @@ export type GoalInjectionAuditInput = {
lane: GoalAnchoringLane;
taskId?: string;
goalsInjected: number;
goalIds?: string[];
truncated?: boolean;
reason?: "no-active-goals" | "injector-empty";
};
@@ -37,6 +38,7 @@ export type GoalRetrievalAuditInput = {
toolName: "fn_goal_list" | "fn_goal_show";
resultCount: number;
goalId?: string;
goalIds?: string[];
notFound?: boolean;
};
@@ -52,6 +54,7 @@ export async function emitGoalAnchoringAudit(auditor: RunAuditor, input: GoalInj
metadata: {
lane: input.lane,
count: input.goalsInjected,
goalIds: input.goalIds ?? [],
...(typeof input.truncated === "boolean" ? { truncated: input.truncated } : {}),
...(input.reason ? { reason: input.reason } : {}),
},
@@ -80,6 +83,7 @@ export function emitGoalRetrievalAudit(
metadata: {
toolName: input.toolName,
count: input.resultCount,
goalIds: input.goalIds ?? [],
notFound: input.notFound ?? false,
},
});

1
packages/engine/src/goal-injection-diagnostics.ts
View File

@@ -144,6 +144,7 @@ export async function resolveAndEmitGoalContext(input: ResolveAndEmitGoalContext
lane: input.lane,
taskId: input.taskId,
goalsInjected: resolution.classification.goalCount,
goalIds: resolution.classification.goalIds,
truncated: resolution.classification.truncated,
reason: resolution.classification.outcome === "no-goals" ? "no-active-goals" : undefined,
});