FN-5758: add cited-goal audit aggregation and runtime route
Add end-to-end goal-citation audit trail plumbing across core, engine, CLI, and dashboard. - extend goal citation extraction with aggregation logic and new core coverage - expose cited-goal runtime data through dashboard routes with dedicated API tests - wire goal-citation audit details into engine diagnostics and CLI extension audit expectations - document the new audit behavior and publish a patch changeset for @runfusion/fusion Files changed: .changeset/fn-5758-goal-citation-audit.md | 9 +++ docs/dashboard-guide.md | 2 + docs/diagnostics.md | 4 ++ .../__tests__/extension-goal-tools-audit.test.ts | 6 +- packages/cli/src/extension.ts | 3 + .../goal-citation-audit-aggregation.test.ts | 63 ++++++++++++++++++ packages/core/src/goal-citation-extractor.ts | 54 ++++++++++++++- packages/core/src/index.ts | 1 + .../src/__tests__/routes-run-cited-goals.test.ts | 77 ++++++++++++++++++++++ packages/dashboard/src/routes.ts | 11 ++++ .../src/routes/register-agent-runtime-routes.ts | 49 ++++++++++++++ .../src/__tests__/goal-anchoring-audit.test.ts | 40 +++++++---- packages/engine/src/goal-anchoring-audit.ts | 4 ++ packages/engine/src/goal-injection-diagnostics.ts | 1 + 14 files changed, 308 insertions(+), 16 deletions(-) Fusion-Task-Id: FN-5758 Fusion-Task-Lineage: b077c0b7-6ca4-489d-8c36-73d80a2afdb2
This commit is contained in:
gsxdsm
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { RunAuditEventInput, TaskStore } from "@fusion/core";
|
||||
import { collectCitedGoalIdsFromAudit, type RunAuditEventInput, type TaskStore } from "@fusion/core";
|
||||
import { createRunAuditor } from "../run-audit.js";
|
||||
import {
|
||||
emitGoalAnchoringAudit,
|
||||
@@ -20,7 +20,7 @@ describe("goal anchoring audit helpers", () => {
|
||||
expect(database).toHaveBeenCalledWith(expect.objectContaining({
|
||||
type: GOAL_INJECTION_APPLIED,
|
||||
target: "FN-1",
|
||||
metadata: expect.objectContaining({ lane: "heartbeat", count: 3 }),
|
||||
metadata: expect.objectContaining({ lane: "heartbeat", count: 3, goalIds: [] }),
|
||||
}));
|
||||
});
|
||||
|
||||
@@ -34,7 +34,7 @@ describe("goal anchoring audit helpers", () => {
|
||||
expect(database).toHaveBeenCalledWith(expect.objectContaining({
|
||||
type: GOAL_INJECTION_SKIPPED,
|
||||
target: "goals",
|
||||
metadata: expect.objectContaining({ reason: "no-active-goals", count: 0 }),
|
||||
metadata: expect.objectContaining({ reason: "no-active-goals", count: 0, goalIds: [] }),
|
||||
}));
|
||||
});
|
||||
|
||||
@@ -43,23 +43,24 @@ describe("goal anchoring audit helpers", () => {
|
||||
await emitGoalAnchoringAudit({ database } as any, {
|
||||
lane: "heartbeat",
|
||||
goalsInjected: 1,
|
||||
goalIds: ["G-ALPHA"],
|
||||
truncated: true,
|
||||
});
|
||||
expect(database).toHaveBeenCalledWith(expect.objectContaining({
|
||||
metadata: expect.objectContaining({ truncated: true }),
|
||||
metadata: expect.objectContaining({ truncated: true, goalIds: ["G-ALPHA"] }),
|
||||
}));
|
||||
});
|
||||
|
||||
it("emits retrieval audit when run context exists", () => {
|
||||
const recordRunAuditEvent = vi.fn();
|
||||
const store = { recordRunAuditEvent } as unknown as TaskStore;
|
||||
emitGoalRetrievalAudit(store, { runId: "r1", agentId: "a1", taskId: "FN-1" }, { toolName: "fn_goal_list", resultCount: 2 });
|
||||
emitGoalRetrievalAudit(store, { runId: "r1", agentId: "a1", taskId: "FN-1" }, { toolName: "fn_goal_list", resultCount: 2, goalIds: ["G-1", "G-2"] });
|
||||
expect(recordRunAuditEvent).toHaveBeenCalledTimes(1);
|
||||
expect(recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({
|
||||
domain: "database",
|
||||
mutationType: GOAL_RETRIEVAL_INVOKED,
|
||||
target: "goals",
|
||||
metadata: expect.objectContaining({ toolName: "fn_goal_list", count: 2, notFound: false }),
|
||||
metadata: expect.objectContaining({ toolName: "fn_goal_list", count: 2, goalIds: ["G-1", "G-2"], notFound: false }),
|
||||
}));
|
||||
});
|
||||
|
||||
@@ -79,17 +80,32 @@ describe("goal anchoring audit helpers", () => {
|
||||
warn.mockRestore();
|
||||
});
|
||||
|
||||
it("persists heartbeat-style events through createRunAuditor", async () => {
|
||||
it("persists goal IDs across injection/retrieval and aggregates cited IDs", async () => {
|
||||
const events: RunAuditEventInput[] = [];
|
||||
const store = { recordRunAuditEvent: vi.fn((input: RunAuditEventInput) => events.push(input)) } as unknown as TaskStore;
|
||||
const auditor = createRunAuditor(store, { runId: "run-1", agentId: "agent-1", taskId: "FN-9", phase: "heartbeat" });
|
||||
|
||||
await emitGoalAnchoringAudit(auditor, { lane: "heartbeat", taskId: "FN-9", goalsInjected: 2 });
|
||||
await emitGoalAnchoringAudit(auditor, { lane: "heartbeat", taskId: "FN-9", goalsInjected: 0, reason: "no-active-goals" });
|
||||
await emitGoalAnchoringAudit(auditor, { lane: "heartbeat", taskId: "FN-9", goalsInjected: 2, goalIds: ["G-A", "G-B"] });
|
||||
await emitGoalAnchoringAudit(auditor, { lane: "heartbeat", taskId: "FN-9", goalsInjected: 0, goalIds: [], reason: "no-active-goals" });
|
||||
emitGoalRetrievalAudit(store, { runId: "run-1", agentId: "agent-1", taskId: "FN-9" }, { toolName: "fn_goal_list", resultCount: 2, goalIds: ["G-A", "G-C"] });
|
||||
emitGoalRetrievalAudit(store, { runId: "run-1", agentId: "agent-1", taskId: "FN-9" }, { toolName: "fn_goal_show", resultCount: 1, goalId: "G-B", goalIds: ["G-B"] });
|
||||
|
||||
const goalEvents = events.filter((event) => String(event.mutationType).startsWith("goal:"));
|
||||
expect(goalEvents).toHaveLength(2);
|
||||
expect(goalEvents[0]).toMatchObject({ mutationType: GOAL_INJECTION_APPLIED, metadata: expect.objectContaining({ count: 2, lane: "heartbeat" }) });
|
||||
expect(goalEvents[1]).toMatchObject({ mutationType: GOAL_INJECTION_SKIPPED, metadata: expect.objectContaining({ count: 0, reason: "no-active-goals" }) });
|
||||
expect(goalEvents).toHaveLength(4);
|
||||
expect(goalEvents[0]).toMatchObject({ mutationType: GOAL_INJECTION_APPLIED, metadata: expect.objectContaining({ count: 2, lane: "heartbeat", goalIds: ["G-A", "G-B"] }) });
|
||||
expect(goalEvents[1]).toMatchObject({ mutationType: GOAL_INJECTION_SKIPPED, metadata: expect.objectContaining({ count: 0, reason: "no-active-goals", goalIds: [] }) });
|
||||
|
||||
const aggregate = collectCitedGoalIdsFromAudit(goalEvents as any);
|
||||
expect(aggregate).toEqual({
|
||||
injectedGoalIds: ["G-A", "G-B"],
|
||||
retrievedGoalIds: ["G-A", "G-C", "G-B"],
|
||||
citedGoalIds: ["G-A", "G-B", "G-C"],
|
||||
});
|
||||
|
||||
for (const event of goalEvents) {
|
||||
expect(JSON.stringify(event.metadata ?? {})).not.toContain("Description:");
|
||||
expect(JSON.stringify(event.metadata ?? {})).not.toContain("title");
|
||||
expect(JSON.stringify(event.metadata ?? {})).not.toContain("goalContext");
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -25,6 +25,7 @@ export type GoalInjectionAuditInput = {
|
||||
lane: GoalAnchoringLane;
|
||||
taskId?: string;
|
||||
goalsInjected: number;
|
||||
goalIds?: string[];
|
||||
truncated?: boolean;
|
||||
reason?: "no-active-goals" | "injector-empty";
|
||||
};
|
||||
@@ -37,6 +38,7 @@ export type GoalRetrievalAuditInput = {
|
||||
toolName: "fn_goal_list" | "fn_goal_show";
|
||||
resultCount: number;
|
||||
goalId?: string;
|
||||
goalIds?: string[];
|
||||
notFound?: boolean;
|
||||
};
|
||||
|
||||
@@ -52,6 +54,7 @@ export async function emitGoalAnchoringAudit(auditor: RunAuditor, input: GoalInj
|
||||
metadata: {
|
||||
lane: input.lane,
|
||||
count: input.goalsInjected,
|
||||
goalIds: input.goalIds ?? [],
|
||||
...(typeof input.truncated === "boolean" ? { truncated: input.truncated } : {}),
|
||||
...(input.reason ? { reason: input.reason } : {}),
|
||||
},
|
||||
@@ -80,6 +83,7 @@ export function emitGoalRetrievalAudit(
|
||||
metadata: {
|
||||
toolName: input.toolName,
|
||||
count: input.resultCount,
|
||||
goalIds: input.goalIds ?? [],
|
||||
notFound: input.notFound ?? false,
|
||||
},
|
||||
});
|
||||
|
||||
@@ -144,6 +144,7 @@ export async function resolveAndEmitGoalContext(input: ResolveAndEmitGoalContext
|
||||
lane: input.lane,
|
||||
taskId: input.taskId,
|
||||
goalsInjected: resolution.classification.goalCount,
|
||||
goalIds: resolution.classification.goalIds,
|
||||
truncated: resolution.classification.truncated,
|
||||
reason: resolution.classification.outcome === "no-goals" ? "no-active-goals" : undefined,
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user