FN-5758: add cited-goal audit aggregation and runtime route
Add end-to-end goal-citation audit trail plumbing across core, engine, CLI, and dashboard. - extend goal citation extraction with aggregation logic and new core coverage - expose cited-goal runtime data through dashboard routes with dedicated API tests - wire goal-citation audit details into engine diagnostics and CLI extension audit expectations - document the new audit behavior and publish a patch changeset for @runfusion/fusion Files changed: .changeset/fn-5758-goal-citation-audit.md | 9 +++ docs/dashboard-guide.md | 2 + docs/diagnostics.md | 4 ++ .../__tests__/extension-goal-tools-audit.test.ts | 6 +- packages/cli/src/extension.ts | 3 + .../goal-citation-audit-aggregation.test.ts | 63 ++++++++++++++++++ packages/core/src/goal-citation-extractor.ts | 54 ++++++++++++++- packages/core/src/index.ts | 1 + .../src/__tests__/routes-run-cited-goals.test.ts | 77 ++++++++++++++++++++++ packages/dashboard/src/routes.ts | 11 ++++ .../src/routes/register-agent-runtime-routes.ts | 49 ++++++++++++++ .../src/__tests__/goal-anchoring-audit.test.ts | 40 +++++++---- packages/engine/src/goal-anchoring-audit.ts | 4 ++ packages/engine/src/goal-injection-diagnostics.ts | 1 + 14 files changed, 308 insertions(+), 16 deletions(-) Fusion-Task-Id: FN-5758 Fusion-Task-Lineage: b077c0b7-6ca4-489d-8c36-73d80a2afdb2
This commit is contained in:
@@ -0,0 +1,63 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { RunAuditEvent } from "../types.js";
|
||||
import { collectCitedGoalIdsFromAudit } from "../goal-citation-extractor.js";
|
||||
|
||||
function event(partial: Partial<RunAuditEvent>): RunAuditEvent {
|
||||
return {
|
||||
id: "e-1",
|
||||
timestamp: new Date().toISOString(),
|
||||
agentId: "agent-1",
|
||||
runId: "run-1",
|
||||
domain: "database",
|
||||
mutationType: "task:update",
|
||||
target: "task",
|
||||
...partial,
|
||||
};
|
||||
}
|
||||
|
||||
describe("collectCitedGoalIdsFromAudit", () => {
|
||||
it("returns empty collections for empty events", () => {
|
||||
expect(collectCitedGoalIdsFromAudit([])).toEqual({
|
||||
injectedGoalIds: [],
|
||||
retrievedGoalIds: [],
|
||||
citedGoalIds: [],
|
||||
});
|
||||
});
|
||||
|
||||
it("collects injection goal ids", () => {
|
||||
const result = collectCitedGoalIdsFromAudit([
|
||||
event({ mutationType: "goal:injection-applied", metadata: { goalIds: ["G-A", "G-B"] } }),
|
||||
event({ mutationType: "prompt:goal-injection", metadata: { goalIds: ["G-B", "G-C"] } }),
|
||||
]);
|
||||
expect(result).toEqual({
|
||||
injectedGoalIds: ["G-A", "G-B", "G-C"],
|
||||
retrievedGoalIds: [],
|
||||
citedGoalIds: ["G-A", "G-B", "G-C"],
|
||||
});
|
||||
});
|
||||
|
||||
it("collects retrieval goal ids from metadata, target, and goalId", () => {
|
||||
const result = collectCitedGoalIdsFromAudit([
|
||||
event({ mutationType: "goal:retrieval-invoked", target: "G-A", metadata: { goalIds: ["G-B"], goalId: "G-C" } }),
|
||||
event({ mutationType: "goal:retrieval-invoked", target: "goals", metadata: { goalId: "G-D" } }),
|
||||
]);
|
||||
expect(result).toEqual({
|
||||
injectedGoalIds: [],
|
||||
retrievedGoalIds: ["G-B", "G-A", "G-C", "G-D"],
|
||||
citedGoalIds: ["G-B", "G-A", "G-C", "G-D"],
|
||||
});
|
||||
});
|
||||
|
||||
it("dedupes and ignores malformed/non-goal ids", () => {
|
||||
const result = collectCitedGoalIdsFromAudit([
|
||||
event({ mutationType: "goal:injection-skipped", metadata: { goalIds: ["G-1", "FN-1", 42, "G-1"] } }),
|
||||
event({ mutationType: "goal:retrieval-invoked", target: "goals", metadata: { goalIds: ["G-1", "G-2", "task-1"], goalId: "G-2" } }),
|
||||
event({ mutationType: "goal:retrieval-invoked", target: "FN-9", metadata: { goalId: "not-a-goal" } }),
|
||||
]);
|
||||
expect(result).toEqual({
|
||||
injectedGoalIds: ["G-1"],
|
||||
retrievedGoalIds: ["G-1", "G-2"],
|
||||
citedGoalIds: ["G-1", "G-2"],
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,4 @@
|
||||
import type { GoalCitationMatch } from "./types.js";
|
||||
import type { GoalCitationMatch, RunAuditEvent } from "./types.js";
|
||||
|
||||
export const GOAL_ID_PATTERN = /\bG-[0-9A-Z]+(?:-[0-9A-Z]+)*\b/g;
|
||||
|
||||
@@ -27,6 +27,58 @@ export function extractGoalCitations(text: string): GoalCitationMatch[] {
|
||||
return matches;
|
||||
}
|
||||
|
||||
const INJECTION_AUDIT_TYPES = new Set(["goal:injection-applied", "goal:injection-skipped", "prompt:goal-injection"]);
|
||||
const RETRIEVAL_AUDIT_TYPE = "goal:retrieval-invoked";
|
||||
const GOAL_ID_EXACT_PATTERN = new RegExp(`^${GOAL_ID_PATTERN.source.replace(/\\b/g, "")}$`);
|
||||
|
||||
function isGoalId(value: string): boolean {
|
||||
return GOAL_ID_EXACT_PATTERN.test(value);
|
||||
}
|
||||
|
||||
function collectGoalIds(ids: Set<string>, values: unknown): void {
|
||||
if (!Array.isArray(values)) return;
|
||||
for (const value of values) {
|
||||
if (typeof value !== "string") continue;
|
||||
if (!isGoalId(value)) continue;
|
||||
ids.add(value);
|
||||
}
|
||||
}
|
||||
|
||||
function collectGoalId(ids: Set<string>, value: unknown): void {
|
||||
if (typeof value !== "string") return;
|
||||
if (!isGoalId(value)) return;
|
||||
ids.add(value);
|
||||
}
|
||||
|
||||
export function collectCitedGoalIdsFromAudit(events: RunAuditEvent[]): {
|
||||
injectedGoalIds: string[];
|
||||
retrievedGoalIds: string[];
|
||||
citedGoalIds: string[];
|
||||
} {
|
||||
const injectedGoalIds = new Set<string>();
|
||||
const retrievedGoalIds = new Set<string>();
|
||||
|
||||
for (const event of events) {
|
||||
const metadata = event.metadata ?? {};
|
||||
if (INJECTION_AUDIT_TYPES.has(event.mutationType)) {
|
||||
collectGoalIds(injectedGoalIds, metadata.goalIds);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (event.mutationType === RETRIEVAL_AUDIT_TYPE) {
|
||||
collectGoalIds(retrievedGoalIds, metadata.goalIds);
|
||||
collectGoalId(retrievedGoalIds, event.target);
|
||||
collectGoalId(retrievedGoalIds, metadata.goalId);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
injectedGoalIds: [...injectedGoalIds],
|
||||
retrievedGoalIds: [...retrievedGoalIds],
|
||||
citedGoalIds: [...new Set([...injectedGoalIds, ...retrievedGoalIds])],
|
||||
};
|
||||
}
|
||||
|
||||
export function buildSnippet(text: string, index: number, max = GOAL_CITATION_SNIPPET_MAX): string {
|
||||
const normalized = String(text ?? "");
|
||||
if (normalized.length === 0 || max <= 0) {
|
||||
|
||||
@@ -736,6 +736,7 @@ export type {
|
||||
export {
|
||||
extractGoalCitations,
|
||||
buildSnippet,
|
||||
collectCitedGoalIdsFromAudit,
|
||||
GOAL_ID_PATTERN,
|
||||
GOAL_CITATION_SNIPPET_MAX,
|
||||
} from "./goal-citation-extractor.js";
|
||||
|
||||
Reference in New Issue
Block a user