FN-5768: add dual-observe workflow parity instrumentation
Add default-off workflow interpreter dual-observe parity instrumentation across core and engine. - add workflow parity comparison primitives and exports in @fusion/core - add engine parity observer seam with experimental flag and public exports - add regression coverage for parity contracts and dual-observe reliability behavior - document the dual-observe parity mode, events, and contract in workflow docs - add a patch changeset for @runfusion/fusion release notes Files changed: .changeset/fn-5768-workflow-parity-observer.md | 9 + docs/workflow-steps.md | 18 ++ .../core/src/__tests__/workflow-parity.test.ts | 143 ++++++++++++++ packages/core/src/index.ts | 18 ++ packages/core/src/workflow-parity.ts | 205 +++++++++++++++++++++ .../workflow-interpreter-dual-observe.test.ts | 144 +++++++++++++++ packages/engine/src/index.ts | 7 + packages/engine/src/workflow-parity-observer.ts | 116 ++++++++++++ 8 files changed, 660 insertions(+) Fusion-Task-Id: FN-5768 Fusion-Task-Lineage: fb4f8111-f48d-482c-b1d9-9106c43c829a
This commit is contained in:
gsxdsm
@@ -0,0 +1,144 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
observeWorkflowParity,
|
||||
WORKFLOW_INTERPRETER_DUAL_OBSERVE_FLAG,
|
||||
} from "../../workflow-parity-observer.js";
|
||||
import type { WorkflowRunObservation } from "@fusion/core";
|
||||
|
||||
const baseObservation: WorkflowRunObservation = {
|
||||
stageTransitions: ["triage", "execute", "review", "merge"],
|
||||
terminalColumn: "done",
|
||||
terminalStatus: null,
|
||||
reviewVerdict: "APPROVE",
|
||||
mergeOutcome: "merged",
|
||||
invariants: {
|
||||
fileScopeGuardOutcome: "pass",
|
||||
squashMergeContractOutcome: "pass",
|
||||
autoMergeTerminalUntilMergedRespected: true,
|
||||
moveTaskHardCancelRespected: true,
|
||||
},
|
||||
};
|
||||
|
||||
describe("FN-5768 workflow interpreter dual-observe", () => {
|
||||
it("is strict no-op when flag is off", async () => {
|
||||
const recordRunAuditEvent = vi.fn();
|
||||
const runShadow = vi.fn();
|
||||
|
||||
await observeWorkflowParity({
|
||||
settings: { experimentalFeatures: {} },
|
||||
store: { recordRunAuditEvent },
|
||||
agentId: "executor",
|
||||
legacy: {
|
||||
taskId: "FN-1",
|
||||
observation: baseObservation,
|
||||
auditEvents: [],
|
||||
},
|
||||
runShadow,
|
||||
});
|
||||
|
||||
expect(runShadow).not.toHaveBeenCalled();
|
||||
expect(recordRunAuditEvent).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("records parity-observed agree=true when observations match", async () => {
|
||||
const recordRunAuditEvent = vi.fn().mockResolvedValue(undefined);
|
||||
|
||||
await observeWorkflowParity({
|
||||
settings: { experimentalFeatures: { [WORKFLOW_INTERPRETER_DUAL_OBSERVE_FLAG]: true } },
|
||||
store: { recordRunAuditEvent },
|
||||
agentId: "executor",
|
||||
legacy: {
|
||||
taskId: "FN-2",
|
||||
observation: baseObservation,
|
||||
auditEvents: [],
|
||||
},
|
||||
runShadow: async () => ({ observation: baseObservation, auditEvents: [] }),
|
||||
});
|
||||
|
||||
expect(recordRunAuditEvent).toHaveBeenCalledTimes(1);
|
||||
expect(recordRunAuditEvent).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
mutationType: "workflow:parity-observed",
|
||||
metadata: expect.objectContaining({ agree: true }),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("records parity drift and keeps authoritative result unchanged", async () => {
|
||||
const recordRunAuditEvent = vi.fn().mockResolvedValue(undefined);
|
||||
const legacyResult = { authoritative: true };
|
||||
|
||||
await observeWorkflowParity({
|
||||
settings: { experimentalFeatures: { [WORKFLOW_INTERPRETER_DUAL_OBSERVE_FLAG]: true } },
|
||||
store: { recordRunAuditEvent },
|
||||
agentId: "executor",
|
||||
legacy: {
|
||||
taskId: "FN-3",
|
||||
observation: baseObservation,
|
||||
auditEvents: [],
|
||||
},
|
||||
runShadow: async () => ({
|
||||
observation: {
|
||||
...baseObservation,
|
||||
terminalColumn: "in-review",
|
||||
},
|
||||
auditEvents: [],
|
||||
}),
|
||||
});
|
||||
|
||||
expect(legacyResult).toEqual({ authoritative: true });
|
||||
expect(recordRunAuditEvent).toHaveBeenCalledTimes(2);
|
||||
expect(recordRunAuditEvent).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.objectContaining({ mutationType: "workflow:parity-observed" }),
|
||||
);
|
||||
expect(recordRunAuditEvent).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expect.objectContaining({
|
||||
mutationType: "workflow:parity-drift",
|
||||
metadata: expect.objectContaining({
|
||||
agree: false,
|
||||
diffs: expect.arrayContaining([expect.objectContaining({ field: "terminalColumn" })]),
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("captures shadow errors fail-soft without rethrow", async () => {
|
||||
const recordRunAuditEvent = vi.fn().mockResolvedValue(undefined);
|
||||
|
||||
await expect(
|
||||
observeWorkflowParity({
|
||||
settings: { experimentalFeatures: { [WORKFLOW_INTERPRETER_DUAL_OBSERVE_FLAG]: true } },
|
||||
store: { recordRunAuditEvent },
|
||||
agentId: "executor",
|
||||
legacy: {
|
||||
taskId: "FN-4",
|
||||
observation: baseObservation,
|
||||
auditEvents: [],
|
||||
},
|
||||
runShadow: async () => {
|
||||
throw new Error("shadow exploded");
|
||||
},
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
|
||||
expect(recordRunAuditEvent).toHaveBeenCalledTimes(2);
|
||||
expect(recordRunAuditEvent).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.objectContaining({
|
||||
mutationType: "workflow:parity-observed",
|
||||
metadata: expect.objectContaining({ agree: false }),
|
||||
}),
|
||||
);
|
||||
expect(recordRunAuditEvent).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expect.objectContaining({
|
||||
mutationType: "workflow:parity-drift",
|
||||
metadata: expect.objectContaining({
|
||||
diffs: expect.arrayContaining([expect.objectContaining({ field: "shadow.error" })]),
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -84,6 +84,13 @@ export {
|
||||
export {
|
||||
generateSyntheticRunId,
|
||||
} from "./run-audit.js";
|
||||
export {
|
||||
observeWorkflowParity,
|
||||
WORKFLOW_INTERPRETER_DUAL_OBSERVE_FLAG,
|
||||
type WorkflowParityObserverInput,
|
||||
type WorkflowParityObserverLegacyRunResult,
|
||||
type WorkflowParityObserverShadowRunResult,
|
||||
} from "./workflow-parity-observer.js";
|
||||
export {
|
||||
auditSquashMerge,
|
||||
formatSquashAuditReport,
|
||||
|
||||
116
packages/engine/src/workflow-parity-observer.ts
Normal file
116
packages/engine/src/workflow-parity-observer.ts
Normal file
@@ -0,0 +1,116 @@
|
||||
import {
|
||||
compareWorkflowRunAudits,
|
||||
compareWorkflowRunObservations,
|
||||
isExperimentalFeatureEnabled,
|
||||
WORKFLOW_PARITY_DRIFT_MUTATION,
|
||||
WORKFLOW_PARITY_OBSERVED_MUTATION,
|
||||
type RunAuditEvent,
|
||||
type Settings,
|
||||
type TaskStore,
|
||||
type WorkflowParityDiff,
|
||||
type WorkflowRunObservation,
|
||||
} from "@fusion/core";
|
||||
import { generateSyntheticRunId } from "./run-audit.js";
|
||||
|
||||
export const WORKFLOW_INTERPRETER_DUAL_OBSERVE_FLAG = "workflowInterpreterDualObserve" as const;
|
||||
|
||||
export interface WorkflowParityObserverLegacyRunResult {
|
||||
taskId: string;
|
||||
observation: WorkflowRunObservation;
|
||||
auditEvents: RunAuditEvent[];
|
||||
}
|
||||
|
||||
export interface WorkflowParityObserverShadowRunResult {
|
||||
observation: WorkflowRunObservation;
|
||||
auditEvents: RunAuditEvent[];
|
||||
}
|
||||
|
||||
export interface WorkflowParityObserverInput {
|
||||
settings: Pick<Settings, "experimentalFeatures"> | undefined;
|
||||
store: Pick<TaskStore, "recordRunAuditEvent">;
|
||||
agentId: string;
|
||||
legacy: WorkflowParityObserverLegacyRunResult;
|
||||
runShadow: () => Promise<WorkflowParityObserverShadowRunResult>;
|
||||
}
|
||||
|
||||
function buildErrorDiff(error: unknown): WorkflowParityDiff {
|
||||
return {
|
||||
field: "shadow.error",
|
||||
legacy: null,
|
||||
interpreter: error instanceof Error ? error.message : String(error),
|
||||
category: "audit",
|
||||
severity: "error",
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Observe-only parity seam. Never mutates/blocks authoritative legacy behavior.
|
||||
*/
|
||||
export async function observeWorkflowParity(input: WorkflowParityObserverInput): Promise<void> {
|
||||
if (!isExperimentalFeatureEnabled(input.settings, WORKFLOW_INTERPRETER_DUAL_OBSERVE_FLAG)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const { store, agentId, legacy } = input;
|
||||
const runId = generateSyntheticRunId("workflow-shadow", legacy.taskId);
|
||||
|
||||
try {
|
||||
const shadow = await input.runShadow();
|
||||
const observationReport = compareWorkflowRunObservations(legacy.observation, shadow.observation);
|
||||
const auditReport = compareWorkflowRunAudits(legacy.auditEvents, shadow.auditEvents);
|
||||
const diffs = [...observationReport.diffs, ...auditReport.diffs];
|
||||
const agree = diffs.length === 0;
|
||||
|
||||
await store.recordRunAuditEvent?.({
|
||||
taskId: legacy.taskId,
|
||||
agentId,
|
||||
runId,
|
||||
domain: "database",
|
||||
mutationType: WORKFLOW_PARITY_OBSERVED_MUTATION,
|
||||
target: legacy.taskId,
|
||||
metadata: {
|
||||
agree,
|
||||
},
|
||||
});
|
||||
|
||||
if (!agree) {
|
||||
await store.recordRunAuditEvent?.({
|
||||
taskId: legacy.taskId,
|
||||
agentId,
|
||||
runId,
|
||||
domain: "database",
|
||||
mutationType: WORKFLOW_PARITY_DRIFT_MUTATION,
|
||||
target: legacy.taskId,
|
||||
metadata: {
|
||||
agree,
|
||||
diffs,
|
||||
},
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
await store.recordRunAuditEvent?.({
|
||||
taskId: legacy.taskId,
|
||||
agentId,
|
||||
runId,
|
||||
domain: "database",
|
||||
mutationType: WORKFLOW_PARITY_OBSERVED_MUTATION,
|
||||
target: legacy.taskId,
|
||||
metadata: {
|
||||
agree: false,
|
||||
},
|
||||
});
|
||||
|
||||
await store.recordRunAuditEvent?.({
|
||||
taskId: legacy.taskId,
|
||||
agentId,
|
||||
runId,
|
||||
domain: "database",
|
||||
mutationType: WORKFLOW_PARITY_DRIFT_MUTATION,
|
||||
target: legacy.taskId,
|
||||
metadata: {
|
||||
agree: false,
|
||||
diffs: [buildErrorDiff(error)],
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user