feat(FN-3293): add stabilization docs to test audit report

Documentation for test stabilization was finalized by updating the test audit report with 2 additional lines.

Fusion-Task-Id: FN-3293
This commit is contained in:
Fusion
2026-05-04 11:38:40 -07:00
committed by gsxdsm
parent 4b6c11ff0a
commit c9e776cb34
15 changed files with 1314 additions and 51 deletions

View File

@@ -157,6 +157,8 @@ function createMockStore(taskOverrides: Partial<Task> = {}, allTasks: Task[] = [
emit: vi.fn(),
on: vi.fn(),
clearStaleBaseBranchReferences: vi.fn().mockReturnValue([]),
getVerificationCacheHit: vi.fn().mockReturnValue(null),
recordVerificationCachePass: vi.fn(),
} as unknown as TaskStore;
}
@@ -4249,6 +4251,178 @@ describe("aiMergeTask — deterministic merge verification", () => {
);
expect(verificationCalls).toHaveLength(0);
});
it("skips test and build commands when a cache hit is found for the current tree sha", async () => {
const treeSha = "cachedtreeshaabc1234567890";
mockedExecSync.mockImplementation((cmd: any) => {
const cmdStr = String(cmd);
if (cmdStr.includes("rev-parse --verify")) return Buffer.from("abc123");
if (cmdStr === "git rev-parse HEAD" || cmdStr.startsWith("git rev-parse HEAD ")) return "mergedcommit123";
if (cmdStr.includes("git log")) return "- feat: something" as any;
if (cmdStr.includes("merge-base")) return Buffer.from("abc123");
if (cmdStr.includes("git diff") && cmdStr.includes("--stat")) return "1 file changed" as any;
if (cmdStr.includes("merge --squash")) return Buffer.from("");
if (cmdStr.includes("diff --cached --quiet")) return "1" as any;
if (cmdStr.includes("diff --cached")) return "0" as any;
if (cmdStr.includes("show --shortstat")) return "3 files changed, 10 insertions(+), 2 deletions(-)" as any;
if (cmdStr.includes("branch -d") || cmdStr.includes("branch -D")) return Buffer.from("");
if (cmdStr.includes("worktree remove")) return Buffer.from("");
// Return the fake tree sha when rev-parse HEAD^{tree} is called
if (cmdStr.includes("HEAD^{tree}")) return Buffer.from(treeSha + "\n");
return Buffer.from("");
});
mockedCreateFnAgent.mockResolvedValue({
session: {
prompt: vi.fn().mockResolvedValue(undefined),
dispose: vi.fn(),
},
} as any);
const store = createMockStore(
{ id: "FN-050", worktree: "/tmp/root/.worktrees/KB-050" },
[{ id: "FN-050", worktree: "/tmp/root/.worktrees/KB-050", column: "in-review" } as Task],
);
// Simulate a cache hit for this tree sha
const cacheHit = { recordedAt: "2026-05-01T00:00:00.000Z", taskId: "FN-049" };
(store.getVerificationCacheHit as ReturnType<typeof vi.fn>).mockReturnValue(cacheHit);
(store.getSettings as ReturnType<typeof vi.fn>).mockResolvedValue({
...DEFAULT_SETTINGS,
testCommand: "vitest run",
buildCommand: "pnpm build",
});
const result = await aiMergeTask(store, "/tmp/root", "FN-050");
expect(result.merged).toBe(true);
// No actual test/build commands should have run
const runCalls = mockedExecSync.mock.calls.filter(
(call) => String(call[0]).includes("vitest run") || String(call[0]).includes("pnpm build"),
);
expect(runCalls).toHaveLength(0);
// The cache skip message should appear in the task log
const logCalls = (store.logEntry as ReturnType<typeof vi.fn>).mock.calls;
const cacheMsg = logCalls.find((call: any[]) =>
typeof call[1] === "string" && call[1].includes("Skipping deterministic verification — cached pass"),
);
expect(cacheMsg).toBeTruthy();
expect(cacheMsg![1]).toContain(treeSha.slice(0, 7));
expect(cacheMsg![1]).toContain("FN-049");
// getVerificationCacheHit should have been called with the tree sha and commands
expect(store.getVerificationCacheHit).toHaveBeenCalledWith(treeSha, "vitest run", "pnpm build");
});
it("runs commands and records a cache pass when no cache hit exists", async () => {
const treeSha = "freshtreedead0000beef";
mockedExecSync.mockImplementation((cmd: any) => {
const cmdStr = String(cmd);
if (cmdStr.includes("rev-parse --verify")) return Buffer.from("abc123");
if (cmdStr === "git rev-parse HEAD" || cmdStr.startsWith("git rev-parse HEAD ")) return "mergedcommit123";
if (cmdStr.includes("git log")) return "- feat: something" as any;
if (cmdStr.includes("merge-base")) return Buffer.from("abc123");
if (cmdStr.includes("git diff") && cmdStr.includes("--stat")) return "1 file changed" as any;
if (cmdStr.includes("merge --squash")) return Buffer.from("");
if (cmdStr.includes("vitest run")) return Buffer.from("");
if (cmdStr.includes("diff --cached --quiet")) return "1" as any;
if (cmdStr.includes("diff --cached")) return "0" as any;
if (cmdStr.includes("show --shortstat")) return "3 files changed, 10 insertions(+), 2 deletions(-)" as any;
if (cmdStr.includes("branch -d") || cmdStr.includes("branch -D")) return Buffer.from("");
if (cmdStr.includes("worktree remove")) return Buffer.from("");
if (cmdStr.includes("HEAD^{tree}")) return Buffer.from(treeSha + "\n");
return Buffer.from("");
});
mockedCreateFnAgent.mockResolvedValue({
session: {
prompt: vi.fn().mockResolvedValue(undefined),
dispose: vi.fn(),
},
} as any);
const store = createMockStore(
{ id: "FN-050", worktree: "/tmp/root/.worktrees/KB-050" },
[{ id: "FN-050", worktree: "/tmp/root/.worktrees/KB-050", column: "in-review" } as Task],
);
// No cache hit — returns null (default mock)
(store.getVerificationCacheHit as ReturnType<typeof vi.fn>).mockReturnValue(null);
(store.getSettings as ReturnType<typeof vi.fn>).mockResolvedValue({
...DEFAULT_SETTINGS,
testCommand: "vitest run",
});
const result = await aiMergeTask(store, "/tmp/root", "FN-050");
expect(result.merged).toBe(true);
// The test command should have been executed
const testRuns = mockedExecSync.mock.calls.filter(
(call) => String(call[0]).includes("vitest run"),
);
expect(testRuns.length).toBeGreaterThan(0);
// recordVerificationCachePass should have been called with the tree sha
expect(store.recordVerificationCachePass).toHaveBeenCalledWith(
treeSha, "vitest run", "", "FN-050",
);
});
it("gracefully skips cache lookup when git rev-parse HEAD^{tree} fails", async () => {
mockedExecSync.mockImplementation((cmd: any) => {
const cmdStr = String(cmd);
if (cmdStr.includes("rev-parse --verify")) return Buffer.from("abc123");
if (cmdStr === "git rev-parse HEAD" || cmdStr.startsWith("git rev-parse HEAD ")) return "mergedcommit123";
if (cmdStr.includes("git log")) return "- feat: something" as any;
if (cmdStr.includes("merge-base")) return Buffer.from("abc123");
if (cmdStr.includes("git diff") && cmdStr.includes("--stat")) return "1 file changed" as any;
if (cmdStr.includes("merge --squash")) return Buffer.from("");
if (cmdStr.includes("vitest run")) return Buffer.from("");
if (cmdStr.includes("diff --cached --quiet")) return "1" as any;
if (cmdStr.includes("diff --cached")) return "0" as any;
if (cmdStr.includes("show --shortstat")) return "3 files changed, 10 insertions(+), 2 deletions(-)" as any;
if (cmdStr.includes("branch -d") || cmdStr.includes("branch -D")) return Buffer.from("");
if (cmdStr.includes("worktree remove")) return Buffer.from("");
// Simulate git failure for tree sha resolution
if (cmdStr.includes("HEAD^{tree}")) {
const err = new Error("not a git repository") as any;
err.status = 128;
throw err;
}
return Buffer.from("");
});
mockedCreateFnAgent.mockResolvedValue({
session: {
prompt: vi.fn().mockResolvedValue(undefined),
dispose: vi.fn(),
},
} as any);
const store = createMockStore(
{ id: "FN-050", worktree: "/tmp/root/.worktrees/KB-050" },
[{ id: "FN-050", worktree: "/tmp/root/.worktrees/KB-050", column: "in-review" } as Task],
);
(store.getSettings as ReturnType<typeof vi.fn>).mockResolvedValue({
...DEFAULT_SETTINGS,
testCommand: "vitest run",
});
// Should not throw — merge should complete normally
const result = await aiMergeTask(store, "/tmp/root", "FN-050");
expect(result.merged).toBe(true);
// Cache methods should never have been called
expect(store.getVerificationCacheHit).not.toHaveBeenCalled();
expect(store.recordVerificationCachePass).not.toHaveBeenCalled();
// The test command should still have run
const testRuns = mockedExecSync.mock.calls.filter(
(call) => String(call[0]).includes("vitest run"),
);
expect(testRuns.length).toBeGreaterThan(0);
});
});
describe("shouldSyncDependenciesForMerge", () => {
@@ -6672,8 +6846,8 @@ describe("aiMergeTask — in-merge verification fix", () => {
name: "VerificationError",
});
// Verify that fix agent was spawned (2 calls: merger + fix)
expect(mockedCreateFnAgent).toHaveBeenCalledTimes(2);
// Verify that fix agent was spawned (3 calls: summarizer + merger + fix)
expect(mockedCreateFnAgent).toHaveBeenCalledTimes(3);
// Verify the fix agent was called with correct options
const fixAgentCall = mockedCreateFnAgent.mock.calls[1];
@@ -6897,8 +7071,8 @@ describe("aiMergeTask — in-merge verification fix", () => {
name: "VerificationError",
});
// Verify fix agent was NOT spawned (only merger)
expect(mockedCreateFnAgent).toHaveBeenCalledTimes(1);
// Verify fix agent was NOT spawned (summarizer + merger only)
expect(mockedCreateFnAgent).toHaveBeenCalledTimes(2);
// Verify no fix attempt was logged
const logCalls = (store.logEntry as ReturnType<typeof vi.fn>).mock.calls;
@@ -7123,8 +7297,8 @@ describe("aiMergeTask — in-merge verification fix", () => {
name: "VerificationError",
});
// Should have 3 fix attempts (capped at 3) + 1 merger = 4 calls
expect(mockedCreateFnAgent).toHaveBeenCalledTimes(4);
// Should have 3 fix attempts (capped at 3) + summarizer + merger = 5 calls
expect(mockedCreateFnAgent).toHaveBeenCalledTimes(5);
});
it("default verificationFixRetries (omitted) results in 3 fix attempts", async () => {
@@ -7174,8 +7348,8 @@ describe("aiMergeTask — in-merge verification fix", () => {
name: "VerificationError",
});
// Should have 3 fix attempts (default) + 1 merger = 4 calls
expect(mockedCreateFnAgent).toHaveBeenCalledTimes(4);
// Should have 3 fix attempts (default) + summarizer + merger = 5 calls
expect(mockedCreateFnAgent).toHaveBeenCalledTimes(5);
// Verify the log shows 3 fix attempts (2 log entries per attempt: start + failure)
const logCalls = (store.logEntry as ReturnType<typeof vi.fn>).mock.calls;

View File

@@ -361,6 +361,30 @@ function rethrowIfMergeAborted(error: unknown): void {
}
}
/**
* Run execSync and always return a trimmed UTF-8 string.
* execSync may return a Buffer, string, or null depending on the encoding option;
* this helper normalises all three cases.
*/
function execSyncText(command: string, options: Parameters<typeof execSync>[1]): string {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const output: any = execSync(command, options);
if (output == null) return "";
if (typeof output === "string") return output.trim();
return (output as Buffer).toString("utf-8").trim();
}
/** Extra environment variables injected into verification child processes to boost concurrency. */
const VERIFICATION_EXTRA_ENV: NodeJS.ProcessEnv = Object.fromEntries(
(
[
["FUSION_TEST_TOTAL_WORKERS", "8"],
["FUSION_TEST_CONCURRENCY", "4"],
["FUSION_TEST_WORKSPACE_CONCURRENCY", "4"],
] as [string, string][]
).filter(([key]) => !(key in process.env)),
);
async function runDeterministicVerification(
store: TaskStore,
rootDir: string,
@@ -384,6 +408,41 @@ async function runDeterministicVerification(
const hasTestCommand = !!normalizedTestCommand;
const hasBuildCommand = !!normalizedBuildCommand;
// ── Tree-hash verification cache (Layer 1) ─────────────────────────────
const effectiveTestCommand = normalizedTestCommand ?? "";
const effectiveBuildCommand = normalizedBuildCommand ?? "";
let treeSha: string | null = null;
try {
treeSha = execSync("git rev-parse HEAD^{tree}", { cwd: rootDir, stdio: "pipe" })
.toString()
.trim();
} catch (err) {
mergerLog.warn(`${taskId}: could not resolve tree sha — skipping verification cache: ${String(err)}`);
}
if (treeSha) {
const cacheHit = store.getVerificationCacheHit(treeSha, effectiveTestCommand, effectiveBuildCommand);
if (cacheHit) {
const sha7 = treeSha.slice(0, 7);
const msg = `Skipping deterministic verification — cached pass for tree ${sha7} (recorded at ${cacheHit.recordedAt}, by ${cacheHit.taskId ?? "unknown"})`;
mergerLog.log(`${taskId}: ${msg}`);
await store.logEntry(taskId, msg);
await store.appendAgentLog(taskId, msg, "text", undefined, "merger");
const syntheticResult: VerificationCommandResult = {
command: "",
exitCode: 0,
stdout: "",
stderr: "",
success: true,
cached: true,
};
if (hasTestCommand) result.testResult = { ...syntheticResult, command: effectiveTestCommand };
if (hasBuildCommand) result.buildResult = { ...syntheticResult, command: effectiveBuildCommand };
return result;
}
}
// ── End cache lookup ───────────────────────────────────────────────────
// Build source indicator for logging
const testSourceLabel = testSource === "inferred" ? " [inferred]" : "";
const buildSourceLabel = buildSource === "inferred" ? " [inferred]" : "";
@@ -461,6 +520,18 @@ async function runDeterministicVerification(
mergerLog.log(`${taskId}: deterministic verification passed`);
await store.logEntry(taskId, "Deterministic merge verification passed");
await store.appendAgentLog(taskId, "Deterministic merge verification passed", "text", undefined, "merger");
// ── Record cache pass ──────────────────────────────────────────────────
if (treeSha) {
try {
store.recordVerificationCachePass(treeSha, effectiveTestCommand, effectiveBuildCommand, taskId);
mergerLog.log(`${taskId}: Recorded verification pass for tree ${treeSha.slice(0, 7)}`);
await store.logEntry(taskId, `Recorded verification pass for tree ${treeSha.slice(0, 7)}`);
} catch (err) {
mergerLog.warn(`${taskId}: could not record verification cache pass: ${String(err)}`);
}
}
return result;
}
@@ -473,7 +544,7 @@ async function runVerificationCommand(
signal?: AbortSignal,
): Promise<VerificationCommandResult> {
throwIfAborted(signal, taskId);
return runVerificationCommandShared(store, rootDir, taskId, command, type, signal, mergerLog, "merger");
return runVerificationCommandShared(store, rootDir, taskId, command, type, signal, mergerLog, "merger", VERIFICATION_EXTRA_ENV);
}
/**
@@ -1887,7 +1958,7 @@ function parsePushRemoteTarget(rootDir: string, pushRemote?: string): { remote:
let branch = branchTokens.join(" ").trim();
if (!branch) {
branch = execSync("git symbolic-ref --short HEAD", {
branch = execSyncText("git symbolic-ref --short HEAD", {
cwd: rootDir,
encoding: "utf-8",
stdio: "pipe",
@@ -2331,7 +2402,7 @@ export async function aiMergeTask(
result.error = `Branch '${branch}' not found — moving to done without merge`;
// Best-effort: try to capture current HEAD commitSha even though branch is missing
try {
const commitSha = execSync("git rev-parse HEAD", {
const commitSha = execSyncText("git rev-parse HEAD", {
cwd: rootDir,
stdio: "pipe",
encoding: "utf-8",
@@ -2360,12 +2431,12 @@ export async function aiMergeTask(
// causing feature code to be committed to the wrong lineage.
try {
throwIfAborted(options.signal, taskId);
const currentBranch = execSync("git symbolic-ref --short HEAD", {
const currentBranch = execSyncText("git symbolic-ref --short HEAD", {
cwd: rootDir,
encoding: "utf-8",
stdio: "pipe",
}).trim();
const mainBranch = execSync("git rev-parse --abbrev-ref origin/HEAD", {
const mainBranch = execSyncText("git rev-parse --abbrev-ref origin/HEAD", {
cwd: rootDir,
encoding: "utf-8",
stdio: "pipe",
@@ -3371,7 +3442,7 @@ export async function aiMergeTask(
// 5b. Collect merge details and store on task
try {
const commitSha = execSync("git rev-parse HEAD", {
const commitSha = execSyncText("git rev-parse HEAD", {
cwd: rootDir,
stdio: "pipe",
encoding: "utf-8",
@@ -3644,7 +3715,7 @@ export async function aiMergeTask(
async function tryFastForwardFromOrigin(rootDir: string, taskId: string): Promise<void> {
let currentBranch: string;
try {
currentBranch = execSync("git rev-parse --abbrev-ref HEAD", {
currentBranch = execSyncText("git rev-parse --abbrev-ref HEAD", {
cwd: rootDir,
encoding: "utf-8",
stdio: "pipe",
@@ -3665,7 +3736,7 @@ async function tryFastForwardFromOrigin(rootDir: string, taskId: string): Promis
let behind = 0;
let ahead = 0;
try {
const counts = execSync(`git rev-list --left-right --count "origin/${currentBranch}...HEAD"`, {
const counts = execSyncText(`git rev-list --left-right --count "origin/${currentBranch}...HEAD"`, {
cwd: rootDir,
encoding: "utf-8",
stdio: "pipe",
@@ -3903,7 +3974,7 @@ async function executeMergeAttempt(
// If only auto-resolvable conflicts (or all were resolved), commit directly
if (complex.length === 0) {
// All conflicts auto-resolved, commit with fallback message
const staged = execSync("git diff --cached --quiet 2>&1; echo $?", {
const staged = execSyncText("git diff --cached --quiet 2>&1; echo $?", {
cwd: rootDir,
encoding: "utf-8",
}).trim();
@@ -4025,7 +4096,7 @@ async function executeMergeAttempt(
}
// Check for conflicts
const conflictedOutput = execSync("git diff --name-only --diff-filter=U", {
const conflictedOutput = execSyncText("git diff --name-only --diff-filter=U", {
cwd: rootDir,
encoding: "utf-8",
}).trim();
@@ -4206,7 +4277,7 @@ async function attemptWithSideStrategy(
});
// Check if there are still conflicts (some types can't be auto-resolved)
const conflictedOutput = execSync("git diff --name-only --diff-filter=U", {
const conflictedOutput = execSyncText("git diff --name-only --diff-filter=U", {
cwd: rootDir,
encoding: "utf-8",
}).trim();
@@ -4217,7 +4288,7 @@ async function attemptWithSideStrategy(
}
// Check if there's anything staged
const staged = execSync("git diff --cached --quiet 2>&1; echo $?", {
const staged = execSyncText("git diff --cached --quiet 2>&1; echo $?", {
cwd: rootDir,
encoding: "utf-8",
}).trim();
@@ -4584,7 +4655,7 @@ async function runAiAgentForCommit(params: AiAgentParams): Promise<{ success: bo
}
// Verify commit happened
const staged = execSync("git diff --cached --quiet 2>&1; echo $?", {
const staged = execSyncText("git diff --cached --quiet 2>&1; echo $?", {
cwd: rootDir,
encoding: "utf-8",
}).trim();

View File

@@ -20,6 +20,8 @@ export interface VerificationCommandResult {
stdout: string;
stderr: string;
success: boolean;
/** True when this result was satisfied from the verification cache rather than running the command. */
cached?: boolean;
}
/** Result of running all verification commands */
@@ -40,7 +42,7 @@ export interface VerificationResult {
*/
export async function execWithProcessGroup(
command: string,
options: { cwd: string; timeout: number; maxBuffer: number; signal?: AbortSignal },
options: { cwd: string; timeout: number; maxBuffer: number; signal?: AbortSignal; env?: NodeJS.ProcessEnv },
): Promise<{ stdout: string; stderr: string; bufferOverflow: boolean; aborted?: boolean }> {
return new Promise((resolve, reject) => {
if (options.signal?.aborted) {
@@ -58,6 +60,7 @@ export async function execWithProcessGroup(
shell: true,
detached: useProcessGroup,
stdio: ["ignore", "pipe", "pipe"],
...(options.env !== undefined && { env: { ...process.env, ...options.env } }),
});
let stdout = "";
@@ -310,6 +313,8 @@ export async function runVerificationCommand(
log?: { log: (message: string, ...args: unknown[]) => void; error: (message: string, ...args: unknown[]) => void; warn: (message: string, ...args: unknown[]) => void },
/** Optional agent label for store log entries (e.g. "merger", "executor") */
agentLabel?: string,
/** Optional extra environment variables to inject into the child process (merged over process.env). */
extraEnv?: NodeJS.ProcessEnv,
): Promise<VerificationCommandResult> {
const logger = log ?? { log: console.log, error: console.error, warn: console.warn };
const label = (agentLabel ?? "merger") as AgentRole;
@@ -340,6 +345,7 @@ export async function runVerificationCommand(
timeout: VERIFICATION_COMMAND_TIMEOUT_MS,
maxBuffer: VERIFICATION_COMMAND_MAX_BUFFER,
signal,
...(extraEnv !== undefined && { env: extraEnv }),
});
if (signal?.aborted) {