feat(engine): scope verification to changed files + scope-aware timeout

Diff-proportional verification (deriveFileScopedPnpmTestCommand) + scope-aware
verification timeout, so merge/step checks finish in seconds. Propagated to this
worktree directly.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
gsxdsm
2026-06-25 15:27:51 -07:00
parent 3513d5f6f8
commit 744aa2cc05
5 changed files with 468 additions and 1 deletions

View File

@@ -0,0 +1,7 @@
---
"@runfusion/fusion": minor
---
summary: Verification now runs only the tests affected by a task's changed files, so merge/step checks finish in seconds.
category: feature
dev: New deriveFileScopedPnpmTestCommand maps changed test files (and co-located tests of changed source) to a per-package `pnpm --filter <pkg> exec vitest run <files>` command; inferDefaultTestCommand uses it (overriding even an explicit testCommand) when the new project setting scopeVerificationToChangedFiles (default true) is on and git context is available, falling back to the configured command when no tests resolve. The thin merge-gate suite remains the cross-cutting safety net.

View File

@@ -375,6 +375,11 @@ export const DEFAULT_PROJECT_SETTINGS = {
// project opts into a single default budget.
buildTimeoutMs: 300_000,
verificationCommandTimeoutMs: undefined,
// FNXC:Verification 2026-06-25-00:00: default-on file-scoped verification —
// run only the branch diff's own test files so merge verification stays
// proportional to the change; the thin merge gate carries cross-cutting
// coverage. Falls back to package/explicit command when no tests resolve.
scopeVerificationToChangedFiles: true,
ephemeralAgentsEnabled: true,
agentProvisioning: {},
sandboxProvisioning: {},

View File

@@ -4040,6 +4040,19 @@ export interface ProjectSettings {
* When set, this millisecond value overrides both fn_run_verification scope defaults (package 300s, workspace 900s); when unset, the legacy per-scope defaults still apply.
*/
verificationCommandTimeoutMs?: number;
/**
* FNXC:Verification 2026-06-25-00:00:
* When true (default), merge/executor verification is narrowed to ONLY the
* test files implicated by the task's branch diff — changed `*.test`/`*.spec`
* files plus the co-located tests of changed source files — run via
* `pnpm --filter <pkg> exec vitest run <files> --silent=passed-only
* --reporter=dot`. This keeps verification proportional to the change
* (seconds-to-<2min) and relies on the thin merge gate for cross-cutting
* coverage. Applies to BOTH explicit and inferred test commands. When no test
* files resolve from the diff, verification falls back to the existing
* package-scoped/explicit command. Set false to always run the broader
* package/full command. Default: true. */
scopeVerificationToChangedFiles?: boolean;
/** When enabled, AI-generated task specifications require manual approval
* before the task can move from triage to todo. Tasks with approved specs
* remain in triage with status "awaiting-approval" until a user approves

View File

@@ -0,0 +1,256 @@
import { describe, it, expect, vi, beforeEach } from "vitest";
// FNXC:Verification 2026-06-25-00:00:
// Unit coverage for diff-proportional verification scoping. We mock node:fs
// (existsSync/readFileSync/readdirSync) and node:child_process.execSync the same
// way merger-verification.test.ts does, so we can drive `git diff` output and
// on-disk test-file presence deterministically without a real repo.
// merger.ts (and its transitive imports) reference exec/execFile/spawn at module
// load via promisify, so the mock must provide them even though these tests only
// exercise the synchronous `git diff` path through execSync.
vi.mock("node:child_process", () => ({
execSync: vi.fn(),
exec: vi.fn(),
execFile: vi.fn(),
spawn: vi.fn(),
}));
vi.mock("node:fs", () => ({
existsSync: vi.fn().mockReturnValue(true),
readFileSync: vi.fn(),
readdirSync: vi.fn().mockReturnValue([]),
}));
import {
deriveFileScopedPnpmTestCommand,
inferDefaultTestCommand,
} from "../merger.js";
import { execSync } from "node:child_process";
import { existsSync, readFileSync, readdirSync } from "node:fs";
const mockedExecSync = vi.mocked(execSync);
const mockedExistsSync = vi.mocked(existsSync);
const mockedReadFileSync = vi.mocked(readFileSync);
const mockedReaddirSync = vi.mocked(readdirSync);
/**
* Wire a single-package ("packages/engine" → "@fusion/engine") workspace, with
* a configurable git-diff output and a set of test files that "exist" on disk.
*/
function setupSinglePackageWorkspace(opts: {
diff: string;
existingTestFiles: string[];
packages?: Array<{ dir: string; name: string }>;
}): void {
const packages = opts.packages ?? [{ dir: "engine", name: "@fusion/engine" }];
const existing = new Set(opts.existingTestFiles.map((p) => `/tmp/root/${p}`));
mockedExistsSync.mockImplementation((p: any) => {
const path = String(p);
if (path.endsWith("pnpm-workspace.yaml")) return true;
if (path.endsWith("pnpm-lock.yaml")) return true;
// package.json existence for resolveWorkspacePackageRoots + name reads
if (path.endsWith("package.json")) return true;
return existing.has(path);
});
mockedReaddirSync.mockReturnValue(
packages.map((pkg) => ({ name: pkg.dir, isDirectory: () => true })) as any,
);
mockedReadFileSync.mockImplementation((p: any) => {
const path = String(p);
if (path.endsWith("pnpm-workspace.yaml")) return `packages:\n - "packages/*"\n`;
for (const pkg of packages) {
if (path.endsWith(`packages/${pkg.dir}/package.json`)) {
return JSON.stringify({ name: pkg.name });
}
}
return JSON.stringify({ name: "unknown" });
});
mockedExecSync.mockImplementation((cmd: any) => {
if (String(cmd).includes("git diff --name-only")) return opts.diff;
return "";
});
}
describe("deriveFileScopedPnpmTestCommand", () => {
beforeEach(() => {
vi.clearAllMocks();
mockedExistsSync.mockReturnValue(false);
mockedReaddirSync.mockReturnValue([] as any);
});
it("includes a changed test file directly", () => {
setupSinglePackageWorkspace({
diff: "packages/engine/src/__tests__/foo.test.ts\n",
existingTestFiles: [],
});
const result = deriveFileScopedPnpmTestCommand("/tmp/root", "main", "fusion/fn-1");
expect(result).toBe(
`pnpm --filter "@fusion/engine" exec vitest run "src/__tests__/foo.test.ts" --silent=passed-only --reporter=dot`,
);
});
it("maps a changed source file to its co-located __tests__ test", () => {
setupSinglePackageWorkspace({
diff: "packages/engine/src/foo.ts\n",
existingTestFiles: ["packages/engine/src/__tests__/foo.test.ts"],
});
const result = deriveFileScopedPnpmTestCommand("/tmp/root", "main", "fusion/fn-1");
expect(result).toContain(`--filter "@fusion/engine"`);
expect(result).toContain(`"src/__tests__/foo.test.ts"`);
});
it("maps a changed source file to a sibling .test file", () => {
setupSinglePackageWorkspace({
diff: "packages/engine/src/bar.ts\n",
existingTestFiles: ["packages/engine/src/bar.test.ts"],
});
const result = deriveFileScopedPnpmTestCommand("/tmp/root", "main", "fusion/fn-1");
expect(result).toContain(`"src/bar.test.ts"`);
});
it("excludes a changed source file with no co-located test", () => {
setupSinglePackageWorkspace({
diff: "packages/engine/src/no-test.ts\n",
existingTestFiles: [], // no test exists on disk
});
const result = deriveFileScopedPnpmTestCommand("/tmp/root", "main", "fusion/fn-1");
expect(result).toBeNull();
});
it("returns null when the diff resolves to no test files at all", () => {
setupSinglePackageWorkspace({
diff: "README.md\npackages/engine/src/untested.ts\n",
existingTestFiles: [],
});
const result = deriveFileScopedPnpmTestCommand("/tmp/root", "main", "fusion/fn-1");
expect(result).toBeNull();
});
it("joins multiple packages with ` && ` and quotes names + paths", () => {
setupSinglePackageWorkspace({
diff: "packages/engine/src/__tests__/a.test.ts\npackages/dashboard/src/__tests__/b.test.ts\n",
existingTestFiles: [],
packages: [
{ dir: "engine", name: "@fusion/engine" },
{ dir: "dashboard", name: "@fusion/dashboard" },
],
});
const result = deriveFileScopedPnpmTestCommand("/tmp/root", "main", "fusion/fn-1");
expect(result).not.toBeNull();
expect(result).toContain(" && ");
// Package roots are sorted, so dashboard precedes engine.
expect(result).toBe(
`pnpm --filter "@fusion/dashboard" exec vitest run "src/__tests__/b.test.ts" --silent=passed-only --reporter=dot` +
` && ` +
`pnpm --filter "@fusion/engine" exec vitest run "src/__tests__/a.test.ts" --silent=passed-only --reporter=dot`,
);
});
it("dedupes when a source file and its test both change", () => {
setupSinglePackageWorkspace({
diff: "packages/engine/src/foo.ts\npackages/engine/src/__tests__/foo.test.ts\n",
existingTestFiles: ["packages/engine/src/__tests__/foo.test.ts"],
});
const result = deriveFileScopedPnpmTestCommand("/tmp/root", "main", "fusion/fn-1");
const occurrences = (result ?? "").split(`"src/__tests__/foo.test.ts"`).length - 1;
expect(occurrences).toBe(1);
});
it("returns null when git diff fails", () => {
setupSinglePackageWorkspace({
diff: "packages/engine/src/__tests__/foo.test.ts\n",
existingTestFiles: [],
});
mockedExecSync.mockImplementation(() => {
throw new Error("git failure");
});
expect(deriveFileScopedPnpmTestCommand("/tmp/root", "main", "fusion/fn-1")).toBeNull();
});
it("returns null when there is no pnpm-workspace.yaml", () => {
mockedExistsSync.mockReturnValue(false);
mockedReadFileSync.mockImplementation(() => {
throw new Error("ENOENT");
});
expect(deriveFileScopedPnpmTestCommand("/tmp/root", "main", "fusion/fn-1")).toBeNull();
});
});
describe("inferDefaultTestCommand — scopeToChangedFiles", () => {
beforeEach(() => {
vi.clearAllMocks();
mockedExistsSync.mockReturnValue(false);
mockedReaddirSync.mockReturnValue([] as any);
});
it("overrides an explicit command with the file-scoped command when tests resolve", () => {
setupSinglePackageWorkspace({
diff: "packages/engine/src/__tests__/foo.test.ts\n",
existingTestFiles: [],
});
const result = inferDefaultTestCommand(
"/tmp/root",
"pnpm -r test", // explicit whole-repo command
undefined,
"main",
"fusion/fn-1",
true, // scopeToChangedFiles
);
expect(result?.testSource).toBe("inferred-scoped");
expect(result?.command).toBe(
`pnpm --filter "@fusion/engine" exec vitest run "src/__tests__/foo.test.ts" --silent=passed-only --reporter=dot`,
);
});
it("falls back to the explicit command when no test files resolve", () => {
setupSinglePackageWorkspace({
diff: "packages/engine/src/untested.ts\n",
existingTestFiles: [],
});
const result = inferDefaultTestCommand(
"/tmp/root",
"pnpm -r test",
undefined,
"main",
"fusion/fn-1",
true,
);
expect(result?.testSource).toBe("explicit");
expect(result?.command).toBe("pnpm -r test");
});
it("preserves existing behavior when scopeToChangedFiles is false", () => {
setupSinglePackageWorkspace({
diff: "packages/engine/src/__tests__/foo.test.ts\n",
existingTestFiles: [],
});
const result = inferDefaultTestCommand(
"/tmp/root",
"pnpm -r test",
undefined,
"main",
"fusion/fn-1",
false, // disabled
);
expect(result?.testSource).toBe("explicit");
expect(result?.command).toBe("pnpm -r test");
});
it("file-scopes even an inferred (non-explicit) command when tests resolve", () => {
setupSinglePackageWorkspace({
diff: "packages/engine/src/foo.ts\n",
existingTestFiles: ["packages/engine/src/__tests__/foo.test.ts"],
});
const result = inferDefaultTestCommand(
"/tmp/root",
undefined,
undefined,
"main",
"fusion/fn-1",
true,
);
expect(result?.testSource).toBe("inferred-scoped");
expect(result?.command).toContain(`exec vitest run "src/__tests__/foo.test.ts"`);
});
});

View File

@@ -53,7 +53,7 @@ export {
import { existsSync, readFileSync, readdirSync, writeFileSync, unlinkSync, renameSync } from "node:fs";
import { createHash } from "node:crypto";
import { join } from "node:path";
import { join, dirname, basename } from "node:path";
import {
computeLockfileHash,
getConfiguredWorktreeInitCommand,
@@ -1101,6 +1101,161 @@ export function deriveScopedPnpmTestCommand(rootDir: string, baseBranch: string,
return `pnpm ${filters} test`;
}
/**
* Matches a Vitest/Jest-style test or spec file by extension.
* @internal
*/
const TEST_FILE_RE = /\.(test|spec)\.(ts|tsx|js|jsx)$/;
/**
* Derive a verification command that runs ONLY the test files implicated by the
* branch diff, so merge verification scales with the change instead of the
* repository.
*
* For each file changed between `baseBranch` and `branch`:
* - A changed test/spec file (`*.test.ts` / `*.spec.tsx` / …) is run directly.
* - A changed source file resolves to its co-located test, if one exists on
* disk: `<dir>/__tests__/<name>.test.{ts,tsx}` or the sibling
* `<dir>/<name>.test.{ts,tsx}`.
* Resolved test files are grouped by their owning pnpm workspace package and run
* via `pnpm --filter <pkg> exec vitest run <relPaths…> --silent=passed-only
* --reporter=dot`. Multiple packages are joined with ` && `.
*
* Returns `null` when scoping can't be established (no workspace, no git
* context, or — importantly — when NO test files resolve from the diff). The
* caller treats `null` as "fall back to the broader command".
*
* FNXC:Verification 2026-06-25-00:00:
* Merge/executor verification must complete in seconds-to-<2min by running only
* the diff's own tests, not a whole-package or full-suite command. This relies
* on the thin, trusted merge gate (`pnpm test:gate`) to carry cross-cutting
* coverage; per-branch verification only needs to prove the branch's own tests
* still pass. When a diff touches source with no co-located test (or only
* non-source files), file-scoping yields nothing and we deliberately return
* null so the caller falls back to the existing package-scoped/explicit command
* rather than verifying nothing. Package names come from workspace package.json
* files and test paths come from `git diff`, so every shell argument is quoted
* via `quoteArg`.
*
* @internal Exported for testing only.
*/
export function deriveFileScopedPnpmTestCommand(
rootDir: string,
baseBranch: string,
branch: string,
): string | null {
// 1. Read and parse pnpm-workspace.yaml + resolve package roots.
let workspaceContent: string;
try {
workspaceContent = readFileSync(join(rootDir, "pnpm-workspace.yaml"), "utf-8");
} catch {
return null;
}
const globs = parsePnpmWorkspaceGlobs(workspaceContent);
if (globs.length === 0) return null;
const packageRoots = resolveWorkspacePackageRoots(rootDir, globs);
if (packageRoots.length === 0) return null;
// 2. Get the changed files between base and the branch tip.
let changedFilesOutput: string;
try {
changedFilesOutput = execSync(
`git diff --name-only ${quoteArg(baseBranch)}...${quoteArg(branch)}`,
{ cwd: rootDir, stdio: "pipe", encoding: "utf-8" },
).toString();
} catch {
return null;
}
const changedFiles = changedFilesOutput
.split("\n")
.map((f) => f.trim())
.filter(Boolean);
if (changedFiles.length === 0) return null;
// 3. Resolve a set of repo-relative test files from the diff.
const resolvedTests = new Set<string>();
for (const file of changedFiles) {
if (TEST_FILE_RE.test(file)) {
// A changed test/spec file is run directly.
resolvedTests.add(file);
continue;
}
// A changed source file maps to a co-located test if one exists on disk.
const dir = dirname(file);
const stem = basename(file).replace(/\.(ts|tsx|js|jsx)$/, "");
if (!stem) continue;
const candidates = [
`${dir}/__tests__/${stem}.test.ts`,
`${dir}/__tests__/${stem}.test.tsx`,
`${dir}/${stem}.test.ts`,
`${dir}/${stem}.test.tsx`,
];
for (const candidate of candidates) {
// dirname("foo.ts") === "." → normalize the leading "./".
const normalized = candidate.startsWith("./") ? candidate.slice(2) : candidate;
if (existsSync(join(rootDir, normalized))) {
resolvedTests.add(normalized);
}
}
}
if (resolvedTests.size === 0) return null;
// 4. Group resolved test files by their owning workspace package.
const byPackage = new Map<string, { name: string; tests: Set<string> }>();
for (const testFile of resolvedTests) {
// Find the longest package root that is a prefix of this test file.
let bestRoot: string | null = null;
let bestLen = -1;
for (const pkgRoot of packageRoots) {
const prefix = pkgRoot.endsWith("/") ? pkgRoot : `${pkgRoot}/`;
if (testFile === pkgRoot || testFile.startsWith(prefix)) {
if (pkgRoot.length > bestLen) {
bestLen = pkgRoot.length;
bestRoot = pkgRoot;
}
}
}
if (bestRoot === null) continue;
const relPath = testFile.slice(bestRoot.length + 1);
if (!relPath) continue;
// Defensively skip any path quoting can't safely contain.
if (relPath.includes("\n") || relPath.includes("\0")) continue;
let entry = byPackage.get(bestRoot);
if (!entry) {
// Read the package name from package.json (fall back to the root path).
let name = bestRoot;
try {
const parsed = JSON.parse(
readFileSync(join(rootDir, bestRoot, "package.json"), "utf-8"),
) as { name?: string };
if (parsed.name) name = parsed.name;
} catch {
// keep the relative root path as the filter
}
entry = { name, tests: new Set<string>() };
byPackage.set(bestRoot, entry);
}
entry.tests.add(relPath);
}
if (byPackage.size === 0) return null;
// 5. Compose one scoped vitest invocation per package, joined with ` && `.
const segments: string[] = [];
for (const root of Array.from(byPackage.keys()).sort()) {
const entry = byPackage.get(root);
if (!entry) continue;
const quotedPaths = Array.from(entry.tests)
.sort()
.map((p) => quoteArg(p));
if (quotedPaths.length === 0) continue;
segments.push(
`pnpm --filter ${quoteArg(entry.name)} exec vitest run ${quotedPaths.join(" ")} --silent=passed-only --reporter=dot`,
);
}
if (segments.length === 0) return null;
return segments.join(" && ");
}
/**
* Infer a default test command based on project files.
* Returns the command and whether it was explicitly configured or inferred.
@@ -1115,6 +1270,17 @@ export function deriveScopedPnpmTestCommand(rootDir: string, baseBranch: string,
* provided, the command is automatically scoped to the packages touched by the
* branch diff. testSource will be "inferred-scoped" in that case.
*
* FNXC:Verification 2026-06-25-00:00:
* When `scopeToChangedFiles` is true (project setting
* `scopeVerificationToChangedFiles`, default true) AND git context is present,
* verification is first narrowed to the diff's own test FILES via
* `deriveFileScopedPnpmTestCommand` — for BOTH explicit and inferred commands,
* so even a configured whole-package `testCommand` gets file-scoped. This keeps
* per-branch verification proportional to the change; cross-cutting coverage is
* owned by the thin merge gate. If file-scoping yields nothing (no resolvable
* tests) or the setting is off, the original behavior is preserved exactly:
* explicit command as-is, else package-scoped inference, else unscoped fallback.
*
* Returns null if no test command can be inferred.
*/
export function inferDefaultTestCommand(
@@ -1123,7 +1289,24 @@ export function inferDefaultTestCommand(
explicitBuildCommand?: string,
baseBranch?: string,
branch?: string,
scopeToChangedFiles?: boolean,
): InferredTestCommand | null {
// File-scoped verification: try first for BOTH explicit and inferred cases.
// Only narrows when the setting is on, git context exists, and at least one
// test file resolves from the diff; otherwise falls through to existing logic.
if (scopeToChangedFiles && baseBranch?.trim() && branch?.trim()) {
try {
const fileScoped = deriveFileScopedPnpmTestCommand(rootDir, baseBranch.trim(), branch.trim());
if (fileScoped) {
mergerLog.log(`Scoped verification to changed test files: ${fileScoped}`);
const fileScopedBuildSource = explicitBuildCommand?.trim() ? "explicit" : undefined;
return { command: fileScoped, testSource: "inferred-scoped", buildSource: fileScopedBuildSource };
}
} catch {
// Fall through to existing explicit/inferred behavior.
}
}
// If explicit test command is set, use it (no inference needed)
if (explicitTestCommand?.trim()) {
return {
@@ -9579,6 +9762,9 @@ export async function aiMergeTask(
explicitBuildCommand,
mergeTarget.branch,
branch,
// FNXC:Verification 2026-06-25-00:00: default-on, file-scope verification to
// the branch diff so merge verification stays proportional to the change.
settings.scopeVerificationToChangedFiles !== false,
);
const effectiveTestCommand = inferredTest?.command || explicitTestCommand;
const effectiveTestSource = inferredTest?.testSource;