Merge origin/main into feat/validator-behavioral-verification

Resolve conflicts from concurrent main work: - db.ts: renumber behavioral-verification migration 115 -> 118 after main's workflow-work-items (115), graphResumeRetryCount (116), autoMergeProvenance (117); SCHEMA_VERSION = 118. Update all schema-version test assertions to 118 (keep SCHEMA_VERSION constant where used). - vitest.config / test-quarantine.json: union both branches' quarantine sets (FN-6206 behavioral-verification files + main's merger-ai-cleanup/merger-ai/ QuickEntryBox/soft-delete-blocker-residue/routes-settings entries). - settings-sync routes: keep workflowId-qualified field keys + persisted global pull; drop duplicate workflowApplyResult from auto-merge interleave. - Keep behavioral-verification changes in mission-execution-loop imports, useNodeSettingsSync, save-split, driver, and sync tests. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-13 15:48:54 -07:00
parent 1bb8cea9f9 6626fe2598
commit 4797eac348
427 changed files with 26535 additions and 3424 deletions
--- a/scripts/tests/agents-md-invariants.test.mjs
+++ b/scripts/tests/agents-md-invariants.test.mjs
@@ -10,8 +10,6 @@ const agentsPath = resolve(rootDir, "AGENTS.md");
 const agents = readFileSync(agentsPath, "utf8");

 const requiredAnchors = [
-  "STANDING DIRECTIVE: Buttons Are Frozen",
-  "Buttons Are Frozen (2026-05-13)",
  "Port 4040",
  "pnpm release --yes",
  "@runfusion/fusion",
--- a/scripts/tests/ci-test-shard.test.mjs
+++ b/scripts/tests/ci-test-shard.test.mjs
@@ -554,6 +554,38 @@ test("U6: enumerateDashboardLanes reads lanes from a fixture package.json shape"
  assert.deepEqual(lanes, ["test:quality:app:a", "test:quality:app:b", "test:quality:api"]);
 });

+test("U6: enumerateDashboardLanes expands run-quality-tests delegators to package leaf lanes", () => {
+  const scripts = {
+    test: "node scripts/run-quality-tests.mjs",
+    "test:quality:app": "node scripts/run-quality-tests.mjs --group app",
+    "test:quality:app:a": "node scripts/run-vitest-with-heap.mjs run --project app-a",
+    "test:quality:app:b": "node scripts/run-vitest-with-heap.mjs run --project app-b --shard=1/2",
+    "test:quality:app:aggregate": "pnpm run test:quality:app:a && pnpm run test:quality:app:b",
+    "test:quality:api": "node scripts/run-quality-tests.mjs --group=api",
+    "test:quality:api:a": "node scripts/run-vitest-with-heap.mjs run --project api-a",
+    "test:quality:api:delegator": "node scripts/run-quality-tests.mjs --group api",
+    "test:quality:misc": "node scripts/run-vitest-with-heap.mjs run --project misc",
+    "test:deep": "vitest run --project deep",
+  };
+
+  assert.deepEqual(enumerateDashboardLanes(scripts, "test"), [
+    "test:quality:app:a",
+    "test:quality:app:b",
+    "test:quality:api:a",
+    "test:quality:misc",
+  ]);
+  assert.deepEqual(enumerateDashboardLanes(scripts, "test:quality:app"), [
+    "test:quality:app:a",
+    "test:quality:app:b",
+  ]);
+  assert.deepEqual(enumerateDashboardLanes(scripts, "test:quality:api"), ["test:quality:api:a"]);
+});
+
+test("U6: enumerateDashboardLanes preserves single-leaf fallback for non-delegating scripts", () => {
+  assert.deepEqual(enumerateDashboardLanes({ test: "node custom-runner.mjs" }, "test"), ["test"]);
+  assert.deepEqual(enumerateDashboardLanes({}, "test"), []);
+});
+
 test("U6: laneProjectNames extracts --project targets including = and space forms", () => {
  assert.deepEqual(laneProjectNames("vitest run --project foo --project=bar baz"), ["foo", "bar"]);
 });
--- a/scripts/tests/test-changed.test.mjs
+++ b/scripts/tests/test-changed.test.mjs
@@ -29,6 +29,7 @@ import {
  __setCleanupRmSyncForTests,
  emitModeDecision,
  pruneFusionTestHomes,
+  pruneFusionTestWorkers,
  buildForwardDependencyMap,
  collectTransitiveDependencies,
  computeOwnHash,
@@ -951,6 +952,164 @@ test("pruneFusionTestHomes: bounded — removes at most maxEntries per call", ()
  }
 });

+test("pruneFusionTestWorkers: bounded — removes at most maxEntries per call", () => {
+  const created = [];
+  try {
+    for (let i = 0; i < 5; i++) {
+      const dir = path.join(tmpdir(), `fusion-test-workers-prune-budget-${process.pid}-${i}`);
+      mkdirSync(dir, { recursive: true });
+      created.push(dir);
+    }
+    // Cap at 2 → at least 3 of ours survive this call.
+    pruneFusionTestWorkers(2);
+    const survivors = created.filter((dir) => existsSync(dir));
+    assert.ok(survivors.length >= 3, `expected >=3 survivors with cap=2, got ${survivors.length}`);
+  } finally {
+    for (const dir of created) rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+function createNonEmptyPruneRoot(prefix, label) {
+  const root = mkdtempSync(path.join(tmpdir(), `${prefix}${label}-${process.pid}-`));
+  const childDir = path.join(root, `w-${process.pid}-busy`);
+  mkdirSync(childDir, { recursive: true });
+  writeFileSync(path.join(childDir, "busy.txt"), "busy\n");
+  return root;
+}
+
+function capturePruneWarnings(fn) {
+  const warnings = [];
+  const originalWarn = console.warn;
+  console.warn = (msg) => warnings.push(String(msg));
+  try {
+    fn(warnings);
+  } finally {
+    console.warn = originalWarn;
+  }
+  return warnings;
+}
+
+function withTransientPruneFailure(root, pruneFn) {
+  const error = Object.assign(new Error("simulated ENOTEMPTY"), { code: "ENOTEMPTY" });
+  let calls = 0;
+  __setCleanupRmSyncForTests((target, options) => {
+    if (target === root) {
+      calls += 1;
+      if (calls === 1) throw error;
+    }
+    return rmSync(target, options);
+  });
+
+  try {
+    const warnings = capturePruneWarnings(() => pruneFn(64, { retries: 3, delayMs: 0 }));
+    assert.equal(existsSync(root), false);
+    assert.equal(calls, 2);
+    assert.deepEqual(warnings, []);
+  } finally {
+    __setCleanupRmSyncForTests(null);
+    rmSync(root, { recursive: true, force: true });
+  }
+}
+
+function withPersistentPruneFailure(root, pruneFn) {
+  const error = Object.assign(new Error("simulated EBUSY"), { code: "EBUSY" });
+  let calls = 0;
+  __setCleanupRmSyncForTests((target, options) => {
+    if (target === root) {
+      calls += 1;
+      throw error;
+    }
+    return rmSync(target, options);
+  });
+
+  try {
+    const warnings = capturePruneWarnings(() => pruneFn(1024, { retries: 3, delayMs: 0 }));
+    assert.equal(existsSync(root), true);
+    assert.equal(calls, 3);
+    assert.equal(warnings.length, 1);
+    assert.match(warnings[0], /failed to prune leftover/);
+    assert.match(warnings[0], /after 3 attempts/);
+  } finally {
+    __setCleanupRmSyncForTests(null);
+    rmSync(root, { recursive: true, force: true });
+  }
+}
+
+test("pruneFusionTestWorkers: skips active per-invocation worker roots", () => {
+  const root = createNonEmptyPruneRoot("fusion-test-workers-", "active");
+  try {
+    writeFileSync(path.join(root, ".fusion-test-worker-root-owner"), `${process.pid}\n`);
+    pruneFusionTestWorkers(1024);
+    assert.equal(existsSync(root), true, "active worker root must not be pruned");
+  } finally {
+    rmSync(root, { recursive: true, force: true });
+  }
+});
+
+test("pruneFusionTestWorkers: skips markerless roots with live redirect sinks", () => {
+  const root = mkdtempSync(path.join(tmpdir(), `fusion-test-workers-active-redir-${process.pid}-`));
+  try {
+    mkdirSync(path.join(root, `redir-${process.pid}`), { recursive: true });
+    writeFileSync(path.join(root, `redir-${process.pid}`, "payload.txt"), "active\n");
+    pruneFusionTestWorkers(1024);
+    assert.equal(existsSync(root), true, "live redir-pid root must not be pruned");
+  } finally {
+    rmSync(root, { recursive: true, force: true });
+  }
+});
+
+test("pruneFusionTestWorkers: reclaims non-empty root after transient ENOTEMPTY", () => {
+  const root = createNonEmptyPruneRoot("fusion-test-workers-", "transient");
+  withTransientPruneFailure(root, pruneFusionTestWorkers);
+});
+
+test("pruneFusionTestWorkers: persistent busy root warns once after bounded retries", () => {
+  const root = createNonEmptyPruneRoot("fusion-test-workers-", "persistent");
+  withPersistentPruneFailure(root, pruneFusionTestWorkers);
+});
+
+test("pruneFusionTestHomes: reclaims non-empty root after transient ENOTEMPTY", () => {
+  const root = createNonEmptyPruneRoot("fusion-test-home-root-", "transient");
+  withTransientPruneFailure(root, pruneFusionTestHomes);
+});
+
+test("pruneFusionTestHomes: persistent busy root warns once after bounded retries", () => {
+  const root = createNonEmptyPruneRoot("fusion-test-home-root-", "persistent");
+  withPersistentPruneFailure(root, pruneFusionTestHomes);
+});
+
+function withEnoentPruneSuccess(root, pruneFn) {
+  let calls = 0;
+  __setCleanupRmSyncForTests((target, options) => {
+    if (target === root) {
+      calls += 1;
+      rmSync(root, { recursive: true, force: true });
+      throw Object.assign(new Error("simulated ENOENT"), { code: "ENOENT" });
+    }
+    return rmSync(target, options);
+  });
+
+  try {
+    const warnings = capturePruneWarnings(() => pruneFn(1024, { retries: 3, delayMs: 0 }));
+    assert.equal(existsSync(root), false);
+    assert.equal(calls, 1);
+    assert.deepEqual(warnings, []);
+  } finally {
+    __setCleanupRmSyncForTests(null);
+    rmSync(root, { recursive: true, force: true });
+  }
+}
+
+test("pruneFusionTestWorkers: ENOENT during prune is success without warning", () => {
+  const root = createNonEmptyPruneRoot("fusion-test-workers-", "enoent");
+  withEnoentPruneSuccess(root, pruneFusionTestWorkers);
+});
+
+test("pruneFusionTestHomes: ENOENT during prune is success without warning", () => {
+  const root = createNonEmptyPruneRoot("fusion-test-home-root-", "enoent");
+  withEnoentPruneSuccess(root, pruneFusionTestHomes);
+});
+
 // ---------------------------------------------------------------------------
 // U4: real-git-fixture integration (dirty working tree + transitive deps).
 //
@@ -1357,3 +1516,18 @@ test("pruneFusionTestHomes: only targets the fusion-test-home-root- prefix", ()
    rmSync(foreign, { recursive: true, force: true });
  }
 });
+
+test("pruneFusionTestWorkers: only targets the fusion-test-workers- prefix", () => {
+  const ours = path.join(tmpdir(), `fusion-test-workers-prune-prefix-${process.pid}`);
+  const foreign = path.join(tmpdir(), `not-ours-workers-prune-prefix-${process.pid}`);
+  mkdirSync(ours, { recursive: true });
+  mkdirSync(foreign, { recursive: true });
+  try {
+    pruneFusionTestWorkers();
+    assert.equal(existsSync(ours), false, "orphaned worker root should be pruned");
+    assert.equal(existsSync(foreign), true, "foreign dir must be left untouched");
+  } finally {
+    rmSync(ours, { recursive: true, force: true });
+    rmSync(foreign, { recursive: true, force: true });
+  }
+});
--- a/scripts/ci-test-shard.mjs
+++ b/scripts/ci-test-shard.mjs
@@ -565,12 +565,40 @@ export function enumerateDashboardLanes(scripts, entryScript = "test") {
    while ((match = re.exec(command)) !== null) names.push(match[1]);
    return names;
  };
+  const delegatedGroup = (command) => {
+    const match = command.match(/--group(?:=|\s+)(app|api)\b/);
+    return match?.[1] ?? null;
+  };
+  const isQualityLeaf = ([name, command]) => (
+    name.startsWith("test:quality:")
+    && command.includes("--project")
+    && !command.includes("run-quality-tests")
+    && referencedRuns(command).length === 0
+  );
+  const pushLane = (lane) => {
+    if (seen.has(lane)) return;
+    seen.add(lane);
+    lanes.push(lane);
+  };
+  const expandQualityDelegation = (command) => {
+    // The dashboard package's quality-test runner owns the current lane manifest;
+    // expand delegators back to real package.json leaf scripts so CI can shard them.
+    const group = delegatedGroup(command);
+    const prefix = group ? `test:quality:${group}:` : "test:quality:";
+    for (const [name, leafCommand] of Object.entries(scripts ?? {})) {
+      if (name.startsWith(prefix) && isQualityLeaf([name, leafCommand])) pushLane(name);
+    }
+  };

  const visit = (scriptName) => {
    if (seen.has(scriptName)) return;
    seen.add(scriptName);
    const command = scripts?.[scriptName];
    if (typeof command !== "string") return;
+    if (command.includes("run-quality-tests")) {
+      expandQualityDelegation(command);
+      return;
+    }
    const children = referencedRuns(command);
    if (children.length === 0) {
      // Leaf: a lane that actually invokes a test runner.
--- a/scripts/lib/test-quarantine.json
+++ b/scripts/lib/test-quarantine.json
@@ -16,10 +16,35 @@
      "reason": "Flake: same mock-contention mode as the sibling changeset-file test above (vi.mock('node:child_process') not taking under concurrent load). FN-6206.",
      "quarantinedAt": "2026-06-10"
    },
+    {
+      "file": "packages/engine/src/__tests__/merger-ai-cleanup-active-session.test.ts",
+      "reason": "Flake: pruneExistingAiMergeWorktrees skips active-session paths — active-session temp AI merge dir was unexpectedly pruned during pnpm --filter @fusion/engine test in FN-6206 verification, while the same file passed standalone. Root cause suspected: realpathSync resolution mismatch or readdirSync mock interaction with activeSessionRegistry singleton under concurrent engine suite load. Discovered during FN-6206.",
+      "quarantinedAt": "2026-06-10"
+    },
    {
      "file": "packages/engine/src/__tests__/merger-ai-cleanup.test.ts",
      "reason": "Flake observed during FN-6206 verification: `pruneExistingAiMergeWorktrees skips active-session paths` failed in full `pnpm --filter @fusion/engine test` runs while the file passed standalone, indicating suite-order/concurrency sensitivity. Follow-up FN-6207.",
      "quarantinedAt": "2026-06-10"
+    },
+    {
+      "file": "packages/engine/src/__tests__/merger-ai.test.ts",
+      "reason": "Flake observed during FN-6238 verification: full `pnpm --filter @fusion/engine test` failed in two merger-ai tests with git ENOENT / unable to read current working directory after a temp checkout disappeared, while the file passed standalone (23/23). Follow-up FN-6248.",
+      "quarantinedAt": "2026-06-11"
+    },
+    {
+      "file": "packages/dashboard/app/components/__tests__/QuickEntryBox.test.tsx",
+      "reason": "Flake observed during FN-6239 verification: broad `pnpm test` in dashboard backfill shard 4/4 could not find `quick-entry-priority-button` immediately after a successful task creation, while the named test passed standalone. Indicates suite-order/concurrency sensitivity unrelated to QuickChatFAB coverage.",
+      "quarantinedAt": "2026-06-11"
+    },
+    {
+      "file": "packages/engine/src/__tests__/reliability-interactions/soft-delete-blocker-residue.test.ts",
+      "reason": "Flake observed during FN-6294 verification and reproduced during FN-6319 broad `pnpm --filter @fusion/engine test`: `clearStaleBlockedBy handles missed task:deleted event with soft-deleted-blocker reason` failed because the log entry was absent, while the same file passed standalone and the narrow three-file reproduction passed. Product-code cross-check: `clearStaleBlockedBy` still has the soft-deleted-blocker branch and soft-delete-deadlock-scan-exclusion.test.ts covers it via a deterministic store double, indicating suite-order/concurrency sensitivity in this reliability-interactions fixture rather than a confirmed product bug.",
+      "quarantinedAt": "2026-06-12"
+    },
+    {
+      "file": "packages/dashboard/src/__tests__/routes-settings.test.ts",
+      "reason": "Flake observed during FN-6354 broad `pnpm test`: `GET /api/memory/audit > preserves extraction metadata across extract then audit requests` received HTTP 503 instead of 200 in the dashboard api:curated lane, while the same named test passed standalone immediately afterward. FN-6354 only changed the task-detail Chat composer UI/tests, so this is classified as unrelated suite-order/concurrency sensitivity in the dashboard API quality lane.",
+      "quarantinedAt": "2026-06-13"
    }
  ]
 }
--- a/scripts/test-changed.mjs
+++ b/scripts/test-changed.mjs
@@ -169,8 +169,94 @@ export function shouldRunIsolationGuard(env = process.env) {
 // can't spend unbounded time rm-rf'ing a tmpdir that accumulated thousands of
 // stale homes — and so the cache-fresh fast path can skip it entirely.
 const PRUNE_MAX_ENTRIES = 64;
+let cleanupRmSync = rmSync;
+const PRUNE_REMOVE_RETRIES = 3;
+const PRUNE_REMOVE_DELAY_MS = 75;
+const PRUNE_DIAGNOSTIC_CHILD_LIMIT = 8;
+const FUSION_WORKER_ROOT_OWNER_FILE = ".fusion-test-worker-root-owner";

-export function pruneFusionTestHomes(maxEntries = PRUNE_MAX_ENTRIES) {
+function isEnoentError(err) {
+  return Boolean(err && typeof err === "object" && "code" in err && err.code === "ENOENT");
+}
+
+function isProcessAlive(pid) {
+  if (!Number.isInteger(pid) || pid <= 0) return false;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (error) {
+    return error && typeof error === "object" && error.code === "EPERM";
+  }
+}
+
+function readWorkerRootOwnerPid(rootPath) {
+  try {
+    const raw = readFileSync(path.join(rootPath, FUSION_WORKER_ROOT_OWNER_FILE), "utf8").trim();
+    const pid = Number.parseInt(raw, 10);
+    return Number.isInteger(pid) && pid > 0 ? pid : null;
+  } catch {
+    return null;
+  }
+}
+
+function isActiveFusionWorkerRoot(rootPath) {
+  const ownerPid = readWorkerRootOwnerPid(rootPath);
+  if (ownerPid !== null && isProcessAlive(ownerPid)) return true;
+
+  // Backward-compatible guard for worker roots created before the owner marker
+  // landed, or marker writes that failed: an alive redir-<pid> child means a
+  // Vitest worker still owns temp workspaces beneath this root.
+  try {
+    for (const child of readdirSync(rootPath, { withFileTypes: true })) {
+      if (!child.isDirectory()) continue;
+      const match = /^redir-(\d+)$/.exec(child.name);
+      if (match && isProcessAlive(Number.parseInt(match[1], 10))) return true;
+    }
+  } catch {
+    // If we cannot inspect it, fall through to normal best-effort pruning.
+  }
+  return false;
+}
+
+function listImmediateChildrenForPruneWarning(rootPath) {
+  try {
+    const children = readdirSync(rootPath).slice(0, PRUNE_DIAGNOSTIC_CHILD_LIMIT);
+    if (children.length === 0) return "";
+    const suffix = children.length === PRUNE_DIAGNOSTIC_CHILD_LIMIT ? ", ..." : "";
+    return `; remaining children: ${children.join(", ")}${suffix}`;
+  } catch {
+    return "";
+  }
+}
+
+function removePrunedRootWithRetry(rawPath, { retries = PRUNE_REMOVE_RETRIES, delayMs = PRUNE_REMOVE_DELAY_MS } = {}) {
+  if (!existsSync(rawPath)) return true;
+
+  let lastError = null;
+  for (let attempt = 1; attempt <= retries; attempt++) {
+    try {
+      // FN-6371/FN-6360: macOS can report a transient ENOTEMPTY/EBUSY while
+      // child handles inside an orphaned fusion-test-* root are still closing.
+      // Keep this a short bounded retry (not a long live-root deletion loop) and
+      // keep the surrounding scan single-level/prefix-capped.
+      cleanupRmSync(rawPath, { recursive: true, force: true });
+      return true;
+    } catch (err) {
+      if (isEnoentError(err)) return true;
+      lastError = err;
+      if (attempt < retries) {
+        sleepMsSync(delayMs);
+      }
+    }
+  }
+
+  const message = lastError instanceof Error ? lastError.message : String(lastError);
+  const children = listImmediateChildrenForPruneWarning(rawPath);
+  console.warn(`[test-changed] failed to prune leftover ${rawPath} after ${retries} attempts: ${message}${children}`);
+  return false;
+}
+
+function pruneFusionTestRoots(prefix, maxEntries = PRUNE_MAX_ENTRIES, retryOptions = {}) {
  let tmpEntries = [];
  try {
    tmpEntries = readdirSync(tmpdir(), { withFileTypes: true });
@@ -178,26 +264,30 @@ export function pruneFusionTestHomes(maxEntries = PRUNE_MAX_ENTRIES) {
    return;
  }

-  let removed = 0;
+  let processed = 0;
  for (const entry of tmpEntries) {
-    if (removed >= maxEntries) break;
-    if (!entry.isDirectory() || !entry.name.startsWith("fusion-test-home-root-")) continue;
+    if (processed >= maxEntries) break;
+    if (!entry.isDirectory() || !entry.name.startsWith(prefix)) continue;
+    processed++;
    const rawPath = path.join(tmpdir(), entry.name);
    try {
      realpathSync(rawPath);
    } catch {
      // Keep raw path fallback.
    }
-    try {
-      rmSync(rawPath, { recursive: true, force: true });
-      removed++;
-    } catch (err) {
-      const message = err instanceof Error ? err.message : String(err);
-      console.warn(`[test-changed] failed to prune leftover ${rawPath}: ${message}`);
-    }
+    if (isActiveFusionWorkerRoot(rawPath)) continue;
+    removePrunedRootWithRetry(rawPath, retryOptions);
  }
 }

+export function pruneFusionTestHomes(maxEntries = PRUNE_MAX_ENTRIES, retryOptions = {}) {
+  pruneFusionTestRoots("fusion-test-home-root-", maxEntries, retryOptions);
+}
+
+export function pruneFusionTestWorkers(maxEntries = PRUNE_MAX_ENTRIES, retryOptions = {}) {
+  pruneFusionTestRoots("fusion-test-workers-", maxEntries, retryOptions);
+}
+
 function runMaybeIsolated(command, commandArgs, options = {}) {
  const enabled = shouldRunIsolationGuard();
  const env = options.env ?? process.env;
@@ -210,6 +300,7 @@ function runMaybeIsolated(command, commandArgs, options = {}) {
      onBeforeAfterCheck();
    }
    pruneFusionTestHomes();
+    pruneFusionTestWorkers();
    if (enabled) runIsolationCheck(false, env);
  }
 }
@@ -916,8 +1007,6 @@ const isolatedHomesToCleanup = new Set();
 // unconditionally, even if cleanup's rm silently failed.
 export const knownIsolatedHomeBasenames = new Set();

-let cleanupRmSync = rmSync;
-
 export function __setCleanupRmSyncForTests(nextRmSync) {
  cleanupRmSync = typeof nextRmSync === "function" ? nextRmSync : rmSync;
 }