feat(FN-4039): recover worktree db scratch bootstrap on hydration failure

Codifies the worktree database open-path hydration failure with a recovery bootstrap mechanism, documented the behavior in storage docs, and added test coverage for the executor hydration degradation path — all gated behind a patch changeset for `@runfusion/fusion`.

Fusion-Task-Id: FN-4039
This commit is contained in:
Fusion
2026-05-11 18:09:35 -07:00
committed by gsxdsm
parent c501e00fdf
commit a8b904c993
5 changed files with 72 additions and 6 deletions

View File

@@ -0,0 +1,5 @@
---
"@runfusion/fusion": patch
---
Harden per-worktree DB hydration so missing `.fusion/` scratch state is bootstrapped and retried before degrading with `unable to open database file`.

View File

@@ -378,4 +378,6 @@ Expected executor log entry on success:
Hydrated worktree DB: 4 tasks, 12 task_documents
```
Failure policy is strict non-blocking: hydration warnings are logged, but worktree creation/execution continues. Canonical task data remains the root project TaskStore DB; if an agent needs non-hydrated rows immediately, `fn_task_show` remains the canonical fallback path.
A concrete recovered failure mode now covered by tests: when a worktree directory exists but its local `.fusion/` scratch state is missing, opening `DatabaseSync(<worktree>/.fusion/fusion.db)` can fail with `unable to open database file`. Hydration now performs destination bootstrap (`mkdir -p .fusion` + schema init) and retries the destination open once before degrading.
Failure policy remains strict non-blocking for genuinely unrecoverable cases: hydration warnings are logged, but worktree creation/execution continues. Examples that still intentionally degrade include source DB missing, destination write-permission failures, and irreconcilable schema/open errors after bootstrap retry. Canonical task data remains the root project TaskStore DB; if an agent needs non-hydrated rows immediately, `fn_task_show` remains the canonical fallback path.

View File

@@ -2154,6 +2154,31 @@ describe("worktree DB hydration", () => {
expect(mockedHydrateWorktreeDb).toHaveBeenCalledTimes(1);
});
it("logs degraded hydration reason and continues execution", async () => {
mockedHydrateWorktreeDb.mockResolvedValueOnce({
tasksCopied: 0,
documentsCopied: 0,
degraded: true,
reason: "unable to open database file",
});
mockedExistsSync.mockReturnValue(false);
const store = createMockStore();
const executor = new TaskExecutor(store, "/tmp/test");
await executor.execute(makeTask());
expect((store.logEntry as ReturnType<typeof vi.fn>).mock.calls).toEqual(
expect.arrayContaining([
[
"FN-HYD",
"Worktree DB hydration degraded: unable to open database file",
undefined,
expect.objectContaining({ agentId: "executor" }),
],
]),
);
expect(mockedCreateFnAgent).toHaveBeenCalled();
});
it("hydration failure does not abort execute", async () => {
mockedHydrateWorktreeDb.mockRejectedValueOnce(new Error("boom"));
mockedExistsSync.mockReturnValue(false);

View File

@@ -129,6 +129,22 @@ describe("hydrateWorktreeDb", () => {
expect(after).toEqual(before);
});
it("bootstraps worktree db when .fusion scratch dir is missing", async () => {
const root = makeProject("h-open-root-");
const worktree = mkdtempSync(join(tmpdir(), "h-open-dst-"));
cleanup.push(root, worktree);
insertTask(root, "FN-1");
const warn = vi.fn();
const store = { getTask: vi.fn(async () => ({ id: "FN-1", dependencies: [] })) };
const result = await hydrateWorktreeDb({ rootDir: root, worktreePath: worktree, taskId: "FN-1", store: store as any, logger: { warn } });
expect(result.degraded).toBe(false);
expect(result.tasksCopied).toBe(1);
expect(existsSync(join(worktree, ".fusion", "fusion.db"))).toBe(true);
expect(warn).not.toHaveBeenCalledWith(expect.stringContaining("unable to open database file"));
});
it("degrades on write failure", async () => {
const root = makeProject("h-denied-");
const worktree = makeProject("h-denied-dst-");

View File

@@ -1,6 +1,6 @@
import { existsSync } from "node:fs";
import { existsSync, mkdirSync } from "node:fs";
import { join } from "node:path";
import { DatabaseSync, TaskStore } from "@fusion/core";
import { Database, DatabaseSync, type TaskStore } from "@fusion/core";
export interface HydrateWorktreeDbParams {
rootDir: string;
@@ -59,8 +59,26 @@ async function resolveDependencyIds(taskId: string, store: Pick<TaskStore, "getT
}
function ensureWorktreeSchema(worktreePath: string): void {
const schemaStore = new TaskStore(worktreePath);
schemaStore.close();
const fusionDir = join(worktreePath, ".fusion");
mkdirSync(fusionDir, { recursive: true });
const db = new Database(fusionDir);
db.init();
db.close();
}
function isRecoverableOpenError(error: unknown): boolean {
const message = error instanceof Error ? error.message : String(error);
return message.includes("unable to open database file");
}
function openWorktreeDbWithRecovery(dstDbPath: string, worktreePath: string): DatabaseSync {
try {
return new DatabaseSync(dstDbPath);
} catch (error) {
if (!isRecoverableOpenError(error)) throw error;
ensureWorktreeSchema(worktreePath);
return new DatabaseSync(dstDbPath);
}
}
export async function hydrateWorktreeDb({
@@ -95,7 +113,7 @@ export async function hydrateWorktreeDb({
}
srcDb = new DatabaseSync(srcDbPath);
dstDb = new DatabaseSync(dstDbPath);
dstDb = openWorktreeDbWithRecovery(dstDbPath, worktreePath);
dstDb.exec("PRAGMA journal_mode = WAL");