feat(FN-4039): recover worktree db scratch bootstrap on hydration failure
Codifies the worktree database open-path hydration failure with a recovery bootstrap mechanism, documented the behavior in storage docs, and added test coverage for the executor hydration degradation path — all gated behind a patch changeset for `@runfusion/fusion`. Fusion-Task-Id: FN-4039
This commit is contained in:
5
.changeset/fn-4039-worktree-hydration.md
Normal file
5
.changeset/fn-4039-worktree-hydration.md
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
---
|
||||||
|
"@runfusion/fusion": patch
|
||||||
|
---
|
||||||
|
|
||||||
|
Harden per-worktree DB hydration so missing `.fusion/` scratch state is bootstrapped and retried before degrading with `unable to open database file`.
|
||||||
@@ -378,4 +378,6 @@ Expected executor log entry on success:
|
|||||||
Hydrated worktree DB: 4 tasks, 12 task_documents
|
Hydrated worktree DB: 4 tasks, 12 task_documents
|
||||||
```
|
```
|
||||||
|
|
||||||
Failure policy is strict non-blocking: hydration warnings are logged, but worktree creation/execution continues. Canonical task data remains the root project TaskStore DB; if an agent needs non-hydrated rows immediately, `fn_task_show` remains the canonical fallback path.
|
A concrete recovered failure mode now covered by tests: when a worktree directory exists but its local `.fusion/` scratch state is missing, opening `DatabaseSync(<worktree>/.fusion/fusion.db)` can fail with `unable to open database file`. Hydration now performs destination bootstrap (`mkdir -p .fusion` + schema init) and retries the destination open once before degrading.
|
||||||
|
|
||||||
|
Failure policy remains strict non-blocking for genuinely unrecoverable cases: hydration warnings are logged, but worktree creation/execution continues. Examples that still intentionally degrade include source DB missing, destination write-permission failures, and irreconcilable schema/open errors after bootstrap retry. Canonical task data remains the root project TaskStore DB; if an agent needs non-hydrated rows immediately, `fn_task_show` remains the canonical fallback path.
|
||||||
|
|||||||
@@ -2154,6 +2154,31 @@ describe("worktree DB hydration", () => {
|
|||||||
expect(mockedHydrateWorktreeDb).toHaveBeenCalledTimes(1);
|
expect(mockedHydrateWorktreeDb).toHaveBeenCalledTimes(1);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("logs degraded hydration reason and continues execution", async () => {
|
||||||
|
mockedHydrateWorktreeDb.mockResolvedValueOnce({
|
||||||
|
tasksCopied: 0,
|
||||||
|
documentsCopied: 0,
|
||||||
|
degraded: true,
|
||||||
|
reason: "unable to open database file",
|
||||||
|
});
|
||||||
|
mockedExistsSync.mockReturnValue(false);
|
||||||
|
const store = createMockStore();
|
||||||
|
const executor = new TaskExecutor(store, "/tmp/test");
|
||||||
|
await executor.execute(makeTask());
|
||||||
|
|
||||||
|
expect((store.logEntry as ReturnType<typeof vi.fn>).mock.calls).toEqual(
|
||||||
|
expect.arrayContaining([
|
||||||
|
[
|
||||||
|
"FN-HYD",
|
||||||
|
"Worktree DB hydration degraded: unable to open database file",
|
||||||
|
undefined,
|
||||||
|
expect.objectContaining({ agentId: "executor" }),
|
||||||
|
],
|
||||||
|
]),
|
||||||
|
);
|
||||||
|
expect(mockedCreateFnAgent).toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
it("hydration failure does not abort execute", async () => {
|
it("hydration failure does not abort execute", async () => {
|
||||||
mockedHydrateWorktreeDb.mockRejectedValueOnce(new Error("boom"));
|
mockedHydrateWorktreeDb.mockRejectedValueOnce(new Error("boom"));
|
||||||
mockedExistsSync.mockReturnValue(false);
|
mockedExistsSync.mockReturnValue(false);
|
||||||
|
|||||||
@@ -129,6 +129,22 @@ describe("hydrateWorktreeDb", () => {
|
|||||||
expect(after).toEqual(before);
|
expect(after).toEqual(before);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("bootstraps worktree db when .fusion scratch dir is missing", async () => {
|
||||||
|
const root = makeProject("h-open-root-");
|
||||||
|
const worktree = mkdtempSync(join(tmpdir(), "h-open-dst-"));
|
||||||
|
cleanup.push(root, worktree);
|
||||||
|
insertTask(root, "FN-1");
|
||||||
|
|
||||||
|
const warn = vi.fn();
|
||||||
|
const store = { getTask: vi.fn(async () => ({ id: "FN-1", dependencies: [] })) };
|
||||||
|
const result = await hydrateWorktreeDb({ rootDir: root, worktreePath: worktree, taskId: "FN-1", store: store as any, logger: { warn } });
|
||||||
|
|
||||||
|
expect(result.degraded).toBe(false);
|
||||||
|
expect(result.tasksCopied).toBe(1);
|
||||||
|
expect(existsSync(join(worktree, ".fusion", "fusion.db"))).toBe(true);
|
||||||
|
expect(warn).not.toHaveBeenCalledWith(expect.stringContaining("unable to open database file"));
|
||||||
|
});
|
||||||
|
|
||||||
it("degrades on write failure", async () => {
|
it("degrades on write failure", async () => {
|
||||||
const root = makeProject("h-denied-");
|
const root = makeProject("h-denied-");
|
||||||
const worktree = makeProject("h-denied-dst-");
|
const worktree = makeProject("h-denied-dst-");
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import { existsSync } from "node:fs";
|
import { existsSync, mkdirSync } from "node:fs";
|
||||||
import { join } from "node:path";
|
import { join } from "node:path";
|
||||||
import { DatabaseSync, TaskStore } from "@fusion/core";
|
import { Database, DatabaseSync, type TaskStore } from "@fusion/core";
|
||||||
|
|
||||||
export interface HydrateWorktreeDbParams {
|
export interface HydrateWorktreeDbParams {
|
||||||
rootDir: string;
|
rootDir: string;
|
||||||
@@ -59,8 +59,26 @@ async function resolveDependencyIds(taskId: string, store: Pick<TaskStore, "getT
|
|||||||
}
|
}
|
||||||
|
|
||||||
function ensureWorktreeSchema(worktreePath: string): void {
|
function ensureWorktreeSchema(worktreePath: string): void {
|
||||||
const schemaStore = new TaskStore(worktreePath);
|
const fusionDir = join(worktreePath, ".fusion");
|
||||||
schemaStore.close();
|
mkdirSync(fusionDir, { recursive: true });
|
||||||
|
const db = new Database(fusionDir);
|
||||||
|
db.init();
|
||||||
|
db.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
function isRecoverableOpenError(error: unknown): boolean {
|
||||||
|
const message = error instanceof Error ? error.message : String(error);
|
||||||
|
return message.includes("unable to open database file");
|
||||||
|
}
|
||||||
|
|
||||||
|
function openWorktreeDbWithRecovery(dstDbPath: string, worktreePath: string): DatabaseSync {
|
||||||
|
try {
|
||||||
|
return new DatabaseSync(dstDbPath);
|
||||||
|
} catch (error) {
|
||||||
|
if (!isRecoverableOpenError(error)) throw error;
|
||||||
|
ensureWorktreeSchema(worktreePath);
|
||||||
|
return new DatabaseSync(dstDbPath);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function hydrateWorktreeDb({
|
export async function hydrateWorktreeDb({
|
||||||
@@ -95,7 +113,7 @@ export async function hydrateWorktreeDb({
|
|||||||
}
|
}
|
||||||
|
|
||||||
srcDb = new DatabaseSync(srcDbPath);
|
srcDb = new DatabaseSync(srcDbPath);
|
||||||
dstDb = new DatabaseSync(dstDbPath);
|
dstDb = openWorktreeDbWithRecovery(dstDbPath, worktreePath);
|
||||||
|
|
||||||
dstDb.exec("PRAGMA journal_mode = WAL");
|
dstDb.exec("PRAGMA journal_mode = WAL");
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user