root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

FN-5704: escalate self-owned execution limbo instead of endless resume

Browse Source 
Escalate reclaimable in-progress resume limbo into an explicit failure path to stop infinite self-healing loops.

- add resume-limbo escalation handling in self-healing/executor flow so self-owned stuck execution is failed and surfaced
- extend core task/run-audit types and retry-reset/store behavior to persist and expose the new escalation state
- add reliability interaction coverage for reclaim self-owned resume limbo escalation and update related schema/store/CLI/plugin tests and docs

Files changed:
 AGENTS.md                                          |   1 +
 docs/architecture.md                               |   2 +
 packages/cli/src/commands/__tests__/task.test.ts   |   2 +
 packages/core/src/__tests__/db-migrate.test.ts     |  12 +-
 packages/core/src/__tests__/db.test.ts             |  34 ++---
 packages/core/src/__tests__/goals-schema.test.ts   |   2 +-
 packages/core/src/__tests__/insight-store.test.ts  |  10 +-
 packages/core/src/__tests__/mission-store.test.ts  |   2 +-
 packages/core/src/__tests__/run-audit.test.ts      |   2 +-
 packages/core/src/__tests__/secrets-schema.test.ts |   6 +-
 .../core/src/__tests__/store-merge-queue.test.ts   |   2 +-
 packages/core/src/__tests__/task-documents.test.ts |   2 +-
 packages/core/src/db.ts                            |  13 +-
 packages/core/src/manual-retry-reset.ts            |   1 +
 packages/core/src/store.ts                         |  37 ++++-
 packages/core/src/types.ts                         |  11 ++
 ...laim-self-owned-resume-limbo-escalation.test.ts | 168 +++++++++++++++++++++
 packages/engine/src/executor.ts                    |   5 +
 packages/engine/src/run-audit.ts                   |   1 +
 packages/engine/src/self-healing.ts                |  72 +++++++++
 .../src/store/__tests__/roadmap-store.test.ts      |   4 +-
 21 files changed, 345 insertions(+), 44 deletions(-)

Fusion-Task-Id: FN-5704

Fusion-Task-Lineage: c8f73d16-d95b-450f-a514-e7d3c2f7aebe
This commit is contained in:
gsxdsm
2026-05-29 19:37:54 -07:00
parent 129e17d16c
commit 0dbb1cd6f9
21 changed files with 345 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
packages/core/src/__tests__/db-migrate.test.ts
View File

@@ -715,7 +715,7 @@ describe("schema migration", () => {
const row = db.prepare("SELECT deletedAt FROM tasks WHERE id = 'FN-legacy'").get() as { deletedAt: string | null };
expect(row.deletedAt).toBeNull();
expect(db.getSchemaVersion()).toBe(98);
expect(db.getSchemaVersion()).toBe(99);
db.close();
});
@@ -748,7 +748,7 @@ describe("schema migration", () => {
{ id: "WS-001", mode: "prompt", gateMode: "advisory" },
{ id: "WS-002", mode: "script", gateMode: "advisory" },
]);
expect(db.getSchemaVersion()).toBe(98);
expect(db.getSchemaVersion()).toBe(99);
db.close();
});
@@ -798,7 +798,7 @@ describe("schema migration", () => {
reviewerContextRetryCount: 0,
reviewerFallbackRetryCount: 0,
});
expect(db.getSchemaVersion()).toBe(98);
expect(db.getSchemaVersion()).toBe(99);
db.close();
});
@@ -827,7 +827,7 @@ describe("schema migration", () => {
const columns = db.prepare("PRAGMA table_info(milestones)").all() as Array<{ name: string }>;
expect(columns.map((column) => column.name)).toContain("acceptanceCriteria");
expect(db.getSchemaVersion()).toBe(98);
expect(db.getSchemaVersion()).toBe(99);
db.close();
});
@@ -868,7 +868,7 @@ describe("schema migration", () => {
const missionColumns = db.prepare("PRAGMA table_info(missions)").all() as Array<{ name: string }>;
expect(missionColumns.map((column) => column.name)).toContain("autoMerge");
expect(db.getSchemaVersion()).toBe(98);
expect(db.getSchemaVersion()).toBe(99);
db.close();
});
@@ -902,7 +902,7 @@ describe("schema migration", () => {
{ id: "WS-002", mode: "script", enabled: 1, gateMode: "advisory" },
{ id: "WS-003", mode: "prompt", enabled: 0, gateMode: "advisory" },
]);
expect(db.getSchemaVersion()).toBe(98);
expect(db.getSchemaVersion()).toBe(99);
db.close();
});

34
packages/core/src/__tests__/db.test.ts
View File

@@ -330,7 +330,7 @@ describe("Database", () => {
});
it("seeds schema version", () => {
expect(db.getSchemaVersion()).toBe(98);
expect(db.getSchemaVersion()).toBe(99);
});
it("includes tokenUsageCacheWriteTokens on freshly initialized tasks table", () => {
@@ -389,7 +389,7 @@ describe("Database", () => {
it("is idempotent - calling init() twice does not fail", () => {
expect(() => db.init()).not.toThrow();
expect(db.getSchemaVersion()).toBe(98);
expect(db.getSchemaVersion()).toBe(99);
});
it("does not overwrite existing config on re-init", () => {
// Update the config
@@ -1459,7 +1459,7 @@ describe("schema migrations", () => {
db.init();
// Verify version bumped to 29 (includes v1→v2 through v26→v29)
expect(db.getSchemaVersion()).toBe(98);
expect(db.getSchemaVersion()).toBe(99);
// Verify new columns exist and existing data is intact
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
@@ -1484,11 +1484,11 @@ describe("schema migrations", () => {
const db = new Database(fusionDir);
db.init();
expect(db.getSchemaVersion()).toBe(98);
expect(db.getSchemaVersion()).toBe(99);
// Re-init should not fail
db.init();
expect(db.getSchemaVersion()).toBe(98);
expect(db.getSchemaVersion()).toBe(99);
db.close();
});
@@ -1523,7 +1523,7 @@ describe("schema migrations", () => {
db.init();
expect(db.getSchemaVersion()).toBe(98);
expect(db.getSchemaVersion()).toBe(99);
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
expect(cols.map((col) => col.name)).toContain("priority");
@@ -1564,7 +1564,7 @@ describe("schema migrations", () => {
db.init();
expect(db.getSchemaVersion()).toBe(98);
expect(db.getSchemaVersion()).toBe(99);
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
const colNames = cols.map((col) => col.name);
@@ -1636,7 +1636,7 @@ describe("schema migrations", () => {
db.init();
expect(db.getSchemaVersion()).toBe(98);
expect(db.getSchemaVersion()).toBe(99);
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
const colNames = cols.map((col) => col.name);
@@ -1876,7 +1876,7 @@ describe("schema migrations", () => {
db.init();
expect(db.getSchemaVersion()).toBe(98);
expect(db.getSchemaVersion()).toBe(99);
const cols = db.prepare("PRAGMA table_info(chat_messages)").all() as Array<{ name: string }>;
expect(cols.map((col) => col.name)).toContain("attachments");
@@ -1950,7 +1950,7 @@ describe("schema migrations", () => {
db.init();
expect(db.getSchemaVersion()).toBe(98);
expect(db.getSchemaVersion()).toBe(99);
const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name = 'agentRatings'").all() as Array<{ name: string }>;
expect(tables).toEqual([{ name: "agentRatings" }]);
@@ -1974,7 +1974,7 @@ describe("schema migrations", () => {
db.init();
expect(db.getSchemaVersion()).toBe(98);
expect(db.getSchemaVersion()).toBe(99);
const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name = 'mission_events'").all() as Array<{ name: string }>;
expect(tables).toEqual([{ name: "mission_events" }]);
@@ -2078,7 +2078,7 @@ describe("schema migrations", () => {
db.init();
// Verify version bumped to 29
expect(db.getSchemaVersion()).toBe(98);
expect(db.getSchemaVersion()).toBe(99);
// Verify new columns exist and existing data is intact
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
@@ -2297,7 +2297,7 @@ describe("schema migrations", () => {
localDb.init();
expect(localDb.getSchemaVersion()).toBe(98);
expect(localDb.getSchemaVersion()).toBe(99);
const columns = localDb.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
expect(columns.map((column) => column.name)).toContain("tokenUsageCacheWriteTokens");
@@ -2608,7 +2608,7 @@ describe("createDatabase factory", () => {
const db = createDatabase(fusionDir);
db.init();
expect(db.getSchemaVersion()).toBe(98);
expect(db.getSchemaVersion()).toBe(99);
expect(db.getLastModified()).toBeGreaterThan(0);
db.close();
@@ -2762,7 +2762,7 @@ describe("migration v77 task token budget columns", () => {
migrated = new Database(fusion);
migrated.init();
expect(migrated.getSchemaVersion()).toBe(98);
expect(migrated.getSchemaVersion()).toBe(99);
const rows = migrated.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
const names = new Set(rows.map((row) => row.name));
expect(names.has("tokenBudgetSoftAlertedAt")).toBe(true);
@@ -2808,7 +2808,7 @@ describe("migration v67 drops orphan project auth tables", () => {
migrated = new Database(fusion);
migrated.init();
expect(migrated.getSchemaVersion()).toBe(98);
expect(migrated.getSchemaVersion()).toBe(99);
const tables = migrated
.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'project_auth_%'")
.all() as Array<{ name: string }>;
@@ -2835,7 +2835,7 @@ describe("migration v67 drops orphan project auth tables", () => {
try {
fresh.init();
expect(fresh.getSchemaVersion()).toBe(98);
expect(fresh.getSchemaVersion()).toBe(99);
const tables = fresh
.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'project_auth_%'")
.all() as Array<{ name: string }>;

2
packages/core/src/__tests__/goals-schema.test.ts
View File

@@ -91,6 +91,6 @@ describe("goals schema", () => {
});
it("reports schema version 92", () => {
expect(db.getSchemaVersion()).toBe(98);
expect(db.getSchemaVersion()).toBe(99);
});
});

10
packages/core/src/__tests__/insight-store.test.ts
View File

@@ -1000,7 +1000,7 @@ describe("Migration: pre-33 DB upgrade", () => {
// Step 1: Create a fresh database at v33 (runs all migrations up to 33)
const db1 = createDatabase(legacyDir);
db1.init();
expect(db1.getSchemaVersion()).toBe(98);
expect(db1.getSchemaVersion()).toBe(99);
db1.close();
// Step 2: Manually downgrade to version 32 and drop insight tables
@@ -1035,7 +1035,7 @@ describe("Migration: pre-33 DB upgrade", () => {
expect(tableNamesBefore).not.toContain("project_insight_runs");
// Now run init — this triggers the v32→v33 migration
db3.init();
expect(db3.getSchemaVersion()).toBe(98);
expect(db3.getSchemaVersion()).toBe(99);
// Step 4: Verify insight tables exist after migration
const tablesAfter = db3.prepare(
@@ -1066,12 +1066,12 @@ describe("Migration: pre-33 DB upgrade", () => {
try {
const db1 = createDatabase(testDir);
db1.init();
expect(db1.getSchemaVersion()).toBe(98);
expect(db1.getSchemaVersion()).toBe(99);
db1.close();
const db2 = createDatabase(testDir);
expect(() => db2.init()).not.toThrow();
expect(db2.getSchemaVersion()).toBe(98);
expect(db2.getSchemaVersion()).toBe(99);
db2.close();
} finally {
rmSync(testDir, { recursive: true, force: true });
@@ -1085,7 +1085,7 @@ describe("Migration: pre-33 DB upgrade", () => {
// Step 1: Create a fresh DB and run migrations
const db1 = createDatabase(compatDir);
db1.init();
expect(db1.getSchemaVersion()).toBe(98);
expect(db1.getSchemaVersion()).toBe(99);
// Step 2: Strip lifecycle and cancelledAt columns by recreating the
// table without them. This simulates a DB that was created before the

2
packages/core/src/__tests__/mission-store.test.ts
View File

@@ -3200,7 +3200,7 @@ describe("MissionStore", () => {
describe("Loop State & Validator Run Schema (v31)", () => {
it("schema version is 40 after migration", () => {
expect(db.getSchemaVersion()).toBe(98);
expect(db.getSchemaVersion()).toBe(99);
});
it("mission_features table has loop state columns", () => {

2
packages/core/src/__tests__/run-audit.test.ts
View File

@@ -584,7 +584,7 @@ describe("Run Audit", () => {
});
it("schema version is bumped to 40", () => {
expect(db.getSchemaVersion()).toBe(98);
expect(db.getSchemaVersion()).toBe(99);
});
});
});

6
packages/core/src/__tests__/secrets-schema.test.ts
View File

@@ -42,7 +42,7 @@ describe("secrets schema migrations", () => {
const version = db
.prepare("SELECT value FROM __meta WHERE key = 'schemaVersion'")
.get() as { value: string };
expect(version.value).toBe("98");
expect(version.value).toBe("99");
} finally {
db.close();
rmSync(dir, { recursive: true, force: true });
@@ -105,7 +105,7 @@ describe("secrets schema migrations", () => {
const version = db
.prepare("SELECT value FROM __meta WHERE key = 'schemaVersion'")
.get() as { value: string };
expect(version.value).toBe("98");
expect(version.value).toBe("99");
} finally {
db.close();
rmSync(dir, { recursive: true, force: true });
@@ -155,7 +155,7 @@ describe("secrets schema migrations", () => {
.prepare("SELECT value FROM __meta WHERE key = 'schemaVersion'")
.get() as { value: string };
expect(projectVersion.value).toBe("98");
expect(projectVersion.value).toBe("99");
expect(centralVersion.value).toBe("13");
} finally {
projectDb.close();

2
packages/core/src/__tests__/store-merge-queue.test.ts
View File

@@ -60,7 +60,7 @@ describe("TaskStore merge queue", () => {
expect.arrayContaining(["idx_mergeQueue_lease_ready", "idx_mergeQueue_leaseExpiresAt"]),
);
expect(store.getDatabase().getSchemaVersion()).toBe(98);
expect(store.getDatabase().getSchemaVersion()).toBe(99);
});
it("migrates a legacy v88 database and preserves task rows", async () => {

2
packages/core/src/__tests__/task-documents.test.ts
View File

@@ -51,7 +51,7 @@ describe("TaskStore task documents", () => {
expect(tableNames.has("task_documents")).toBe(true);
expect(tableNames.has("task_document_revisions")).toBe(true);
expect(db.getSchemaVersion()).toBe(98);
expect(db.getSchemaVersion()).toBe(99);
const index = db
.prepare(