From b61311baa8060c5903a5a2972a3e682e47ee462b Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Sat, 18 Jul 2026 19:31:30 -0700 Subject: [PATCH] FN-8305: add durable PostgreSQL symbol locks Introduce durable project-scoped symbol locks backed by PostgreSQL. - Add normalized lease-based lock acquisition, renewal, release, and reconciliation APIs with audit events. - Add PostgreSQL schema migrations and self-healing reconciliation coverage. - Document the lock model and test migration and lock behavior. Files changed: AGENTS.md | 1 + docs/architecture.md | 1 + docs/storage.md | 7 + .../src/__tests__/postgres/schema-applier.test.ts | 115 +++++++++- packages/core/src/__tests__/symbol-locks.test.ts | 91 ++++++++ packages/core/src/index.ts | 17 ++ .../core/src/postgres/migrations/0000_initial.sql | 25 +++ .../src/postgres/migrations/0025_symbol_locks.sql | 63 ++++++ packages/core/src/postgres/schema-applier.ts | 18 +- packages/core/src/postgres/schema/project.ts | 29 +++ packages/core/src/store.ts | 23 ++ packages/core/src/symbol-lock-types.ts | 60 +++++ packages/core/src/task-store/symbol-locks.ts | 244 +++++++++++++++++++++ .../__tests__/symbol-lock-reconciliation.test.ts | 19 ++ packages/engine/src/self-healing.ts | 35 +++ 15 files changed, 745 insertions(+), 3 deletions(-) Fusion-Task-Id: FN-8305 Fusion-Task-Lineage: efd95c73-23e3-4359-8204-dfad374a39bc Co-authored-by: Fusion (runfusion.ai) --- AGENTS.md | 1 + docs/architecture.md | 1 + docs/storage.md | 7 + .../__tests__/postgres/schema-applier.test.ts | 115 ++++++++- .../core/src/__tests__/symbol-locks.test.ts | 91 +++++++ packages/core/src/index.ts | 17 ++ .../src/postgres/migrations/0000_initial.sql | 25 ++ .../postgres/migrations/0025_symbol_locks.sql | 63 +++++ packages/core/src/postgres/schema-applier.ts | 18 +- packages/core/src/postgres/schema/project.ts | 29 +++ packages/core/src/store.ts | 23 ++ packages/core/src/symbol-lock-types.ts | 60 +++++ packages/core/src/task-store/symbol-locks.ts | 244 ++++++++++++++++++ .../symbol-lock-reconciliation.test.ts | 19 ++ packages/engine/src/self-healing.ts | 35 +++ 15 files changed, 745 insertions(+), 3 deletions(-) create mode 100644 packages/core/src/__tests__/symbol-locks.test.ts create mode 100644 packages/core/src/postgres/migrations/0025_symbol_locks.sql create mode 100644 packages/core/src/symbol-lock-types.ts create mode 100644 packages/core/src/task-store/symbol-locks.ts create mode 100644 packages/engine/src/__tests__/symbol-lock-reconciliation.test.ts diff --git a/AGENTS.md b/AGENTS.md index e8b853a6fa..00580da989 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -280,6 +280,7 @@ Scoped exception (FN-5819): shared-branch-group members (`branchContext.assignme - FN-7998: executor emits `task:execution-escalation-retry` when its opt-in, single alternate model/node attempt is persisted after FN-7996 exhaustion, and `task:execution-escalation-exhausted` when that attempt also reaches the terminal park. Metadata remains ids/counts/outcomes-only (`taskId`, graph node id, target booleans, and prior retry count); no model identifiers or prose are persisted in run-audit. - FN-8004: `agent:heartbeat-move-skipped-soft-delete` records a heartbeat move that races a soft-deleted task without parking the durable agent. Metadata remains ids/timestamps/source only (`agentId`, optional `taskId`/`deletedAt`, `moveAttemptedAt`, optional `source`); it never stores error prose. - FN-8141: the executor's `fn_task_done(outcome="blocked", reason=..., blockedBy?=[...])` honest-blocked exit emits `task:execution-blocked-parked` when an executor parks a genuinely-impossible task `failed` (`error = "BLOCKED: "`) instead of laundering it to `done` by skipping steps. It bypasses the completion/verdict/bulk-completion gates (blocked is not a completion claim), leaves steps in their true statuses, preserves worktree/branch, records `blockedBy` as real `task.dependencies` edges so the task requeues behind the blocker, and does NOT hand off to review — the parked row is honored by the executor's `status === "failed"` post-loop branch and is not auto-recovered into in-review by `recoverStrandedCompletedTodoTasks` (steps are not all done/skipped and `task.error` is set). Metadata stays ids/outcomes-only (`taskId`, `blockedBy` ids, `hasReason` boolean — never the reason prose). +- FN-8305: durable symbol-lock operations emit `symbol-lock:acquired`, `symbol-lock:acquire-conflict`, `symbol-lock:renewed`, `symbol-lock:released`, `symbol-lock:reconcile-stale`, and deduplicated `symbol-lock:reconcile-stale-no-action`. Metadata is ids/counts/outcomes-only; normalized opaque symbol keys are permitted IDs, while raw symbol prose is not. ## Reference docs (deeper detail) diff --git a/docs/architecture.md b/docs/architecture.md index c45c7e9307..9c99ef679f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -679,6 +679,7 @@ Runtime action-gate flow (v1): - `SelfHealingManager` (`self-healing.ts`) — auto-unpause/maintenance recovery actions - Batch 1 maintenance includes `reconcile-orphaned-task-dirs` (FN-6783), a paused-safe housekeeping step that calls `TaskStore.reconcileOrphanedTaskDirs()` so valid live `.fusion/tasks/{ID}/task.json` records missing from PostgreSQL become visible without waiting for process restart. The guard skips any ID already present in active, soft-deleted, archived, or tombstoned storage and emits `task:reconcile-orphaned-task-dir` only for recovered rows. - Batch 1 maintenance also includes `reconcile-phantom-committed-reservations` (FN-7069), which calls `TaskStore.reconcilePhantomCommittedReservations()` for committed task-ID reservations that have no live/soft-deleted/archived task row and no `.fusion/tasks/{ID}/task.json`. The sweep prunes orphaned `activityLog` rows and `agents`/cascaded `agentRuns`, preserves `runAuditEvents`, and keeps the reservation `committed` per FN-5105 so the ID is permanently reserved rather than resurrected or handed out again. + - Startup recovery and Batch 1 both call `reconcileStaleSymbolLocks()` (FN-8305). It expires only project-scoped held locks whose lease elapsed or whose owner task is terminal/missing, preserves live owners, and does not alter task lifecycle, scheduler admission, worktrees, semaphores, or verification. The audit surface is `symbol-lock:reconcile-stale` plus a deduplicated `symbol-lock:reconcile-stale-no-action` idle signal. - Batch 1 maintenance now includes one `fts-maintenance` step for both search indexes. The live `tasks_fts` branch still runs `merge` every tick, `optimize` every 4th tick, and `rebuild` above `32 MiB` or `1 MiB × live task count`. The archive `archived_tasks_fts` branch is lighter because archive writes are mostly append-only: `merge` every 8th tick, `optimize` every 24th tick, and `rebuild` above `64 MiB` or `512 KiB × archived row count`. Each branch is independently guarded by `fts5Available` and emits `task:fts-maintenance` run-audit telemetry with distinct `target` values (`tasks_fts` vs `archived_tasks_fts`). - AI merge clean-room worktrees are created under the configured worktrees directory's hidden container, `/.ai-merge/`, as `fusion-ai-merge-fn--` detached worktrees. When that container is repo-local, its relative path is added to the repo's local git exclude when possible (alongside the legacy `.fusion/ai-merge/` entry) so an in-flight clean room does not dirty the integration checkout. After `git worktree add` and before the merge/review loop, `runAiMerge` bootstraps the clean room with the shared merge dependency-sync helper: a configured `worktreeInitCommand` is authoritative and always runs, while unset settings infer `pnpm`/`npm`/`yarn`/`bun` installs from lockfiles and can skip only when the `node_modules/.fusion-install-marker` hash still matches. Failures and aborts hard-stop the AI merge before merge agents or verification run, and `merge:ai-deps-sync` records the command, skip state, and duration. Inline cleanup runs from `runAiMerge`'s clean-room `finally` for successful lands, empty/no-op finalization, concurrent-advance retries, and thrown/aborted merges. Cleanup canonicalizes the path, attempts `git worktree remove --force`, always falls back to filesystem removal, then runs `git worktree prune` so stale or partial registrations (including `git worktree add` failures) do not dangle. Cleanup emits `merge:ai-worktree-cleanup` audit events for git-remove, fs-rm, and prune phases; benign already-absent/de-registered paths are treated as idempotent success, while genuine filesystem-removal failures are logged/audited with `success: false` rather than silently swallowed. - Worktrees-dir sweeps that list direct children of `` (pool idle scan, orphan cleanup/reap, self-healing unregistered-orphan reap, and cap enforcement) must exclude the `.ai-merge` container by name; those one-level sweeps never inspect or recycle clean rooms beneath it. Batch 1 sweeps stale AI merge clean-room worktrees under the new `/.ai-merge/` root and still scans legacy `.fusion/ai-merge/` plus legacy `tmpdir()` locations for pre-relocation leftovers; candidates are bounded to names starting with `fusion-ai-merge-`. `runAiMerge` registers each live clean-room worktree in `activeSessionRegistry` with kind `ai-merge` as soon as the directory exists and keeps both raw and canonical paths registered for the duration of the merge, so the dedicated periodic sweep and pre-merge prune defer when either path is active (including concurrent same-task merge attempts). The default age gate is 2 hours; task-aware cleanup uses a 10-minute grace period for `done`/`archived` tasks and for genuinely missing/deleted task rows, and every removal path is clamped by the same 10-minute minimum-age floor so a freshly created worktree is never reaped. Transient `getTask` lookup failures (for example PostgreSQL availability/transaction errors) are not treated as deletion evidence; they log a warning, emit `lookup-error` only if eventually removed, and retain the conservative 2-hour gate. The sweep canonicalizes paths before checking `activeSessionRegistry`, attempts `git worktree remove --force ` before filesystem removal, runs `git worktree prune` after cleanup attempts, and emits `worktree:tempdir-sweep` run-audit telemetry for removal attempts and failures. Fresh directories, active-session paths, and individual removal failures are skipped/logged without aborting the maintenance cycle. diff --git a/docs/storage.md b/docs/storage.md index 6bbd35a4c3..255ed90958 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -17,6 +17,13 @@ See the [2026-07-14 PostgreSQL runtime cutover review](./postgres-migration-revi - Startup/store-open allocator reconciliation bumps each active prefix sequence to `max(current nextSequence, max(tasks suffix)+1, max(archivedTasks suffix)+1, max(reservation sequence)+1)` so stale allocator rows self-heal before local task creation resumes. - Create-class task persistence is intentionally non-destructive: new tasks use plain `INSERT` semantics, while upserts remain update-only. If counters drift and a reserved ID still collides, the create fails and the existing PostgreSQL row / task directory stays intact. A `committed` distributed reservation is valid only with a matching durable task row/directory; failed creates burn the reservation as `aborted` instead of leaving a committed-reservation-without-task phantom. +## Durable symbol locks (FN-8305) + +- `project.symbol_locks` is the project-scoped, lease-based admission seam for later mission-lineage scheduling. Its composite `(project_id, symbol_key)` identity permits only one current lock row per normalized symbol in a project; ownership records task ID plus optional mission, feature, lineage, node, and agent IDs. +- Lock acquire is all-or-nothing over normalized keys. Held unexpired rows owned by another task return their owner as a conflict, while expired/released rows may be reclaimed. Renewal and release are owner-scoped and release is idempotent. +- The `0000_initial.sql` baseline defines the table and indexes only. The later `0025_symbol_locks.sql` migration enables and forces RLS, creates `fusion_project_isolation`, and attaches `fusion_assign_project_id` after `0006_project_ownership.sql` creates that function/policy machinery. Both fresh full-applier and upgrade paths therefore end with the same project-isolation contract. +- Startup and Batch 1 self-healing expire locks when their lease elapsed or the owner task is terminal/missing. They never move a task or alter scheduler, worktree, semaphore, or verification state. Run-audit events are `symbol-lock:acquired`, `symbol-lock:acquire-conflict`, `symbol-lock:renewed`, `symbol-lock:released`, `symbol-lock:reconcile-stale`, and deduplicated `symbol-lock:reconcile-stale-no-action`; metadata uses only counts/outcomes and normalized opaque keys. + ## Soft-deleted tasks (FN-5105) - User-initiated `TaskStore.deleteTask` is a **soft delete**: the task row stays in `tasks` and `deletedAt` is set. diff --git a/packages/core/src/__tests__/postgres/schema-applier.test.ts b/packages/core/src/__tests__/postgres/schema-applier.test.ts index ac12ac5d51..c15c3c590c 100644 --- a/packages/core/src/__tests__/postgres/schema-applier.test.ts +++ b/packages/core/src/__tests__/postgres/schema-applier.test.ts @@ -65,6 +65,8 @@ import { PROJECT_OWNERSHIP_SCHEMA_VERSION, SESSION_ADVISOR_ENABLED_SCHEMA_VERSION, SQLITE_SCHEMA_PARITY_VERSION, + SYMBOL_LOCKS_SCHEMA_VERSION, + TASK_VERIFICATION_REQUEST_VERSION, } from "../../postgres/schema-applier.js"; import { rekeyFallbackProjectPartition } from "../../postgres/migration-stamping.js"; import type { PluginSchemaInitHook } from "../../postgres/plugin-schema-hook.js"; @@ -171,6 +173,11 @@ describe("schema-applier: immutable migration identities", () => { expect(Number(SCHEMA_BASELINE_VERSION)).toBeGreaterThanOrEqual(Number(TASK_PROPOSAL_CLAIM_VERSION)); }); + it("registers durable symbol locks at the next free migration version", () => { + expect(SYMBOL_LOCKS_SCHEMA_VERSION).toBe("0025"); + expect(Number(SCHEMA_BASELINE_VERSION)).toBeGreaterThanOrEqual(Number(SYMBOL_LOCKS_SCHEMA_VERSION)); + }); + }); /* @@ -266,6 +273,79 @@ async function teardownDb(ctx: TestContext | null): Promise { } } +/* +FNXC:SymbolLock 2026-07-30-15:15: +The baseline declares symbol_locks but cannot attach its ownership trigger before +0006 defines fusion_assign_project_id. Both a full fresh apply and an upgraded +installation must therefore prove 0025 leaves the final table forced-RLS with +its policy and trigger, including actual second-project read/write isolation. +*/ +async function assertSymbolLocksOwnershipContract(ctx: TestContext): Promise { + const catalog = (await ctx.db.execute(sql` + SELECT c.relrowsecurity AS rls, c.relforcerowsecurity AS forced, + EXISTS ( + SELECT 1 FROM pg_policies + WHERE schemaname = 'project' AND tablename = 'symbol_locks' + AND policyname = 'fusion_project_isolation' + ) AS policy, + EXISTS ( + SELECT 1 FROM pg_trigger + WHERE tgrelid = 'project.symbol_locks'::regclass + AND tgname = 'fusion_assign_project_id' AND NOT tgisinternal + ) AS trigger + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'project' AND c.relname = 'symbol_locks' + `)) as unknown as Array<{ rls: boolean; forced: boolean; policy: boolean; trigger: boolean }>; + expect(catalog).toEqual([{ rls: true, forced: true, policy: true, trigger: true }]); + + const role = `fusion_symbol_lock_isolation_${process.pid}_${Math.random().toString(36).slice(2, 8)}`; + await ctx.db.execute(sql.raw(` + CREATE ROLE ${role} NOLOGIN; + GRANT USAGE ON SCHEMA project TO ${role}; + GRANT SELECT, INSERT, UPDATE, DELETE ON project.symbol_locks TO ${role}; + `)); + try { + for (const projectId of ["project-a", "project-b"]) { + await ctx.db.transaction(async (tx) => { + await tx.execute(sql.raw(`SET LOCAL ROLE ${role}`)); + await tx.execute(sql`SELECT set_config('fusion.project_id', ${projectId}, true)`); + await tx.execute(sql` + INSERT INTO project.symbol_locks( + symbol_key, owner_task_id, status, acquired_at, renewed_at, expires_at, created_at, updated_at + ) VALUES ( + 'pkg/shared.ts#export', ${`FN-${projectId}`}, 'held', + '2026-07-30T15:15:00.000Z', '2026-07-30T15:15:00.000Z', + '2026-07-30T16:15:00.000Z', '2026-07-30T15:15:00.000Z', '2026-07-30T15:15:00.000Z' + ) + `); + }); + } + await ctx.db.transaction(async (tx) => { + await tx.execute(sql.raw(`SET LOCAL ROLE ${role}`)); + await tx.execute(sql`SELECT set_config('fusion.project_id', 'project-b', true)`); + const visible = (await tx.execute(sql` + SELECT project_id, owner_task_id FROM project.symbol_locks ORDER BY project_id + `)) as unknown as Array<{ project_id: string; owner_task_id: string }>; + expect(visible).toEqual([{ project_id: "project-b", owner_task_id: "FN-project-b" }]); + const stolen = (await tx.execute(sql` + UPDATE project.symbol_locks SET owner_task_id = 'stolen' + WHERE project_id = 'project-a' RETURNING owner_task_id + `)) as unknown as Array<{ owner_task_id: string }>; + expect(stolen).toEqual([]); + }); + } finally { + await ctx.db.execute(sql.raw(`DROP OWNED BY ${role}; DROP ROLE ${role};`)); + } + const rows = (await ctx.db.execute(sql` + SELECT project_id, owner_task_id FROM project.symbol_locks ORDER BY project_id + `)) as unknown as Array<{ project_id: string; owner_task_id: string }>; + expect(rows).toEqual([ + { project_id: "project-a", owner_task_id: "FN-project-a" }, + { project_id: "project-b", owner_task_id: "FN-project-b" }, + ]); +} + /** * FNXC:PostgresSchema 2026-06-24-06:30: * Complete enumeration of every CREATE INDEX name from the SQLite final @@ -539,9 +619,10 @@ pgDescribe("schema-applier: VAL-SCHEMA-001 final-schema parity (table counts)", // Project: 87 typed core tables + 2 lossless legacy preservation tables // + 1 import_translation_cache (FNXC:GitHubImportTranslate 2026-07-15-09:30) // + 1 configuration_revisions (FNXC:ConfigVersioning 2026-07-18-14:00) - // + 2 ideation_sessions/ideation_candidates (FNXC:Ideation 2026-07-18-13:25 / FN-8295). + // + 2 ideation_sessions/ideation_candidates (FNXC:Ideation 2026-07-18-13:25 / FN-8295) + // + 1 task_verification_requests + 1 durable symbol_locks table (FN-8305). // Plugin tables are added separately by the hook. - expect(bySchema.project).toBe(93); + expect(bySchema.project).toBe(95); expect(bySchema.central).toBe(18); expect(bySchema.archive).toBe(1); }); @@ -694,6 +775,26 @@ pgDescribe("schema-applier: VAL-SCHEMA-001 final-schema parity (table counts)", expect(rlsGaps).toEqual([]); }); + it("applies symbol-lock RLS, ownership policy, and trigger after a full fresh sequence", async () => { + ctx = await setupFreshDb(); + await applySchemaBaseline(ctx.db, { pluginHooks: [] }); + await assertSymbolLocksOwnershipContract(ctx); + }); + + it("repairs an upgraded installation missing the 0025 symbol-lock migration", async () => { + ctx = await setupFreshDb(); + await applySchemaBaseline(ctx.db, { pluginHooks: [] }); + // Preserve the 0000 marker while removing the later table/marker: this is + // the pre-0025 upgraded-install shape, where replay must not rely on baseline SQL. + await ctx.db.execute(sql.raw(` + DROP TABLE project.symbol_locks; + DELETE FROM public.fusion_schema_migrations WHERE version = '0025'; + `)); + expect((await applySchemaBaseline(ctx.db, { pluginHooks: [] })).applied).toBe(true); + expect(await getAppliedMigrations(ctx.db)).toContain(SYMBOL_LOCKS_SCHEMA_VERSION); + await assertSymbolLocksOwnershipContract(ctx); + }); + /* FNXC:ProjectDataIsolation 2026-07-14-12:10: Exercise the user-visible invariant through a non-superuser role: agents created in one project are invisible and immutable from another project even when application SQL omits project predicates. @@ -1169,6 +1270,8 @@ pgDescribe("schema-applier: automation project-isolation upgrade", () => { CONFIGURATION_REVISIONS_VERSION, IDEATION_SCHEMA_VERSION, RESEARCH_FEATURE_PROVENANCE_VERSION, + TASK_VERIFICATION_REQUEST_VERSION, + SYMBOL_LOCKS_SCHEMA_VERSION, ]); expect((await applySchemaBaseline(ctx.db, { pluginHooks: [] })).applied).toBe(false); }); @@ -1218,6 +1321,8 @@ pgDescribe("schema-applier: automation project-isolation upgrade", () => { CONFIGURATION_REVISIONS_VERSION, IDEATION_SCHEMA_VERSION, RESEARCH_FEATURE_PROVENANCE_VERSION, + TASK_VERIFICATION_REQUEST_VERSION, + SYMBOL_LOCKS_SCHEMA_VERSION, ]); }); @@ -1400,6 +1505,8 @@ pgDescribe("schema-applier: automation project-isolation upgrade", () => { CONFIGURATION_REVISIONS_VERSION, IDEATION_SCHEMA_VERSION, RESEARCH_FEATURE_PROVENANCE_VERSION, + TASK_VERIFICATION_REQUEST_VERSION, + SYMBOL_LOCKS_SCHEMA_VERSION, ]); }); @@ -1463,6 +1570,8 @@ pgDescribe("schema-applier: automation project-isolation upgrade", () => { CONFIGURATION_REVISIONS_VERSION, IDEATION_SCHEMA_VERSION, RESEARCH_FEATURE_PROVENANCE_VERSION, + TASK_VERIFICATION_REQUEST_VERSION, + SYMBOL_LOCKS_SCHEMA_VERSION, ]); }); @@ -1526,6 +1635,8 @@ pgDescribe("schema-applier: automation project-isolation upgrade", () => { CONFIGURATION_REVISIONS_VERSION, IDEATION_SCHEMA_VERSION, RESEARCH_FEATURE_PROVENANCE_VERSION, + TASK_VERIFICATION_REQUEST_VERSION, + SYMBOL_LOCKS_SCHEMA_VERSION, ]); }); }); diff --git a/packages/core/src/__tests__/symbol-locks.test.ts b/packages/core/src/__tests__/symbol-locks.test.ts new file mode 100644 index 0000000000..78b4015ed4 --- /dev/null +++ b/packages/core/src/__tests__/symbol-locks.test.ts @@ -0,0 +1,91 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; +import { and, eq } from "drizzle-orm"; +import * as schema from "../postgres/schema/index.js"; +import { TaskStore } from "../store.js"; +import { extractSymbolLockIdentity, normalizeSymbolLockKey, symbolLocksConflict } from "../task-store/symbol-locks.js"; +import { createSharedPgTaskStoreTestHarness, pgDescribe, type SharedPgTaskStoreHarness } from "../__test-utils__/pg-test-harness.js"; + +function storeForProject(harness: SharedPgTaskStoreHarness, projectId: string): TaskStore { + return new TaskStore(harness.rootDir(), undefined, { + asyncLayer: { ...harness.layer(), projectId }, + }); +} + +describe("symbol lock normalization", () => { + it("normalizes equivalent symbol references deterministically", () => { + expect(normalizeSymbolLockKey(" Pkg\\File.ts # Exported.member ")).toBe("pkg/file.ts#exported.member"); + expect(extractSymbolLockIdentity("project-a", "pkg/file.ts#Exported.member")).toEqual({ projectId: "project-a", normalizedSymbol: "pkg/file.ts#exported.member", symbolKey: "pkg/file.ts#exported.member" }); + expect(symbolLocksConflict("pkg\\file.ts#Thing", " pkg/file.ts # thing ")).toBe(true); + }); +}); + +pgDescribe("TaskStore durable symbol locks", () => { + const h: SharedPgTaskStoreHarness = createSharedPgTaskStoreTestHarness({ prefix: "fusion_symbol_lock" }); + beforeAll(h.beforeAll); afterAll(h.afterAll); beforeEach(h.beforeEach); afterEach(h.afterEach); + + it("acquires all symbols atomically and returns the contending owner without partial acquisition", async () => { + const store = h.store(); + const first = await store.acquireSymbolLocks(["pkg/a.ts#A"], { ownerTaskId: "FN-owner", agentId: "agent-a" }, 60_000); + expect(first.acquired).toBe(true); + const second = await store.acquireSymbolLocks(["pkg/a.ts#A", "pkg/b.ts#B"], { ownerTaskId: "FN-other" }, 60_000); + expect(second).toMatchObject({ acquired: false, locks: [], conflicts: [{ ownerTaskId: "FN-owner", symbolKey: "pkg/a.ts#a" }] }); + expect(await store.inspectSymbolLockConflicts(["pkg/b.ts#B"])).toEqual([]); + }); + + /* + FNXC:SymbolLock 2026-07-30-15:10: + Scheduler admission must partition every symbol-lock operation by project. + Two projects may hold the same canonical key, while inspect, reconciliation, + and their structured audit records must never make either project contend with + or expose the other's lock. + */ + it("isolates acquire, inspect, reconciliation, and audit metadata by project", async () => { + const projectA = storeForProject(h, "project-a"); + const projectB = storeForProject(h, "project-b"); + + expect((await projectA.acquireSymbolLocks(["pkg/shared.ts#Export"], { ownerTaskId: "FN-project-a" }, 60_000)).acquired).toBe(true); + expect((await projectB.acquireSymbolLocks(["pkg/shared.ts#Export"], { ownerTaskId: "FN-project-b" }, 60_000)).acquired).toBe(true); + expect(await projectA.inspectSymbolLockConflicts(["pkg/shared.ts#Export"])).toMatchObject([ + { ownerTaskId: "FN-project-a", symbolKey: "pkg/shared.ts#export" }, + ]); + expect(await projectB.inspectSymbolLockConflicts(["pkg/shared.ts#Export"])).toMatchObject([ + { ownerTaskId: "FN-project-b", symbolKey: "pkg/shared.ts#export" }, + ]); + + // The missing B task makes only B's lock stale; A remains invisible to B's sweep. + expect((await projectB.reconcileStaleSymbolLocks()).reconciled).toEqual(["pkg/shared.ts#export"]); + expect(await projectA.inspectSymbolLockConflicts(["pkg/shared.ts#Export"])).toMatchObject([ + { ownerTaskId: "FN-project-a", symbolKey: "pkg/shared.ts#export" }, + ]); + + const audits = await h.adminDb().select({ taskId: schema.project.runAuditEvents.taskId, metadata: schema.project.runAuditEvents.metadata }) + .from(schema.project.runAuditEvents) + .where(eq(schema.project.runAuditEvents.mutationType, "symbol-lock:acquired")); + expect(audits).toEqual(expect.arrayContaining([ + { taskId: "FN-project-a", metadata: { count: 1, symbolKeys: ["pkg/shared.ts#export"], outcome: "acquired" } }, + { taskId: "FN-project-b", metadata: { count: 1, symbolKeys: ["pkg/shared.ts#export"], outcome: "acquired" } }, + ])); + }); + + it("renews and releases only the caller's unexpired locks", async () => { + const store = h.store(); + await store.acquireSymbolLocks(["pkg/a.ts#A"], { ownerTaskId: "FN-owner" }, 60_000); + expect(await store.renewSymbolLocks(["pkg/a.ts#A"], "FN-other", 60_000)).toEqual({ renewed: [], lost: ["pkg/a.ts#a"] }); + expect((await store.releaseSymbolLocks(["pkg/a.ts#A"], "FN-other")).released).toEqual([]); + expect((await store.releaseSymbolLocks(["pkg/a.ts#A"], "FN-owner")).released).toEqual(["pkg/a.ts#a"]); + expect((await store.releaseSymbolLocks(["pkg/a.ts#A"], "FN-owner")).released).toEqual([]); + }); + + it("allows expired locks to be acquired and reconciles terminal owners", async () => { + const store = h.store(); const layer = h.layer(); const projectId = layer.projectId?.trim() || "__legacy_unscoped__"; + await store.acquireSymbolLocks(["pkg/expired.ts#A"], { ownerTaskId: "FN-dead" }, 60_000); + await layer.db.update(schema.project.symbolLocks).set({ expiresAt: new Date(Date.now() - 1_000).toISOString() }).where(and(eq(schema.project.symbolLocks.projectId, projectId), eq(schema.project.symbolLocks.symbolKey, "pkg/expired.ts#a"))); + expect((await store.acquireSymbolLocks(["pkg/expired.ts#A"], { ownerTaskId: "FN-new" }, 60_000)).acquired).toBe(true); + const owner = await store.createTask({ description: "terminal symbol lock owner" }); + await store.acquireSymbolLocks(["pkg/terminal.ts#A"], { ownerTaskId: owner.id }, 60_000); + await layer.db.update(schema.project.tasks).set({ column: "done" }).where(and( + eq(schema.project.tasks.projectId, projectId), eq(schema.project.tasks.id, owner.id), + )); + expect((await store.reconcileStaleSymbolLocks()).reconciled).toContain("pkg/terminal.ts#a"); + }); +}); diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 6f4db08c0c..9f2c71b3d9 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1,6 +1,23 @@ export { COLUMNS, DEFAULT_COLUMN, isColumn, normalizeColumn, COLUMN_LABELS, COLUMN_DESCRIPTIONS, VALID_TRANSITIONS, DEFAULT_SETTINGS, DEFAULT_GLOBAL_SETTINGS, DEFAULT_PROJECT_SETTINGS, GLOBAL_SETTINGS_KEYS, PROJECT_SETTINGS_KEYS, isGlobalSettingsKey, isProjectSettingsKey, isMergeRequestContractShadowEnabled, resolvePersistAgentThinkingLog, THINKING_LEVELS, THEME_MODES, COLOR_THEMES, SUPPORTED_LOCALES, DEFAULT_LOCALE, isLocale, AGENT_PERMISSIONS, PERMANENT_AGENT_ACTION_CATEGORIES, AGENT_PERMISSION_POLICY_ACTION_CATEGORIES, AGENT_PROVISIONING_APPROVAL_MODES, SANDBOX_PROVISIONING_APPROVAL_MODES, AGENT_PERMISSION_POLICY_PRESET_IDS, LEGACY_AGENT_PERMISSION_POLICY_ACTION_CATEGORY_ALIASES, APPROVAL_REQUEST_STATUSES, APPROVAL_REQUEST_AUDIT_EVENT_TYPES, normalizeApprovalRequestActionCategory, isValidApprovalRequestTransition, agentToConfigSnapshot, diffConfigSnapshots, isEphemeralAgent, hasAgentIdentity, CheckoutConflictError, DEFAULT_HEARTBEAT_PROCEDURE_PATH, getDefaultHeartbeatProcedurePath, EXECUTION_MODES, DEFAULT_EXECUTION_MODE, PLANNER_OVERSIGHT_LEVELS, DEFAULT_PLANNER_OVERSIGHT_LEVEL, TASK_PRIORITIES, DEFAULT_TASK_PRIORITY, WORKFLOW_WORK_ITEM_KINDS, WORKFLOW_WORK_ITEM_STATES, HIGH_FANOUT_BLOCKER_TODO_THRESHOLD, STALE_HIGH_FANOUT_BLOCKER_AGE_THRESHOLD_MS, REVIEW_ARTIFACTS_MODES, LIVE_DEMO_ARTIFACT_MIME_TYPE, isReviewArtifact, parseReviewArtifactsModeOverride, resolveReviewArtifactsMode, classifyReviewArtifactTask, isReviewArtifactGenerationEligible, DASHBOARD_USER_ID, normalizeMessageParticipant, validateMessageMetadata, resolveEphemeralTaskCreationPolicy, validateDockerNodeConfig, sanitizeDockerNodeConfigForResponse, normalizeMergeIntegrationWorktreeMode, normalizeMergeAdvanceAutoSyncMode, DEFAULT_GITLAB_API_BASE_URL, DEFAULT_GITLAB_INSTANCE_URL, resolveGitlabConfig, resolveGitlabEnabled, MERGE_ADVANCE_AUTO_SYNC_MODES, normalizeMergeConflictStrategy, normalizeMergeStrategyOverlapBehavior, normalizePostMergeAuditMode, POST_MERGE_AUDIT_MODES, normalizeMergeAuditAutoRecovery, MERGE_AUDIT_AUTO_RECOVERY_MODES, normalizeMergerMode, MERGER_MODES, normalizeAutoRecovery, AUTO_RECOVERY_MODES, buildResearchDocumentKey, REPO_OVERRIDE_RE, SHARED_STATE_SNAPSHOT_VERSION, sanitizeCliAgentSettings, sanitizeCliAgentsSettings, sanitizeMcpServers, CLI_AGENT_ADAPTER_IDS, CLI_AGENT_AUTONOMY_MODES, isMcpSecretRef, OVERSEER_INTERVENTION_MUTATION } from "./types.js"; export type { Column, ColumnId, IssueInfo, IssueState, TaskSourceIssue, TaskGitLabTracking, TaskGitLabTrackedItem, GitLabTrackedItemKind, PrInfo, PrConflictState, PrConflictDiagnostics, PrCheckState, PrCheckStatus, PrStatus, BranchGroup, BranchGroupCreateInput, BranchGroupUpdate, BranchGroupPrState, Task, TaskTokenUsage, TaskTokenUsagePerModel, TaskAttachment, TaskComment, TaskCommentInput, TaskDocument, TaskDocumentRevision, TaskDocumentCreateInput, TaskDocumentWithTask, ArtifactType, Artifact, ArtifactCreateInput, ArtifactWithTask, TaskCreateInput, TaskSource, SourceType, TaskDetail, RetrySummary, InboxTask, TodoList, TodoItem, TodoListCreateInput, TodoListUpdateInput, TodoItemCreateInput, TodoItemUpdateInput, TodoListWithItems, AgentLogEntry, AgentLogType, AgentRole, BoardConfig, DistributedTaskIdReserveInput, DistributedTaskIdReserveResult, DistributedTaskIdCommitInput, DistributedTaskIdCommitResult, DistributedTaskIdAbortInput, DistributedTaskIdAbortResult, DistributedTaskIdStateInput, DistributedTaskIdStateResult, AutostashOrphanRecord, AutostashOutcome, MergeDetails, MergeResult, MergeIntegrationWorktreeMode, MergeAdvanceAutoSyncMode, MergeConflictStrategy, CanonicalMergeConflictStrategy, MergeStrategyOverlapBehavior, PostMergeAuditMode, MergeAuditAutoRecoveryMode, MergerMode, MergerSettings, AutoRecoveryMode, AutoRecoveryFailureClass, AutoRecoverySettings, DirectMergeCommitStrategy, Settings, GlobalSettings, ProjectSettings, ReportMode, ReportActionType, SecretsEnvConfig, WebSearchBackend, ResearchEnabledSources, ResearchGlobalDefaults, ResearchProjectLimits, ResearchProjectSettings, SandboxBackendName, SandboxFailureMode, SandboxPolicy, SandboxProjectSettings, EvalFollowUpPolicy, EvalProjectSettings, ResolvedEvalSettings, SettingsScope, DaemonTokenSettings, TaskStep, StepStatus, TaskLogEntry, RunMutationContext, ActivityLogEntry, ActivityEventType, ThinkingLevel, ThemeMode, ColorTheme, Locale, ExecutionMode, PlannerOversightLevel, ReviewArtifactsMode, ReviewArtifactTaskClassification, TaskPriority, MergeQueueEntry, MergeQueueEnqueueOptions, MergeQueueAcquireOptions, MergeQueueReleaseOutcome, MergeRequestState, MergeRequestRecord, MergeRequestWorkflowProjectionOptions, CompletionHandoffMarker, WorkflowWorkItem, WorkflowWorkItemDueFilter, WorkflowWorkItemKind, WorkflowWorkItemState, WorkflowWorkItemTransitionPatch, WorkflowWorkItemUpsertInput, HandoffEvidence, HandoffToReviewOptions, UnavailableNodePolicy, OwningNodeHandoffPolicy, PlanningQuestion, PlanningSummary, PlanningResponse, PlanningQuestionType, ArchivedTaskEntry, BatchStatusRequest, BatchStatusResponse, BatchStatusEntry, BatchStatusResult, GithubIssueAction, ModelPreset, WorkflowStep, WorkflowStepMode, WorkflowStepGateMode, WorkflowStepPhase, WorkflowStepInput, WorkflowStepResult, WorkflowStepTemplate, Agent, OrgTreeNode, AgentState, AgentDetail, AgentCreateInput, AgentUpdateInput, AgentApiKey, AgentApiKeyCreateResult, AgentCapability, AgentPromptTemplate, AgentPromptsConfig, AgentPermission, PermanentAgentActionCategory, PermanentAgentSensitiveActionCategory, PermanentAgentGatingContext, AgentPermissionPolicy, AgentPermissionPolicyRules, AgentPermissionPolicyToolRules, AgentPermissionPolicyActionCategory, AgentProvisioningApprovalMode, SandboxProvisioningApprovalMode, LegacyAgentPermissionPolicyActionCategory, ApprovalRequestActionCategoryInput, ApprovalRequestActionCategory, AgentPermissionPolicyDisposition, AgentPermissionPolicyPresetId, ApprovalRequestStatus, ApprovalRequestAuditEventType, ApprovalRequestActorSnapshot, ApprovalRequestTargetAction, ApprovalRequestAuditEvent, ApprovalRequest, ApprovalRequestCreateInput, ApprovalRequestDecisionInput, ApprovalRequestCompletionInput, ApprovalRequestListInput, TaskAssignSource, AgentAccessState, AgentHeartbeatConfig, AgentBudgetConfig, AgentBudgetStatus, InstructionsBundleConfig, MessageResponseMode, AgentHeartbeatEvent, AgentHeartbeatRun, BlockedStateSnapshot, HeartbeatInvocationSource, AgentTaskSession, AgentRating, AgentRatingSummary, AgentRatingInput, AgentConfigSnapshot, RevisionFieldDiff, AgentConfigRevision, AgentStats, ReflectionTrigger, ReflectionMetrics, AgentReflection, AgentPerformanceSummary, NtfyNotificationEvent, NotificationEvent, NotificationPayload, NotificationProviderConfig, CustomProvider, SteeringComment, ParticipantType, MessageType, Message, MessageCreateInput, MessageFilter, MessageMetadata, ProposedTaskMetadata, EphemeralTaskCreationPolicy, MessageReplyReference, Mailbox, CheckoutLease, CheckoutClaimPrecondition, TaskClaimRow, CentralClaimStore, RunAuditDomain, RunAuditEvent, RunAuditEventInput, RunAuditEventFilter, AgentMemoryInclusionMode, HeartbeatPromptTemplate, HeartbeatScopeDisciplineMode, WorktrunkSettings, WorktrunkOnFailure, TaskBranchContext, CliAgentSettings, McpSecretRef, McpSensitiveValue, McpStdioTransport, McpSseTransport, McpStreamableHttpTransport, McpTransport, McpServerDefinition, McpServersSettings, GitlabConfigSettingsSource, ResolvedGitlabConfig, ResolveGitlabConfigInput, GitlabAuthTokenType, PlannerOversightStage, PlannerInterventionAction, PlannerInterventionOutcome, PlannerInterventionSourceLink, PlannerInterventionEntry, ExecutorOverseerSignalMemory, BackupSettingsMigrationCandidate, BackupSettingsMigrationConflict } from "./types.js"; export type { NativeStructureRef, NativeStructureOpenTarget, NativeStructurePreviewPayload, NativeStructureUnavailablePayload, NativeStructurePreviewResult } from "./types.js"; +export type { + SymbolLockStatus, + SymbolLockIdentity, + SymbolLockOwner, + SymbolLockLease, + SymbolLock, + SymbolLockConflict, + AcquireSymbolLocksResult, + RenewSymbolLocksResult, + ReleaseSymbolLocksResult, + ReconcileStaleSymbolLocksResult, +} from "./symbol-lock-types.js"; +export { + normalizeSymbolLockKey, + extractSymbolLockIdentity, + symbolLocksConflict, +} from "./task-store/symbol-locks.js"; export { AGENT_VALID_TRANSITIONS, DUPLICATE_OF_METADATA_KEY, assertNotWorkspaceTaskMerge, isWorkspaceTask, WorkspaceTaskMergeError } from "./types.js"; export { resolveEntryPointBranchAssignment, diff --git a/packages/core/src/postgres/migrations/0000_initial.sql b/packages/core/src/postgres/migrations/0000_initial.sql index 6617624aa1..dae8fc52e2 100644 --- a/packages/core/src/postgres/migrations/0000_initial.sql +++ b/packages/core/src/postgres/migrations/0000_initial.sql @@ -243,6 +243,31 @@ CREATE INDEX IF NOT EXISTS "idxDistributedTaskIdReservationsPrefixStatus" CREATE INDEX IF NOT EXISTS "idxDistributedTaskIdReservationsExpiry" ON project.distributed_task_id_reservations(status, expires_at); +-- FNXC:SymbolLock 2026-07-30-14:10: baseline creates the table only; 0025 +-- applies RLS after 0006 defines the ownership trigger and policy machinery. +CREATE TABLE IF NOT EXISTS project.symbol_locks ( + project_id text NOT NULL DEFAULT current_setting('fusion.project_id', true), + symbol_key text NOT NULL, + owner_task_id text NOT NULL, + mission_id text, + feature_id text, + lineage_id text, + node_id text, + agent_id text, + status text NOT NULL, + acquired_at text NOT NULL, + renewed_at text NOT NULL, + expires_at text NOT NULL, + created_at text NOT NULL, + updated_at text NOT NULL, + PRIMARY KEY (project_id, symbol_key), + CONSTRAINT symbol_locks_status_check CHECK (status IN ('held', 'released', 'expired')) +); +CREATE INDEX IF NOT EXISTS "idxSymbolLocksOwner" + ON project.symbol_locks(project_id, owner_task_id); +CREATE INDEX IF NOT EXISTS "idxSymbolLocksExpiry" + ON project.symbol_locks(status, expires_at); + CREATE TABLE IF NOT EXISTS project.workflow_steps ( id text PRIMARY KEY, template_id text, diff --git a/packages/core/src/postgres/migrations/0025_symbol_locks.sql b/packages/core/src/postgres/migrations/0025_symbol_locks.sql new file mode 100644 index 0000000000..45b82a9644 --- /dev/null +++ b/packages/core/src/postgres/migrations/0025_symbol_locks.sql @@ -0,0 +1,63 @@ +-- FNXC:SymbolLock 2026-07-30-14:10: +-- Durable mission-lineage admission locks require an upgraded-install table and +-- the complete project ownership contract. The baseline intentionally omits +-- this block because 0006 creates the trigger function and policy machinery. +CREATE TABLE IF NOT EXISTS project.symbol_locks ( + project_id text NOT NULL DEFAULT current_setting('fusion.project_id', true), + symbol_key text NOT NULL, + owner_task_id text NOT NULL, + mission_id text, + feature_id text, + lineage_id text, + node_id text, + agent_id text, + status text NOT NULL, + acquired_at text NOT NULL, + renewed_at text NOT NULL, + expires_at text NOT NULL, + created_at text NOT NULL, + updated_at text NOT NULL, + PRIMARY KEY (project_id, symbol_key), + CONSTRAINT symbol_locks_status_check CHECK (status IN ('held', 'released', 'expired')) +); +CREATE INDEX IF NOT EXISTS "idxSymbolLocksOwner" + ON project.symbol_locks(project_id, owner_task_id); +CREATE INDEX IF NOT EXISTS "idxSymbolLocksExpiry" + ON project.symbol_locks(status, expires_at); + +ALTER TABLE project.symbol_locks ENABLE ROW LEVEL SECURITY; +ALTER TABLE project.symbol_locks FORCE ROW LEVEL SECURITY; +DROP POLICY IF EXISTS fusion_project_isolation ON project.symbol_locks; +CREATE POLICY fusion_project_isolation ON project.symbol_locks + USING ( + current_setting('fusion.project_bypass', true) = 'on' + OR project_id = current_setting('fusion.project_id', true) + ) + WITH CHECK ( + current_setting('fusion.project_bypass', true) = 'on' + OR project_id = current_setting('fusion.project_id', true) + ); +DROP TRIGGER IF EXISTS fusion_assign_project_id ON project.symbol_locks; +CREATE TRIGGER fusion_assign_project_id + BEFORE INSERT OR UPDATE OF project_id ON project.symbol_locks + FOR EACH ROW EXECUTE FUNCTION project.fusion_assign_project_id(); + +-- FNXC:TaskVerificationRequest 2026-07-30-14:30: 0024 was already a +-- published migration when its post-0006 ownership omission was discovered; +-- repair existing 0024 installations here instead of mutating its identity. +ALTER TABLE project.task_verification_requests ENABLE ROW LEVEL SECURITY; +ALTER TABLE project.task_verification_requests FORCE ROW LEVEL SECURITY; +DROP POLICY IF EXISTS fusion_project_isolation ON project.task_verification_requests; +CREATE POLICY fusion_project_isolation ON project.task_verification_requests + USING ( + current_setting('fusion.project_bypass', true) = 'on' + OR project_id = current_setting('fusion.project_id', true) + ) + WITH CHECK ( + current_setting('fusion.project_bypass', true) = 'on' + OR project_id = current_setting('fusion.project_id', true) + ); +DROP TRIGGER IF EXISTS fusion_assign_project_id ON project.task_verification_requests; +CREATE TRIGGER fusion_assign_project_id + BEFORE INSERT OR UPDATE OF project_id ON project.task_verification_requests + FOR EACH ROW EXECUTE FUNCTION project.fusion_assign_project_id(); diff --git a/packages/core/src/postgres/schema-applier.ts b/packages/core/src/postgres/schema-applier.ts index 7fb5e4376e..579c4c0535 100644 --- a/packages/core/src/postgres/schema-applier.ts +++ b/packages/core/src/postgres/schema-applier.ts @@ -33,7 +33,7 @@ import { acquireSchemaMutationLocks } from "./advisory-locks.js"; FNXC:GitHubImportTranslate 2026-07-17-23:48: Advances to 0019 for the import-translation legacy-partition backfill. Per-migration identities above stay fixed; only this latest-version marker moves. */ -export const SCHEMA_BASELINE_VERSION = "0024"; +export const SCHEMA_BASELINE_VERSION = "0025"; const INITIAL_SCHEMA_VERSION = "0000"; const AUTOMATION_ISOLATION_SCHEMA_VERSION = "0001"; const ANALYTICS_ISOLATION_SCHEMA_VERSION = "0002"; @@ -114,6 +114,8 @@ export const IDEATION_SCHEMA_VERSION = "0022"; export const RESEARCH_FEATURE_PROVENANCE_VERSION = "0023"; /** FNXC:TaskVerificationRequest 2026-07-30-00:00: upgrades need the project-scoped chat-to-executor verification queue. */ export const TASK_VERIFICATION_REQUEST_VERSION = "0024"; +/** FNXC:SymbolLock 2026-07-30-14:10: upgraded projects need the durable lock table and RLS contract before scheduler admission can use it. */ +export const SYMBOL_LOCKS_SCHEMA_VERSION = "0025"; /** Bookkeeping table for the fresh Drizzle migration history. */ export const MIGRATION_BOOKKEEPING_TABLE = "fusion_schema_migrations"; @@ -225,6 +227,7 @@ const CONFIGURATION_REVISIONS_MIGRATION_PATH = join(MIGRATIONS_DIR, "0021_config const IDEATION_MIGRATION_PATH = join(MIGRATIONS_DIR, "0022_ideation.sql"); const RESEARCH_FEATURE_PROVENANCE_MIGRATION_PATH = join(MIGRATIONS_DIR, "0023_research_feature_provenance.sql"); const TASK_VERIFICATION_REQUEST_MIGRATION_PATH = join(MIGRATIONS_DIR, "0024_task_verification_request.sql"); +const SYMBOL_LOCKS_MIGRATION_PATH = join(MIGRATIONS_DIR, "0025_symbol_locks.sql"); /** * Ensure the migration bookkeeping table exists. Lives in the public schema so @@ -318,6 +321,7 @@ export async function applySchemaBaseline( const ideationAlreadyApplied = applied.includes(IDEATION_SCHEMA_VERSION); const researchFeatureProvenanceAlreadyApplied = applied.includes(RESEARCH_FEATURE_PROVENANCE_VERSION); const taskVerificationRequestAlreadyApplied = applied.includes(TASK_VERIFICATION_REQUEST_VERSION); + const symbolLocksAlreadyApplied = applied.includes(SYMBOL_LOCKS_SCHEMA_VERSION); let schemaChanged = false; if (!baselineAlreadyApplied) { @@ -695,6 +699,18 @@ export async function applySchemaBaseline( schemaChanged = true; } + /* + FNXC:SymbolLock 2026-07-30-14:10: + Migration files are manually registered. Apply 0025 independently so both + fresh and upgraded installations gain forced RLS after 0006 owns its setup. + */ + if (!symbolLocksAlreadyApplied) { + const migrationSql = await readFile(SYMBOL_LOCKS_MIGRATION_PATH, "utf8"); + await tx.execute(sql.raw(migrationSql)); + await tx.execute(sql`INSERT INTO public.${sql.identifier(MIGRATION_BOOKKEEPING_TABLE)} (version) VALUES (${SYMBOL_LOCKS_SCHEMA_VERSION}) ON CONFLICT (version) DO NOTHING`); + schemaChanged = true; + } + if (!importTranslationCacheLegacyPartitionBackfillAlreadyApplied) { const migrationSql = await readFile(IMPORT_TRANSLATION_CACHE_LEGACY_PARTITION_BACKFILL_MIGRATION_PATH, "utf8"); await tx.execute(sql.raw(migrationSql)); diff --git a/packages/core/src/postgres/schema/project.ts b/packages/core/src/postgres/schema/project.ts index b8d5c5eaa7..262b449431 100644 --- a/packages/core/src/postgres/schema/project.ts +++ b/packages/core/src/postgres/schema/project.ts @@ -521,6 +521,35 @@ export const distributedTaskIdReservations = projectSchema.table("distributed_ta index("idxDistributedTaskIdReservationsExpiry").on(t.status, t.expiresAt), ]); +// ── Durable symbol locks ───────────────────────────────────────────── +/* +FNXC:SymbolLock 2026-07-30-14:10: +Mission-lineage admission needs one project-scoped row per canonical symbol. +A composite primary key retains released/expired ownership history while the +atomic store seam may reclaim those rows as held without cross-project conflict. +*/ +export const symbolLocks = projectSchema.table("symbol_locks", { + projectId: text("project_id").notNull().default(sql`current_setting('fusion.project_id', true)`), + symbolKey: text("symbol_key").notNull(), + ownerTaskId: text("owner_task_id").notNull(), + missionId: text("mission_id"), + featureId: text("feature_id"), + lineageId: text("lineage_id"), + nodeId: text("node_id"), + agentId: text("agent_id"), + status: text("status").notNull(), + acquiredAt: text("acquired_at").notNull(), + renewedAt: text("renewed_at").notNull(), + expiresAt: text("expires_at").notNull(), + createdAt: text("created_at").notNull(), + updatedAt: text("updated_at").notNull(), +}, (t) => [ + primaryKey({ columns: [t.projectId, t.symbolKey] }), + check("symbol_locks_status_check", sql`${t.status} IN ('held', 'released', 'expired')`), + index("idxSymbolLocksOwner").on(t.projectId, t.ownerTaskId), + index("idxSymbolLocksExpiry").on(t.status, t.expiresAt), +]); + // ── Workflow step definitions ──────────────────────────────────────── export const workflowSteps = projectSchema.table("workflow_steps", { id: text("id").primaryKey(), diff --git a/packages/core/src/store.ts b/packages/core/src/store.ts index fef29e1c02..7e920dac9e 100644 --- a/packages/core/src/store.ts +++ b/packages/core/src/store.ts @@ -113,6 +113,8 @@ import { saveWorkflowRunBranchImpl, clearNearDuplicateReferencesToImpl, selectNe import { taskToArchiveEntryImpl, deleteTaskBackendImpl, archiveTaskBackendImpl, unarchiveTaskImpl, restoreFromArchiveImpl, listArchivedTasksImpl } from "./task-store/archive-lifecycle-2.js"; import { pruneOperationalLogsAsync, pruneAgentLogFilesAsync, type OperationalLogPruneResult } from "./task-store/async-maintenance.js"; import { reconcilePhantomCommittedReservationsAsync } from "./task-store/async-phantom-reservations.js"; +import { acquireSymbolLocksAsync, inspectSymbolLockConflictsAsync, reconcileStaleSymbolLocksAsync, releaseSymbolLocksAsync, renewSymbolLocksAsync } from "./task-store/symbol-locks.js"; +import type { AcquireSymbolLocksResult, ReconcileStaleSymbolLocksResult, ReleaseSymbolLocksResult, RenewSymbolLocksResult, SymbolLockConflict, SymbolLockOwner } from "./symbol-lock-types.js"; import { queryRunAuditEvents } from "./task-store/async-audit.js"; import { isValidMergeRequestTransitionImpl, enqueueMergeQueueSyncInternalImpl, releaseMergeQueueLeaseImpl, collectMergeDetailsImpl, applyPrMergedTransitionImpl } from "./task-store/merge-queue-ops-2.js"; import { upsertWorkflowWorkItemImpl, transitionWorkflowWorkItemImpl, acquireWorkflowWorkItemLeaseImpl } from "./task-store/workflow-workitems-ops-2.js"; @@ -704,6 +706,27 @@ export class TaskStore extends EventEmitter { return reconcileOrphanedTaskDirsImpl(this, opts); } + /** + * FNXC:SymbolLock 2026-07-30-14:10: + * The TaskStore owns project binding, so callers cannot accidentally inspect + * or mutate an identically named symbol in another project's partition. + */ + async acquireSymbolLocks(symbols: readonly string[], owner: SymbolLockOwner, leaseMs: number): Promise { + return acquireSymbolLocksAsync(this, symbols, owner, leaseMs); + } + async renewSymbolLocks(symbols: readonly string[], ownerTaskId: string, leaseMs: number): Promise { + return renewSymbolLocksAsync(this, symbols, ownerTaskId, leaseMs); + } + async releaseSymbolLocks(symbols: readonly string[], ownerTaskId: string): Promise { + return releaseSymbolLocksAsync(this, symbols, ownerTaskId); + } + async inspectSymbolLockConflicts(symbols: readonly string[]): Promise { + return inspectSymbolLockConflictsAsync(this, symbols); + } + async reconcileStaleSymbolLocks(): Promise { + return reconcileStaleSymbolLocksAsync(this); + } + /** Reconcile committed reservations whose task and archive representations are absent. */ async reconcilePhantomCommittedReservations(): Promise<{ reconciled: string[]; diff --git a/packages/core/src/symbol-lock-types.ts b/packages/core/src/symbol-lock-types.ts new file mode 100644 index 0000000000..a6e4559561 --- /dev/null +++ b/packages/core/src/symbol-lock-types.ts @@ -0,0 +1,60 @@ +/** + * FNXC:SymbolLock 2026-07-30-14:00: + * Mission-lineage scheduler admission needs a durable, project-scoped seam for + * protecting code symbols before later scheduling phases consume it. These + * opaque normalized keys deliberately avoid storing raw planning prose. + */ +export type SymbolLockStatus = "held" | "released" | "expired"; + +export interface SymbolLockIdentity { + projectId: string; + symbolKey: string; + normalizedSymbol: string; +} + +export interface SymbolLockOwner { + ownerTaskId: string; + missionId?: string; + featureId?: string; + lineageId?: string; + nodeId?: string; + agentId?: string; +} + +export interface SymbolLockLease { + status: SymbolLockStatus; + acquiredAt: string; + renewedAt: string; + expiresAt: string; +} + +export interface SymbolLock extends SymbolLockIdentity, SymbolLockOwner, SymbolLockLease {} + +export interface SymbolLockConflict { + symbolKey: string; + ownerTaskId: string; + missionId?: string; + featureId?: string; + lineageId?: string; + nodeId?: string; + agentId?: string; + expiresAt: string; +} + +export type AcquireSymbolLocksResult = + | { acquired: true; locks: SymbolLock[]; conflicts: [] } + | { acquired: false; locks: []; conflicts: SymbolLockConflict[] }; + +export interface RenewSymbolLocksResult { + renewed: string[]; + lost: string[]; +} + +export interface ReleaseSymbolLocksResult { + released: string[]; +} + +export interface ReconcileStaleSymbolLocksResult { + reconciled: string[]; + skipped: string[]; +} diff --git a/packages/core/src/task-store/symbol-locks.ts b/packages/core/src/task-store/symbol-locks.ts new file mode 100644 index 0000000000..794fa6d641 --- /dev/null +++ b/packages/core/src/task-store/symbol-locks.ts @@ -0,0 +1,244 @@ +import { and, eq, gt, inArray, sql } from "drizzle-orm"; +import * as schema from "../postgres/schema/index.js"; +import { projectOwnershipPartition, recordRunAuditEventWithinTransaction } from "../postgres/data-layer.js"; +import type { DbTransaction } from "../postgres/data-layer.js"; +import type { TaskStore } from "../store.js"; +import type { + AcquireSymbolLocksResult, + ReconcileStaleSymbolLocksResult, + ReleaseSymbolLocksResult, + RenewSymbolLocksResult, + SymbolLock, + SymbolLockConflict, + SymbolLockIdentity, + SymbolLockOwner, +} from "../symbol-lock-types.js"; + +/** + * FNXC:SymbolLock 2026-07-30-14:00: + * Symbol references are an admission key, not a user-facing label. Normalize + * whitespace, path separators, and casing deterministically so equivalent + * `pkg/file.ts#Exported.member` references contend across agents. + */ +export function normalizeSymbolLockKey(rawSymbol: string): string { + const normalized = rawSymbol + .trim() + .replaceAll("\\", "/") + .replace(/\s+/g, "") + .replace(/\/+/g, "/") + .replace(/:+/g, ":") + .replace(/#+/g, "#") + .toLowerCase(); + if (!normalized) throw new Error("Symbol lock key must not be empty"); + return normalized; +} + +/** Extracts the canonical project-scoped identity from a raw symbol reference. */ +export function extractSymbolLockIdentity(projectId: string, rawSymbol: string): SymbolLockIdentity { + const normalizedSymbol = normalizeSymbolLockKey(rawSymbol); + return { projectId, normalizedSymbol, symbolKey: normalizedSymbol }; +} + +/** Equivalent canonical symbols contend; callers must normalize before storage. */ +export function symbolLocksConflict(left: string, right: string): boolean { + return normalizeSymbolLockKey(left) === normalizeSymbolLockKey(right); +} + +function symbolKeys(symbols: readonly string[]): string[] { + return [...new Set(symbols.map(normalizeSymbolLockKey))].sort(); +} + +function toLock(row: typeof schema.project.symbolLocks.$inferSelect): SymbolLock { + return { + projectId: row.projectId, + symbolKey: row.symbolKey, + normalizedSymbol: row.symbolKey, + ownerTaskId: row.ownerTaskId, + missionId: row.missionId ?? undefined, + featureId: row.featureId ?? undefined, + lineageId: row.lineageId ?? undefined, + nodeId: row.nodeId ?? undefined, + agentId: row.agentId ?? undefined, + status: row.status as SymbolLock["status"], + acquiredAt: row.acquiredAt, + renewedAt: row.renewedAt, + expiresAt: row.expiresAt, + }; +} + +function toConflict(row: typeof schema.project.symbolLocks.$inferSelect): SymbolLockConflict { + return { + symbolKey: row.symbolKey, + ownerTaskId: row.ownerTaskId, + missionId: row.missionId ?? undefined, + featureId: row.featureId ?? undefined, + lineageId: row.lineageId ?? undefined, + nodeId: row.nodeId ?? undefined, + agentId: row.agentId ?? undefined, + expiresAt: row.expiresAt, + }; +} + +/** + * FNXC:SymbolLock 2026-07-30-14:10: + * Advisory transaction locks serialize same-symbol admissions even when no row + * exists yet. Acquiring all sorted keys before inspection preserves the + * all-or-nothing contract without making later scheduler behavior responsible + * for PostgreSQL race recovery. + */ +async function lockSymbolKeys(tx: DbTransaction, projectId: string, keys: readonly string[]): Promise { + for (const key of keys) { + await tx.execute(sql`SELECT pg_advisory_xact_lock(hashtextextended(${`${projectId}:${key}`}, 0))`); + } +} + +export async function acquireSymbolLocksAsync( + store: TaskStore, + symbols: readonly string[], + owner: SymbolLockOwner, + leaseMs: number, +): Promise { + const layer = store.getAsyncLayer(); + if (!layer) throw new Error("Durable symbol locks require an AsyncDataLayer"); + if (!Number.isFinite(leaseMs) || leaseMs <= 0) throw new Error("Symbol lock leaseMs must be positive"); + const projectId = projectOwnershipPartition(layer.projectId); + const keys = symbolKeys(symbols); + if (keys.length === 0) return { acquired: true, locks: [], conflicts: [] }; + + return layer.transactionImmediate(async (tx) => { + await lockSymbolKeys(tx, projectId, keys); + const now = new Date(); + const nowIso = now.toISOString(); + const rows = await tx.select().from(schema.project.symbolLocks).where(and( + eq(schema.project.symbolLocks.projectId, projectId), + inArray(schema.project.symbolLocks.symbolKey, keys), + )); + const conflicts = rows.filter((row) => row.status === "held" && row.expiresAt > nowIso && row.ownerTaskId !== owner.ownerTaskId); + if (conflicts.length > 0) { + await recordRunAuditEventWithinTransaction(tx, { + taskId: owner.ownerTaskId, agentId: owner.agentId ?? "symbol-lock", runId: `symbol-lock:${owner.ownerTaskId}`, + domain: "symbol-lock", mutationType: "symbol-lock:acquire-conflict", target: owner.ownerTaskId, + metadata: { count: conflicts.length, symbolKeys: conflicts.map((row) => row.symbolKey).sort(), outcome: "conflict" }, + }); + return { acquired: false, locks: [], conflicts: conflicts.map(toConflict) }; + } + const expiresAt = new Date(now.getTime() + leaseMs).toISOString(); + for (const key of keys) { + const existing = rows.find((row) => row.symbolKey === key); + const values = { + projectId, symbolKey: key, ownerTaskId: owner.ownerTaskId, missionId: owner.missionId ?? null, + featureId: owner.featureId ?? null, lineageId: owner.lineageId ?? null, nodeId: owner.nodeId ?? null, + agentId: owner.agentId ?? null, status: "held", acquiredAt: nowIso, renewedAt: nowIso, + expiresAt, createdAt: nowIso, updatedAt: nowIso, + }; + if (existing) { + await tx.update(schema.project.symbolLocks).set(values).where(and( + eq(schema.project.symbolLocks.projectId, projectId), eq(schema.project.symbolLocks.symbolKey, key), + )); + } else { + await tx.insert(schema.project.symbolLocks).values(values); + } + } + const held = await tx.select().from(schema.project.symbolLocks).where(and( + eq(schema.project.symbolLocks.projectId, projectId), inArray(schema.project.symbolLocks.symbolKey, keys), + )); + await recordRunAuditEventWithinTransaction(tx, { + taskId: owner.ownerTaskId, agentId: owner.agentId ?? "symbol-lock", runId: `symbol-lock:${owner.ownerTaskId}`, + domain: "symbol-lock", mutationType: "symbol-lock:acquired", target: owner.ownerTaskId, + metadata: { count: held.length, symbolKeys: keys, outcome: "acquired" }, + }); + return { acquired: true, locks: held.map(toLock), conflicts: [] }; + }); +} + +export async function renewSymbolLocksAsync(store: TaskStore, symbols: readonly string[], ownerTaskId: string, leaseMs: number): Promise { + const layer = store.getAsyncLayer(); + if (!layer) throw new Error("Durable symbol locks require an AsyncDataLayer"); + if (!Number.isFinite(leaseMs) || leaseMs <= 0) throw new Error("Symbol lock leaseMs must be positive"); + const projectId = projectOwnershipPartition(layer.projectId); + const keys = symbolKeys(symbols); + const now = new Date(); const nowIso = now.toISOString(); const expiresAt = new Date(now.getTime() + leaseMs).toISOString(); + return layer.transactionImmediate(async (tx) => { + await lockSymbolKeys(tx, projectId, keys); + const rows = keys.length === 0 ? [] : await tx.select().from(schema.project.symbolLocks).where(and(eq(schema.project.symbolLocks.projectId, projectId), inArray(schema.project.symbolLocks.symbolKey, keys))); + const renewable = rows.filter((row) => row.ownerTaskId === ownerTaskId && row.status === "held" && row.expiresAt > nowIso).map((row) => row.symbolKey); + // FNXC:SymbolLock 2026-07-30-14:45: recheck expiry in the write predicate + // so a delayed renewal cannot revive a lease that became expired after read. + const renewed = renewable.length === 0 ? [] : (await tx.update(schema.project.symbolLocks) + .set({ renewedAt: nowIso, expiresAt, updatedAt: nowIso }) + .where(and( + eq(schema.project.symbolLocks.projectId, projectId), + eq(schema.project.symbolLocks.ownerTaskId, ownerTaskId), + inArray(schema.project.symbolLocks.symbolKey, renewable), + eq(schema.project.symbolLocks.status, "held"), + gt(schema.project.symbolLocks.expiresAt, nowIso), + )) + .returning({ symbolKey: schema.project.symbolLocks.symbolKey })) + .map((row) => row.symbolKey); + const lost = keys.filter((key) => !renewed.includes(key)); + await recordRunAuditEventWithinTransaction(tx, { taskId: ownerTaskId, agentId: "symbol-lock", runId: `symbol-lock:${ownerTaskId}`, domain: "symbol-lock", mutationType: "symbol-lock:renewed", target: ownerTaskId, metadata: { count: renewed.length, lostCount: lost.length, symbolKeys: renewed, outcome: "renewed" } }); + return { renewed, lost }; + }); +} + +export async function releaseSymbolLocksAsync(store: TaskStore, symbols: readonly string[], ownerTaskId: string): Promise { + const layer = store.getAsyncLayer(); + if (!layer) throw new Error("Durable symbol locks require an AsyncDataLayer"); + const projectId = projectOwnershipPartition(layer.projectId); const keys = symbolKeys(symbols); const nowIso = new Date().toISOString(); + return layer.transactionImmediate(async (tx) => { + await lockSymbolKeys(tx, projectId, keys); + const released = keys.length === 0 ? [] : await tx.update(schema.project.symbolLocks).set({ status: "released", updatedAt: nowIso }).where(and(eq(schema.project.symbolLocks.projectId, projectId), eq(schema.project.symbolLocks.ownerTaskId, ownerTaskId), eq(schema.project.symbolLocks.status, "held"), inArray(schema.project.symbolLocks.symbolKey, keys))).returning({ symbolKey: schema.project.symbolLocks.symbolKey }); + await recordRunAuditEventWithinTransaction(tx, { taskId: ownerTaskId, agentId: "symbol-lock", runId: `symbol-lock:${ownerTaskId}`, domain: "symbol-lock", mutationType: "symbol-lock:released", target: ownerTaskId, metadata: { count: released.length, symbolKeys: released.map((row) => row.symbolKey).sort(), outcome: "released" } }); + return { released: released.map((row) => row.symbolKey) }; + }); +} + +export async function inspectSymbolLockConflictsAsync(store: TaskStore, symbols: readonly string[]): Promise { + const layer = store.getAsyncLayer(); + if (!layer) throw new Error("Durable symbol locks require an AsyncDataLayer"); + const keys = symbolKeys(symbols); if (!keys.length) return []; + const projectId = projectOwnershipPartition(layer.projectId); const nowIso = new Date().toISOString(); + const rows = await layer.db.select().from(schema.project.symbolLocks).where(and(eq(schema.project.symbolLocks.projectId, projectId), eq(schema.project.symbolLocks.status, "held"), inArray(schema.project.symbolLocks.symbolKey, keys))); + return rows.filter((row) => row.expiresAt > nowIso).map(toConflict); +} + +export async function reconcileStaleSymbolLocksAsync(store: TaskStore): Promise { + const layer = store.getAsyncLayer(); + if (!layer) return { reconciled: [], skipped: [] }; + const projectId = projectOwnershipPartition(layer.projectId); const nowIso = new Date().toISOString(); + const held = await layer.db.select().from(schema.project.symbolLocks).where(and(eq(schema.project.symbolLocks.projectId, projectId), eq(schema.project.symbolLocks.status, "held"))); + const stale: Array<{ symbolKey: string; ownerTaskId: string; expiresAt: string }> = []; + const skipped: string[] = []; + for (const lock of held) { + const owner = await store.getTask(lock.ownerTaskId, { includeDeleted: true }).catch(() => undefined); + const terminal = !owner || owner.deletedAt != null || owner.column === "done" || owner.column === "archived" || owner.status === "failed"; + if (lock.expiresAt <= nowIso || terminal) { + stale.push({ symbolKey: lock.symbolKey, ownerTaskId: lock.ownerTaskId, expiresAt: lock.expiresAt }); + } else { + skipped.push(lock.symbolKey); + } + } + if (!stale.length) return { reconciled: [], skipped }; + const reconciled = await layer.transactionImmediate(async (tx) => { + // FNXC:SymbolLock 2026-07-30-14:40: stale detection happens before this + // transaction, so CAS on the observed owner and lease prevents a sweep from + // expiring a lock that was reclaimed by another task in the interim. + await lockSymbolKeys(tx, projectId, stale.map((lock) => lock.symbolKey).sort()); + const expired: string[] = []; + for (const lock of stale) { + const updated = await tx.update(schema.project.symbolLocks) + .set({ status: "expired", updatedAt: nowIso }) + .where(and( + eq(schema.project.symbolLocks.projectId, projectId), + eq(schema.project.symbolLocks.symbolKey, lock.symbolKey), + eq(schema.project.symbolLocks.ownerTaskId, lock.ownerTaskId), + eq(schema.project.symbolLocks.expiresAt, lock.expiresAt), + eq(schema.project.symbolLocks.status, "held"), + )) + .returning({ symbolKey: schema.project.symbolLocks.symbolKey }); + expired.push(...updated.map((row) => row.symbolKey)); + } + return expired; + }); + return { reconciled, skipped }; +} diff --git a/packages/engine/src/__tests__/symbol-lock-reconciliation.test.ts b/packages/engine/src/__tests__/symbol-lock-reconciliation.test.ts new file mode 100644 index 0000000000..237876719d --- /dev/null +++ b/packages/engine/src/__tests__/symbol-lock-reconciliation.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it, vi } from "vitest"; +import { SelfHealingManager } from "../self-healing.js"; +import type { TaskStore } from "@fusion/core"; + +describe("SelfHealingManager symbol-lock reconciliation", () => { + it("audits stale lock reclamation and dedupes an idle no-action sweep", async () => { + const reconcileStaleSymbolLocks = vi.fn() + .mockResolvedValueOnce({ reconciled: ["pkg/a.ts#a"], skipped: ["pkg/live.ts#a"] }) + .mockResolvedValue({ reconciled: [], skipped: ["pkg/live.ts#a"] }); + const recordRunAuditEvent = vi.fn().mockResolvedValue(undefined); + const manager = new SelfHealingManager({ reconcileStaleSymbolLocks, recordRunAuditEvent } as unknown as TaskStore, { rootDir: "/tmp/symbol-lock-test" }); + await expect(manager.reconcileStaleSymbolLocks()).resolves.toBe(1); + await expect(manager.reconcileStaleSymbolLocks()).resolves.toBe(0); + await expect(manager.reconcileStaleSymbolLocks()).resolves.toBe(0); + expect(recordRunAuditEvent).toHaveBeenCalledTimes(2); + expect(recordRunAuditEvent.mock.calls[0]?.[0]).toMatchObject({ mutationType: "symbol-lock:reconcile-stale", metadata: { count: 1, outcome: "reconciled" } }); + expect(recordRunAuditEvent.mock.calls[1]?.[0]).toMatchObject({ mutationType: "symbol-lock:reconcile-stale-no-action", metadata: { count: 0, outcome: "no-action" } }); + }); +}); diff --git a/packages/engine/src/self-healing.ts b/packages/engine/src/self-healing.ts index 4dbd9a9013..7a9529ec5f 100644 --- a/packages/engine/src/self-healing.ts +++ b/packages/engine/src/self-healing.ts @@ -751,6 +751,8 @@ export class SelfHealingManager { * because the task leaves the promotable state before returning to it. */ private strandedCompletedFailureProvenanceWarned = new Set(); + /* FNXC:SymbolLock 2026-07-30-14:20: idle symbol-lock sweeps emit one no-action audit until a stale lock re-arms the diagnostic. */ + private symbolLockNoActionAudited = false; private metaResolvedSkipAuditMemo = new Map(); private metaStalledSkipAuditMemo = new Map(); private preservedQueuedOverlapLogged = new Map(); @@ -1396,6 +1398,7 @@ export class SelfHealingManager { { name: "clear-stale-blocked-by", fn: () => this.clearStaleBlockedBy().then(() => undefined) }, { name: "reconcile-self-defeating-deps", fn: () => this.reconcileSelfDefeatingDependencies().then(() => undefined) }, { name: "reconcile-dependency-blocking-leases", fn: () => this.reconcileDependencyBlockingLeases().then(() => undefined) }, + { name: "reconcile-stale-symbol-locks", fn: () => this.reconcileStaleSymbolLocks().then(() => undefined) }, { name: "reconcile-completed-blocked", fn: () => this.reconcileCompletedBlockedTasks().then(() => undefined) }, { name: "reconcile-in-review-unmet-dependencies", fn: () => this.reconcileInReviewUnmetDependencies().then(() => undefined) }, { name: "reconcile-engine-downtime-active-timing", fn: () => this.reconcileEngineDowntimeActiveTiming().then(() => undefined) }, @@ -2512,6 +2515,10 @@ export class SelfHealingManager { return result; }, }, + { + name: "reconcile-stale-symbol-locks", + fn: () => this.reconcileStaleSymbolLocks(), + }, { name: "reconcile-phantom-committed-reservations", fn: async () => { @@ -5658,6 +5665,34 @@ export class SelfHealingManager { } } + /** + * FNXC:SymbolLock 2026-07-30-14:20: + * Crash recovery expires only locks whose owner is terminal/missing or whose + * lease elapsed. It is intentionally orthogonal to scheduler admission and + * never changes a task, worktree, semaphore, or verification state. + */ + async reconcileStaleSymbolLocks(): Promise { + const result = await this.store.reconcileStaleSymbolLocks(); + if (result.reconciled.length > 0) { + this.symbolLockNoActionAudited = false; + await this.store.recordRunAuditEvent({ + agentId: "self-healing", runId: "symbol-lock-reconcile", domain: "database", + mutationType: "symbol-lock:reconcile-stale", target: "symbol-locks", + metadata: { count: result.reconciled.length, symbolKeys: result.reconciled, outcome: "reconciled" }, + }); + return result.reconciled.length; + } + if (!this.symbolLockNoActionAudited) { + this.symbolLockNoActionAudited = true; + await this.store.recordRunAuditEvent({ + agentId: "self-healing", runId: "symbol-lock-reconcile", domain: "database", + mutationType: "symbol-lock:reconcile-stale-no-action", target: "symbol-locks", + metadata: { count: 0, outcome: "no-action" }, + }); + } + return 0; + } + async reconcileDependencyBlockingLeases(): Promise { const settings = await this.store.getSettings(); if (settings.globalPause || settings.enginePaused) return 0;