From 5573d042a92dee40748ee4a10d8912d2ae17e4bd Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Mon, 10 Aug 2026 04:46:10 -0700 Subject: [PATCH] FN-8922: add persistent memory recall store Add PostgreSQL-backed recall records for durable project memory retrieval. - add recall record types, storage, search, deduplication, and prompt instructions - migrate project schemas and expose recall APIs from core memory modules - document plugin and storage contracts, with unit and PostgreSQL coverage Files changed: .changeset/fn-8922-memory-recall-store.md | 7 + docs/memory-plugin-contract.md | 10 + docs/storage.md | 4 + .../__tests__/memory-recall-dedup-search.test.ts | 10 + .../__tests__/memory-recall-instructions.test.ts | 62 +++++ .../postgres/memory-recall-store.pg.test.ts | 296 +++++++++++++++++++++ .../src/__tests__/postgres/schema-applier.test.ts | 15 +- packages/core/src/index.ts | 1 + packages/core/src/memory/index.ts | 1 + packages/core/src/memory/project-memory.ts | 33 ++- packages/core/src/memory/recall/index.ts | 10 + packages/core/src/memory/recall/recall-dedup.ts | 35 +++ .../core/src/memory/recall/recall-instructions.ts | 59 ++++ packages/core/src/memory/recall/recall-search.ts | 20 ++ packages/core/src/memory/recall/recall-store.ts | 51 ++++ packages/core/src/memory/recall/recall-types.ts | 14 + .../0052_fn_8922_memory_recall_records.sql | 17 ++ packages/core/src/postgres/schema-applier.ts | 13 +- packages/core/src/postgres/schema/project.ts | 10 +- 19 files changed, 648 insertions(+), 20 deletions(-) Fusion-Task-Id: FN-8922 Fusion-Task-Lineage: bb98f048-6fcf-46d9-bd1f-09294c2dc005 Co-authored-by: Fusion (runfusion.ai) --- .changeset/fn-8922-memory-recall-store.md | 7 + docs/memory-plugin-contract.md | 10 + docs/storage.md | 4 + .../memory-recall-dedup-search.test.ts | 10 + .../memory-recall-instructions.test.ts | 62 ++++ .../postgres/memory-recall-store.pg.test.ts | 296 ++++++++++++++++++ .../__tests__/postgres/schema-applier.test.ts | 15 +- packages/core/src/index.ts | 1 + packages/core/src/memory/index.ts | 1 + packages/core/src/memory/project-memory.ts | 33 +- packages/core/src/memory/recall/index.ts | 10 + .../core/src/memory/recall/recall-dedup.ts | 35 +++ .../src/memory/recall/recall-instructions.ts | 59 ++++ .../core/src/memory/recall/recall-search.ts | 20 ++ .../core/src/memory/recall/recall-store.ts | 51 +++ .../core/src/memory/recall/recall-types.ts | 14 + .../0052_fn_8922_memory_recall_records.sql | 17 + packages/core/src/postgres/schema-applier.ts | 13 +- packages/core/src/postgres/schema/project.ts | 10 +- 19 files changed, 648 insertions(+), 20 deletions(-) create mode 100644 .changeset/fn-8922-memory-recall-store.md create mode 100644 packages/core/src/__tests__/memory-recall-dedup-search.test.ts create mode 100644 packages/core/src/__tests__/memory-recall-instructions.test.ts create mode 100644 packages/core/src/__tests__/postgres/memory-recall-store.pg.test.ts create mode 100644 packages/core/src/memory/recall/index.ts create mode 100644 packages/core/src/memory/recall/recall-dedup.ts create mode 100644 packages/core/src/memory/recall/recall-instructions.ts create mode 100644 packages/core/src/memory/recall/recall-search.ts create mode 100644 packages/core/src/memory/recall/recall-store.ts create mode 100644 packages/core/src/memory/recall/recall-types.ts create mode 100644 packages/core/src/postgres/migrations/0052_fn_8922_memory_recall_records.sql diff --git a/.changeset/fn-8922-memory-recall-store.md b/.changeset/fn-8922-memory-recall-store.md new file mode 100644 index 0000000000..331a8cec79 --- /dev/null +++ b/.changeset/fn-8922-memory-recall-store.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": minor +--- + +summary: Add durable project recall for decisions, preferences, and solutions. +category: feature +dev: Adds project.memory_recall_records migration, advisory-lock dedup, optional rank-only vector provider with store-side limits; no MCP surface yet. diff --git a/docs/memory-plugin-contract.md b/docs/memory-plugin-contract.md index 6e4f0517c4..20207ec67a 100644 --- a/docs/memory-plugin-contract.md +++ b/docs/memory-plugin-contract.md @@ -341,3 +341,13 @@ If backend contract behavior changes in source, update these docs in the same ch --- *Last updated: 2026-04-19* + +## Recall layer + +The project-scoped recall store records durable `decision`, `preference`, and `solution` entries with content, tags, source provenance (`taskId`, `agentId`, `sessionId`, and origin), timestamps, and optional knowledge-graph node ids. `appendRecall` normalizes content (trim/lowercase/whitespace collapse/trailing punctuation removal) and rejects an exact normalized hash or Jaccard token similarity of at least `0.9` among the 200 most-recent same-kind records. That candidate window is intentionally bounded: an older exact twin is not visible to the in-memory classifier. + +Writes hold a transaction advisory lock keyed by `(project, kind)` across candidate lookup and insert, which serializes the read-then-write near-duplicate decision without contending unrelated kinds or projects. The named `(project_id, kind, content_hash)` constraint is only an exact-hash backstop: `ON CONFLICT DO NOTHING` keeps the transaction usable for its in-transaction re-read when an exact twin is outside the bounded window (or a bypassing importer races the write). A raising unique insert would abort the transaction, and the constraint cannot catch near duplicates. + +`searchRecall` uses deterministic keyword scoring and one shared `clampRecallSearchLimit` (default 10, maximum 50) for keyword, vector, degradation, and list paths. A caller may supply a per-call `RecallVectorSearchProvider` to rank—never fetch, write, or filter—the already project-scoped, kind/tag-filtered candidate set. There is no provider registry, setting, default implementation, or embedding dependency. `mode: "vector"` is returned only for a successful provider result containing a resolvable candidate; missing, throwing, empty, or unknown-only providers degrade to keyword mode while `capabilities.vector` remains true when a provider was supplied. Provider limits are advisory: the store discards unknown ids, keeps each duplicate id's highest score, ranks, then applies the same clamped limit after ranking. + +Prompt builders may append a `### Recalled Context` section capped at 800 UTF-8 bytes; the budget includes its separator, heading, lines, and trailing newline, and never truncates pre-existing instructions. This task adds no MCP/tool surface, automatic capture, consolidation, agent pre-steering, or knowledge-graph integration; those remain later work. diff --git a/docs/storage.md b/docs/storage.md index 043d883eab..5679853874 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -829,3 +829,7 @@ Automatic admission locks the project-scoped feature row, records a running row `engineLastActiveAt` is engine liveness bookkeeping, not operator configuration. It is a non-versioned project-settings key: revision diffs and stored snapshots omit it, while the live project setting remains written normally. Project-settings rollback overlays live non-versioned values over both modern stripped snapshots and legacy snapshots, so rollback cannot delete or resurrect a stale heartbeat. `appendConfigurationRevision` deliberately remains an unfiltered raw writer for migrations and legacy fixtures; a heartbeat-only rollback is rejected as already restored without writing. Revision listing defaults to 100 rows and clamps `limit` to 1–500. The API accepts `limit` and `offset` and returns `hasMore`, determined by fetching `limit + 1` rows rather than a count query. Rows are ordered `createdAt DESC, sequence DESC`. Since history is append-only, rows appended between offset page requests can shift offsets and be observed twice; they are not silently skipped backwards. + +### `project.memory_recall_records` + +Project-scoped structured recall records for durable decisions, preferences, and solutions. The table uses the composite `(project_id, id)` key, row-level security, created-at indexes, and a named `(project_id, kind, content_hash)` exact-hash backstop. diff --git a/packages/core/src/__tests__/memory-recall-dedup-search.test.ts b/packages/core/src/__tests__/memory-recall-dedup-search.test.ts new file mode 100644 index 0000000000..da9b6cf633 --- /dev/null +++ b/packages/core/src/__tests__/memory-recall-dedup-search.test.ts @@ -0,0 +1,10 @@ +import { describe, expect, it } from "vitest"; +import { RECALL_DEDUP_SIMILARITY_THRESHOLD, classifyRecallDuplicate, normalizeRecallContent, recallContentHash, recallDedupLockKey } from "../memory/recall/recall-dedup.js"; +import { applyRecallVectorRanking, clampRecallSearchLimit, RECALL_SEARCH_DEFAULT_LIMIT, RECALL_SEARCH_MAX_LIMIT, resolveRecallCapabilities, searchRecallKeyword } from "../memory/recall/recall-search.js"; +import type { RecallRecord } from "../memory/recall/recall-types.js"; +const record = (id: string, content: string, createdAt = "2026-01-01T00:00:00.000Z"): RecallRecord => ({ id, projectId: "p", kind: "decision", content, contentHash: recallContentHash("decision", content), source: { origin: "manual" }, tags: [], graphNodeIds: [], createdAt, updatedAt: createdAt }); +describe("memory recall pure helpers", () => { + it("normalizes and classifies bounded duplicate candidates", () => { expect(normalizeRecallContent(" HELLO world!! ")).toBe("hello world"); expect(recallContentHash("decision", "hello world")).toBe(recallContentHash("decision", " HELLO world!")); const original = record("a", "one two three four five six seven eight nine ten eleven twelve thirteen fourteen fifteen sixteen seventeen eighteen nineteen twenty"); const near = record("b", "one two three four five six seven eight nine ten eleven twelve thirteen fourteen fifteen sixteen seventeen eighteen nineteen changed"); expect(classifyRecallDuplicate(near, [original])?.record.id).toBe("a"); expect(classifyRecallDuplicate(near, [original])?.similarity).toBeGreaterThanOrEqual(RECALL_DEDUP_SIMILARITY_THRESHOLD); expect(classifyRecallDuplicate(near, [])).toBeUndefined(); }); + it("uses stable scoped locks and clamps all malformed limits", () => { expect(recallDedupLockKey("a", "decision")).toBe(recallDedupLockKey("a", "decision")); expect(recallDedupLockKey("a", "decision")).not.toBe(recallDedupLockKey("a", "solution")); expect(recallDedupLockKey("a", "decision")).not.toBe(recallDedupLockKey("b", "decision")); for (const value of [undefined, null, NaN, 0, -5, 2.7]) expect(clampRecallSearchLimit(value)).toBe(RECALL_SEARCH_DEFAULT_LIMIT); expect(clampRecallSearchLimit(10_000)).toBe(RECALL_SEARCH_MAX_LIMIT); expect(clampRecallSearchLimit(3)).toBe(3); }); + it("scores deterministic keyword and provider rankings after ranking", () => { const rows = [record("b", "alpha", "2026-01-01T00:00:00.000Z"), record("a", "alpha", "2026-01-01T00:00:00.000Z"), record("c", "alpha")]; expect(searchRecallKeyword("alpha", rows, 2).map((x) => x.record.id)).toEqual(["a", "b"]); const ranked = applyRecallVectorRanking(rows, [{ recordId: "a", score: 1 }, { recordId: "b", score: 5 }, { recordId: "c", score: 4 }, { recordId: "unknown", score: 99 }, { recordId: "b", score: 6 }], 2); expect(ranked.map((x) => x.record.id)).toEqual(["b", "c"]); expect(new Set(ranked.map((x) => x.record.id)).size).toBe(ranked.length); expect(applyRecallVectorRanking(rows, Array.from({ length: 1000 }, (_, i) => ({ recordId: rows[i % rows.length]!.id, score: i })), 50)).toHaveLength(3); const provider = { id: "stub", search: async () => [] }; expect(resolveRecallCapabilities()).toMatchObject({ vector: false }); expect(resolveRecallCapabilities(provider)).toMatchObject({ vector: true }); }); +}); diff --git a/packages/core/src/__tests__/memory-recall-instructions.test.ts b/packages/core/src/__tests__/memory-recall-instructions.test.ts new file mode 100644 index 0000000000..fc1636e008 --- /dev/null +++ b/packages/core/src/__tests__/memory-recall-instructions.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from "vitest"; +import { + buildExecutionMemoryInstructions, + buildReviewerMemoryInstructions, + buildTriageMemoryInstructions, +} from "../memory/project-memory.js"; +import { + formatRecallInstructionSection, + MAX_RECALL_INJECTION_BYTES, +} from "../memory/recall/recall-instructions.js"; +import type { RecallSearchHit } from "../memory/recall/recall-types.js"; + +const settings = [ + { memoryBackendType: "file" }, + { memoryBackendType: "qmd" }, + { memoryBackendType: "readonly" }, +] as const; +const hits: RecallSearchHit[] = Array.from({ length: 200 }, (_, index) => ({ + score: index, + matchedTerms: [], + record: { + id: String(index), projectId: "p", kind: "decision", + content: `決定 ${"🌱".repeat(100)}`, contentHash: String(index), + source: { origin: "manual" }, tags: [], graphNodeIds: [], + createdAt: "2026-01-01", updatedAt: "2026-01-01", + }, +})); + +const builders = [ + buildTriageMemoryInstructions, + buildExecutionMemoryInstructions, + buildReviewerMemoryInstructions, +] as const; + +describe("recall instruction injection", () => { + it("preserves every existing builder output when recall is omitted", () => { + for (const builder of builders) { + for (const backend of settings) { + expect(builder("/tmp", backend)).toBe(builder("/tmp", backend, undefined)); + } + expect(builder("/tmp", { memoryEnabled: false })).toBe(""); + expect(builder("/tmp", { memoryEnabled: false }, hits)).toBe(""); + } + }); + + it("bounds only appended recall context without splitting UTF-8", () => { + const section = formatRecallInstructionSection(hits); + expect(section).toContain("### Recalled Context"); + expect(Buffer.byteLength(section, "utf8")).toBeLessThanOrEqual(MAX_RECALL_INJECTION_BYTES); + expect(Buffer.from(section, "utf8").toString("utf8")).toBe(section); + expect(section).not.toContain("\uFFFD"); + + for (const builder of builders) { + const baseline = builder("/tmp", { memoryBackendType: "file" }); + const withRecall = builder("/tmp", { memoryBackendType: "file" }, hits); + expect(withRecall).toContain(baseline); + expect(Buffer.byteLength(withRecall, "utf8") - Buffer.byteLength(baseline, "utf8")) + .toBeLessThanOrEqual(MAX_RECALL_INJECTION_BYTES); + expect(withRecall).toContain("### Recalled Context"); + } + }); +}); diff --git a/packages/core/src/__tests__/postgres/memory-recall-store.pg.test.ts b/packages/core/src/__tests__/postgres/memory-recall-store.pg.test.ts new file mode 100644 index 0000000000..c4948ee7b7 --- /dev/null +++ b/packages/core/src/__tests__/postgres/memory-recall-store.pg.test.ts @@ -0,0 +1,296 @@ +/* +FNXC:MemoryRecall 2026-08-10-11:03: +Recall persistence is project-scoped and its near-duplicate decision is made under a PostgreSQL +transaction advisory lock. These integration assertions exercise the real migration and database +path rather than a mocked store, including concurrent writers and untrusted vector rankings. +*/ +import { afterAll, afterEach, beforeAll, beforeEach, expect, it } from "vitest"; +import { and, eq, sql } from "drizzle-orm"; +import { + createSharedPgTaskStoreTestHarness, + pgDescribe, + type SharedPgTaskStoreHarness, +} from "../../__test-utils__/pg-test-harness.js"; +import { + appendRecall, + getRecallRecord, + listRecall, + searchRecall, + setRecallAppendTestHooksForTest, +} from "../../memory/recall/recall-store.js"; +import { + RECALL_DEDUP_CANDIDATE_LIMIT, + recallContentHash, +} from "../../memory/recall/recall-dedup.js"; +import { + RECALL_SEARCH_DEFAULT_LIMIT, + RECALL_SEARCH_MAX_LIMIT, +} from "../../memory/recall/recall-search.js"; +import { project } from "../../postgres/schema/index.js"; +import { createAsyncDataLayer, type AsyncDataLayer } from "../../postgres/data-layer.js"; +import { createConnectionSetFromUrl } from "../../postgres/connection.js"; +import type { ResolvedBackend } from "../../postgres/backend-resolver.js"; + +pgDescribe("memory recall store (PostgreSQL)", () => { + const h: SharedPgTaskStoreHarness = createSharedPgTaskStoreTestHarness({ + prefix: "fusion_memory_recall", projectId: "recall-project-a", + }); + const layerA = () => Object.assign(h.layer(), { projectId: "recall-project-a" }); + const layerB = () => ({ ...h.layer(), projectId: "recall-project-b" }); + + beforeAll(h.beforeAll); + beforeEach(h.beforeEach); + afterEach(async () => { + setRecallAppendTestHooksForTest(undefined); + await h.afterEach(); + }); + afterAll(h.afterAll); + + async function independentLayer(projectId: string): Promise { + const backend: ResolvedBackend = { + mode: "external", runtimeUrl: h.testUrl(), migrationUrl: h.testUrl(), migrationUrlOverridden: false, + }; + const connections = await createConnectionSetFromUrl(backend, { poolMax: 1, connectTimeoutSeconds: 5 }); + return createAsyncDataLayer(connections, { projectId }); + } + + function deferred() { + let resolve!: () => void; + const promise = new Promise((resolvePromise) => { resolve = resolvePromise; }); + return { promise, resolve }; + } + + async function assertSerializedAppend(firstContent: string, secondContent: string): Promise { + const firstLayer = await independentLayer("recall-project-a"); + const secondLayer = await independentLayer("recall-project-a"); + const reachedCandidateRead = deferred(); + const releaseFirst = deferred(); + let held = false; + setRecallAppendTestHooksForTest({ afterCandidateRead: async () => { + if (!held) { + held = true; + reachedCandidateRead.resolve(); + await releaseFirst.promise; + } + } }); + try { + const first = appendRecall(firstLayer, { + kind: "decision", content: firstContent, source: { origin: "manual" }, + }); + await reachedCandidateRead.promise; + let secondSettled = false; + const second = appendRecall(secondLayer, { + kind: "decision", content: secondContent, source: { origin: "manual" }, + }).finally(() => { secondSettled = true; }); + // The second independent connection cannot reach its candidate read until the first lock releases. + await new Promise((resolve) => setTimeout(resolve, 30)); + expect(secondSettled).toBe(false); + releaseFirst.resolve(); + const [firstResult, secondResult] = await Promise.all([first, second]); + expect([firstResult.status, secondResult.status].sort()).toEqual(["created", "duplicate"]); + const records = await listRecall(layerA(), { kinds: ["decision"], limit: 50 }); + expect(records).toHaveLength(1); + const created = firstResult.status === "created" ? firstResult.record : secondResult.record; + const duplicate = firstResult.status === "duplicate" ? firstResult : secondResult; + expect(duplicate.duplicateOf.id).toBe(created.id); + } finally { + releaseFirst.resolve(); + await Promise.all([firstLayer.close(), secondLayer.close()]); + } + } + + it("applies the migration and preserves project-scoped typed records", async () => { + const tables = await h.adminDb().execute(sql` + SELECT table_name FROM information_schema.tables + WHERE table_schema = 'project' AND table_name = 'memory_recall_records' + `); + expect(tables).toHaveLength(1); + + for (const kind of ["decision", "preference", "solution"] as const) { + const result = await appendRecall(layerA(), { + kind, content: `${kind} content`, source: { + taskId: "FN-1", agentId: "agent-1", sessionId: "session-1", origin: "manual", + }, tags: ["memory", kind], graphNodeIds: kind === "decision" ? ["node-1"] : [], + }); + expect(result.status).toBe("created"); + if (result.status === "created") { + expect(result.record.source).toMatchObject({ taskId: "FN-1", origin: "manual" }); + expect(result.record.tags).toContain(kind); + expect(result.record.graphNodeIds).toEqual(kind === "decision" ? ["node-1"] : []); + expect(result.record.createdAt).toBeTruthy(); + expect(result.record.updatedAt).toBeTruthy(); + } + } + const records = await listRecall(layerA(), { limit: 50 }); + expect(records).toHaveLength(3); + expect(await searchRecall(layerB(), "content", { limit: 50 })).toMatchObject({ hits: [] }); + expect(await listRecall(layerB(), { limit: 50 })).toEqual([]); + expect(await getRecallRecord(layerB(), records[0]!.id)).toBeNull(); + }); + + it("serializes forced concurrent near-duplicate writers", async () => { + const original = "one two three four five six seven eight nine ten eleven twelve thirteen fourteen fifteen sixteen seventeen eighteen nineteen twenty"; + const restatement = "one two three four five six seven eight nine ten eleven twelve thirteen fourteen fifteen sixteen seventeen eighteen nineteen changed"; + await assertSerializedAppend(original, restatement); + }); + + it("serializes forced concurrent byte-identical writers", async () => { + await assertSerializedAppend("identical concurrent decision", "identical concurrent decision"); + }); + + it("uses ON CONFLICT recovery when the exact twin is outside the bounded candidate window", async () => { + const content = "window evicted exact duplicate"; + const original = await appendRecall(layerA(), { + kind: "decision", content, source: { origin: "manual" }, + }); + expect(original.status).toBe("created"); + if (original.status !== "created") return; + await h.adminDb().insert(project.memoryRecallRecords).values( + Array.from({ length: RECALL_DEDUP_CANDIDATE_LIMIT }, (_, index) => { + const filler = `unrelated filler ${index}`; + return { + projectId: "recall-project-a", id: `filler-${index}`, kind: "decision", content: filler, + contentHash: recallContentHash("decision", filler), source: { origin: "manual" }, tags: [], graphNodeIds: [], + createdAt: `2030-01-01T00:00:${String(index % 60).padStart(2, "0")}.000Z`, updatedAt: "2030-01-01T00:00:00.000Z", + }; + }), + ); + const duplicate = await appendRecall(layerA(), { + kind: "decision", content, source: { origin: "manual" }, + }); + expect(duplicate).toMatchObject({ + status: "duplicate", similarity: 1, duplicateOf: { id: original.record.id }, + }); + const rows = await h.adminDb().select({ total: sql`count(*)::int` }) + .from(project.memoryRecallRecords) + .where(eq(project.memoryRecallRecords.projectId, "recall-project-a")); + expect(rows[0]?.total).toBe(RECALL_DEDUP_CANDIDATE_LIMIT + 1); + }); + + it("demonstrates the unguarded near-duplicate race on two independent connections", async () => { + const firstLayer = await independentLayer("recall-project-a"); + const secondLayer = await independentLayer("recall-project-a"); + const bothRead = deferred(); + let readerCount = 0; + const content = ["unguarded alpha beta gamma", "unguarded alpha beta changed"]; + const unguardedInsert = (layer: AsyncDataLayer, index: number) => layer.transactionImmediate(async (tx) => { + await tx.select({ id: project.memoryRecallRecords.id }).from(project.memoryRecallRecords) + .where(and(eq(project.memoryRecallRecords.projectId, "recall-project-a"), eq(project.memoryRecallRecords.kind, "decision"))); + readerCount += 1; + if (readerCount === 2) bothRead.resolve(); + await bothRead.promise; + await tx.insert(project.memoryRecallRecords).values({ + projectId: "recall-project-a", id: `unguarded-${index}`, kind: "decision", content: content[index]!, + contentHash: recallContentHash("decision", content[index]!), source: { origin: "manual" }, tags: [], graphNodeIds: [], + createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(), + }); + }); + try { + await Promise.all([unguardedInsert(firstLayer, 0), unguardedInsert(secondLayer, 1)]); + const rows = await listRecall(layerA(), { kinds: ["decision"], limit: 50 }); + expect(rows).toHaveLength(2); + } finally { + await Promise.all([firstLayer.close(), secondLayer.close()]); + } + }); + + it("scopes writer locks by project and kind while reads remain non-blocking", async () => { + const decisionLayer = await independentLayer("recall-project-a"); + const otherProjectLayer = await independentLayer("recall-project-b"); + const reachedCandidateRead = deferred(); + const releaseDecision = deferred(); + let held = false; + setRecallAppendTestHooksForTest({ afterCandidateRead: async () => { + if (!held) { + held = true; + reachedCandidateRead.resolve(); + await releaseDecision.promise; + } + } }); + try { + const heldAppend = appendRecall(decisionLayer, { + kind: "decision", content: "held decision", source: { origin: "manual" }, + }); + await reachedCandidateRead.promise; + await expect(Promise.race([ + searchRecall(layerA(), "held", { limit: 2 }), + new Promise((_, reject) => setTimeout(() => reject(new Error("read was serialized")), 500)), + ])).resolves.toMatchObject({ hits: [] }); + const [solution, otherProject] = await Promise.all([ + appendRecall(layerA(), { kind: "solution", content: "other kind", source: { origin: "manual" } }), + appendRecall(otherProjectLayer, { kind: "decision", content: "other project", source: { origin: "manual" } }), + ]); + expect(solution.status).toBe("created"); + expect(otherProject.status).toBe("created"); + releaseDecision.resolve(); + expect((await heldAppend).status).toBe("created"); + } finally { + releaseDecision.resolve(); + await Promise.all([decisionLayer.close(), otherProjectLayer.close()]); + } + }); + + it("filters searches and re-enforces vector provider result limits", async () => { + for (let index = 0; index < 8; index += 1) { + await appendRecall(layerA(), { + kind: index % 2 ? "solution" : "decision", content: `alpha solution ${index}`, + source: { origin: "manual" }, tags: index % 2 ? ["alpha", "solution"] : ["alpha", "decision"], + }); + } + const keyword = await searchRecall(layerA(), "alpha", { kinds: ["solution"], tags: ["alpha"], limit: 2 }); + expect(keyword.mode).toBe("keyword"); + expect(keyword.hits).toHaveLength(2); + expect(keyword.hits.every((hit) => hit.record.kind === "solution")).toBe(true); + expect((await listRecall(layerA())).length).toBeLessThanOrEqual(RECALL_SEARCH_DEFAULT_LIMIT); + expect((await listRecall(layerA(), { limit: 10_000 })).length).toBeLessThanOrEqual(RECALL_SEARCH_MAX_LIMIT); + + let overReturningIds: string[] = []; + const provider = { + id: "over-returning", + search: async ({ candidates }: { candidates: readonly { id: string }[] }) => { + const matches = candidates.slice(0, 5).map((record, index) => ({ recordId: record.id, score: index + 1 })); + overReturningIds = matches.map((match) => match.recordId); + expect(matches.length).toBeGreaterThan(2); + return [...matches, { recordId: "unknown", score: 99 }, matches[0]!]; + }, + }; + const vector = await searchRecall(layerA(), "alpha", { limit: 2, vector: provider }); + expect(vector).toMatchObject({ mode: "vector", capabilities: { vector: true } }); + expect(vector.hits.map((hit) => hit.record.id)).toEqual(overReturningIds.slice(-2).reverse()); + expect(vector.hits).toHaveLength(2); + + const degraded = await searchRecall(layerA(), "alpha", { + limit: 2, vector: { id: "failing", search: async () => { throw new Error("offline"); } }, + }); + expect(degraded).toMatchObject({ mode: "keyword", capabilities: { vector: true } }); + expect(degraded.hits.length).toBeLessThanOrEqual(2); + + const unknownOnly = await searchRecall(layerA(), "alpha", { + limit: 2, vector: { id: "unknown-only", search: async () => [{ recordId: "missing", score: 1 }] }, + }); + expect(unknownOnly).toMatchObject({ mode: "keyword", capabilities: { vector: true } }); + + let candidateIds: string[] = []; + const pathological = await searchRecall(layerA(), "alpha", { + limit: 5, + vector: { + id: "pathological", + search: async ({ candidates }: { candidates: readonly { id: string }[] }) => { + candidateIds = candidates.map((candidate) => candidate.id); + return [ + ...candidates.flatMap((candidate, index) => [ + { recordId: candidate.id, score: index }, + { recordId: candidate.id, score: index + 100 }, + ]), + ...Array.from({ length: 20 }, (_, index) => ({ recordId: `unknown-${index}`, score: index + 500 })), + ]; + }, + }, + }); + expect(pathological).toMatchObject({ mode: "vector", capabilities: { vector: true } }); + expect(pathological.hits.length).toBeLessThanOrEqual(5); + expect(pathological.hits.length).toBeLessThanOrEqual(candidateIds.length); + expect(pathological.hits.every((hit) => candidateIds.includes(hit.record.id))).toBe(true); + expect(new Set(pathological.hits.map((hit) => hit.record.id)).size).toBe(pathological.hits.length); + }); +}); diff --git a/packages/core/src/__tests__/postgres/schema-applier.test.ts b/packages/core/src/__tests__/postgres/schema-applier.test.ts index 0405a7e2c6..83ef029dd0 100644 --- a/packages/core/src/__tests__/postgres/schema-applier.test.ts +++ b/packages/core/src/__tests__/postgres/schema-applier.test.ts @@ -94,6 +94,7 @@ import { AGENT_ACTIVITY_EVENTS_VERSION, SPEC_LOCK_DRIFT_REPORT_VERSION, SPEC_LOCK_SOURCE_REVISION_BIGINT_VERSION, + MEMORY_RECALL_RECORDS_VERSION, } from "../../postgres/schema-applier.js"; import { ProjectPartitionRekeyError, rekeyFallbackProjectPartition } from "../../postgres/migration-stamping.js"; import type { PluginSchemaInitHook } from "../../postgres/plugin-schema-hook.js"; @@ -121,7 +122,8 @@ describe("schema-applier: immutable migration identities", () => { expect(AGENT_ACTIVITY_EVENTS_VERSION).toBe("0049"); expect(SPEC_LOCK_DRIFT_REPORT_VERSION).toBe("0050"); expect(SPEC_LOCK_SOURCE_REVISION_BIGINT_VERSION).toBe("0051"); - expect(SCHEMA_BASELINE_VERSION).toBe("0051"); + expect(MEMORY_RECALL_RECORDS_VERSION).toBe("0052"); + expect(SCHEMA_BASELINE_VERSION).toBe("0052"); }); it("keeps monitor and approval isolation assigned to version 0003", () => { @@ -736,7 +738,7 @@ pgDescribe("schema-applier: VAL-SCHEMA-001 final-schema parity (table counts)", ctx = null; }); - it("creates all 112 project tables, 17 central tables, 1 archive table", async () => { + it("creates all 113 project tables, 17 central tables, 1 archive table", async () => { ctx = await setupFreshDb(); // FNXC:PostgresCutover 2026-07-05-15:55: apply the BASELINE only. // applySchemaBaseline now runs the plugin schema-init hooks by default, @@ -757,10 +759,10 @@ pgDescribe("schema-applier: VAL-SCHEMA-001 final-schema parity (table counts)", outbox tables; 0041 adds 4 lifecycle consumer tables; 0043 adds the durable unplanned-dispatch refusal marker (100 → 105); later baseline additions bring the count to 106; and 0048 adds GitHub check state (106 → 107); 0049 adds the agent-activity outbox and counter (→ 109); - 0050 adds immutable lock, evidence, and report history (109 → 112). Plugin tables are added separately + 0050 adds immutable lock, evidence, and report history (109 → 112); 0052 adds recall records (→ 113). Plugin tables are added separately by the schema-init hook and are excluded here. */ - expect(bySchema.project).toBe(112); + expect(bySchema.project).toBe(113); /* FNXC:CapacityModel 2026-07-29-08:10 (drop the cross-project cap — table half): 17, not 18: `central.global_concurrency` is dropped by migration 0037. A fresh @@ -1788,6 +1790,7 @@ pgDescribe("schema-applier: automation project-isolation upgrade", () => { AGENT_ACTIVITY_EVENTS_VERSION, SPEC_LOCK_DRIFT_REPORT_VERSION, SPEC_LOCK_SOURCE_REVISION_BIGINT_VERSION, + MEMORY_RECALL_RECORDS_VERSION, ]); expect((await applySchemaBaseline(ctx.db, { pluginHooks: [] })).applied).toBe(false); }); @@ -1865,6 +1868,7 @@ pgDescribe("schema-applier: automation project-isolation upgrade", () => { AGENT_ACTIVITY_EVENTS_VERSION, SPEC_LOCK_DRIFT_REPORT_VERSION, SPEC_LOCK_SOURCE_REVISION_BIGINT_VERSION, + MEMORY_RECALL_RECORDS_VERSION, ]); }); @@ -2075,6 +2079,7 @@ pgDescribe("schema-applier: automation project-isolation upgrade", () => { AGENT_ACTIVITY_EVENTS_VERSION, SPEC_LOCK_DRIFT_REPORT_VERSION, SPEC_LOCK_SOURCE_REVISION_BIGINT_VERSION, + MEMORY_RECALL_RECORDS_VERSION, ]); }); @@ -2166,6 +2171,7 @@ pgDescribe("schema-applier: automation project-isolation upgrade", () => { AGENT_ACTIVITY_EVENTS_VERSION, SPEC_LOCK_DRIFT_REPORT_VERSION, SPEC_LOCK_SOURCE_REVISION_BIGINT_VERSION, + MEMORY_RECALL_RECORDS_VERSION, ]); }); @@ -2257,6 +2263,7 @@ pgDescribe("schema-applier: automation project-isolation upgrade", () => { AGENT_ACTIVITY_EVENTS_VERSION, SPEC_LOCK_DRIFT_REPORT_VERSION, SPEC_LOCK_SOURCE_REVISION_BIGINT_VERSION, + MEMORY_RECALL_RECORDS_VERSION, ]); }); }); diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index dd727f9f68..fe12f8134d 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -2779,3 +2779,4 @@ export type { AgentActivityEventType, AgentActivityAttribution, AgentActivityIdP export { AGENT_ACTIVITY_EVENT_TYPES, AGENT_ACTIVITY_ATTRIBUTIONS, AGENT_ACTIVITY_LANE_SENTINELS, AGENT_ACTIVITY_GENERATED_ID_PATTERNS, AGENT_ACTIVITY_HANDOFF_REASONS, AGENT_ACTIVITY_TOOL_NAMES, AGENT_ACTIVITY_WORKFLOW_STEP_IDS, AGENT_ACTIVITY_METADATA_SCHEMA, AGENT_ACTIVITY_METADATA_KEYS, isAgentActivityEventType } from "./types/agents/agents.js"; export { appendAgentActivityEvent, queryAgentActivityEvents, getMaxAgentActivitySeq, pruneAgentActivityEvents } from "./task-store/async/async-agent-activity.js"; export { makeAgentActivityEventId, resolveAgentActivityAttribution, agentIdExistsInRoster, formatAgentActivitySummary, sanitizeAgentActivityMetadata } from "./task-store/agent-activity-outbox.js"; +export * from "./memory/recall/index.js"; diff --git a/packages/core/src/memory/index.ts b/packages/core/src/memory/index.ts index 990644494b..594d41b885 100644 --- a/packages/core/src/memory/index.ts +++ b/packages/core/src/memory/index.ts @@ -8,3 +8,4 @@ export * from "./memory-compaction.js"; export * from "./memory-dreams.js"; export * from "./memory-insights.js"; export * from "./project-memory.js"; +export * from "./recall/index.js"; diff --git a/packages/core/src/memory/project-memory.ts b/packages/core/src/memory/project-memory.ts index 58a9576882..e93b65217f 100644 --- a/packages/core/src/memory/project-memory.ts +++ b/packages/core/src/memory/project-memory.ts @@ -33,6 +33,8 @@ import { type MemoryGetOptions, type MemoryGetResult, } from "./memory-backend.js"; +import { appendRecallInstructionSection } from "./recall/recall-instructions.js"; +import type { RecallSearchHit } from "./recall/recall-types.js"; // ── Default Scaffold ───────────────────────────────────────────────── @@ -337,6 +339,7 @@ export async function getProjectMemory( export function buildTriageMemoryInstructions( rootDir: string, settings?: MemorySettings, + recallHits?: readonly RecallSearchHit[], ): string { void rootDir; // Parameter kept for future use (e.g., checking file existence) const ctx = resolveMemoryInstructionContext(settings); @@ -348,7 +351,7 @@ export function buildTriageMemoryInstructions( if (!ctx.capabilities.writable) { // Read-only backend: consult memory for context but don't mention file path - return ` + return appendRecallInstructionSection(` ## Project Memory This project has a memory system that stores durable project learnings. @@ -356,13 +359,13 @@ This project has a memory system that stores durable project learnings. **Before writing the specification:** 1. Consult the project memory for relevant context 2. Incorporate any useful learnings into your specification -`; +`, recallHits); } // Writable backend (file or qmd) if (ctx.instructionPathHint) { // File backend: mention the explicit path - return ` + return appendRecallInstructionSection(` ## Project Memory This project has OpenClaw-style memory files: @@ -379,11 +382,11 @@ This project has OpenClaw-style memory files: - Do **not** write \`.fusion/memory/MEMORY.md\`, \`.fusion/memory/YYYY-MM-DD.md\`, or any other memory files directly when \`fn_memory_append\` is available Do not read all memory directly by default. If memory is irrelevant, skip it. -`; +`, recallHits); } // QMD/non-file writable backend: generic instructions without specific path - return ` + return appendRecallInstructionSection(` ## Project Memory This project has a memory system that stores durable project learnings. @@ -398,7 +401,7 @@ This project has a memory system that stores durable project learnings. - Do **not** write memory files directly when \`fn_memory_append\` is available **If the memory contains useful context for this task, reference it in the specification.** -`; +`, recallHits); } /** @@ -423,6 +426,7 @@ This project has a memory system that stores durable project learnings. export function buildExecutionMemoryInstructions( rootDir: string, settings?: MemorySettings, + recallHits?: readonly RecallSearchHit[], ): string { void rootDir; // Parameter kept for future use (e.g., checking file existence) const ctx = resolveMemoryInstructionContext(settings); @@ -434,7 +438,7 @@ export function buildExecutionMemoryInstructions( if (!ctx.capabilities.writable) { // Read-only backend: consult memory for context but no update instructions - return ` + return appendRecallInstructionSection(` ## Project Memory This project has a memory system that stores durable project learnings. @@ -442,13 +446,13 @@ This project has a memory system that stores durable project learnings. **At the start of execution:** 1. Consult the project memory for relevant context 2. Apply any useful learnings to your implementation -`; +`, recallHits); } // Writable backend (file or qmd) if (ctx.instructionPathHint) { // File backend: mention the explicit path with full read/write instructions - return ` + return appendRecallInstructionSection(` ## Project Memory This project has OpenClaw-style memory files: @@ -479,11 +483,11 @@ This project has OpenClaw-style memory files: - Use \`- \` prefix for list items - Keep entries concise and actionable - Example: \`- The API layer uses Zod schemas for all request validation\` -`; +`, recallHits); } // QMD/non-file writable backend: generic instructions without specific path - return ` + return appendRecallInstructionSection(` ## Project Memory This project has a memory system that stores durable project learnings accumulated from past task runs. @@ -504,12 +508,13 @@ This project has a memory system that stores durable project learnings accumulat 4. **If nothing durable was learned, skip the memory update entirely** — do not append trivial or task-specific notes 5. **Avoid task-specific trivia** in project scope (for example: personal reminders, one-off scratch thoughts, individual communication preferences) 6. Consolidate when possible: refine an existing memory entry instead of adding duplicates. -`; +`, recallHits); } export function buildReviewerMemoryInstructions( rootDir: string, settings?: MemorySettings, + recallHits?: readonly RecallSearchHit[], ): string { void rootDir; const ctx = resolveMemoryInstructionContext(settings); @@ -517,7 +522,7 @@ export function buildReviewerMemoryInstructions( return ""; } - return ` + return appendRecallInstructionSection(` ## Project Memory This project has a memory system that stores durable project learnings. @@ -528,7 +533,7 @@ This project has a memory system that stores durable project learnings. 3. Treat documented durable conventions and pitfalls as review evidence when deciding APPROVE, REVISE, or RETHINK 4. Do not update memory during review; reviewer memory access is read-only 5. Skip memory reads when they are not relevant to the reviewed plan or code -`; +`, recallHits); } /** diff --git a/packages/core/src/memory/recall/index.ts b/packages/core/src/memory/recall/index.ts new file mode 100644 index 0000000000..601c7857d4 --- /dev/null +++ b/packages/core/src/memory/recall/index.ts @@ -0,0 +1,10 @@ +export * from "./recall-types.js"; +export * from "./recall-dedup.js"; +export * from "./recall-search.js"; +export { + appendRecall, + deleteRecallRecord, + getRecallRecord, + listRecall, + searchRecall, +} from "./recall-store.js"; diff --git a/packages/core/src/memory/recall/recall-dedup.ts b/packages/core/src/memory/recall/recall-dedup.ts new file mode 100644 index 0000000000..80594d2b0a --- /dev/null +++ b/packages/core/src/memory/recall/recall-dedup.ts @@ -0,0 +1,35 @@ +import { createHash } from "node:crypto"; +import type { RecallKind, RecallRecord } from "./recall-types.js"; + +export const RECALL_DEDUP_SIMILARITY_THRESHOLD = 0.9; +export const RECALL_DEDUP_CANDIDATE_LIMIT = 200; + +/* +FNXC:MemoryRecall 2026-08-10-11:03: +Dedup compares normalized same-kind content in a bounded recent window. An older exact twin can fall +outside that window, so the database exact-hash constraint remains a required backstop. +*/ +export function normalizeRecallContent(content: string): string { + return content.trim().toLowerCase().replace(/\s+/g, " ").replace(/[.!?,;:]+$/g, ""); +} +export function recallContentHash(kind: RecallKind, content: string): string { + return createHash("sha256").update(`${kind}\0${normalizeRecallContent(content)}`).digest("hex"); +} +function tokens(content: string): Set { return new Set(normalizeRecallContent(content).split(/[^\p{L}\p{N}_-]+/u).filter(Boolean)); } +export function recallSimilarity(a: string, b: string): number { + const left = tokens(a), right = tokens(b); const union = new Set([...left, ...right]); + if (!union.size) return 1; + let common = 0; for (const token of left) if (right.has(token)) common += 1; + return common / union.size; +} +export function classifyRecallDuplicate(candidate: Pick, existingRecords: readonly RecallRecord[]) { + for (const record of existingRecords) { + if (record.kind !== candidate.kind) continue; + if (record.contentHash === candidate.contentHash) return { record, similarity: 1 }; + const similarity = recallSimilarity(candidate.content, record.content); + if (similarity >= RECALL_DEDUP_SIMILARITY_THRESHOLD) return { record, similarity }; + } + return undefined; +} +/* FNXC:MemoryRecall 2026-08-10-11:03: Classification is read-then-write and unsafe without serializing this key across its candidate read and insert. */ +export function recallDedupLockKey(projectId: string, kind: RecallKind): string { return `fusion:memory-recall:${projectId}:${kind}`; } diff --git a/packages/core/src/memory/recall/recall-instructions.ts b/packages/core/src/memory/recall/recall-instructions.ts new file mode 100644 index 0000000000..b6474dbae1 --- /dev/null +++ b/packages/core/src/memory/recall/recall-instructions.ts @@ -0,0 +1,59 @@ +import type { RecallSearchHit } from "./recall-types.js"; + +/* +FNXC:MemoryRecall 2026-08-10-11:03: +This mirrors engine agent-memory-index's UTF-8 clamp. The 800-byte budget covers only the complete +recall section appended to a prompt, including its heading, lines, separator, and trailing newline; +it never truncates pre-existing prompt instructions. +*/ +export const MAX_RECALL_INJECTION_BYTES = 800; +const RECALL_LINE_MAX_BYTES = 360; + +/** Clamp on code points so UTF-8 characters are never split. */ +function clampUtf8(value: string, maxBytes: number): string { + if (Buffer.byteLength(value, "utf8") <= maxBytes) return value; + let output = ""; + for (const character of value) { + if (Buffer.byteLength(output + character, "utf8") > maxBytes) break; + output += character; + } + return output; +} + +function formatRecallInstructionSectionWithinBudget( + hits: readonly RecallSearchHit[], + maxBytes: number, +): string { + if (!hits.length || maxBytes <= 0) return ""; + const lines = ["### Recalled Context"]; + for (const hit of hits) { + const line = clampUtf8(`- [${hit.record.kind}] ${hit.record.content.trim()}`, RECALL_LINE_MAX_BYTES); + const proposed = `${lines.join("\n")}\n${line}\n`; + if (Buffer.byteLength(proposed, "utf8") > maxBytes) break; + lines.push(line); + } + // Do not inject a heading with no usable recalled content. + return lines.length === 1 ? "" : `${lines.join("\n")}\n`; +} + +/** Render a standalone recall section within the recall-only byte budget. */ +export function formatRecallInstructionSection(hits: readonly RecallSearchHit[]): string { + return formatRecallInstructionSectionWithinBudget(hits, MAX_RECALL_INJECTION_BYTES); +} + +/** + * Append recall without charging its separator to existing instructions. + * The section formatter receives the remaining budget, so the complete appended delta is bounded. + */ +export function appendRecallInstructionSection( + instructions: string, + hits?: readonly RecallSearchHit[], +): string { + if (!instructions || !hits?.length) return instructions; + const separator = instructions.endsWith("\n") ? "\n" : "\n\n"; + const section = formatRecallInstructionSectionWithinBudget( + hits, + MAX_RECALL_INJECTION_BYTES - Buffer.byteLength(separator, "utf8"), + ); + return section ? `${instructions}${separator}${section}` : instructions; +} diff --git a/packages/core/src/memory/recall/recall-search.ts b/packages/core/src/memory/recall/recall-search.ts new file mode 100644 index 0000000000..7cb85c0656 --- /dev/null +++ b/packages/core/src/memory/recall/recall-search.ts @@ -0,0 +1,20 @@ +import type { RecallCapabilities, RecallRecord, RecallSearchHit } from "./recall-types.js"; +export const RECALL_SEARCH_DEFAULT_LIMIT = 10; +export const RECALL_SEARCH_MAX_LIMIT = 50; +export interface RecallVectorMatch { recordId: string; score: number; } +export interface RecallVectorSearchProvider { readonly id: string; search(input: { query: string; candidates: readonly RecallRecord[]; limit: number }): Promise; } +export interface RecallSearchOptions { kinds?: readonly RecallRecord["kind"][]; tags?: readonly string[]; limit?: number | null; vector?: RecallVectorSearchProvider; } +/* FNXC:MemoryRecall 2026-08-10-11:03: All hit-producing paths use this one clamp; malformed, zero, and non-integral values resolve to the default, preserving bounded deterministic reads. */ +export function clampRecallSearchLimit(requested?: number | null): number { return typeof requested !== "number" || !Number.isFinite(requested) || !Number.isInteger(requested) || requested <= 0 ? RECALL_SEARCH_DEFAULT_LIMIT : Math.min(requested, RECALL_SEARCH_MAX_LIMIT); } +export function resolveRecallCapabilities(provider?: RecallVectorSearchProvider): RecallCapabilities { return { keyword: true, vector: Boolean(provider), persistent: true }; } +function terms(value: string): string[] { return [...new Set(value.toLowerCase().split(/[^\p{L}\p{N}_-]+/u).filter(Boolean))]; } +export function searchRecallKeyword(query: string, candidates: readonly RecallRecord[], limit: number): RecallSearchHit[] { + const queryTerms = terms(query); + return candidates.map((record) => { const content = new Set(terms(record.content)); const tags = new Set(record.tags.flatMap(terms)); const matchedTerms = queryTerms.filter((term) => content.has(term) || tags.has(term)); const score = queryTerms.reduce((sum, term) => sum + (content.has(term) ? 1 : 0) + (tags.has(term) ? 2 : 0), 0); return { record, score, matchedTerms }; }).filter((hit) => hit.score > 0).sort((a,b) => b.score-a.score || b.record.createdAt.localeCompare(a.record.createdAt) || a.record.id.localeCompare(b.record.id)).slice(0, limit); +} +/* FNXC:MemoryRecall 2026-08-10-11:03: Providers rank untrusted candidate ids only. Store-side deduplication and post-ranking slicing prevent excess, duplicate, or unknown matches from escaping the common cap. */ +export function applyRecallVectorRanking(candidates: readonly RecallRecord[], matches: readonly RecallVectorMatch[], limit: number): RecallSearchHit[] { + const byId = new Map(candidates.map((record) => [record.id, record])); const best = new Map(); + for (const match of matches) if (byId.has(match.recordId) && Number.isFinite(match.score) && (!best.has(match.recordId) || best.get(match.recordId)!.score < match.score)) best.set(match.recordId, match); + return [...best.values()].map((match) => ({ record: byId.get(match.recordId)!, score: match.score, matchedTerms: [] })).sort((a,b) => b.score-a.score || b.record.createdAt.localeCompare(a.record.createdAt) || a.record.id.localeCompare(b.record.id)).slice(0, limit); +} diff --git a/packages/core/src/memory/recall/recall-store.ts b/packages/core/src/memory/recall/recall-store.ts new file mode 100644 index 0000000000..3db09c8ca0 --- /dev/null +++ b/packages/core/src/memory/recall/recall-store.ts @@ -0,0 +1,51 @@ +import { randomUUID } from "node:crypto"; +import { and, desc, eq, inArray, sql } from "drizzle-orm"; +import type { AsyncDataLayer } from "../../postgres/data-layer.js"; +import { project } from "../../postgres/schema/index.js"; +import { MemoryBackendError } from "../memory-backend.js"; +import { RECALL_DEDUP_CANDIDATE_LIMIT, classifyRecallDuplicate, recallContentHash, recallDedupLockKey } from "./recall-dedup.js"; +import { applyRecallVectorRanking, clampRecallSearchLimit, resolveRecallCapabilities, searchRecallKeyword, type RecallSearchOptions } from "./recall-search.js"; +import type { RecallAppendInput, RecallAppendResult, RecallRecord, RecallSearchResult } from "./recall-types.js"; + +type RecallRow = typeof project.memoryRecallRecords.$inferSelect; + +type RecallAppendTestHooks = { + afterCandidateRead?: () => void | Promise; +}; + +// Intentionally omitted from the recall barrel: PG tests use this to force the lock interleaving. +let appendRecallTestHooks: RecallAppendTestHooks | undefined; +export function setRecallAppendTestHooksForTest(hooks: RecallAppendTestHooks | undefined): void { + appendRecallTestHooks = hooks; +} + +const map = (row: RecallRow): RecallRecord => ({ ...row, kind: row.kind as RecallRecord["kind"], source: row.source as RecallRecord["source"], tags: row.tags as string[], graphNodeIds: row.graphNodeIds as string[] }); +function projectId(layer: AsyncDataLayer): string { if (!layer.projectId) throw new MemoryBackendError("BACKEND_UNAVAILABLE", "Recall requires AsyncDataLayer.projectId", "recall"); return layer.projectId; } + +/* +FNXC:MemoryRecall 2026-08-10-11:03: +The advisory transaction lock is first, spans candidate read and insert, and is namespaced by project/kind. +It is the sole lock in this path, preventing read-then-write near-duplicate races without serializing other kinds. +ON CONFLICT DO NOTHING preserves transaction usability: raising 23505 would poison a re-select; the branch is reachable +when an exact twin falls outside the bounded candidate window or a bypassing writer races this transaction. +*/ +export async function appendRecall(layer: AsyncDataLayer, input: RecallAppendInput): Promise { + const pid = projectId(layer); const now = new Date().toISOString(); const contentHash = recallContentHash(input.kind, input.content); const table = project.memoryRecallRecords; + return layer.transactionImmediate(async (tx) => { + await tx.execute(sql`SELECT pg_advisory_xact_lock(hashtext(${recallDedupLockKey(pid, input.kind)}))`); + const candidates = (await tx.select().from(table).where(and(eq(table.projectId, pid), eq(table.kind, input.kind))).orderBy(desc(table.createdAt)).limit(RECALL_DEDUP_CANDIDATE_LIMIT)).map(map); + await appendRecallTestHooks?.afterCandidateRead?.(); + const duplicate = classifyRecallDuplicate({ kind: input.kind, content: input.content, contentHash }, candidates); + if (duplicate) return { status: "duplicate", duplicateOf: duplicate.record, similarity: duplicate.similarity }; + const inserted = await tx.insert(table).values({ projectId: pid, id: randomUUID(), kind: input.kind, content: input.content, contentHash, source: input.source, tags: input.tags ?? [], graphNodeIds: input.graphNodeIds ?? [], createdAt: now, updatedAt: now }).onConflictDoNothing({ target: [table.projectId, table.kind, table.contentHash] }).returning(); + if (inserted[0]) return { status: "created", record: map(inserted[0]) }; + const exact = await tx.select().from(table).where(and(eq(table.projectId, pid), eq(table.kind, input.kind), eq(table.contentHash, contentHash))).limit(1); + if (!exact[0]) throw new MemoryBackendError("CONFLICT", "Recall exact-hash conflict disappeared before recovery", "recall"); + return { status: "duplicate", duplicateOf: map(exact[0]), similarity: 1 }; + }); +} +export async function getRecallRecord(layer: AsyncDataLayer, id: string): Promise { const pid=projectId(layer); const row=await layer.db.select().from(project.memoryRecallRecords).where(and(eq(project.memoryRecallRecords.projectId,pid),eq(project.memoryRecallRecords.id,id))).limit(1); return row[0] ? map(row[0]) : null; } +export async function listRecall(layer: AsyncDataLayer, options?: { kinds?: RecallRecord["kind"][]; limit?: number | null }): Promise { const pid=projectId(layer), t=project.memoryRecallRecords, limit=clampRecallSearchLimit(options?.limit); const rows=await layer.db.select().from(t).where(and(eq(t.projectId,pid), options?.kinds?.length ? inArray(t.kind, options.kinds) : undefined)).orderBy(desc(t.createdAt)).limit(limit); return rows.map(map); } +export async function deleteRecallRecord(layer: AsyncDataLayer, id: string): Promise { const pid=projectId(layer); const rows=await layer.db.delete(project.memoryRecallRecords).where(and(eq(project.memoryRecallRecords.projectId,pid),eq(project.memoryRecallRecords.id,id))).returning({ id: project.memoryRecallRecords.id }); return Boolean(rows[0]); } +/* FNXC:MemoryRecall 2026-08-10-11:03: Vector providers only rank store-filtered candidates; failures degrade to deterministic keyword hits while the shared clamp remains authoritative after ranking. */ +export async function searchRecall(layer: AsyncDataLayer, query: string, options?: RecallSearchOptions): Promise { const pid=projectId(layer), t=project.memoryRecallRecords, limit=clampRecallSearchLimit(options?.limit); let rows=(await layer.db.select().from(t).where(and(eq(t.projectId,pid), options?.kinds?.length ? inArray(t.kind, [...options.kinds]) : undefined)).orderBy(desc(t.createdAt))).map(map); if(options?.tags?.length) rows=rows.filter((r)=>options.tags!.every((tag)=>r.tags.includes(tag))); const keyword=searchRecallKeyword(query,rows,limit), capabilities=resolveRecallCapabilities(options?.vector); if(!options?.vector) return {hits:keyword,mode:"keyword",capabilities}; try { const matches=await options.vector.search({query,candidates:rows,limit}); const hits=applyRecallVectorRanking(rows,matches,limit); return hits.length ? {hits,mode:"vector",capabilities} : {hits:keyword,mode:"keyword",capabilities}; } catch { return {hits:keyword,mode:"keyword",capabilities}; } } diff --git a/packages/core/src/memory/recall/recall-types.ts b/packages/core/src/memory/recall/recall-types.ts new file mode 100644 index 0000000000..b8138358b2 --- /dev/null +++ b/packages/core/src/memory/recall/recall-types.ts @@ -0,0 +1,14 @@ +/* +FNXC:MemoryRecall 2026-08-10-11:03: +Recall records retain durable decisions, preferences, and solutions separately from markdown memory so +callers can filter provenance without coupling this layer to capture automation or an MCP surface. +*/ +export type RecallKind = "decision" | "preference" | "solution"; +export type RecallOrigin = "chat" | "council" | "deep-research" | "task-completion" | "manual" | "other"; +export interface RecallSource { origin: RecallOrigin; taskId?: string; agentId?: string; sessionId?: string; } +export interface RecallRecord { id: string; projectId: string; kind: RecallKind; content: string; contentHash: string; source: RecallSource; tags: string[]; graphNodeIds: string[]; createdAt: string; updatedAt: string; } +export interface RecallAppendInput { kind: RecallKind; content: string; source: RecallSource; tags?: string[]; graphNodeIds?: string[]; } +export type RecallAppendResult = { status: "created"; record: RecallRecord } | { status: "duplicate"; duplicateOf: RecallRecord; similarity: number }; +export interface RecallCapabilities { keyword: true; vector: boolean; persistent: true; } +export interface RecallSearchHit { record: RecallRecord; score: number; matchedTerms: string[]; } +export interface RecallSearchResult { hits: RecallSearchHit[]; mode: "keyword" | "vector"; capabilities: RecallCapabilities; } diff --git a/packages/core/src/postgres/migrations/0052_fn_8922_memory_recall_records.sql b/packages/core/src/postgres/migrations/0052_fn_8922_memory_recall_records.sql new file mode 100644 index 0000000000..ebf2878857 --- /dev/null +++ b/packages/core/src/postgres/migrations/0052_fn_8922_memory_recall_records.sql @@ -0,0 +1,17 @@ +/* FNXC:MemoryRecall 2026-08-10-11:03: Exact hashes are a last-resort backstop; near-duplicate safety is provided by appendRecall's advisory lock. */ +CREATE TABLE IF NOT EXISTS project.memory_recall_records ( + project_id text NOT NULL DEFAULT current_setting('fusion.project_id', true), id text NOT NULL, + kind text NOT NULL, content text NOT NULL, content_hash text NOT NULL, source jsonb NOT NULL, + tags jsonb NOT NULL DEFAULT '[]'::jsonb, graph_node_ids jsonb NOT NULL DEFAULT '[]'::jsonb, + created_at text NOT NULL, updated_at text NOT NULL, + PRIMARY KEY (project_id, id), + CONSTRAINT memory_recall_records_project_kind_hash_key UNIQUE (project_id, kind, content_hash), + CONSTRAINT memory_recall_records_kind_check CHECK (kind IN ('decision','preference','solution')) +); +CREATE INDEX IF NOT EXISTS "idxMemoryRecallRecordsKindCreated" ON project.memory_recall_records(project_id, kind, created_at DESC); +CREATE INDEX IF NOT EXISTS "idxMemoryRecallRecordsCreated" ON project.memory_recall_records(project_id, created_at DESC); +ALTER TABLE project.memory_recall_records ENABLE ROW LEVEL SECURITY; ALTER TABLE project.memory_recall_records FORCE ROW LEVEL SECURITY; +DROP POLICY IF EXISTS fusion_project_isolation ON project.memory_recall_records; +CREATE POLICY fusion_project_isolation ON project.memory_recall_records USING (current_setting('fusion.project_bypass', true) = 'on' OR project_id = current_setting('fusion.project_id', true)) WITH CHECK (current_setting('fusion.project_bypass', true) = 'on' OR project_id = current_setting('fusion.project_id', true)); +DROP TRIGGER IF EXISTS fusion_assign_project_id ON project.memory_recall_records; +CREATE TRIGGER fusion_assign_project_id BEFORE INSERT OR UPDATE OF project_id ON project.memory_recall_records FOR EACH ROW EXECUTE FUNCTION project.fusion_assign_project_id(); diff --git a/packages/core/src/postgres/schema-applier.ts b/packages/core/src/postgres/schema-applier.ts index 8b28749d50..6a4d766c6b 100644 --- a/packages/core/src/postgres/schema-applier.ts +++ b/packages/core/src/postgres/schema-applier.ts @@ -59,7 +59,8 @@ capacity-model table drop that landed while this PR was open. /* FNXC:PrMergeEventDrivenChecks 2026-08-09-14:35: 0048 registers project-scoped GitHub CI check state. */ /** FNXC:AgentActivityStream 2026-08-09-21:32: 0049 follows the landed 0048 GitHub check-state migration so upgraded projects receive the durable activity outbox. */ /* FNXC:SpecLock 2026-08-09-18:17: 0050 stores immutable plan history and 0051 widens source revisions before Date.now()-based writes. */ -export const SCHEMA_BASELINE_VERSION = "0051"; +/* FNXC:MemoryRecall 2026-08-10-11:03: Explicit baseline registration prevents the recall migration from being silently skipped. */ +export const SCHEMA_BASELINE_VERSION = "0052"; /** FNXC:SymbolLock 2026-07-20-10:00: upgrades need durable task declarations before admission resolves symbols. */ export const TASK_DECLARED_SYMBOLS_VERSION = "0028"; const INITIAL_SCHEMA_VERSION = "0000"; @@ -204,6 +205,8 @@ export const AGENT_ACTIVITY_EVENTS_VERSION = "0049"; export const SPEC_LOCK_DRIFT_REPORT_VERSION = "0050"; /** FNXC:SpecLock 2026-08-09-18:17: widen source revisions before Date.now()-based current-plan writes overflow integer storage. */ export const SPEC_LOCK_SOURCE_REVISION_BIGINT_VERSION = "0051"; +/** FNXC:MemoryRecall 2026-08-10-11:03: explicit migration bookkeeping for project recall rows. */ +export const MEMORY_RECALL_RECORDS_VERSION = "0052"; /** SECURITY DEFINER helper that only inserts LEGACY_ADOPTION_DRAINED_MARKER. */ export const LEGACY_ADOPTION_DRAINED_MARKER_FUNCTION = "fusion_mark_legacy_adoption_drained"; @@ -429,6 +432,7 @@ const GITHUB_CHECK_STATES_MIGRATION_PATH = join(MIGRATIONS_DIR, "0048_fn_8903_gi const AGENT_ACTIVITY_EVENTS_MIGRATION_PATH = join(MIGRATIONS_DIR, "0049_fn_8864_agent_activity_events.sql"); const SPEC_LOCK_DRIFT_REPORT_MIGRATION_PATH = join(MIGRATIONS_DIR, "0050_spec_lock_drift_report.sql"); const SPEC_LOCK_SOURCE_REVISION_BIGINT_MIGRATION_PATH = join(MIGRATIONS_DIR, "0051_spec_lock_source_revision_bigint.sql"); +const MEMORY_RECALL_RECORDS_MIGRATION_PATH = join(MIGRATIONS_DIR, "0052_fn_8922_memory_recall_records.sql"); /** * Ensure the migration bookkeeping table exists. Lives in the public schema so @@ -550,6 +554,7 @@ export async function applySchemaBaseline( const agentActivityEventsAlreadyApplied = applied.includes(AGENT_ACTIVITY_EVENTS_VERSION); const specLockDriftReportAlreadyApplied = applied.includes(SPEC_LOCK_DRIFT_REPORT_VERSION); const specLockSourceRevisionBigintAlreadyApplied = applied.includes(SPEC_LOCK_SOURCE_REVISION_BIGINT_VERSION); + const memoryRecallRecordsAlreadyApplied = applied.includes(MEMORY_RECALL_RECORDS_VERSION); assertBinaryNotOlderThanDatabase(applied); let schemaChanged = false; @@ -1208,6 +1213,12 @@ export async function applySchemaBaseline( await tx.execute(sql`INSERT INTO public.${sql.identifier(MIGRATION_BOOKKEEPING_TABLE)} (version) VALUES (${SPEC_LOCK_SOURCE_REVISION_BIGINT_VERSION}) ON CONFLICT (version) DO NOTHING`); schemaChanged = true; } + if (!memoryRecallRecordsAlreadyApplied) { + const migrationSql = await readFile(MEMORY_RECALL_RECORDS_MIGRATION_PATH, "utf8"); + await tx.execute(sql.raw(migrationSql)); + await tx.execute(sql`INSERT INTO public.${sql.identifier(MIGRATION_BOOKKEEPING_TABLE)} (version) VALUES (${MEMORY_RECALL_RECORDS_VERSION}) ON CONFLICT (version) DO NOTHING`); + schemaChanged = true; + } return { applied: schemaChanged, pluginHooksRun: pluginHooks.length }; }); } diff --git a/packages/core/src/postgres/schema/project.ts b/packages/core/src/postgres/schema/project.ts index 0c79deaa8e..4df5e269b7 100644 --- a/packages/core/src/postgres/schema/project.ts +++ b/packages/core/src/postgres/schema/project.ts @@ -620,6 +620,14 @@ export const agentActivityEvents = projectSchema.table("agent_activity_events", index("idxAgentActivityEventsSeq").on(t.projectId, t.seq), index("idxAgentActivityEventsAgentSeq").on(t.projectId, t.agentId, t.seq), index("idxAgentActivityEventsTaskSeq").on(t.projectId, t.taskId, t.seq), index("idxAgentActivityEventsTypeSeq").on(t.projectId, t.type, t.seq), ]); +/* FNXC:MemoryRecall 2026-08-10-11:03: Project-scoped recall keeps structured durable context isolated by the same composite key and RLS contract as other project rows. */ +export const memoryRecallRecords = projectSchema.table("memory_recall_records", { + projectId: text("project_id").notNull().default(sql`current_setting('fusion.project_id', true)`), + id: text("id").notNull(), kind: text("kind").notNull(), content: text("content").notNull(), contentHash: text("content_hash").notNull(), + source: jsonb("source").notNull(), tags: jsonb("tags").notNull().default(sql`'[]'::jsonb`), graphNodeIds: jsonb("graph_node_ids").notNull().default(sql`'[]'::jsonb`), + createdAt: text("created_at").notNull(), updatedAt: text("updated_at").notNull(), +}, (t) => [primaryKey({ columns: [t.projectId, t.id] }), unique("memory_recall_records_project_kind_hash_key").on(t.projectId, t.kind, t.contentHash), index("idxMemoryRecallRecordsKindCreated").on(t.projectId, t.kind, t.createdAt), index("idxMemoryRecallRecordsCreated").on(t.projectId, t.createdAt)]); + export const agentActivityEventSeq = projectSchema.table("agent_activity_event_seq", { projectId: text("project_id").notNull().default(sql`current_setting('fusion.project_id', true)`), lastSeq: bigint("last_seq", { mode: "bigint" }).notNull().default(sql`0`), }, (t) => [primaryKey({ columns: [t.projectId] })]); @@ -2461,6 +2469,6 @@ export const projectTableNames = [ "mission_validator_runs", "mission_validator_failures", "mission_fix_feature_lineage", "verification_cache", "import_translation_cache", "approval_requests", - "approval_request_audit_events", "agent_activity_events", "agent_activity_event_seq", "chat_rooms", "chat_room_members", + "approval_request_audit_events", "agent_activity_events", "agent_activity_event_seq", "memory_recall_records", "chat_rooms", "chat_room_members", "chat_room_messages", "chat_token_usage", ] as const;