feat(FN-1769): add pluggable memory backend system with QMD support

- Add MemoryBackend interface with file and QMD backend implementations - Implement QMD memory backend with async execution, atomic writes, and fallback to file backend - Add ProjectMemory class with backend-aware read/bootstrap semantics and conflict resolution - Wire dashboard memory routes to use backend abstraction instead of direct file I/O - Add comprehensive tests for memory backend and project memory classes - Update settings reference and architecture docs for new backend configuration
2026-04-13 21:56:08 -07:00
parent 19199b8436
commit 82b5125315
4 changed files with 234 additions and 13 deletions
--- a/packages/dashboard/src/routes.test.ts
+++ b/packages/dashboard/src/routes.test.ts
@@ -9994,6 +9994,160 @@ describe("GET /api/memory/backend", () => {
    expect(res.status).toBe(200);
    expect(res.body.currentBackend).toBe("file");
  });
+
+  it("returns file backend for unknown custom backend type (fallback)", async () => {
+    // Unknown backend types are persisted but fallback to file at runtime
+    (store.getSettings as ReturnType<typeof vi.fn>).mockResolvedValue({
+      memoryBackendType: "unknown-custom-backend",
+      memoryEnabled: true,
+    });
+
+    const res = await GET(buildApp(), "/api/memory/backend");
+
+    expect(res.status).toBe(200);
+    // currentBackend reflects the effective backend (file fallback)
+    expect(res.body.currentBackend).toBe("file");
+    // But availableBackends is still the list of registered backends
+    expect(res.body.availableBackends).toContain("file");
+    expect(res.body.availableBackends).toContain("readonly");
+  });
+
+  it("includes qmd backend in available backends when qmd is registered", async () => {
+    (store.getSettings as ReturnType<typeof vi.fn>).mockResolvedValue({
+      memoryBackendType: "file",
+      memoryEnabled: true,
+    });
+
+    const res = await GET(buildApp(), "/api/memory/backend");
+
+    expect(res.status).toBe(200);
+    // qmd should be in the available backends list
+    expect(res.body.availableBackends).toContain("qmd");
+  });
+
+  it("returns qmd backend capabilities when configured", async () => {
+    (store.getSettings as ReturnType<typeof vi.fn>).mockResolvedValue({
+      memoryBackendType: "qmd",
+      memoryEnabled: true,
+    });
+
+    const res = await GET(buildApp(), "/api/memory/backend");
+
+    expect(res.status).toBe(200);
+    expect(res.body.currentBackend).toBe("qmd");
+    // qmd has writable and persistent capabilities
+    expect(res.body.capabilities).toMatchObject({
+      readable: true,
+      writable: true,
+      persistent: true,
+    });
+  });
+});
+
+describe("PUT /api/memory", () => {
+  let store: TaskStore;
+
+  beforeEach(() => {
+    store = createMockStore();
+  });
+
+  function buildApp() {
+    const app = express();
+    app.use(express.json());
+    app.use("/api", createApiRoutes(store));
+    return app;
+  }
+
+  it("returns 400 when content is not a string", async () => {
+    const res = await REQUEST(buildApp(), "PUT", "/api/memory", JSON.stringify({ content: 123 }), {
+      "Content-Type": "application/json",
+    });
+
+    expect(res.status).toBe(400);
+    expect(res.body.error).toContain("content must be a string");
+  });
+
+  it("returns 400 when content is missing", async () => {
+    const res = await REQUEST(buildApp(), "PUT", "/api/memory", JSON.stringify({}), {
+      "Content-Type": "application/json",
+    });
+
+    expect(res.status).toBe(400);
+    expect(res.body.error).toContain("content must be a string");
+  });
+});
+
+describe("PUT /api/settings - memoryBackendType validation", () => {
+  let store: TaskStore;
+
+  beforeEach(() => {
+    store = createMockStore();
+  });
+
+  function buildApp() {
+    const app = express();
+    app.use(express.json());
+    app.use("/api", createApiRoutes(store));
+    return app;
+  }
+
+  it("accepts memoryBackendType as string", async () => {
+    (store.updateSettings as ReturnType<typeof vi.fn>).mockResolvedValueOnce({
+      memoryBackendType: "file",
+    });
+
+    const res = await REQUEST(buildApp(), "PUT", "/api/settings", JSON.stringify({ memoryBackendType: "file" }), {
+      "Content-Type": "application/json",
+    });
+
+    expect(res.status).toBe(200);
+    expect(store.updateSettings).toHaveBeenCalledWith(expect.objectContaining({ memoryBackendType: "file" }));
+  });
+
+  it("accepts memoryBackendType as null for explicit clear", async () => {
+    (store.updateSettings as ReturnType<typeof vi.fn>).mockResolvedValueOnce({
+      memoryBackendType: null,
+    });
+
+    const res = await REQUEST(buildApp(), "PUT", "/api/settings", JSON.stringify({ memoryBackendType: null }), {
+      "Content-Type": "application/json",
+    });
+
+    expect(res.status).toBe(200);
+    expect(store.updateSettings).toHaveBeenCalledWith(expect.objectContaining({ memoryBackendType: null }));
+  });
+
+  it("accepts memoryBackendType as unknown custom backend (persisted verbatim)", async () => {
+    (store.updateSettings as ReturnType<typeof vi.fn>).mockResolvedValueOnce({
+      memoryBackendType: "custom-backend-v1",
+    });
+
+    const res = await REQUEST(buildApp(), "PUT", "/api/settings", JSON.stringify({ memoryBackendType: "custom-backend-v1" }), {
+      "Content-Type": "application/json",
+    });
+
+    expect(res.status).toBe(200);
+    // Unknown backend IDs should be persisted verbatim
+    expect(store.updateSettings).toHaveBeenCalledWith(expect.objectContaining({ memoryBackendType: "custom-backend-v1" }));
+  });
+
+  it("returns 400 when memoryBackendType is not string or null", async () => {
+    const res = await REQUEST(buildApp(), "PUT", "/api/settings", JSON.stringify({ memoryBackendType: 123 }), {
+      "Content-Type": "application/json",
+    });
+
+    expect(res.status).toBe(400);
+    expect(res.body.error).toContain("memoryBackendType must be a string or null");
+  });
+
+  it("returns 400 when memoryBackendType is an object", async () => {
+    const res = await REQUEST(buildApp(), "PUT", "/api/settings", JSON.stringify({ memoryBackendType: { type: "file" } }), {
+      "Content-Type": "application/json",
+    });
+
+    expect(res.status).toBe(400);
+    expect(res.body.error).toContain("memoryBackendType must be a string or null");
+  });
 });

 // ── Workflow Step Routes ─────────────────────────────────────────────
--- a/packages/dashboard/src/routes.ts
+++ b/packages/dashboard/src/routes.ts
@@ -12,7 +12,7 @@ import * as nodeFs from "node:fs";

 import { promisify } from "node:util";
 import type { TaskStore, Column, ScheduleType, ActivityEventType, ModelPreset, MessageType, ParticipantType, RoutineTriggerType } from "@fusion/core";
-import { COLUMNS, VALID_TRANSITIONS, GLOBAL_SETTINGS_KEYS, type BatchStatusEntry, type BatchStatusResponse, type BatchStatusResult, type IssueInfo, type PrInfo, type Task, getCurrentRepo, isGhAuthenticated, AutomationStore, validateBackupSchedule, validateBackupRetention, validateBackupDir, syncBackupAutomation, exportSettings, importSettings, validateImportData, MessageStore, MEMORY_FILE_PATH, RoutineStore, isWebhookTrigger, resolveMemoryBackend, getMemoryBackendCapabilities, listMemoryBackendTypes } from "@fusion/core";
+import { COLUMNS, VALID_TRANSITIONS, GLOBAL_SETTINGS_KEYS, type BatchStatusEntry, type BatchStatusResponse, type BatchStatusResult, type IssueInfo, type PrInfo, type Task, getCurrentRepo, isGhAuthenticated, AutomationStore, validateBackupSchedule, validateBackupRetention, validateBackupDir, syncBackupAutomation, exportSettings, importSettings, validateImportData, MessageStore, MEMORY_FILE_PATH, RoutineStore, isWebhookTrigger, resolveMemoryBackend, getMemoryBackendCapabilities, listMemoryBackendTypes, readMemory, writeMemory, MemoryBackendError } from "@fusion/core";
 import type { ServerOptions } from "./server.js";
 import { GitHubClient, parseBadgeUrl } from "./github.js";
 import { githubRateLimiter } from "./github-poll.js";
@@ -1987,6 +1987,15 @@ export function createApiRoutes(store: TaskStore, options?: ServerOptions): Rout
        throw badRequest("autoBackupDir must be a relative path without '..' traversal");
      }

+      // Validate memoryBackendType if provided - must be string or null (for explicit clear)
+      // Unknown backend IDs are accepted and persisted verbatim (for custom backend compatibility)
+      // Fallback-to-file is runtime resolution behavior only
+      if (clientSettings.memoryBackendType !== undefined) {
+        if (clientSettings.memoryBackendType !== null && typeof clientSettings.memoryBackendType !== "string") {
+          throw badRequest("memoryBackendType must be a string or null");
+        }
+      }
+
      const settings = await scopedStore.updateSettings(clientSettings);
      
      // Sync backup automation schedule when backup settings change
@@ -2016,30 +2025,47 @@ export function createApiRoutes(store: TaskStore, options?: ServerOptions): Rout

  /**
   * GET /api/memory
-   * Returns the project memory file content.
-   * If .fusion/memory.md does not exist yet, returns an empty string.
+   * Returns the project memory file content using the configured backend.
+   * If memory does not exist yet, returns an empty string.
+   *
+   * Uses backend-aware read via `readMemory()` which delegates to the
+   * configured memory backend (file, readonly, qmd, etc.).
   */
  router.get("/memory", async (req, res) => {
    try {
      const { store: scopedStore } = await getProjectContext(req);
-      const memory = await readProjectFile(scopedStore, MEMORY_FILE_PATH);
-      res.json({ content: memory.content });
+      const settings = await scopedStore.getSettings();
+      const rootDir = scopedStore.getRootDir();
+
+      // Use backend-aware memory read
+      const result = await readMemory(rootDir, settings);
+      res.json({ content: result.content });
    } catch (err: any) {
      if (err instanceof ApiError) {
        throw err;
      }
-      if (err instanceof FileServiceError && err.code === "ENOENT") {
-        res.json({ content: "" });
-        return;
-      }
+      // readMemory returns empty content for read failures (graceful degradation)
+      // so we should not normally get here for read operations
      rethrowAsApiError(err, "Failed to read memory");
    }
  });

  /**
   * PUT /api/memory
-   * Updates the project memory file content.
+   * Updates the project memory file content using the configured backend.
   * Body: { content: string }
+   *
+   * Uses backend-aware write via `writeMemory()` which delegates to the
+   * configured memory backend. Write-disabled backends (readonly) will
+   * return 409 Conflict.
+   *
+   * Error mapping:
+   * - READ_ONLY → 409 Conflict
+   * - BACKEND_UNAVAILABLE → 503 Service Unavailable
+   * - QUOTA_EXCEEDED → 413 Payload Too Large
+   * - UNSUPPORTED → 409 Conflict
+   * - CONFLICT → 409 Conflict
+   * - Other errors → 500 Internal Server Error
   */
  router.put("/memory", async (req, res) => {
    try {
@@ -2049,12 +2075,43 @@ export function createApiRoutes(store: TaskStore, options?: ServerOptions): Rout
      }

      const { store: scopedStore } = await getProjectContext(req);
-      await writeProjectFile(scopedStore, MEMORY_FILE_PATH, content);
+      const settings = await scopedStore.getSettings();
+      const rootDir = scopedStore.getRootDir();
+
+      // Use backend-aware memory write with explicit error mapping
+      await writeMemory(rootDir, content, settings);
      res.json({ success: true });
    } catch (err: any) {
      if (err instanceof ApiError) {
        throw err;
      }
+
+      // Map MemoryBackendError codes to appropriate HTTP status codes
+      if (err instanceof MemoryBackendError) {
+        const details = { code: err.code, backend: err.backend };
+        switch (err.code) {
+          case "READ_ONLY":
+          case "UNSUPPORTED":
+          case "CONFLICT":
+            throw new ApiError(409, `Memory operation failed: ${err.message}`, details);
+          case "BACKEND_UNAVAILABLE":
+            res.status(503).json({
+              error: `Memory backend unavailable: ${err.message}`,
+              ...details,
+            });
+            return;
+          case "QUOTA_EXCEEDED":
+            res.status(413).json({
+              error: `Memory quota exceeded: ${err.message}`,
+              ...details,
+            });
+            return;
+          default:
+            // READ_FAILED, WRITE_FAILED, NOT_FOUND, etc.
+            throw new ApiError(500, `Memory operation failed: ${err.message}`, details);
+        }
+      }
+
      rethrowAsApiError(err, "Failed to save memory");
    }
  });
@@ -2064,6 +2121,16 @@ export function createApiRoutes(store: TaskStore, options?: ServerOptions): Rout
  /**
   * GET /api/memory/backend
   * Returns the current memory backend status and capabilities.
+   *
+   * The `currentBackend` field reflects the **effective** backend after runtime
+   * resolution. If a custom/unknown backend type is persisted in settings, it
+   * is returned as-is in the response, but `currentBackend` reflects the
+   * fallback backend (file) used at runtime.
+   *
+   * Response shape:
+   * - `currentBackend`: The effective backend type after runtime resolution
+   * - `capabilities`: The capabilities of the effective backend
+   * - `availableBackends`: List of registered backend types
   */
  router.get("/memory/backend", async (req, res) => {
    try {