FN-5911: move agent logs to per-task JSONL storage

Store agent logs in per-task JSONL files instead of the legacy SQLite table.

- add a file-backed agent log store with JSONL append/read/prune helpers and task-scoped source refs
- migrate legacy SQLite agentLogEntries data into task files, rewrite goal citation references, and preserve soft-deleted logs for forensics
- update task store, settings, docs, dashboard route coverage, and add regression tests for migration, retention, and log access

Files changed:
 .changeset/fn-5911-agent-log-jsonl.md              |   5 +
 AGENTS.md                                          |   2 +-
 docs/diagnostics.md                                |   2 +-
 docs/settings-reference.md                         |   2 +
 docs/soft-delete-verification-matrix.md            |   7 +-
 docs/storage.md                                    |   8 +-
 .../src/__tests__/agent-log-file-store.test.ts     | 123 ++++++
 .../core/src/__tests__/agent-log-migration.test.ts | 186 +++++++++
 .../core/src/__tests__/agent-log-retention.test.ts | 208 ++++++++++
 packages/core/src/__tests__/db-migrate.test.ts     |  14 +-
 packages/core/src/__tests__/db.test.ts             |  39 +-
 .../src/__tests__/goal-citations-store.test.ts     |  38 +-
 packages/core/src/__tests__/goals-schema.test.ts   |   2 +-
 packages/core/src/__tests__/insight-store.test.ts  |  10 +-
 .../src/__tests__/merge-request-record.test.ts     |   2 +-
 packages/core/src/__tests__/mission-store.test.ts  |   2 +-
 packages/core/src/__tests__/run-audit.test.ts      |   2 +-
 packages/core/src/__tests__/secrets-schema.test.ts |   6 +-
 .../src/__tests__/soft-delete-agent-logs.test.ts   |  71 ++--
 .../src/__tests__/store-agent-log-file.test.ts     |  91 +++++
 .../core/src/__tests__/store-merge-queue.test.ts   |   2 +-
 packages/core/src/__tests__/store-test-helpers.ts  |  43 +-
 packages/core/src/__tests__/store-upsert.test.ts   |  37 +-
 packages/core/src/__tests__/task-documents.test.ts |   2 +-
 packages/core/src/agent-log-constants.ts           |  25 ++
 packages/core/src/agent-log-file-store.ts          | 267 ++++++++++++
 packages/core/src/db.ts                            |  51 ++-
 packages/core/src/settings-schema.ts               |   1 +
 packages/core/src/store.ts                         | 453 ++++++++++++---------
 packages/core/src/types.ts                         |  10 +-
 .../__tests__/agent-log-routes.integration.test.ts |  48 +++
 .../src/__tests__/evaluator-evidence.test.ts       |  47 ++-
 packages/engine/src/self-healing.ts                |  12 +
 .../src/store/__tests__/roadmap-store.test.ts      |   4 +-
 34 files changed, 1477 insertions(+), 345 deletions(-)

Fusion-Task-Id: FN-5911

Fusion-Task-Lineage: 07c42f3a-87cf-4558-8f01-ac8460b5558b
This commit is contained in:
gsxdsm
2026-06-03 01:10:18 -07:00
parent 1962bea0fc
commit d72cb2ab2b
34 changed files with 1478 additions and 346 deletions

View File

@@ -1,4 +1,7 @@
import { describe, expect, it, vi } from "vitest";
import { describe, expect, it, vi, beforeEach, afterEach } from "vitest";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import * as core from "@fusion/core";
import { collectTaskEvaluationEvidence } from "../evaluator-evidence.js";
@@ -30,6 +33,23 @@ function makeStore(overrides: Partial<core.TaskStore> = {}): core.TaskStore {
}
describe("collectTaskEvaluationEvidence", () => {
let integrationRootDir: string | null = null;
let integrationStore: core.TaskStore | null = null;
beforeEach(() => {
integrationRootDir = null;
integrationStore = null;
});
afterEach(() => {
integrationStore?.close();
integrationStore = null;
if (integrationRootDir) {
rmSync(integrationRootDir, { recursive: true, force: true });
integrationRootDir = null;
}
});
it("collects fixed source groups with bounded excerpts", async () => {
const store = makeStore({
getTaskDocuments: vi.fn().mockResolvedValue([{ key: "plan", content: "x".repeat(900), revision: 1, author: "agent", updatedAt: "2026-01-01T00:01:00.000Z" }]),
@@ -123,6 +143,31 @@ describe("collectTaskEvaluationEvidence", () => {
expect(evidence.agentLogs.at(-1)?.excerpt).toContain("entry-29");
});
it("reads file-backed agent logs through the TaskStore evidence seam", async () => {
integrationRootDir = mkdtempSync(join(tmpdir(), "fusion-evaluator-evidence-"));
const globalDir = join(integrationRootDir, ".fusion-global-settings");
integrationStore = new core.TaskStore(integrationRootDir, globalDir, { inMemoryDb: true });
await integrationStore.init();
const task = await integrationStore.createTask({ description: "Collect evaluator evidence from file-backed logs" });
await integrationStore.appendAgentLog(task.id, "first line", "text", undefined, "executor");
await integrationStore.appendAgentLog(task.id, "tool finished", "tool_result", "ok", "executor");
const detail = await integrationStore.getTask(task.id);
const evidence = await collectTaskEvaluationEvidence({
store: integrationStore,
task: detail,
runId: "ER-file-backed",
cwd: integrationRootDir,
});
expect(evidence.agentLogs).toHaveLength(2);
expect(evidence.agentLogs.map((entry) => entry.label)).toEqual(["text", "tool_result"]);
expect(evidence.agentLogs.map((entry) => entry.excerpt)).toEqual(["first line", "tool finished — ok"]);
expect(evidence.agentLogs.map((entry) => entry.agentId)).toEqual(["executor", "executor"]);
});
it("truncates task metadata summary when oversized", async () => {
const evidence = await collectTaskEvaluationEvidence({
store: makeStore(),

View File

@@ -1657,6 +1657,18 @@ export class SelfHealingManager {
log.log(`Maintenance batch 1 step "prune-operational-logs" succeeded — deleted=${deletedTotal}${detail ? ` (${detail})` : ""}`);
},
},
{
name: "prune-agent-log-files",
fn: async () => {
const days = Number(settings.agentLogFileRetentionDays ?? 0);
if (!Number.isFinite(days) || days <= 0) {
log.log("Maintenance batch 1 step \"prune-agent-log-files\" skipped — agentLogFileRetentionDays is not enabled");
return;
}
const { prunedFiles, prunedEntries, freedBytes } = this.store.pruneAgentLogFiles(days);
log.log(`Maintenance batch 1 step "prune-agent-log-files" succeeded — files=${prunedFiles} entries=${prunedEntries} bytes=${freedBytes}`);
},
},
{ name: "checkpoint-wal", fn: () => Promise.resolve(this.checkpointWal()) },
{ name: "enforce-worktree-cap", fn: () => this.enforceWorktreeCap() },
];