FN-5911: move agent logs to per-task JSONL storage
Store agent logs in per-task JSONL files instead of the legacy SQLite table. - add a file-backed agent log store with JSONL append/read/prune helpers and task-scoped source refs - migrate legacy SQLite agentLogEntries data into task files, rewrite goal citation references, and preserve soft-deleted logs for forensics - update task store, settings, docs, dashboard route coverage, and add regression tests for migration, retention, and log access Files changed: .changeset/fn-5911-agent-log-jsonl.md | 5 + AGENTS.md | 2 +- docs/diagnostics.md | 2 +- docs/settings-reference.md | 2 + docs/soft-delete-verification-matrix.md | 7 +- docs/storage.md | 8 +- .../src/__tests__/agent-log-file-store.test.ts | 123 ++++++ .../core/src/__tests__/agent-log-migration.test.ts | 186 +++++++++ .../core/src/__tests__/agent-log-retention.test.ts | 208 ++++++++++ packages/core/src/__tests__/db-migrate.test.ts | 14 +- packages/core/src/__tests__/db.test.ts | 39 +- .../src/__tests__/goal-citations-store.test.ts | 38 +- packages/core/src/__tests__/goals-schema.test.ts | 2 +- packages/core/src/__tests__/insight-store.test.ts | 10 +- .../src/__tests__/merge-request-record.test.ts | 2 +- packages/core/src/__tests__/mission-store.test.ts | 2 +- packages/core/src/__tests__/run-audit.test.ts | 2 +- packages/core/src/__tests__/secrets-schema.test.ts | 6 +- .../src/__tests__/soft-delete-agent-logs.test.ts | 71 ++-- .../src/__tests__/store-agent-log-file.test.ts | 91 +++++ .../core/src/__tests__/store-merge-queue.test.ts | 2 +- packages/core/src/__tests__/store-test-helpers.ts | 43 +- packages/core/src/__tests__/store-upsert.test.ts | 37 +- packages/core/src/__tests__/task-documents.test.ts | 2 +- packages/core/src/agent-log-constants.ts | 25 ++ packages/core/src/agent-log-file-store.ts | 267 ++++++++++++ packages/core/src/db.ts | 51 ++- packages/core/src/settings-schema.ts | 1 + packages/core/src/store.ts | 453 ++++++++++++--------- packages/core/src/types.ts | 10 +- .../__tests__/agent-log-routes.integration.test.ts | 48 +++ .../src/__tests__/evaluator-evidence.test.ts | 47 ++- packages/engine/src/self-healing.ts | 12 + .../src/store/__tests__/roadmap-store.test.ts | 4 +- 34 files changed, 1477 insertions(+), 345 deletions(-) Fusion-Task-Id: FN-5911 Fusion-Task-Lineage: 07c42f3a-87cf-4558-8f01-ac8460b5558b
This commit is contained in:
@@ -1,4 +1,7 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { describe, expect, it, vi, beforeEach, afterEach } from "vitest";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import * as core from "@fusion/core";
|
||||
import { collectTaskEvaluationEvidence } from "../evaluator-evidence.js";
|
||||
|
||||
@@ -30,6 +33,23 @@ function makeStore(overrides: Partial<core.TaskStore> = {}): core.TaskStore {
|
||||
}
|
||||
|
||||
describe("collectTaskEvaluationEvidence", () => {
|
||||
let integrationRootDir: string | null = null;
|
||||
let integrationStore: core.TaskStore | null = null;
|
||||
|
||||
beforeEach(() => {
|
||||
integrationRootDir = null;
|
||||
integrationStore = null;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
integrationStore?.close();
|
||||
integrationStore = null;
|
||||
if (integrationRootDir) {
|
||||
rmSync(integrationRootDir, { recursive: true, force: true });
|
||||
integrationRootDir = null;
|
||||
}
|
||||
});
|
||||
|
||||
it("collects fixed source groups with bounded excerpts", async () => {
|
||||
const store = makeStore({
|
||||
getTaskDocuments: vi.fn().mockResolvedValue([{ key: "plan", content: "x".repeat(900), revision: 1, author: "agent", updatedAt: "2026-01-01T00:01:00.000Z" }]),
|
||||
@@ -123,6 +143,31 @@ describe("collectTaskEvaluationEvidence", () => {
|
||||
expect(evidence.agentLogs.at(-1)?.excerpt).toContain("entry-29");
|
||||
});
|
||||
|
||||
it("reads file-backed agent logs through the TaskStore evidence seam", async () => {
|
||||
integrationRootDir = mkdtempSync(join(tmpdir(), "fusion-evaluator-evidence-"));
|
||||
const globalDir = join(integrationRootDir, ".fusion-global-settings");
|
||||
integrationStore = new core.TaskStore(integrationRootDir, globalDir, { inMemoryDb: true });
|
||||
await integrationStore.init();
|
||||
|
||||
const task = await integrationStore.createTask({ description: "Collect evaluator evidence from file-backed logs" });
|
||||
await integrationStore.appendAgentLog(task.id, "first line", "text", undefined, "executor");
|
||||
await integrationStore.appendAgentLog(task.id, "tool finished", "tool_result", "ok", "executor");
|
||||
|
||||
const detail = await integrationStore.getTask(task.id);
|
||||
const evidence = await collectTaskEvaluationEvidence({
|
||||
store: integrationStore,
|
||||
task: detail,
|
||||
runId: "ER-file-backed",
|
||||
cwd: integrationRootDir,
|
||||
});
|
||||
|
||||
expect(evidence.agentLogs).toHaveLength(2);
|
||||
expect(evidence.agentLogs.map((entry) => entry.label)).toEqual(["text", "tool_result"]);
|
||||
expect(evidence.agentLogs.map((entry) => entry.excerpt)).toEqual(["first line", "tool finished — ok"]);
|
||||
expect(evidence.agentLogs.map((entry) => entry.agentId)).toEqual(["executor", "executor"]);
|
||||
|
||||
});
|
||||
|
||||
it("truncates task metadata summary when oversized", async () => {
|
||||
const evidence = await collectTaskEvaluationEvidence({
|
||||
store: makeStore(),
|
||||
|
||||
@@ -1657,6 +1657,18 @@ export class SelfHealingManager {
|
||||
log.log(`Maintenance batch 1 step "prune-operational-logs" succeeded — deleted=${deletedTotal}${detail ? ` (${detail})` : ""}`);
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "prune-agent-log-files",
|
||||
fn: async () => {
|
||||
const days = Number(settings.agentLogFileRetentionDays ?? 0);
|
||||
if (!Number.isFinite(days) || days <= 0) {
|
||||
log.log("Maintenance batch 1 step \"prune-agent-log-files\" skipped — agentLogFileRetentionDays is not enabled");
|
||||
return;
|
||||
}
|
||||
const { prunedFiles, prunedEntries, freedBytes } = this.store.pruneAgentLogFiles(days);
|
||||
log.log(`Maintenance batch 1 step "prune-agent-log-files" succeeded — files=${prunedFiles} entries=${prunedEntries} bytes=${freedBytes}`);
|
||||
},
|
||||
},
|
||||
{ name: "checkpoint-wal", fn: () => Promise.resolve(this.checkpointWal()) },
|
||||
{ name: "enforce-worktree-cap", fn: () => this.enforceWorktreeCap() },
|
||||
];
|
||||
|
||||
Reference in New Issue
Block a user