fix(FN-3175): reduce dashboard log stalls

This commit is contained in:
gsxdsm
2026-05-02 08:05:33 -07:00
parent 207d9d2524
commit d787a16a79
16 changed files with 209 additions and 43 deletions

View File

@@ -57,7 +57,7 @@ describe("Database", () => {
const journalSizeLimit = db.prepare("PRAGMA journal_size_limit").get() as { journal_size_limit: number };
expect(synchronous.synchronous).toBe(1); // NORMAL
expect(autoCheckpoint.wal_autocheckpoint).toBe(100);
expect(autoCheckpoint.wal_autocheckpoint).toBe(1000);
expect(journalSizeLimit.journal_size_limit).toBe(4_194_304);
});
@@ -267,6 +267,13 @@ describe("Database", () => {
expect(typeof result.log).toBe("number");
expect(typeof result.checkpointed).toBe("number");
});
it("supports explicit truncate checkpoints when requested", () => {
const result = db.walCheckpoint("TRUNCATE");
expect(result).toHaveProperty("busy");
expect(result).toHaveProperty("log");
expect(result).toHaveProperty("checkpointed");
});
});
describe("transactions", () => {

View File

@@ -4158,6 +4158,25 @@ Task with acceptance criteria
expect(events[1]).toMatchObject({ text: "tool", type: "tool", detail: "read file", agent: "executor" });
});
it("truncates oversized tool detail before persisting and emitting", async () => {
const task = await createTestTask();
const events: any[] = [];
const oversizedDetail = "X".repeat(5000);
const truncationMarker = "[tool output truncated to keep dashboard log views responsive]";
store.on("agent:log", (entry) => events.push(entry));
await store.appendAgentLogBatch([
{ taskId: task.id, text: "Bash", type: "tool_result", detail: oversizedDetail, agent: "executor" },
]);
const logs = await store.getAgentLogs(task.id);
expect(logs).toHaveLength(1);
expect(logs[0].detail).toContain(truncationMarker);
expect(logs[0].detail!.match(/\[tool output truncated to keep dashboard log views responsive\]/g)).toHaveLength(1);
expect(logs[0].detail!.length).toBeLessThan(oversizedDetail.length);
expect(events[0].detail).toBe(logs[0].detail);
});
it("appendAgentLogBatch with empty entries is a no-op", async () => {
const task = await createTestTask();
@@ -4304,6 +4323,28 @@ Task with acceptance criteria
expect(logs[0].detail!.length).toBe(longDetail.length);
});
it("clips oversized historical tool detail at read time", async () => {
const task = await createTestTask();
const oversizedDetail = "Y".repeat(7000);
const truncationMarker = "[tool output truncated to keep dashboard log views responsive]";
insertLogEntryWithTimestamp(
store,
task.id,
"Bash",
"tool_result",
"2026-04-24T12:00:00.000Z",
oversizedDetail,
"executor",
);
const logs = await store.getAgentLogs(task.id);
expect(logs).toHaveLength(1);
expect(logs[0].detail).toContain(truncationMarker);
expect(logs[0].detail!.match(/\[tool output truncated to keep dashboard log views responsive\]/g)).toHaveLength(1);
expect(logs[0].detail!.length).toBeLessThan(oversizedDetail.length);
});
it("appendAgentLog persists and reads back the agent field", async () => {
const task = await createTestTask();

View File

@@ -754,8 +754,9 @@ export class Database {
this.db.exec("PRAGMA busy_timeout = 5000");
// In WAL mode NORMAL is nearly as durable as FULL with much lower fsync cost.
this.db.exec("PRAGMA synchronous = NORMAL");
// Checkpoint aggressively to avoid large WAL growth under bursty writes.
this.db.exec("PRAGMA wal_autocheckpoint = 100");
// Let WAL grow to roughly the journal size limit before auto-checkpointing.
// This avoids frequent synchronous checkpoints on log-heavy workloads.
this.db.exec("PRAGMA wal_autocheckpoint = 1000");
// Bound WAL growth between checkpoints/maintenance cycles.
this.db.exec("PRAGMA journal_size_limit = 4194304");
} else {
@@ -2334,11 +2335,15 @@ export class Database {
}
/**
* Run a WAL checkpoint to truncate the WAL file and reclaim disk space.
* Safe to call periodically. Returns checkpoint stats.
* Run a WAL checkpoint and return checkpoint stats.
*
* TRUNCATE remains the default so explicit maintenance/compaction calls keep
* reclaiming disk space as before. Live engine maintenance should opt into
* PASSIVE to avoid forcing a blocking truncate on the shared event loop
* while tasks are actively writing logs.
*/
walCheckpoint(): { busy: number; log: number; checkpointed: number } {
const row = this.db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get() as
walCheckpoint(mode: "PASSIVE" | "TRUNCATE" = "TRUNCATE"): { busy: number; log: number; checkpointed: number } {
const row = this.db.prepare(`PRAGMA wal_checkpoint(${mode})`).get() as
| { busy?: number; log?: number; checkpointed?: number }
| undefined;
return { busy: row?.busy ?? 0, log: row?.log ?? 0, checkpointed: row?.checkpointed ?? 0 };

View File

@@ -168,6 +168,10 @@ const TASK_ACTIVITY_LOG_ENTRY_LIMIT = 1_000;
const TASK_ACTIVITY_LOG_OUTCOME_LIMIT = 4_000;
const ARCHIVE_AGENT_LOG_SNAPSHOT_LIMIT = 25;
const ARCHIVE_AGENT_LOG_SNIPPET_LIMIT = 160;
const AGENT_LOG_TOOL_DETAIL_LIMIT = 4_096;
const AGENT_LOG_TOOL_DETAIL_TRUNCATION_NOTICE =
"\n\n[tool output truncated to keep dashboard log views responsive]";
const AGENT_LOG_TOOL_TYPES = new Set<AgentLogEntry["type"]>(["tool", "tool_result", "tool_error"]);
const storeLog = createLogger("task-store");
/**
@@ -224,6 +228,16 @@ function truncateTaskLogOutcome(outcome: string | undefined): string | undefined
return `${outcome.slice(0, TASK_ACTIVITY_LOG_OUTCOME_LIMIT)}\n... outcome truncated to ${TASK_ACTIVITY_LOG_OUTCOME_LIMIT} characters ...`;
}
function truncateAgentLogDetail(
detail: string | null | undefined,
type: AgentLogEntry["type"],
): string | undefined {
if (detail == null) return undefined;
if (!AGENT_LOG_TOOL_TYPES.has(type)) return detail;
if (detail.length <= AGENT_LOG_TOOL_DETAIL_LIMIT) return detail;
return `${detail.slice(0, AGENT_LOG_TOOL_DETAIL_LIMIT)}${AGENT_LOG_TOOL_DETAIL_TRUNCATION_NOTICE}`;
}
function compactTaskActivityLog(entries: TaskLogEntry[]): TaskLogEntry[] {
const recentEntries = entries.slice(-TASK_ACTIVITY_LOG_ENTRY_LIMIT);
return recentEntries.map((entry) => ({
@@ -4507,19 +4521,20 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
agent?: AgentLogEntry["agent"],
): Promise<void> {
const timestamp = new Date().toISOString();
const normalizedDetail = truncateAgentLogDetail(detail, type);
const entry: AgentLogEntry = {
timestamp,
taskId,
text,
type,
...(detail !== undefined && { detail }),
...(normalizedDetail !== undefined && { detail: normalizedDetail }),
...(agent !== undefined && { agent }),
};
this.db.prepare(`
INSERT INTO agentLogEntries (taskId, timestamp, text, type, detail, agent)
VALUES (?, ?, ?, ?, ?, ?)
`).run(taskId, timestamp, text, type, detail ?? null, agent ?? null);
`).run(taskId, timestamp, text, type, normalizedDetail ?? null, agent ?? null);
this.db.bumpLastModified();
this.emit("agent:log", entry);
@@ -4539,13 +4554,17 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
}
const timestamp = new Date().toISOString();
const normalizedEntries = entries.map((entry) => ({
...entry,
detail: truncateAgentLogDetail(entry.detail, entry.type),
}));
const stmt = this.db.prepare(`
INSERT INTO agentLogEntries (taskId, timestamp, text, type, detail, agent)
VALUES (?, ?, ?, ?, ?, ?)
`);
this.db.transaction(() => {
for (const entry of entries) {
for (const entry of normalizedEntries) {
stmt.run(
entry.taskId,
timestamp,
@@ -4558,7 +4577,7 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
});
this.db.bumpLastModified();
for (const entry of entries) {
for (const entry of normalizedEntries) {
this.emit("agent:log", {
timestamp,
taskId: entry.taskId,
@@ -4571,16 +4590,36 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
}
private mapAgentLogRow(row: Record<string, unknown>): AgentLogEntry {
const type = row.type as AgentLogEntry["type"];
const detail = row.detail != null ? String(row.detail) : undefined;
return {
timestamp: row.timestamp as string,
taskId: row.taskId as string,
text: row.text as string,
type: row.type as AgentLogEntry["type"],
...(row.detail != null && { detail: row.detail as string }),
type,
...(detail !== undefined && { detail }),
...(row.agent != null && { agent: row.agent as AgentLogEntry["agent"] }),
};
}
private getAgentLogSelectClause(): string {
const escapedNotice = AGENT_LOG_TOOL_DETAIL_TRUNCATION_NOTICE.replace(/'/g, "''");
return `
taskId,
timestamp,
text,
type,
CASE
WHEN type IN ('tool', 'tool_result', 'tool_error')
AND detail IS NOT NULL
AND LENGTH(detail) > ${AGENT_LOG_TOOL_DETAIL_LIMIT}
THEN SUBSTR(detail, 1, ${AGENT_LOG_TOOL_DETAIL_LIMIT}) || '${escapedNotice}'
ELSE detail
END AS detail,
agent
`;
}
async addTaskComment(id: string, text: string, author: string): Promise<Task> {
// Delegate to unified addComment method
return this.addComment(id, text, author);
@@ -5180,9 +5219,10 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
* Read historical agent log entries for a task from SQLite.
* Returns entries in chronological order (oldest first).
*
* Each entry's `text` and `detail` fields are returned in full — there is
* no per-entry truncation at the persistence layer. The 500-entry cap
* (`MAX_LOG_ENTRIES`) in the dashboard hooks is a whole-list limit only.
* Tool-oriented detail payloads are clipped server-side to keep historical
* log reads responsive even when agents emit very large command results.
* The 500-entry cap (`MAX_LOG_ENTRIES`) in the dashboard hooks remains a
* whole-list limit only.
*
* @param taskId - The task ID (e.g. "KB-001")
* @param options - Optional pagination options
@@ -5203,10 +5243,12 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
if (limit === 0) return [];
const selectClause = this.getAgentLogSelectClause();
if (limit !== undefined) {
const readCount = offset > 0 ? limit + offset : limit;
const rows = this.db.prepare(`
SELECT * FROM agentLogEntries
SELECT ${selectClause} FROM agentLogEntries
WHERE taskId = ?
ORDER BY timestamp DESC, id DESC
LIMIT ?
@@ -5219,7 +5261,7 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
}
const rows = this.db.prepare(`
SELECT * FROM agentLogEntries
SELECT ${selectClause} FROM agentLogEntries
WHERE taskId = ?
ORDER BY timestamp ASC, id ASC
`).all(taskId) as Array<Record<string, unknown>>;
@@ -5257,8 +5299,9 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
endIso: string | null,
): Promise<AgentLogEntry[]> {
const end = endIso ?? new Date().toISOString();
const selectClause = this.getAgentLogSelectClause();
const rows = this.db.prepare(`
SELECT * FROM agentLogEntries
SELECT ${selectClause} FROM agentLogEntries
WHERE taskId = ? AND timestamp >= ? AND timestamp <= ?
ORDER BY timestamp ASC, id ASC
`).all(taskId, startIso, end) as Array<Record<string, unknown>>;
@@ -5296,8 +5339,12 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
const text = typeof parsed.text === "string" ? parsed.text : "";
const detail = typeof parsed.detail === "string" ? parsed.detail : null;
const agent = typeof parsed.agent === "string" ? parsed.agent : null;
const normalizedDetail = truncateAgentLogDetail(
detail,
type as AgentLogEntry["type"],
);
insertStmt.run(parsedTaskId, timestamp, text, type, detail, agent);
insertStmt.run(parsedTaskId, timestamp, text, type, normalizedDetail ?? null, agent);
imported += 1;
} catch {
// Skip malformed JSONL lines.
@@ -5866,11 +5913,15 @@ ${stepsSection}`;
}
/**
* Run a WAL checkpoint to truncate the WAL file and reclaim disk space.
* Safe to call periodically from the self-healing maintenance timer.
* Run a WAL checkpoint and return checkpoint stats.
*
* The default preserves SQLite's aggressive TRUNCATE behavior for explicit
* maintenance/compaction calls. Live engine maintenance should request
* PASSIVE explicitly to avoid forcing a blocking truncate on the shared
* event loop.
*/
walCheckpoint(): { busy: number; log: number; checkpointed: number } {
return this.db.walCheckpoint();
walCheckpoint(mode?: "PASSIVE" | "TRUNCATE"): { busy: number; log: number; checkpointed: number } {
return this.db.walCheckpoint(mode);
}
getRootDir(): string {

View File

@@ -1445,10 +1445,11 @@ export interface GlobalSettings {
* triggers a vitest auto-kill. Clamped to [50, 99] in the UI.
* Default: 90. */
vitestKillThresholdPct?: number;
/** When true (default), persist detailed tool argument/result payloads in
* task agent logs (`agent.log`) for `tool`, `tool_result`, and
* `tool_error` entries. When false, tool timeline rows are still stored,
* but their verbose `detail` payload is omitted to reduce log size/noise. */
/** When true (default), persist tool argument/result payloads in task agent
* logs for `tool`, `tool_result`, and `tool_error` entries. Very large tool
* payloads may still be clipped server-side to keep dashboard log reads
* responsive. When false, tool timeline rows are still stored, but their
* verbose `detail` payload is omitted to reduce log size/noise. */
persistAgentToolOutput?: boolean;
/** Research defaults shared across all projects.
* Project settings may override these via `researchSettings`. */

View File

@@ -553,7 +553,7 @@ All color themes automatically provide values for these tokens. Adding a new col
The dashboard includes several runtime safeguards to stay responsive during long sessions and on larger boards:
- **Agent log cap**: The UI keeps only the most recent **500 agent log entries per task** in memory. Historical log fetches and live SSE appends are both capped to this window. **Per-entry content is never truncated** — each entry's `text` and `detail` fields survive in full from persistence (`agent.log` JSONL) through the API (`GET /tasks/:id/logs`), SSE streaming (`/api/tasks/:id/logs/stream`), and rendering in `AgentLogViewer` / `AgentDetailView`. The 500-entry limit is a whole-list in-memory cap only.
- **Agent log cap**: The UI keeps only the most recent **500 agent log entries per task** in memory. Historical log fetches and live SSE appends are both capped to this window. Tool-oriented `detail` payloads may be clipped server-side before they reach the dashboard so oversized command output does not stall the shared engine/dashboard event loop. The 500-entry limit is still a whole-list in-memory cap only.
- **Memoized task rendering**: `TaskCard`, `Column`, and worktree grouping are memoized so unrelated SSE updates do not force the whole board to repaint. The board also preserves stable per-column task arrays for unchanged columns.
- **Large-column pagination**: Columns with more than **100 tasks** use incremental client-side pagination, rendering **50 tasks initially** and loading **25 more** at a time. This is applied to active non-archived, non-`in-progress` columns to avoid breaking worktree grouping and archived browsing behavior.
- **Badge update isolation**: Live GitHub PR/issue badge websocket updates are rendered through a dedicated child component so badge freshness is preserved even when task cards are memoized.

View File

@@ -1898,6 +1898,7 @@ export function SettingsModal({
</label>
<div className="settings-field-help">
When disabled, tool rows are still logged but detailed tool payloads are omitted.
Very large tool payloads may still be clipped even when this stays enabled.
</div>
</div>
<div className="form-group">

View File

@@ -10,9 +10,9 @@ const INITIAL_LOAD_LIMIT = 100;
* Cap the total number of log entries to `MAX_LOG_ENTRIES`.
*
* This is a **whole-list cap** — it limits how many entries are kept
* in memory, not the content of any individual entry. Per-entry `text`
* and `detail` fields are never truncated anywhere in the pipeline
* (persistence → API → SSE → hook → rendering).
* in memory, not the content of any individual entry. Tool-oriented
* `detail` payloads may still be clipped server-side to keep the live
* dashboard responsive when agents emit very large command results.
*/
function capLogEntries(entries: AgentLogEntry[]): AgentLogEntry[] {
return entries.length > MAX_LOG_ENTRIES

View File

@@ -4481,6 +4481,34 @@ describe("Attachment routes", () => {
expect(res.body[1].detail).toBe(longDetail);
expect(res.body[1].detail.length).toBe(5000);
});
it("GET /activity — defaults to a bounded limit when none is provided", async () => {
const fakeEntries = [
{
id: "activity-1",
timestamp: "2026-01-01T00:00:00Z",
type: "task:created",
details: "Created task",
},
];
const activityStore = createMockStore({
getActivityLog: vi.fn().mockResolvedValue(fakeEntries),
});
const app = express();
app.use(express.json());
app.use("/api", createApiRoutes(activityStore));
const res = await GET(app, "/api/activity");
expect(res.status).toBe(200);
expect(res.body).toEqual(fakeEntries);
expect(activityStore.getActivityLog).toHaveBeenCalledWith({
limit: 100,
since: undefined,
type: undefined,
});
});
});
// --- Models route tests ---

View File

@@ -2414,8 +2414,9 @@ export function createApiRoutes(store: TaskStore, options?: ServerOptions): Rout
const sinceParam = req.query.since;
const typeParam = req.query.type;
// Parse and validate limit
let limit: number | undefined;
// Parse and validate limit. Omitted limit intentionally defaults to 100
// to match the documented API contract and avoid unbounded history reads.
let limit = 100;
if (limitParam !== undefined) {
const parsed = Number.parseInt(limitParam as string, 10);
if (!Number.isFinite(parsed) || parsed < 0) {

View File

@@ -698,8 +698,9 @@ export function registerTaskWorkflowRoutes(ctx: ApiRoutesContext, deps: TaskWork
});
// Get historical agent logs for a task.
// Per-entry text and detail fields are returned in full — no truncation.
// The 500-entry cap (MAX_LOG_ENTRIES) is a client-side whole-list limit.
// Tool-oriented detail payloads may be clipped server-side to keep the
// dashboard responsive when agents emit very large command results.
// The 500-entry cap (MAX_LOG_ENTRIES) is still a client-side whole-list limit.
// When limit is provided, includes X-Total-Count and X-Has-More headers for pagination.
router.get("/tasks/:id/logs", async (req, res) => {
try {

View File

@@ -688,9 +688,9 @@ export function createServer(store: TaskStore, options?: ServerOptions): ReturnT
// - With projectId: use scoped store from engine or resolver (ensures multi-project isolation)
// - Without projectId: use default store (preserves existing single-project behavior)
//
// Per-entry text and detail fields are serialized in full — there is no
// SSE-level truncation. The 500-entry cap is applied client-side in the
// React hooks (useAgentLogs / useMultiAgentLogs).
// Tool-oriented detail payloads may already be clipped in storage to keep
// live log streaming responsive. The 500-entry cap is applied client-side
// in the React hooks (useAgentLogs / useMultiAgentLogs).
let scopedStore: TaskStore;
try {
scopedStore = await resolveProjectScopedStore(projectId);

View File

@@ -2912,6 +2912,30 @@ describe("maintenance cycle concurrency", () => {
expect((manager as any).maintenanceRunning).toBe(false);
});
it("uses a passive WAL checkpoint during maintenance", async () => {
(vi.spyOn(manager as any, "pruneWorktrees").mockResolvedValue(0) as any);
(vi.spyOn(manager as any, "cleanupOrphans").mockResolvedValue(0) as any);
(vi.spyOn(manager as any, "cleanupOrphanedBranches").mockResolvedValue(0) as any);
(vi.spyOn(manager as any, "enforceWorktreeCap").mockResolvedValue(0) as any);
(vi.spyOn(manager as any, "recoverCompletedTasks").mockResolvedValue(0) as any);
(vi.spyOn(manager as any, "recoverStaleIncompleteReviewTasks").mockResolvedValue(0) as any);
(vi.spyOn(manager as any, "recoverInterruptedMergingTasks").mockResolvedValue(0) as any);
(vi.spyOn(manager as any, "recoverMergeableReviewTasks").mockResolvedValue(0) as any);
(vi.spyOn(manager as any, "recoverMergedReviewTasks").mockResolvedValue(0) as any);
(vi.spyOn(manager as any, "recoverMisclassifiedFailures").mockResolvedValue(0) as any);
(vi.spyOn(manager as any, "recoverNoProgressNoTaskDoneFailures").mockResolvedValue(0) as any);
(vi.spyOn(manager as any, "recoverPartialProgressNoTaskDoneFailures").mockResolvedValue(0) as any);
(vi.spyOn(manager as any, "recoverOrphanedExecutions").mockResolvedValue(0) as any);
(vi.spyOn(manager as any, "recoverApprovedTriageTasks").mockResolvedValue(0) as any);
(vi.spyOn(manager as any, "recoverOrphanedPlanningTasks").mockResolvedValue(0) as any);
(vi.spyOn(manager as any, "recoverGhostReviewTasks").mockResolvedValue(0) as any);
(vi.spyOn(manager as any, "archiveStaleDoneTasks").mockResolvedValue(0) as any);
await (manager as any).runMaintenance();
expect(store.walCheckpoint).toHaveBeenCalledWith("PASSIVE");
});
it("runs batch 1 operations in sequence (with isolation — one failure doesn't block others)", async () => {
let runningCount = 0;
let maxConcurrent = 0;

View File

@@ -179,7 +179,8 @@ export class AgentLogger {
*
* @param name - The tool name
* @param isError - Whether the tool execution resulted in an error
* @param result - Optional result value (persisted in full)
* @param result - Optional result value. Downstream storage may clip very
* large tool payloads to keep dashboard log views responsive.
*/
onToolEnd(name: string, isError: boolean, result?: unknown): void {
const type = isError ? "tool_error" : "tool_result";

View File

@@ -1844,12 +1844,12 @@ export class SelfHealingManager {
}
}
/** Run SQLite WAL checkpoint to reclaim disk space. */
/** Run a best-effort passive WAL checkpoint without forcing live writers to truncate. */
private checkpointWal(): void {
try {
const result = this.store.walCheckpoint();
const result = this.store.walCheckpoint("PASSIVE");
if (result.log > 0) {
log.log(`WAL checkpoint: ${result.checkpointed}/${result.log} pages checkpointed` +
log.log(`WAL checkpoint (passive): ${result.checkpointed}/${result.log} pages checkpointed` +
(result.busy > 0 ? ` (${result.busy} busy)` : ""));
}
} catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err);