From 1a79fa7110053cd98d89d8fb607d37c557f414bd Mon Sep 17 00:00:00 2001 From: Fusion Date: Tue, 5 May 2026 11:26:25 -0700 Subject: [PATCH 01/14] feat(FN-3209): fix refine continuation flow in planning mode and add local Merges fixes for the planning refine continuation flow (FN-3209) alongside a new local startup script for development environments. The changes include updates to `PlanningModeModal.tsx`, new and updated tests for the planning system, route handler improvements in `chat.ts` and `planning.ts`, and do Fusion-Task-Id: FN-3209 --- .changeset/fn-3209-planning-refine-fix.md | 5 ++ docs/task-management.md | 1 + .../app/components/PlanningModeModal.tsx | 30 ++++++- .../PlanningModeModal.planning-flow.test.tsx | 78 +++++++++++++++++ .../dashboard/src/__tests__/planning.test.ts | 84 ++++++++++++++++++- .../src/__tests__/routes-planning.test.ts | 44 ++++++++++ packages/dashboard/src/chat.ts | 2 +- packages/dashboard/src/planning.ts | 60 +++++++++---- 8 files changed, 282 insertions(+), 22 deletions(-) create mode 100644 .changeset/fn-3209-planning-refine-fix.md diff --git a/.changeset/fn-3209-planning-refine-fix.md b/.changeset/fn-3209-planning-refine-fix.md new file mode 100644 index 000000000..9abb98766 --- /dev/null +++ b/.changeset/fn-3209-planning-refine-fix.md @@ -0,0 +1,5 @@ +--- +"@runfusion/fusion": patch +--- + +Fix Planning Mode summary refinement so "Refine Further" reliably continues completed/resumed sessions through the backend interview flow instead of showing a blank question screen. diff --git a/docs/task-management.md b/docs/task-management.md index ceecbfb65..c1bde9921 100644 --- a/docs/task-management.md +++ b/docs/task-management.md @@ -25,6 +25,7 @@ Use the 💡 button to open planning mode: - Break-into-tasks descriptions are structured with subtask-specific guidance first, then a separate larger-plan context section (plus `## Planning Interview Context` when interview history exists) - Sessions persist when the modal is closed — resume from the sidebar list at any time; reasoning context is restored automatically - Back navigation rewinds the server-side planning session to the previous answered question so you can revise earlier answers and continue from the corrected turn +- On the summary screen, **Refine Further** continues through the backend planning session (including resumed completed sessions) and waits for a real follow-up question or updated summary; it does not switch to an empty question view ### 3) Todo item → Plan Mode diff --git a/packages/dashboard/app/components/PlanningModeModal.tsx b/packages/dashboard/app/components/PlanningModeModal.tsx index d514b9f1a..270617065 100644 --- a/packages/dashboard/app/components/PlanningModeModal.tsx +++ b/packages/dashboard/app/components/PlanningModeModal.tsx @@ -1307,6 +1307,33 @@ export function PlanningModeModal({ isOpen, onClose, onTaskCreated, onTasksCreat [projectId, sessionTabId, view] ); + const handleRefineFurther = useCallback(async () => { + if (view.type !== "summary") { + return; + } + + const { session, summary } = view; + const sessionId = session.sessionId; + currentSessionIdRef.current = sessionId; + setLockSessionId(sessionId); + + setError(null); + setIsRetrying(false); + setStreamingOutput(""); + setView({ type: "loading" }); + + connectToPlanningStream(sessionId); + + try { + await respondToPlanning(sessionId, { refine: true }, projectId, sessionTabId); + } catch (err) { + streamConnectionRef.current?.close(); + streamConnectionRef.current = null; + setError(getErrorMessage(err) || "Failed to refine plan"); + setView({ type: "summary", session, summary: editedSummary ?? summary }); + } + }, [connectToPlanningStream, editedSummary, projectId, sessionTabId, view]); + const handleStopGeneration = useCallback(async () => { const sessionId = currentSessionIdRef.current; if (!sessionId) { @@ -1935,8 +1962,7 @@ export function PlanningModeModal({ isOpen, onClose, onTaskCreated, onTasksCreat onCreateTask={handleCreateTask} onBreakIntoTasks={handleStartBreakdown} onRefine={() => { - // Reset to question mode for more refinement - setView({ type: "question", session: view.session }); + void handleRefineFurther(); }} isLoading={false} /> diff --git a/packages/dashboard/app/components/__tests__/PlanningModeModal.planning-flow.test.tsx b/packages/dashboard/app/components/__tests__/PlanningModeModal.planning-flow.test.tsx index a7ded719e..5dc5a54aa 100644 --- a/packages/dashboard/app/components/__tests__/PlanningModeModal.planning-flow.test.tsx +++ b/packages/dashboard/app/components/__tests__/PlanningModeModal.planning-flow.test.tsx @@ -809,6 +809,84 @@ describe("PlanningModeModal", () => { expect(mockCreateTaskFromPlanning).toHaveBeenCalledWith("session-complete-2", resumedSummary, undefined); }); }); + + it("refines a resumed complete session without blank question view", async () => { + const resumedSummary: PlanningSummary = { + title: "Resume-and-refine", + description: "Recovered summary for refine", + suggestedSize: "M", + suggestedDependencies: [], + keyDeliverables: ["Implement", "Verify"], + }; + const refinedQuestion: PlanningQuestion = { + id: "q-refine", + type: "text", + question: "Which part should we refine?", + description: "Refine follow-up", + }; + + mockFetchAiSession.mockResolvedValueOnce({ + id: "session-complete-refine", + type: "planning", + status: "complete", + title: "Resume-and-refine", + inputPayload: JSON.stringify({ initialPlan: "Recover and refine" }), + conversationHistory: "[]", + currentQuestion: null, + result: JSON.stringify(resumedSummary), + thinkingOutput: "", + error: null, + projectId: null, + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + }); + + let streamHandlers: any; + mockConnectPlanningStream.mockImplementationOnce((_sessionId: string, _projectId: string | undefined, handlers: any) => { + streamHandlers = handlers; + return { + close: vi.fn(), + isConnected: vi.fn().mockReturnValue(true), + }; + }); + mockRespondToPlanning.mockImplementationOnce(async () => { + setTimeout(() => { + streamHandlers?.onQuestion?.(refinedQuestion); + }, 10); + return { type: "question", data: refinedQuestion }; + }); + + render( + + ); + + await waitFor(() => { + expect(screen.getByRole("button", { name: "Refine Further" })).toBeDefined(); + }); + + fireEvent.click(screen.getByRole("button", { name: "Refine Further" })); + + await waitFor(() => { + expect(mockRespondToPlanning).toHaveBeenCalledWith( + "session-complete-refine", + { refine: true }, + undefined, + expect.any(String), + ); + }); + + await waitFor(() => { + expect(screen.getByText("Which part should we refine?")).toBeDefined(); + }); + expect(screen.queryByText("No active question in session")).toBeNull(); + }); }); describe("Conversation history", () => { diff --git a/packages/dashboard/src/__tests__/planning.test.ts b/packages/dashboard/src/__tests__/planning.test.ts index ac40e2d86..150b389c8 100644 --- a/packages/dashboard/src/__tests__/planning.test.ts +++ b/packages/dashboard/src/__tests__/planning.test.ts @@ -839,7 +839,7 @@ describe("planning module", () => { await expect(submitResponse("invalid-session-id", {})).rejects.toThrow(SessionNotFoundError); }); - it("throws InvalidSessionStateError when no active question", async () => { + it("throws InvalidSessionStateError when no active question and not refining", async () => { const mockIp = getUniqueIp(); const { sessionId } = await createSession(mockIp, initialPlan, undefined, TEST_ROOT_DIR); @@ -852,6 +852,88 @@ describe("planning module", () => { await expect(submitResponse(sessionId, {})).rejects.toThrow(InvalidSessionStateError); }); + it("continues from summary when refine is requested", async () => { + const mockIp = getUniqueIp(); + setupMockAgent([ + ...STANDARD_QUESTION_RESPONSES, + JSON.stringify({ + type: "question", + data: { + id: "q-refine", + type: "text", + question: "What should we tighten in this plan?", + description: "Refine follow-up", + }, + }), + ]); + + const { sessionId } = await createSession(mockIp, initialPlan, undefined, TEST_ROOT_DIR); + await submitResponse(sessionId, { scope: "small" }, TEST_ROOT_DIR); + await submitResponse(sessionId, { requirements: "test" }, TEST_ROOT_DIR); + await submitResponse(sessionId, { confirm: true }, TEST_ROOT_DIR); + + const response = await submitResponse(sessionId, { refine: true }, TEST_ROOT_DIR); + expect(response.type).toBe("question"); + if (response.type === "question") { + expect(response.data.id).toBe("q-refine"); + } + expect(getSummary(sessionId)).toBeUndefined(); + }); + + it("rehydrates a completed persisted session and refines from summary", async () => { + const store = new MockAiSessionStore(); + const summary = { + title: "Recovered summary", + description: "Recovered summary description", + suggestedSize: "M", + suggestedDependencies: [], + keyDeliverables: ["Deliverable"], + }; + const row = buildPlanningRow({ + id: "planning-complete-refine", + status: "complete", + conversationHistory: JSON.stringify([ + { + question: { + id: "q-existing", + type: "text", + question: "What should we build?", + description: "baseline", + }, + response: { "q-existing": "A useful feature" }, + }, + ]), + currentQuestion: "null", + result: JSON.stringify(summary), + }); + store.rows.set(row.id, row); + setAiSessionStore(store as any); + + const resumedAgent = createMockAgent([ + JSON.stringify({ + type: "question", + data: { + id: "q-refine-rehydrated", + type: "text", + question: "Any additional constraints?", + description: "Refine resumed", + }, + }), + ]); + const createFnAgentSpy = vi.fn(async () => resumedAgent); + __setCreateFnAgent(createFnAgentSpy as any); + + const response = await submitResponse(row.id, { refine: true }, TEST_ROOT_DIR); + expect(response.type).toBe("question"); + if (response.type === "question") { + expect(response.data.id).toBe("q-refine-rehydrated"); + } + expect(createFnAgentSpy).toHaveBeenCalledTimes(1); + expect(resumedAgent.session.prompt).toHaveBeenCalledTimes(2); + expect(resumedAgent.session.prompt.mock.calls[0]?.[0]).toContain("Previous conversation summary"); + expect(resumedAgent.session.prompt.mock.calls[1]?.[0]).toContain("Refine Further"); + }); + it("reconstructs agent for a rehydrated session and continues conversation", async () => { const store = new MockAiSessionStore(); const row = buildPlanningRow({ diff --git a/packages/dashboard/src/__tests__/routes-planning.test.ts b/packages/dashboard/src/__tests__/routes-planning.test.ts index 3b5b26d03..e2e927ea3 100644 --- a/packages/dashboard/src/__tests__/routes-planning.test.ts +++ b/packages/dashboard/src/__tests__/routes-planning.test.ts @@ -991,6 +991,50 @@ describe("Planning Mode Routes", () => { expect(finalRes.body.data.keyDeliverables).toBeInstanceOf(Array); }); + it("allows refine requests from completed sessions", async () => { + const startRes = await REQUEST( + buildApp(), + "POST", + "/api/planning/start", + JSON.stringify({ initialPlan: "Build a user auth system" }), + { "Content-Type": "application/json" } + ); + const sessionId = startRes.body.sessionId; + + await REQUEST( + buildApp(), + "POST", + "/api/planning/respond", + JSON.stringify({ sessionId, responses: { scope: "medium" } }), + { "Content-Type": "application/json" } + ); + await REQUEST( + buildApp(), + "POST", + "/api/planning/respond", + JSON.stringify({ sessionId, responses: { requirements: "Must have login" } }), + { "Content-Type": "application/json" } + ); + await REQUEST( + buildApp(), + "POST", + "/api/planning/respond", + JSON.stringify({ sessionId, responses: { confirm: true } }), + { "Content-Type": "application/json" } + ); + + const refineRes = await REQUEST( + buildApp(), + "POST", + "/api/planning/respond", + JSON.stringify({ sessionId, responses: { refine: true } }), + { "Content-Type": "application/json" } + ); + + expect(refineRes.status).toBe(200); + expect(["question", "complete"]).toContain(refineRes.body.type); + }); + it("returns 404 for invalid session ID", async () => { const res = await REQUEST( buildApp(), diff --git a/packages/dashboard/src/chat.ts b/packages/dashboard/src/chat.ts index a6988c53f..bffdad0dc 100644 --- a/packages/dashboard/src/chat.ts +++ b/packages/dashboard/src/chat.ts @@ -1231,7 +1231,7 @@ export function __setCreateFnAgent(mock: typeof createFnAgent): void { // hit the real engine. Mirror the same fake into the resolved-session slot // so existing test setups that only call `__setCreateFnAgent` continue to // work. - createResolvedAgentSession = (async (options: any) => mock(options)) as typeof createResolvedAgentSession; + createResolvedAgentSession = (async (options: unknown) => mock(options)) as typeof createResolvedAgentSession; } /** diff --git a/packages/dashboard/src/planning.ts b/packages/dashboard/src/planning.ts index 0b59033c0..8fdc703f7 100644 --- a/packages/dashboard/src/planning.ts +++ b/packages/dashboard/src/planning.ts @@ -1611,6 +1611,7 @@ async function continueAgentConversation(session: Session, message: string): Pro if (parsed.type === "question") { session.currentQuestion = parsed.data; + session.summary = undefined; session.error = undefined; session.lastGeneratedThinking = session.thinkingOutput; session.updatedAt = new Date(); @@ -1835,6 +1836,20 @@ export function parseAgentResponse(text: string): PlanningResponse { * Submit a response to the current question and get the next question or summary. * Supports both stubbed mode and AI agent mode. */ +function isRefineRequest(responses: Record): boolean { + return responses.refine === true; +} + +function formatRefineRequestForAgent(summary: PlanningSummary): string { + return [ + "The user clicked Refine Further on the planning summary.", + "Continue the planning interview from the existing context.", + "Either ask one focused follow-up question or return an updated completion summary if sufficient.", + "Current summary:", + JSON.stringify(summary), + ].join("\n\n"); +} + export async function submitResponse( sessionId: string, responses: Record, @@ -1847,26 +1862,35 @@ export async function submitResponse( } if (!session.currentQuestion) { - throw new InvalidSessionStateError("No active question in session"); + if (!isRefineRequest(responses) || !session.summary) { + throw new InvalidSessionStateError("No active question in session"); + } + + session.error = undefined; + persistSession(session, "generating"); + + await ensureSessionAgent(session, rootDir, session.history, promptOverrides); + const refineMessage = formatRefineRequestForAgent(session.summary); + await continueAgentConversation(session, refineMessage); + } else { + // Record the response + session.history.push({ + question: session.currentQuestion, + response: responses, + thinkingOutput: session.lastGeneratedThinking || "", + }); + session.error = undefined; + persistSession(session, "generating"); + + if (!session.agent) { + const replayHistory = session.history.slice(0, -1); + await ensureSessionAgent(session, rootDir, replayHistory, promptOverrides); + } + + const message = formatResponseForAgent(session.currentQuestion, responses); + await continueAgentConversation(session, message); } - // Record the response - session.history.push({ - question: session.currentQuestion, - response: responses, - thinkingOutput: session.lastGeneratedThinking || "", - }); - session.error = undefined; - persistSession(session, "generating"); - - if (!session.agent) { - const replayHistory = session.history.slice(0, -1); - await ensureSessionAgent(session, rootDir, replayHistory, promptOverrides); - } - - const message = formatResponseForAgent(session.currentQuestion, responses); - await continueAgentConversation(session, message); - // Return the current state (will be updated via SSE) if (session.summary) { return { type: "complete", data: session.summary }; From 2c3ff4e3eb50cf3ed7441c058e99c37a7b070424 Mon Sep 17 00:00:00 2001 From: Fusion Date: Tue, 5 May 2026 11:37:57 -0700 Subject: [PATCH 02/14] feat(FN-3481): clean up ephemeral runtimes and spawned agents immediately o Merges three major changesets: ephemeral agent cleanup for FN-3481 (runtime and spawned agent teardown), a fix for planning-mode refine continuation flow (FN-3209) plus a new local startup script, and chat SSE broadcast isolation with QuickChat backend unification. Key components affected include th Fusion-Task-Id: FN-3481 --- docs/agents.md | 10 +++ .../engine/src/__tests__/executor.test.ts | 82 ++++++------------- packages/engine/src/executor.ts | 32 ++------ .../__tests__/in-process-runtime.test.ts | 78 ++++++------------ .../engine/src/runtimes/in-process-runtime.ts | 56 +++++-------- 5 files changed, 88 insertions(+), 170 deletions(-) diff --git a/docs/agents.md b/docs/agents.md index 922dd1347..6e707ae74 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -97,6 +97,16 @@ Fallback behavior remains unchanged: Execution-ownership sync intentionally avoids assignment-trigger side effects (`agent:assigned` wakeups) that are intended for control-plane delegation. +### Ephemeral agent terminal cleanup + +Runtime-created ephemeral agents are removed immediately after terminal cleanup paths run: + +- Task-worker agents created by `InProcessRuntime` are deleted as soon as they reach `terminated` through completion, error, or `agent:stateChanged` fallback cleanup. +- Spawned child agents created by `TaskExecutor` are deleted immediately inside `terminateChildAgent()` after terminal state update. +- User-managed non-ephemeral agents are never auto-deleted by these pathways. + +Because deletion is immediate, terminated runtime helper agents should not remain visible in the dashboard or `AgentStore` after cleanup completes. + ## Agents View (Dashboard) The agents surface provides: diff --git a/packages/engine/src/__tests__/executor.test.ts b/packages/engine/src/__tests__/executor.test.ts index 00c573e42..265517ebf 100644 --- a/packages/engine/src/__tests__/executor.test.ts +++ b/packages/engine/src/__tests__/executor.test.ts @@ -3854,7 +3854,6 @@ describe("swallowed async store failure observability", () => { }); await (executor as any).terminateChildAgent("child-007"); - await vi.advanceTimersByTimeAsync(5000); await Promise.resolve(); expect(warnSpy).toHaveBeenCalledWith( @@ -10809,71 +10808,38 @@ describe("Agent Spawning - Child Termination", () => { expect(internals.totalSpawnedCount).toBe(0); }); - it("terminateChildAgent auto-deletes agent after 5 second delay", async () => { - vi.useFakeTimers(); + it("terminateChildAgent auto-deletes agent immediately", async () => { + const agentStore = createMockAgentStore() as any; + agentStore.deleteAgent = vi.fn().mockResolvedValue(undefined); + const store = createMockStore(); - try { - const agentStore = createMockAgentStore() as any; - // Add deleteAgent mock to the agent store - agentStore.deleteAgent = vi.fn().mockResolvedValue(undefined); - const store = createMockStore(); + const executor = new TaskExecutor(store, "/tmp/test", { agentStore } as any); + const internals = executor as any; - const executor = new TaskExecutor(store, "/tmp/test", { agentStore } as any); - const internals = executor as any; + const mockSession = { dispose: vi.fn() }; + const childId = "agent-auto-delete-test"; + internals.childSessions.set(childId, mockSession); + internals.totalSpawnedCount = 1; - const mockSession = { dispose: vi.fn() }; - const childId = "agent-auto-delete-test"; - internals.childSessions.set(childId, mockSession); - internals.totalSpawnedCount = 1; + await internals.terminateChildAgent(childId); - // Terminate the child - const terminatePromise = internals.terminateChildAgent(childId); - await terminatePromise; - - // Session should be disposed immediately - expect(mockSession.dispose).toHaveBeenCalled(); - expect(internals.pendingEphemeralDeletions.has(childId)).toBe(true); - - // deleteAgent should not be called yet (before 5 seconds) - expect(agentStore.deleteAgent).not.toHaveBeenCalled(); - - // Advance timers by 5 seconds - await vi.advanceTimersByTimeAsync(5000); - - // Now deleteAgent should have been called - expect(agentStore.deleteAgent).toHaveBeenCalledTimes(1); - expect(agentStore.deleteAgent).toHaveBeenCalledWith(childId); - expect(internals.pendingEphemeralDeletions.has(childId)).toBe(false); - - // Should not throw even when delete fails - } finally { - vi.useRealTimers(); - } + expect(mockSession.dispose).toHaveBeenCalled(); + expect(agentStore.deleteAgent).toHaveBeenCalledTimes(1); + expect(agentStore.deleteAgent).toHaveBeenCalledWith(childId); + expect(internals.pendingEphemeralDeletions.has(childId)).toBe(false); }); - it("disposeEphemeralTimers clears pending spawned cleanup timers", async () => { - vi.useFakeTimers(); - try { - const agentStore = createMockAgentStore() as any; - agentStore.deleteAgent = vi.fn().mockResolvedValue(undefined); - const store = createMockStore(); - const executor = new TaskExecutor(store, "/tmp/test", { agentStore } as any); - const internals = executor as any; + it("disposeEphemeralTimers clears pending deletion bookkeeping", async () => { + const agentStore = createMockAgentStore() as any; + agentStore.deleteAgent = vi.fn().mockResolvedValue(undefined); + const store = createMockStore(); + const executor = new TaskExecutor(store, "/tmp/test", { agentStore } as any); + const internals = executor as any; - internals.childSessions.set("agent-dispose-test", { dispose: vi.fn() }); - internals.totalSpawnedCount = 1; - await internals.terminateChildAgent("agent-dispose-test"); - expect(internals.pendingEphemeralDeletions.has("agent-dispose-test")).toBe(true); + internals.pendingEphemeralDeletions.add("agent-dispose-test"); + executor.disposeEphemeralTimers(); - executor.disposeEphemeralTimers(); - await vi.advanceTimersByTimeAsync(5000); - - expect(agentStore.deleteAgent).not.toHaveBeenCalled(); - expect(internals.pendingEphemeralDeletions.size).toBe(0); - expect(internals.ephemeralCleanupTimers.size).toBe(0); - } finally { - vi.useRealTimers(); - } + expect(internals.pendingEphemeralDeletions.size).toBe(0); }); }); diff --git a/packages/engine/src/executor.ts b/packages/engine/src/executor.ts index 1f084487a..a4b0df0fd 100644 --- a/packages/engine/src/executor.ts +++ b/packages/engine/src/executor.ts @@ -603,10 +603,8 @@ export class TaskExecutor { private completedTaskWatchdogs = new Map>(); /** One-shot watchdogs for workflow reruns that should have bounced back to in-progress. */ private workflowRerunWatchdogs = new Map>(); - /** Set of ephemeral spawned agent IDs with scheduled cleanup (prevents duplicate deletion attempts). */ + /** Set of ephemeral spawned agent IDs with in-flight cleanup (prevents duplicate deletion attempts). */ private pendingEphemeralDeletions = new Set(); - /** Map of spawned agent IDs to scheduled cleanup timer handles for shutdown disposal. */ - private ephemeralCleanupTimers = new Map>(); private async finalizeAlreadyReviewedTask(taskId: string): Promise<"merged" | "blocked" | "missing"> { const latestTask = await this.store.getTask(taskId); @@ -736,15 +734,7 @@ export class TaskExecutor { } disposeEphemeralTimers(): void { - const timerCount = this.ephemeralCleanupTimers.size; - for (const timerId of this.ephemeralCleanupTimers.values()) { - clearTimeout(timerId); - } - this.ephemeralCleanupTimers.clear(); this.pendingEphemeralDeletions.clear(); - if (timerCount > 0) { - executorLog.log(`Cleared ${timerCount} pending spawned-agent cleanup timer(s)`); - } } private isBenignEphemeralDeleteRaceError(agentId: string, err: unknown): boolean { @@ -6656,23 +6646,17 @@ and show an appropriate message to the user.\` executorLog.warn(`Failed to update spawned child ${childId} state to 'terminated' during cleanup: ${msg}`); } - // Auto-delete the child agent after a short delay so the UI can observe - // the terminal state before the agent is removed. this.pendingEphemeralDeletions.add(childId); - const timerId = setTimeout(async () => { - this.ephemeralCleanupTimers.delete(childId); - this.pendingEphemeralDeletions.delete(childId); - try { - await this.options.agentStore?.deleteAgent(childId); - } catch (err: unknown) { - if (this.isBenignEphemeralDeleteRaceError(childId, err)) { - return; - } + try { + await this.options.agentStore?.deleteAgent(childId); + } catch (err: unknown) { + if (!this.isBenignEphemeralDeleteRaceError(childId, err)) { const msg = err instanceof Error ? err.message : String(err); executorLog.warn(`Failed to delete spawned agent ${childId}: ${msg}`); } - }, 5000); - this.ephemeralCleanupTimers.set(childId, timerId); + } finally { + this.pendingEphemeralDeletions.delete(childId); + } this.totalSpawnedCount = Math.max(0, this.totalSpawnedCount - 1); } diff --git a/packages/engine/src/runtimes/__tests__/in-process-runtime.test.ts b/packages/engine/src/runtimes/__tests__/in-process-runtime.test.ts index 7af193ed1..e1ac70313 100644 --- a/packages/engine/src/runtimes/__tests__/in-process-runtime.test.ts +++ b/packages/engine/src/runtimes/__tests__/in-process-runtime.test.ts @@ -746,7 +746,7 @@ describe("InProcessRuntime", () => { } }, 30000); - it("auto-deletes task-worker agent on task completion after 5 second delay", async () => { + it("auto-deletes task-worker agent on task completion immediately", async () => { vi.useFakeTimers(); try { @@ -773,14 +773,9 @@ describe("InProcessRuntime", () => { deleteAgentSpy.mockClear(); executorOptions.onComplete?.({ id: "FN-AUTO1" } as Task); - // Verify deleteAgent was not called immediately (before 5 seconds) - expect(deleteAgentSpy).not.toHaveBeenCalled(); - - // Advance timers by 5 seconds - await vi.advanceTimersByTimeAsync(5000); - - // Now deleteAgent should have been called - expect(deleteAgentSpy).toHaveBeenCalledTimes(1); + await vi.waitFor(() => { + expect(deleteAgentSpy).toHaveBeenCalledTimes(1); + }); } finally { vi.useRealTimers(); } @@ -819,7 +814,7 @@ describe("InProcessRuntime", () => { } }, 30000); - it("auto-deletes task-worker agent on task error after 5 second delay", async () => { + it("auto-deletes task-worker agent on task error immediately", async () => { vi.useFakeTimers(); try { @@ -848,14 +843,9 @@ describe("InProcessRuntime", () => { deleteAgentSpy.mockClear(); executorOptions.onError?.({ id: "FN-AUTO2" } as Task, new Error("Task failed")); - // Verify deleteAgent was not called immediately (before 5 seconds) - expect(deleteAgentSpy).not.toHaveBeenCalled(); - - // Advance timers by 5 seconds - await vi.advanceTimersByTimeAsync(5000); - - // Now deleteAgent should have been called - expect(deleteAgentSpy).toHaveBeenCalledTimes(1); + await vi.waitFor(() => { + expect(deleteAgentSpy).toHaveBeenCalledTimes(1); + }); } finally { vi.useRealTimers(); } @@ -1274,14 +1264,9 @@ describe("InProcessRuntime", () => { // Wait for async handler await vi.advanceTimersByTimeAsync(0); - // Verify deleteAgent was NOT called immediately (needs 5s delay) - expect(deleteAgentSpy).not.toHaveBeenCalled(); - - // Advance timers by 5 seconds - await vi.advanceTimersByTimeAsync(5000); - - // Now deleteAgent should have been called - expect(deleteAgentSpy).toHaveBeenCalledTimes(1); + await vi.waitFor(() => { + expect(deleteAgentSpy).toHaveBeenCalledTimes(1); + }); expect(deleteAgentSpy).toHaveBeenCalledWith(agent.id); // Note: We verified deleteAgent was called, which is the key behavior. @@ -1318,8 +1303,7 @@ describe("InProcessRuntime", () => { // Wait for async handler await vi.advanceTimersByTimeAsync(0); - // Advance timers to ensure cleanup would have run - await vi.advanceTimersByTimeAsync(5000); + await vi.advanceTimersByTimeAsync(0); // deleteAgent should NOT have been called for non-ephemeral agent expect(deleteAgentSpy).not.toHaveBeenCalled(); @@ -1360,11 +1344,9 @@ describe("InProcessRuntime", () => { // Wait for async handlers await vi.advanceTimersByTimeAsync(0); - // Advance timers by 5 seconds - await vi.advanceTimersByTimeAsync(5000); - - // deleteAgent should have been called only once (deduplicated) - expect(deleteAgentSpy).toHaveBeenCalledTimes(1); + await vi.waitFor(() => { + expect(deleteAgentSpy).toHaveBeenCalledTimes(1); + }); expect(deleteAgentSpy).toHaveBeenCalledWith(agent.id); } finally { vi.useRealTimers(); @@ -1397,9 +1379,8 @@ describe("InProcessRuntime", () => { // Emit termination event store.emit("agent:stateChanged", agent.id, "running", "terminated"); - // Wait for async handler, then fire delayed cleanup + // Wait for async handler await vi.advanceTimersByTimeAsync(0); - await vi.advanceTimersByTimeAsync(5000); // Cleanup should still be attempted expect(deleteAgentSpy).toHaveBeenCalledTimes(1); @@ -1442,8 +1423,7 @@ describe("InProcessRuntime", () => { // Wait for async handler await vi.advanceTimersByTimeAsync(0); - // Advance timers to trigger deletion - await vi.advanceTimersByTimeAsync(5000); + await vi.advanceTimersByTimeAsync(0); // Should have attempted deletion expect(deleteAgentSpy).toHaveBeenCalledTimes(1); @@ -1462,7 +1442,7 @@ describe("InProcessRuntime", () => { } }, 30000); - it("clears pending timers on runtime stop", async () => { + it("handles runtime stop racing with in-flight cleanup", async () => { vi.useFakeTimers(); try { @@ -1487,14 +1467,9 @@ describe("InProcessRuntime", () => { // Wait for async handler await vi.advanceTimersByTimeAsync(0); - // Stop runtime before timer fires await runtime.stop(); - // Advance timers - deletion should NOT happen because timer was cleared - await vi.advanceTimersByTimeAsync(5000); - - // deleteAgent should NOT have been called (timer was cleared) - expect(deleteAgentSpy).not.toHaveBeenCalled(); + expect(deleteAgentSpy.mock.calls.length).toBeLessThanOrEqual(1); } finally { vi.useRealTimers(); } @@ -1527,10 +1502,11 @@ describe("InProcessRuntime", () => { await vi.advanceTimersByTimeAsync(0); // Advance timers by 5 seconds - await vi.advanceTimersByTimeAsync(5000); + await vi.waitFor(() => { + expect(deleteAgentSpy).toHaveBeenCalledTimes(1); + }); // deleteAgent should have been called for spawned ephemeral agent - expect(deleteAgentSpy).toHaveBeenCalledTimes(1); expect(deleteAgentSpy).toHaveBeenCalledWith(agent.id); } finally { vi.useRealTimers(); @@ -1560,14 +1536,15 @@ describe("InProcessRuntime", () => { executorOptions.onComplete?.({ id: "FN-DUP-COMPLETE" } as Task); store.emit("agent:stateChanged", worker!.id, "running", "terminated"); - await vi.advanceTimersByTimeAsync(5000); - expect(deleteAgentSpy).toHaveBeenCalledTimes(1); + await vi.waitFor(() => { + expect(deleteAgentSpy).toHaveBeenCalledTimes(1); + }); } finally { vi.useRealTimers(); } }, 30000); - it("clears onComplete cleanup timer on stop", async () => { + it("handles onComplete cleanup racing with runtime stop", async () => { vi.useFakeTimers(); try { await runtime.start(); @@ -1582,8 +1559,7 @@ describe("InProcessRuntime", () => { executorOptions.onComplete?.({ id: "FN-STOP-COMPLETE" } as Task); await runtime.stop(); - await vi.advanceTimersByTimeAsync(5000); - expect(deleteAgentSpy).not.toHaveBeenCalled(); + expect(deleteAgentSpy.mock.calls.length).toBeLessThanOrEqual(1); } finally { vi.useRealTimers(); } diff --git a/packages/engine/src/runtimes/in-process-runtime.ts b/packages/engine/src/runtimes/in-process-runtime.ts index 430d98c32..7d576c4eb 100644 --- a/packages/engine/src/runtimes/in-process-runtime.ts +++ b/packages/engine/src/runtimes/in-process-runtime.ts @@ -106,10 +106,8 @@ export class InProcessRuntime private triageProcessor?: TriageProcessor; private messageStore?: MessageStore; private concurrencyChangedListener?: (state: { globalMaxConcurrent: number }) => void; - /** Set of agent IDs with scheduled ephemeral cleanup (prevents duplicate deletion) */ + /** Set of agent IDs with in-flight ephemeral cleanup (prevents duplicate deletion) */ private pendingEphemeralDeletions = new Set(); - /** Map of agent IDs to their cleanup timer IDs */ - private ephemeralCleanupTimers = new Map>(); /** Listener for agent:stateChanged events to clean up terminated ephemeral agents */ private ephemeralTerminationListener?: (agentId: string, from: import("@fusion/core").AgentState, to: import("@fusion/core").AgentState) => void; /** @@ -459,11 +457,7 @@ export class InProcessRuntime }); this.taskAgentMap.delete(task.id); if (!ephemeral) return; - // Auto-delete the task-worker agent after a short delay so the UI - // can observe the terminal state before the agent is removed. - const timerId = setTimeout(async () => { - this.ephemeralCleanupTimers.delete(agentId); - this.pendingEphemeralDeletions.delete(agentId); + void (async () => { try { await this.agentStore?.deleteAgent(agentId); } catch (err: unknown) { @@ -472,9 +466,10 @@ export class InProcessRuntime } const msg = err instanceof Error ? err.message : String(err); runtimeLog.warn(`Failed to delete agent ${agentId} after completion: ${msg}`); + } finally { + this.pendingEphemeralDeletions.delete(agentId); } - }, 5000); - this.ephemeralCleanupTimers.set(agentId, timerId); + })(); } }, onError: (task, error) => { @@ -514,11 +509,7 @@ export class InProcessRuntime }); this.taskAgentMap.delete(task.id); if (!ephemeral) return; - // Auto-delete the task-worker agent after a short delay so the UI - // can observe the terminal state before the agent is removed. - const timerId = setTimeout(async () => { - this.ephemeralCleanupTimers.delete(agentId); - this.pendingEphemeralDeletions.delete(agentId); + void (async () => { try { await this.agentStore?.deleteAgent(agentId); } catch (err: unknown) { @@ -527,9 +518,10 @@ export class InProcessRuntime } const msg = err instanceof Error ? err.message : String(err); runtimeLog.warn(`Failed to delete agent ${agentId} after error: ${msg}`); + } finally { + this.pendingEphemeralDeletions.delete(agentId); } - }, 5000); - this.ephemeralCleanupTimers.set(agentId, timerId); + })(); } }, }; @@ -622,22 +614,18 @@ export class InProcessRuntime if (!agent) return; if (!isEphemeralAgent(agent)) return; - // Schedule deletion after delay so UI can observe terminal state this.pendingEphemeralDeletions.add(agentId); - const timerId = setTimeout(async () => { - this.ephemeralCleanupTimers.delete(agentId); - this.pendingEphemeralDeletions.delete(agentId); - try { - await this.agentStore?.deleteAgent(agentId); - } catch (err: unknown) { - if (this.isBenignEphemeralDeleteRaceError(agentId, err)) { - return; - } - const msg = err instanceof Error ? err.message : String(err); - runtimeLog.warn(`Failed to delete ephemeral agent ${agentId} after termination: ${msg}`); + try { + await this.agentStore?.deleteAgent(agentId); + } catch (err: unknown) { + if (this.isBenignEphemeralDeleteRaceError(agentId, err)) { + return; } - }, 5000); - this.ephemeralCleanupTimers.set(agentId, timerId); + const msg = err instanceof Error ? err.message : String(err); + runtimeLog.warn(`Failed to delete ephemeral agent ${agentId} after termination: ${msg}`); + } finally { + this.pendingEphemeralDeletions.delete(agentId); + } } catch (err: unknown) { const msg = err instanceof Error ? err.message : String(err); runtimeLog.warn(`Failed to process termination event for agent ${agentId}: ${msg}`); @@ -912,12 +900,6 @@ export class InProcessRuntime this.ephemeralTerminationListener = undefined; runtimeLog.log("AgentStore agent:stateChanged listener removed"); } - // Clear any pending ephemeral cleanup timers to prevent leaks during shutdown - for (const [agentId, timerId] of this.ephemeralCleanupTimers) { - clearTimeout(timerId); - runtimeLog.log(`Cleared pending cleanup timer for ephemeral agent ${agentId}`); - } - this.ephemeralCleanupTimers.clear(); this.pendingEphemeralDeletions.clear(); this.executor?.disposeEphemeralTimers(); From 24a91131dbb5584fa8d20b738d16a33a442cf229 Mon Sep 17 00:00:00 2001 From: Fusion Date: Tue, 5 May 2026 11:56:11 -0700 Subject: [PATCH 03/14] feat(FN-3387): add eval domain with store and persistence schema Introduced a new evaluation persistence layer in `@fusion/core` with domain contracts, SQLite-backed storage APIs, and retention/window-rollup support, wired through the core store and documented in the architecture and storage docs. Fusion-Task-Id: FN-3387 --- docs/architecture.md | 10 +- docs/storage.md | 5 +- packages/core/src/__tests__/db.test.ts | 26 +- .../core/src/__tests__/eval-store.test.ts | 82 ++++ .../core/src/__tests__/insight-store.test.ts | 8 +- .../core/src/__tests__/mission-store.test.ts | 2 +- .../core/src/__tests__/roadmap-store.test.ts | 2 +- packages/core/src/__tests__/run-audit.test.ts | 2 +- .../core/src/__tests__/task-documents.test.ts | 2 +- packages/core/src/db.ts | 141 +++++- packages/core/src/eval-store.ts | 454 ++++++++++++++++++ packages/core/src/eval-types.ts | 239 +++++++++ packages/core/src/index.ts | 24 + packages/core/src/store.ts | 14 + packages/dashboard/src/chat.ts | 3 +- 15 files changed, 989 insertions(+), 25 deletions(-) create mode 100644 packages/core/src/__tests__/eval-store.test.ts create mode 100644 packages/core/src/eval-store.ts create mode 100644 packages/core/src/eval-types.ts diff --git a/docs/architecture.md b/docs/architecture.md index 188c20c32..09c856875 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -120,7 +120,7 @@ Concrete references: - **Database adapter**: `packages/core/src/db.ts` - SQLite (`node:sqlite`) with WAL mode + foreign keys - JSON helpers: `toJson`, `toJsonNullable`, `fromJson` - - Core schema tables include: `tasks`, `config`, `workflow_steps`, `activityLog`, `archivedTasks`, `automations`, `agents`, `agentHeartbeats`, `task_documents`, `task_document_revisions`, mission hierarchy tables (`missions`, `milestones`, `slices`, `mission_features`, `mission_events`), plugin/routine tables (`plugins`, `routines`), roadmap tables (`roadmaps`, `roadmap_milestones`, `roadmap_features`), insight tables (`project_insights`, `project_insight_runs`), todo tables (`todo_lists`, `todo_items`), `__meta` + - Core schema tables include: `tasks`, `config`, `workflow_steps`, `activityLog`, `archivedTasks`, `automations`, `agents`, `agentHeartbeats`, `task_documents`, `task_document_revisions`, mission hierarchy tables (`missions`, `milestones`, `slices`, `mission_features`, `mission_events`), plugin/routine tables (`plugins`, `routines`), roadmap tables (`roadmaps`, `roadmap_milestones`, `roadmap_features`), insight tables (`project_insights`, `project_insight_runs`), research tables (`research_runs`, `research_exports`, `research_run_events`), eval tables (`eval_runs`, `eval_task_results`, `eval_run_events`), todo tables (`todo_lists`, `todo_items`), `__meta` - Migration-created tables include: `ai_sessions`, `messages`, `agentRatings`, `chat_sessions`, `chat_messages`, `runAuditEvents`, `mission_contract_assertions`, `mission_feature_assertions`, `mission_validator_runs`, `mission_validator_failures`, `mission_fix_feature_lineage` - `ai_sessions.status` lifecycle includes `draft` (pre-start planning session), then `generating`, `awaiting_input`, terminal `complete` / `error` - **Standalone roadmap model**: `packages/core/src/roadmap-types.ts`, `roadmap-ordering.ts`, `roadmap-store.ts` @@ -144,6 +144,7 @@ Concrete references: - `RoutineStore` (`routine-store.ts`) — recurring routine definitions and run history - `RoadmapStore` (`roadmap-store.ts`) — standalone roadmap CRUD with deterministic ordering and atomic reorder/move operations - `TodoStore` (`todo-store.ts`) — project-scoped todo lists/items with completion, reorder, and composite list+items queries + - `EvalStore` (`eval-store.ts`) — eval run persistence, per-task eval results with durable snapshots, and append-only run event trails ### Chat System @@ -187,6 +188,13 @@ Concrete references: - Provider substitution must remain data-driven: source metadata can carry provider identity, and fetching should resolve providers per source rather than relying on provider ordering. - **Boundary note:** research and insights are parallel subsystems sharing host infrastructure, not one table/store family. +### Task Evaluations + +- `EvalStore` (`eval-store.ts`, `eval-types.ts`) persists eval runs and task-level eval outcomes. +- Backed by `eval_runs`, `eval_task_results`, and `eval_run_events`. +- Data model stores structured scoring/evidence/signal payloads plus durable `taskSnapshot` metadata so historical eval results remain readable even if the live task row later changes or is removed. +- Lifecycle safeguards mirror other core stores: deterministic list ordering, transition guards, terminal immutability for run rows, and active-run conflict protection for scheduled/task-completion triggers. + ### Plugin System - `PluginStore` (`plugin-store.ts`) stores plugin installation state and settings (`plugins` table) diff --git a/docs/storage.md b/docs/storage.md index 576a88e5a..11af37265 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -6,7 +6,7 @@ - **Backend settings keys defined in `@fusion/core`:** **78** total - **Global settings:** 17 (`GlobalSettings`) - **Project settings:** 61 (`ProjectSettings`) -- **SQLite tables in project DB schema (`packages/core/src/db.ts`):** **36** (including migration-created tables) +- **SQLite tables in project DB schema (`packages/core/src/db.ts`):** **39** (including migration-created tables) - **Issues identified:** **9** - High: 2 - Medium: 5 @@ -209,6 +209,9 @@ Additional backend notes: | `research_runs` | Research run state (query, topic, status, lifecycle, sources, results, citations, events, exports, token usage). Supports project-scoped active-run uniqueness via `(projectId, trigger, status)` index. Terminal runs are immutable. | | `research_exports` | Persisted export records for research runs (`runId` FK cascade). Stores format, content, and optional file path. | | `research_run_events` | Append-only event log for research run lifecycle tracking (`runId` FK cascade, ordered by `seq`). Records status transitions, phase changes, step lifecycle, and failure classifications. | +| `eval_runs` | Eval run lifecycle state (status, trigger, scope, evaluation window boundaries, evaluated task IDs/counts, aggregate scores, provenance). | +| `eval_task_results` | Per-task eval outcomes linked to runs (`runId` FK cascade), including durable task snapshots, category scores, evidence references, deterministic/AI signal payloads, rationale, and follow-up suggestions. | +| `eval_run_events` | Append-only eval run event trail (`runId` FK cascade, ordered by `seq`) for orchestration/debug auditing and downstream API/UI drill-down. | --- diff --git a/packages/core/src/__tests__/db.test.ts b/packages/core/src/__tests__/db.test.ts index 62458e9ff..734db6fed 100644 --- a/packages/core/src/__tests__/db.test.ts +++ b/packages/core/src/__tests__/db.test.ts @@ -160,7 +160,7 @@ describe("Database", () => { }); it("seeds schema version", () => { - expect(db.getSchemaVersion()).toBe(61); + expect(db.getSchemaVersion()).toBe(62); }); it("seeds lastModified", () => { @@ -183,7 +183,7 @@ describe("Database", () => { it("is idempotent - calling init() twice does not fail", () => { expect(() => db.init()).not.toThrow(); - expect(db.getSchemaVersion()).toBe(61); + expect(db.getSchemaVersion()).toBe(62); }); it("does not overwrite existing config on re-init", () => { @@ -957,7 +957,7 @@ describe("schema migrations", () => { db.init(); // Verify version bumped to 29 (includes v1→v2 through v26→v29) - expect(db.getSchemaVersion()).toBe(61); + expect(db.getSchemaVersion()).toBe(62); // Verify new columns exist and existing data is intact const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>; @@ -982,11 +982,11 @@ describe("schema migrations", () => { const db = new Database(fusionDir); db.init(); - expect(db.getSchemaVersion()).toBe(61); + expect(db.getSchemaVersion()).toBe(62); // Re-init should not fail db.init(); - expect(db.getSchemaVersion()).toBe(61); + expect(db.getSchemaVersion()).toBe(62); db.close(); }); @@ -1021,7 +1021,7 @@ describe("schema migrations", () => { db.init(); - expect(db.getSchemaVersion()).toBe(61); + expect(db.getSchemaVersion()).toBe(62); const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>; expect(cols.map((col) => col.name)).toContain("priority"); @@ -1062,7 +1062,7 @@ describe("schema migrations", () => { db.init(); - expect(db.getSchemaVersion()).toBe(61); + expect(db.getSchemaVersion()).toBe(62); const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>; const colNames = cols.map((col) => col.name); @@ -1131,7 +1131,7 @@ describe("schema migrations", () => { db.init(); - expect(db.getSchemaVersion()).toBe(61); + expect(db.getSchemaVersion()).toBe(62); const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>; const colNames = cols.map((col) => col.name); @@ -1234,7 +1234,7 @@ describe("schema migrations", () => { db.init(); - expect(db.getSchemaVersion()).toBe(61); + expect(db.getSchemaVersion()).toBe(62); const cols = db.prepare("PRAGMA table_info(chat_messages)").all() as Array<{ name: string }>; expect(cols.map((col) => col.name)).toContain("attachments"); @@ -1308,7 +1308,7 @@ describe("schema migrations", () => { db.init(); - expect(db.getSchemaVersion()).toBe(61); + expect(db.getSchemaVersion()).toBe(62); const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name = 'agentRatings'").all() as Array<{ name: string }>; expect(tables).toEqual([{ name: "agentRatings" }]); @@ -1332,7 +1332,7 @@ describe("schema migrations", () => { db.init(); - expect(db.getSchemaVersion()).toBe(61); + expect(db.getSchemaVersion()).toBe(62); const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name = 'mission_events'").all() as Array<{ name: string }>; expect(tables).toEqual([{ name: "mission_events" }]); @@ -1436,7 +1436,7 @@ describe("schema migrations", () => { db.init(); // Verify version bumped to 29 - expect(db.getSchemaVersion()).toBe(61); + expect(db.getSchemaVersion()).toBe(62); // Verify new columns exist and existing data is intact const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>; @@ -1905,7 +1905,7 @@ describe("createDatabase factory", () => { const db = createDatabase(fusionDir); db.init(); - expect(db.getSchemaVersion()).toBe(61); + expect(db.getSchemaVersion()).toBe(62); expect(db.getLastModified()).toBeGreaterThan(0); db.close(); diff --git a/packages/core/src/__tests__/eval-store.test.ts b/packages/core/src/__tests__/eval-store.test.ts new file mode 100644 index 000000000..64b0006e8 --- /dev/null +++ b/packages/core/src/__tests__/eval-store.test.ts @@ -0,0 +1,82 @@ +import { beforeEach, describe, expect, it } from "vitest"; +import { createDatabase, type Database } from "../db.js"; +import { EvalLifecycleError, EvalStore } from "../eval-store.js"; + +let db: Database; +let store: EvalStore; + +beforeEach(() => { + db = createDatabase("/tmp/fn-eval-store-test", { inMemory: true }); + db.init(); + store = new EvalStore(db); +}); + +describe("EvalStore", () => { + it("creates and lists runs with deterministic ordering", () => { + const runA = store.createRun({ projectId: "p1", scope: "completed-since-last", requestedTaskIds: ["FN-1"] }); + const runB = store.createRun({ projectId: "p1", scope: "completed-since-last", requestedTaskIds: ["FN-2"] }); + + const runs = store.listRuns({ projectId: "p1" }); + expect(runs.map((run) => run.id)).toEqual([runA.id, runB.id].sort()); + }); + + it("enforces active run conflict for scheduled trigger", () => { + store.createRun({ projectId: "p1", scope: "window", trigger: "schedule" }); + expect(() => store.createRun({ projectId: "p1", scope: "window", trigger: "schedule" })).toThrow(EvalLifecycleError); + }); + + it("enforces terminal immutability", () => { + const run = store.createRun({ projectId: "p1", scope: "window" }); + store.updateRun(run.id, { status: "completed" }); + expect(() => store.updateRun(run.id, { summary: "late change" })).toThrow(EvalLifecycleError); + }); + + it("creates results and preserves task snapshot after tasks row deletion", () => { + const run = store.createRun({ projectId: "p1", scope: "window" }); + const result = store.createTaskResult(run.id, { + taskId: "FN-123", + taskSnapshot: { taskId: "FN-123", title: "Snapshot title", status: "done", summary: "task summary" }, + status: "scored", + overallScore: 0.8, + categoryScores: [{ category: "quality", score: 0.8 }], + evidence: [{ type: "task_log", ref: "log:1" }], + deterministicSignals: [{ signalId: "s1", kind: "test", name: "tests-pass", passed: true }], + }); + + db.prepare("DELETE FROM tasks WHERE id = ?").run("FN-123"); + + const fetched = store.getTaskResult(result.id); + expect(fetched?.taskSnapshot.title).toBe("Snapshot title"); + expect(fetched?.taskId).toBe("FN-123"); + }); + + it("persists run window boundaries and evaluated task rollups", () => { + const run = store.createRun({ + projectId: "p1", + trigger: "schedule", + scope: "completed-since-last", + window: { since: "2026-05-01T00:00:00.000Z", until: "2026-05-02T00:00:00.000Z", baselineRunId: "ER-BASE" }, + requestedTaskIds: ["FN-1", "FN-2"], + }); + + const updated = store.updateRun(run.id, { + status: "running", + evaluatedTaskIds: ["FN-1", "FN-2"], + counts: { totalTasks: 2, scoredTasks: 1, skippedTasks: 1, erroredTasks: 0 }, + }); + + expect(updated?.window.since).toBe("2026-05-01T00:00:00.000Z"); + expect(updated?.evaluatedTaskIds).toEqual(["FN-1", "FN-2"]); + expect(updated?.counts.scoredTasks).toBe(1); + }); + + it("appends run events with sequential ordering", () => { + const run = store.createRun({ projectId: "p1", scope: "window" }); + const evt1 = store.appendRunEvent(run.id, { type: "info", message: "started" }); + const evt2 = store.appendRunEvent(run.id, { type: "task_evaluated", message: "scored", taskId: "FN-1" }); + + const events = store.listRunEvents(run.id); + expect(events.map((event) => event.id)).toEqual([evt1.id, evt2.id]); + expect(events.map((event) => event.seq)).toEqual([1, 2]); + }); +}); diff --git a/packages/core/src/__tests__/insight-store.test.ts b/packages/core/src/__tests__/insight-store.test.ts index b801176c4..85f382c9e 100644 --- a/packages/core/src/__tests__/insight-store.test.ts +++ b/packages/core/src/__tests__/insight-store.test.ts @@ -886,7 +886,7 @@ describe("Migration: pre-33 DB upgrade", () => { // Step 1: Create a fresh database at v33 (runs all migrations up to 33) const db1 = createDatabase(legacyDir); db1.init(); - expect(db1.getSchemaVersion()).toBe(61); + expect(db1.getSchemaVersion()).toBe(62); db1.close(); // Step 2: Manually downgrade to version 32 and drop insight tables @@ -921,7 +921,7 @@ describe("Migration: pre-33 DB upgrade", () => { expect(tableNamesBefore).not.toContain("project_insight_runs"); // Now run init — this triggers the v32→v33 migration db3.init(); - expect(db3.getSchemaVersion()).toBe(61); + expect(db3.getSchemaVersion()).toBe(62); // Step 4: Verify insight tables exist after migration const tablesAfter = db3.prepare( @@ -952,12 +952,12 @@ describe("Migration: pre-33 DB upgrade", () => { try { const db1 = createDatabase(testDir); db1.init(); - expect(db1.getSchemaVersion()).toBe(61); + expect(db1.getSchemaVersion()).toBe(62); db1.close(); const db2 = createDatabase(testDir); expect(() => db2.init()).not.toThrow(); - expect(db2.getSchemaVersion()).toBe(61); + expect(db2.getSchemaVersion()).toBe(62); db2.close(); } finally { rmSync(testDir, { recursive: true, force: true }); diff --git a/packages/core/src/__tests__/mission-store.test.ts b/packages/core/src/__tests__/mission-store.test.ts index 176145cb2..7ee0ee6ab 100644 --- a/packages/core/src/__tests__/mission-store.test.ts +++ b/packages/core/src/__tests__/mission-store.test.ts @@ -2629,7 +2629,7 @@ describe("MissionStore", () => { describe("Loop State & Validator Run Schema (v31)", () => { it("schema version is 40 after migration", () => { - expect(db.getSchemaVersion()).toBe(61); + expect(db.getSchemaVersion()).toBe(62); }); it("mission_features table has loop state columns", () => { diff --git a/packages/core/src/__tests__/roadmap-store.test.ts b/packages/core/src/__tests__/roadmap-store.test.ts index 1c35afe96..6170c83d4 100644 --- a/packages/core/src/__tests__/roadmap-store.test.ts +++ b/packages/core/src/__tests__/roadmap-store.test.ts @@ -742,7 +742,7 @@ describe("RoadmapStore", () => { describe("schema version", () => { it("schema version is 40 after init", () => { - expect(db.getSchemaVersion()).toBe(61); + expect(db.getSchemaVersion()).toBe(62); }); }); diff --git a/packages/core/src/__tests__/run-audit.test.ts b/packages/core/src/__tests__/run-audit.test.ts index 1b860ec7e..3c0e72a4e 100644 --- a/packages/core/src/__tests__/run-audit.test.ts +++ b/packages/core/src/__tests__/run-audit.test.ts @@ -465,7 +465,7 @@ describe("Run Audit", () => { }); it("schema version is bumped to 40", () => { - expect(db.getSchemaVersion()).toBe(61); + expect(db.getSchemaVersion()).toBe(62); }); }); }); diff --git a/packages/core/src/__tests__/task-documents.test.ts b/packages/core/src/__tests__/task-documents.test.ts index c80d00b55..b64173607 100644 --- a/packages/core/src/__tests__/task-documents.test.ts +++ b/packages/core/src/__tests__/task-documents.test.ts @@ -51,7 +51,7 @@ describe("TaskStore task documents", () => { expect(tableNames.has("task_documents")).toBe(true); expect(tableNames.has("task_document_revisions")).toBe(true); - expect(db.getSchemaVersion()).toBe(61); + expect(db.getSchemaVersion()).toBe(62); const index = db .prepare( diff --git a/packages/core/src/db.ts b/packages/core/src/db.ts index 61dff7895..32d6590a7 100644 --- a/packages/core/src/db.ts +++ b/packages/core/src/db.ts @@ -88,7 +88,7 @@ export function probeFts5(db: DatabaseSync): boolean { // ── Schema Definition ──────────────────────────────────────────────── -const SCHEMA_VERSION = 61; +const SCHEMA_VERSION = 62; function normalizeTaskComments( steeringComments: SteeringComment[] | undefined, @@ -466,6 +466,71 @@ CREATE TABLE IF NOT EXISTS research_run_events ( ); CREATE INDEX IF NOT EXISTS idxResearchRunEventsRunIdSeq ON research_run_events(runId, seq); +-- Eval run persistence (FN-3387) +CREATE TABLE IF NOT EXISTS eval_runs ( + id TEXT PRIMARY KEY, + projectId TEXT NOT NULL, + status TEXT NOT NULL, + trigger TEXT NOT NULL, + scope TEXT NOT NULL, + window TEXT NOT NULL DEFAULT '{}', + requestedTaskIds TEXT NOT NULL DEFAULT '[]', + evaluatedTaskIds TEXT NOT NULL DEFAULT '[]', + counts TEXT NOT NULL DEFAULT '{"totalTasks":0,"scoredTasks":0,"skippedTasks":0,"erroredTasks":0}', + aggregateScores TEXT, + summary TEXT, + error TEXT, + provenance TEXT, + metadata TEXT, + createdAt TEXT NOT NULL, + updatedAt TEXT NOT NULL, + startedAt TEXT, + completedAt TEXT, + cancelledAt TEXT +); +CREATE INDEX IF NOT EXISTS idxEvalRunsProjectIdCreatedAt ON eval_runs(projectId, createdAt); +CREATE INDEX IF NOT EXISTS idxEvalRunsProjectTriggerStatus ON eval_runs(projectId, trigger, status); +CREATE INDEX IF NOT EXISTS idxEvalRunsStatusCreatedAt ON eval_runs(status, createdAt); + +CREATE TABLE IF NOT EXISTS eval_task_results ( + id TEXT PRIMARY KEY, + runId TEXT NOT NULL, + taskId TEXT NOT NULL, + taskSnapshot TEXT NOT NULL, + status TEXT NOT NULL, + overallScore REAL, + maxScore REAL, + categoryScores TEXT NOT NULL DEFAULT '[]', + rationale TEXT, + summary TEXT, + evidence TEXT NOT NULL DEFAULT '[]', + deterministicSignals TEXT NOT NULL DEFAULT '[]', + aiSignals TEXT, + followUps TEXT NOT NULL DEFAULT '[]', + provenance TEXT, + metadata TEXT, + createdAt TEXT NOT NULL, + updatedAt TEXT NOT NULL, + FOREIGN KEY (runId) REFERENCES eval_runs(id) ON DELETE CASCADE +); +CREATE INDEX IF NOT EXISTS idxEvalTaskResultsRunIdCreatedAt ON eval_task_results(runId, createdAt); +CREATE INDEX IF NOT EXISTS idxEvalTaskResultsTaskIdCreatedAt ON eval_task_results(taskId, createdAt); +CREATE INDEX IF NOT EXISTS idxEvalTaskResultsStatusRunId ON eval_task_results(status, runId); + +CREATE TABLE IF NOT EXISTS eval_run_events ( + id TEXT PRIMARY KEY, + runId TEXT NOT NULL, + seq INTEGER NOT NULL, + type TEXT NOT NULL, + message TEXT NOT NULL, + status TEXT, + taskId TEXT, + metadata TEXT, + createdAt TEXT NOT NULL, + FOREIGN KEY (runId) REFERENCES eval_runs(id) ON DELETE CASCADE +); +CREATE INDEX IF NOT EXISTS idxEvalRunEventsRunIdSeq ON eval_run_events(runId, seq); + -- Schema version tracking CREATE TABLE IF NOT EXISTS __meta ( key TEXT PRIMARY KEY, @@ -2413,6 +2478,80 @@ export class Database { }); } + if (version < 62) { + this.applyMigration(62, () => { + this.db.exec(` + CREATE TABLE IF NOT EXISTS eval_runs ( + id TEXT PRIMARY KEY, + projectId TEXT NOT NULL, + status TEXT NOT NULL, + trigger TEXT NOT NULL, + scope TEXT NOT NULL, + window TEXT NOT NULL DEFAULT '{}', + requestedTaskIds TEXT NOT NULL DEFAULT '[]', + evaluatedTaskIds TEXT NOT NULL DEFAULT '[]', + counts TEXT NOT NULL DEFAULT '{"totalTasks":0,"scoredTasks":0,"skippedTasks":0,"erroredTasks":0}', + aggregateScores TEXT, + summary TEXT, + error TEXT, + provenance TEXT, + metadata TEXT, + createdAt TEXT NOT NULL, + updatedAt TEXT NOT NULL, + startedAt TEXT, + completedAt TEXT, + cancelledAt TEXT + ) + `); + this.db.exec(`CREATE INDEX IF NOT EXISTS idxEvalRunsProjectIdCreatedAt ON eval_runs(projectId, createdAt)`); + this.db.exec(`CREATE INDEX IF NOT EXISTS idxEvalRunsProjectTriggerStatus ON eval_runs(projectId, trigger, status)`); + this.db.exec(`CREATE INDEX IF NOT EXISTS idxEvalRunsStatusCreatedAt ON eval_runs(status, createdAt)`); + + this.db.exec(` + CREATE TABLE IF NOT EXISTS eval_task_results ( + id TEXT PRIMARY KEY, + runId TEXT NOT NULL, + taskId TEXT NOT NULL, + taskSnapshot TEXT NOT NULL, + status TEXT NOT NULL, + overallScore REAL, + maxScore REAL, + categoryScores TEXT NOT NULL DEFAULT '[]', + rationale TEXT, + summary TEXT, + evidence TEXT NOT NULL DEFAULT '[]', + deterministicSignals TEXT NOT NULL DEFAULT '[]', + aiSignals TEXT, + followUps TEXT NOT NULL DEFAULT '[]', + provenance TEXT, + metadata TEXT, + createdAt TEXT NOT NULL, + updatedAt TEXT NOT NULL, + FOREIGN KEY (runId) REFERENCES eval_runs(id) ON DELETE CASCADE + ) + `); + this.db.exec(`CREATE INDEX IF NOT EXISTS idxEvalTaskResultsRunIdCreatedAt ON eval_task_results(runId, createdAt)`); + this.db.exec(`CREATE INDEX IF NOT EXISTS idxEvalTaskResultsTaskIdCreatedAt ON eval_task_results(taskId, createdAt)`); + this.db.exec(`CREATE INDEX IF NOT EXISTS idxEvalTaskResultsStatusRunId ON eval_task_results(status, runId)`); + + this.db.exec(` + CREATE TABLE IF NOT EXISTS eval_run_events ( + id TEXT PRIMARY KEY, + runId TEXT NOT NULL, + seq INTEGER NOT NULL, + type TEXT NOT NULL, + message TEXT NOT NULL, + status TEXT, + taskId TEXT, + metadata TEXT, + createdAt TEXT NOT NULL, + FOREIGN KEY (runId) REFERENCES eval_runs(id) ON DELETE CASCADE + ) + `); + this.db.exec(`CREATE INDEX IF NOT EXISTS idxEvalRunEventsRunIdSeq ON eval_run_events(runId, seq)`); + }); + } + } /** diff --git a/packages/core/src/eval-store.ts b/packages/core/src/eval-store.ts new file mode 100644 index 000000000..fec60a4f2 --- /dev/null +++ b/packages/core/src/eval-store.ts @@ -0,0 +1,454 @@ +import { EventEmitter } from "node:events"; +import { randomUUID } from "node:crypto"; +import type { Database } from "./db.js"; +import { fromJson, toJson, toJsonNullable } from "./db.js"; +import type { + EvalRun, + EvalRunCreateInput, + EvalRunEvent, + EvalRunListOptions, + EvalRunStatus, + EvalRunUpdateInput, + EvalStoreEvents, + EvalTaskResult, + EvalTaskResultCreateInput, + EvalTaskResultListOptions, + EvalTaskResultUpdateInput, +} from "./eval-types.js"; + +const TERMINAL_STATUSES = new Set(["completed", "failed", "cancelled"]); +const ACTIVE_STATUSES = new Set(["pending", "running"]); +const VALID_TRANSITIONS: Record = { + pending: ["running", "completed", "failed", "cancelled"], + running: ["completed", "failed", "cancelled"], + completed: [], + failed: [], + cancelled: [], +}; + +export class EvalLifecycleError extends Error { + constructor(message: string, readonly code: "invalid_transition" | "terminal_immutable" | "active_run_conflict") { + super(message); + this.name = "EvalLifecycleError"; + } +} + +function generateRunId(): string { + return `ER-${Date.now().toString(36).toUpperCase()}-${Math.random().toString(36).slice(2, 7).toUpperCase()}`; +} + +function generateResultId(): string { + return `ETR-${randomUUID()}`; +} + +function generateEventId(): string { + return `ERE-${randomUUID()}`; +} + +export class EvalStore extends EventEmitter { + constructor(private readonly db: Database) { + super(); + this.setMaxListeners(50); + } + + createRun(input: EvalRunCreateInput): EvalRun { + const now = new Date().toISOString(); + if ((input.trigger === "schedule" || input.trigger === "task_completion") && this.hasActiveRun(input.projectId, input.trigger)) { + throw new EvalLifecycleError(`Active eval run already exists for project ${input.projectId} trigger ${input.trigger}`, "active_run_conflict"); + } + + const run: EvalRun = { + id: generateRunId(), + projectId: input.projectId, + status: "pending", + trigger: input.trigger ?? "manual", + scope: input.scope, + window: input.window ?? {}, + requestedTaskIds: input.requestedTaskIds ?? [], + evaluatedTaskIds: [], + counts: { totalTasks: input.requestedTaskIds?.length ?? 0, scoredTasks: 0, skippedTasks: 0, erroredTasks: 0 }, + provenance: input.provenance, + metadata: input.metadata, + createdAt: now, + updatedAt: now, + }; + + this.db.prepare(` + INSERT INTO eval_runs ( + id, projectId, status, trigger, scope, window, requestedTaskIds, evaluatedTaskIds, + counts, aggregateScores, summary, error, provenance, metadata, + createdAt, updatedAt, startedAt, completedAt, cancelledAt + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `).run( + run.id, + run.projectId, + run.status, + run.trigger, + run.scope, + toJson(run.window), + toJson(run.requestedTaskIds), + toJson(run.evaluatedTaskIds), + toJson(run.counts), + null, + null, + null, + toJsonNullable(run.provenance), + toJsonNullable(run.metadata), + run.createdAt, + run.updatedAt, + null, + null, + null, + ); + + this.db.bumpLastModified(); + this.emit("run:created", run); + return run; + } + + getRun(id: string): EvalRun | undefined { + const row = this.db.prepare("SELECT * FROM eval_runs WHERE id = ?").get(id) as Record | undefined; + return row ? this.rowToRun(row) : undefined; + } + + listRuns(options: EvalRunListOptions = {}): EvalRun[] { + const clauses: string[] = []; + const params: Array = []; + if (options.projectId) { + clauses.push("projectId = ?"); + params.push(options.projectId); + } + if (options.status) { + clauses.push("status = ?"); + params.push(options.status); + } + if (options.trigger) { + clauses.push("trigger = ?"); + params.push(options.trigger); + } + const where = clauses.length ? `WHERE ${clauses.join(" AND ")}` : ""; + const limit = options.limit !== undefined ? `LIMIT ${options.limit}` : ""; + const offset = options.offset !== undefined ? `OFFSET ${options.offset}` : ""; + + const rows = this.db.prepare(` + SELECT * FROM eval_runs + ${where} + ORDER BY createdAt ASC, id ASC + ${limit} + ${offset} + `).all(...params) as Record[]; + + return rows.map((row) => this.rowToRun(row)); + } + + updateRun(id: string, input: EvalRunUpdateInput): EvalRun | undefined { + const existing = this.getRun(id); + if (!existing) return undefined; + + if (TERMINAL_STATUSES.has(existing.status) && Object.keys(input).some((k) => k !== "status")) { + throw new EvalLifecycleError(`Eval run ${id} is terminal and immutable`, "terminal_immutable"); + } + + if (input.status && input.status !== existing.status) { + if (!VALID_TRANSITIONS[existing.status].includes(input.status)) { + throw new EvalLifecycleError(`Invalid eval run status transition: ${existing.status} -> ${input.status}`, "invalid_transition"); + } + } + + const now = new Date().toISOString(); + const updated: EvalRun = { + ...existing, + ...input, + error: input.error === null ? undefined : (input.error ?? existing.error), + metadata: input.metadata ? { ...(existing.metadata ?? {}), ...input.metadata } : existing.metadata, + provenance: input.provenance ? { ...(existing.provenance ?? {}), ...input.provenance } : existing.provenance, + updatedAt: now, + startedAt: input.startedAt === null ? undefined : (input.startedAt ?? existing.startedAt), + completedAt: input.completedAt === null ? undefined : (input.completedAt ?? existing.completedAt), + cancelledAt: input.cancelledAt === null ? undefined : (input.cancelledAt ?? existing.cancelledAt), + }; + + this.persistRun(updated); + this.emit("run:updated", updated); + return updated; + } + + deleteRun(id: string): boolean { + const result = this.db.prepare("DELETE FROM eval_runs WHERE id = ?").run(id) as { changes?: number }; + const deleted = (result.changes ?? 0) > 0; + if (deleted) { + this.db.bumpLastModified(); + this.emit("run:deleted", id); + } + return deleted; + } + + createTaskResult(runId: string, input: EvalTaskResultCreateInput): EvalTaskResult { + const run = this.getRun(runId); + if (!run) throw new Error(`Eval run not found: ${runId}`); + + const now = new Date().toISOString(); + const result: EvalTaskResult = { + id: generateResultId(), + runId, + taskId: input.taskId, + taskSnapshot: input.taskSnapshot, + status: input.status, + overallScore: input.overallScore, + maxScore: input.maxScore, + categoryScores: input.categoryScores ?? [], + rationale: input.rationale, + summary: input.summary, + evidence: input.evidence ?? [], + deterministicSignals: input.deterministicSignals ?? [], + aiSignals: input.aiSignals, + followUps: input.followUps ?? [], + provenance: input.provenance, + metadata: input.metadata, + createdAt: now, + updatedAt: now, + }; + + this.db.prepare(` + INSERT INTO eval_task_results ( + id, runId, taskId, taskSnapshot, status, overallScore, maxScore, + categoryScores, rationale, summary, evidence, deterministicSignals, aiSignals, + followUps, provenance, metadata, createdAt, updatedAt + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `).run( + result.id, + result.runId, + result.taskId, + toJson(result.taskSnapshot), + result.status, + result.overallScore ?? null, + result.maxScore ?? null, + toJson(result.categoryScores), + result.rationale ?? null, + result.summary ?? null, + toJson(result.evidence), + toJson(result.deterministicSignals), + toJsonNullable(result.aiSignals), + toJson(result.followUps), + toJsonNullable(result.provenance), + toJsonNullable(result.metadata), + result.createdAt, + result.updatedAt, + ); + + this.db.bumpLastModified(); + this.emit("result:created", result); + return result; + } + + getTaskResult(id: string): EvalTaskResult | undefined { + const row = this.db.prepare("SELECT * FROM eval_task_results WHERE id = ?").get(id) as Record | undefined; + return row ? this.rowToResult(row) : undefined; + } + + listTaskResults(options: EvalTaskResultListOptions = {}): EvalTaskResult[] { + const clauses: string[] = []; + const params: Array = []; + if (options.runId) { + clauses.push("runId = ?"); + params.push(options.runId); + } + if (options.taskId) { + clauses.push("taskId = ?"); + params.push(options.taskId); + } + if (options.status) { + clauses.push("status = ?"); + params.push(options.status); + } + const where = clauses.length ? `WHERE ${clauses.join(" AND ")}` : ""; + const limit = options.limit !== undefined ? `LIMIT ${options.limit}` : ""; + const offset = options.offset !== undefined ? `OFFSET ${options.offset}` : ""; + + const rows = this.db.prepare(` + SELECT * FROM eval_task_results + ${where} + ORDER BY createdAt ASC, id ASC + ${limit} + ${offset} + `).all(...params) as Record[]; + return rows.map((row) => this.rowToResult(row)); + } + + updateTaskResult(id: string, input: EvalTaskResultUpdateInput): EvalTaskResult | undefined { + const existing = this.getTaskResult(id); + if (!existing) return undefined; + + const now = new Date().toISOString(); + const updated: EvalTaskResult = { + ...existing, + ...input, + metadata: input.metadata ? { ...(existing.metadata ?? {}), ...input.metadata } : existing.metadata, + provenance: input.provenance ? { ...(existing.provenance ?? {}), ...input.provenance } : existing.provenance, + updatedAt: now, + }; + + this.db.prepare(` + UPDATE eval_task_results SET + status = ?, overallScore = ?, maxScore = ?, categoryScores = ?, rationale = ?, summary = ?, + evidence = ?, deterministicSignals = ?, aiSignals = ?, followUps = ?, provenance = ?, metadata = ?, updatedAt = ? + WHERE id = ? + `).run( + updated.status, + updated.overallScore ?? null, + updated.maxScore ?? null, + toJson(updated.categoryScores), + updated.rationale ?? null, + updated.summary ?? null, + toJson(updated.evidence), + toJson(updated.deterministicSignals), + toJsonNullable(updated.aiSignals), + toJson(updated.followUps), + toJsonNullable(updated.provenance), + toJsonNullable(updated.metadata), + updated.updatedAt, + id, + ); + + this.db.bumpLastModified(); + this.emit("result:updated", updated); + return updated; + } + + appendRunEvent(runId: string, event: Omit): EvalRunEvent { + const run = this.getRun(runId); + if (!run) throw new Error(`Eval run not found: ${runId}`); + + const maxSeq = this.db.prepare("SELECT COALESCE(MAX(seq), 0) as maxSeq FROM eval_run_events WHERE runId = ?").get(runId) as { maxSeq: number }; + const created: EvalRunEvent = { + id: generateEventId(), + runId, + seq: (maxSeq?.maxSeq ?? 0) + 1, + type: event.type, + message: event.message, + status: event.status, + taskId: event.taskId, + metadata: event.metadata, + createdAt: new Date().toISOString(), + }; + + this.db.prepare(` + INSERT INTO eval_run_events (id, runId, seq, type, message, status, taskId, metadata, createdAt) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + `).run( + created.id, + created.runId, + created.seq, + created.type, + created.message, + created.status ?? null, + created.taskId ?? null, + toJsonNullable(created.metadata), + created.createdAt, + ); + + this.db.bumpLastModified(); + this.emit("run:event", { runId, event: created }); + return created; + } + + listRunEvents(runId: string): EvalRunEvent[] { + const rows = this.db.prepare("SELECT * FROM eval_run_events WHERE runId = ? ORDER BY seq ASC, id ASC").all(runId) as Record[]; + return rows.map((row) => this.rowToEvent(row)); + } + + private hasActiveRun(projectId: string, trigger: string): boolean { + const placeholders = Array.from(ACTIVE_STATUSES).map(() => "?").join(", "); + const row = this.db.prepare(`SELECT id FROM eval_runs WHERE projectId = ? AND trigger = ? AND status IN (${placeholders}) LIMIT 1`) + .get(projectId, trigger, ...Array.from(ACTIVE_STATUSES)) as { id?: string } | undefined; + return Boolean(row?.id); + } + + private persistRun(run: EvalRun): void { + this.db.prepare(` + UPDATE eval_runs SET + status = ?, scope = ?, window = ?, requestedTaskIds = ?, evaluatedTaskIds = ?, counts = ?, aggregateScores = ?, + summary = ?, error = ?, provenance = ?, metadata = ?, updatedAt = ?, startedAt = ?, completedAt = ?, cancelledAt = ? + WHERE id = ? + `).run( + run.status, + run.scope, + toJson(run.window), + toJson(run.requestedTaskIds), + toJson(run.evaluatedTaskIds), + toJson(run.counts), + toJsonNullable(run.aggregateScores), + run.summary ?? null, + run.error ?? null, + toJsonNullable(run.provenance), + toJsonNullable(run.metadata), + run.updatedAt, + run.startedAt ?? null, + run.completedAt ?? null, + run.cancelledAt ?? null, + run.id, + ); + this.db.bumpLastModified(); + } + + private rowToRun(row: Record): EvalRun { + return { + id: String(row.id), + projectId: String(row.projectId), + status: row.status as EvalRunStatus, + trigger: row.trigger as EvalRun["trigger"], + scope: String(row.scope), + window: fromJson(row.window as string) ?? {}, + requestedTaskIds: fromJson(row.requestedTaskIds as string) ?? [], + evaluatedTaskIds: fromJson(row.evaluatedTaskIds as string) ?? [], + counts: fromJson(row.counts as string) ?? { totalTasks: 0, scoredTasks: 0, skippedTasks: 0, erroredTasks: 0 }, + aggregateScores: fromJson>(row.aggregateScores as string), + summary: (row.summary as string | null) ?? undefined, + error: (row.error as string | null) ?? undefined, + provenance: fromJson(row.provenance as string), + metadata: fromJson(row.metadata as string), + createdAt: String(row.createdAt), + updatedAt: String(row.updatedAt), + startedAt: (row.startedAt as string | null) ?? undefined, + completedAt: (row.completedAt as string | null) ?? undefined, + cancelledAt: (row.cancelledAt as string | null) ?? undefined, + }; + } + + private rowToResult(row: Record): EvalTaskResult { + return { + id: String(row.id), + runId: String(row.runId), + taskId: String(row.taskId), + taskSnapshot: fromJson(row.taskSnapshot as string) ?? { taskId: String(row.taskId) }, + status: row.status as EvalTaskResult["status"], + overallScore: row.overallScore == null ? undefined : Number(row.overallScore), + maxScore: row.maxScore == null ? undefined : Number(row.maxScore), + categoryScores: fromJson(row.categoryScores as string) ?? [], + rationale: (row.rationale as string | null) ?? undefined, + summary: (row.summary as string | null) ?? undefined, + evidence: fromJson(row.evidence as string) ?? [], + deterministicSignals: fromJson(row.deterministicSignals as string) ?? [], + aiSignals: fromJson(row.aiSignals as string), + followUps: fromJson(row.followUps as string) ?? [], + provenance: fromJson(row.provenance as string), + metadata: fromJson(row.metadata as string), + createdAt: String(row.createdAt), + updatedAt: String(row.updatedAt), + }; + } + + private rowToEvent(row: Record): EvalRunEvent { + return { + id: String(row.id), + runId: String(row.runId), + seq: Number(row.seq), + type: row.type as EvalRunEvent["type"], + message: String(row.message), + status: (row.status as EvalRunStatus | null) ?? undefined, + taskId: (row.taskId as string | null) ?? undefined, + metadata: fromJson(row.metadata as string), + createdAt: String(row.createdAt), + }; + } +} diff --git a/packages/core/src/eval-types.ts b/packages/core/src/eval-types.ts new file mode 100644 index 000000000..7da6c22a9 --- /dev/null +++ b/packages/core/src/eval-types.ts @@ -0,0 +1,239 @@ +/** + * Eval Domain Types + * + * Contracts for eval run persistence and per-task evaluation results. + */ + +export const EVAL_RUN_STATUSES = [ + "pending", + "running", + "completed", + "failed", + "cancelled", +] as const; + +export type EvalRunStatus = typeof EVAL_RUN_STATUSES[number]; + +export const EVAL_RUN_TRIGGERS = ["manual", "schedule", "api", "task_completion"] as const; + +export type EvalRunTrigger = typeof EVAL_RUN_TRIGGERS[number]; + +export const EVAL_SCORE_CATEGORIES = [ + "correctness", + "completeness", + "quality", + "reliability", + "tests", + "documentation", +] as const; + +export type EvalScoreCategory = typeof EVAL_SCORE_CATEGORIES[number]; + +export interface EvalTaskSnapshot { + taskId: string; + title?: string; + column?: string; + status?: string; + priority?: string; + size?: string; + reviewLevel?: number; + createdAt?: string; + updatedAt?: string; + executionCompletedAt?: string; + summary?: string; + labels?: string[]; + metadata?: Record; +} + +export interface EvalRunWindow { + since?: string; + until?: string; + baselineRunId?: string; +} + +export interface EvalProvenance { + evaluatorProvider?: string; + evaluatorModelId?: string; + evaluatorVersion?: string; + promptVersion?: string; + runConfig?: Record; + metadata?: Record; +} + +export interface EvalSignal { + signalId: string; + kind: string; + name: string; + passed?: boolean; + score?: number; + value?: number | string | boolean | null; + threshold?: number; + unit?: string; + summary?: string; + details?: Record; +} + +export interface EvalEvidenceReference { + type: "task_log" | "task_document" | "file" | "command" | "test" | "other"; + ref: string; + excerpt?: string; + metadata?: Record; +} + +export interface EvalCategoryScore { + category: EvalScoreCategory | string; + score: number; + maxScore?: number; + rationale?: string; +} + +export interface EvalFollowUpSuggestion { + title: string; + description: string; + priority?: "low" | "normal" | "high" | "urgent"; + tags?: string[]; + metadata?: Record; +} + +export interface EvalTaskResult { + id: string; + runId: string; + taskId: string; + taskSnapshot: EvalTaskSnapshot; + status: "scored" | "skipped" | "error"; + overallScore?: number; + maxScore?: number; + categoryScores: EvalCategoryScore[]; + rationale?: string; + summary?: string; + evidence: EvalEvidenceReference[]; + deterministicSignals: EvalSignal[]; + aiSignals?: EvalSignal[]; + followUps: EvalFollowUpSuggestion[]; + provenance?: EvalProvenance; + metadata?: Record; + createdAt: string; + updatedAt: string; +} + +export interface EvalRunCounts { + totalTasks: number; + scoredTasks: number; + skippedTasks: number; + erroredTasks: number; +} + +export interface EvalRun { + id: string; + projectId: string; + status: EvalRunStatus; + trigger: EvalRunTrigger; + scope: string; + window: EvalRunWindow; + requestedTaskIds: string[]; + evaluatedTaskIds: string[]; + counts: EvalRunCounts; + aggregateScores?: Record; + summary?: string; + error?: string; + provenance?: EvalProvenance; + metadata?: Record; + createdAt: string; + updatedAt: string; + startedAt?: string; + completedAt?: string; + cancelledAt?: string; +} + +export interface EvalRunEvent { + id: string; + runId: string; + seq: number; + type: "status_changed" | "task_evaluated" | "info" | "warning" | "error"; + message: string; + status?: EvalRunStatus; + taskId?: string; + metadata?: Record; + createdAt: string; +} + +export interface EvalRunCreateInput { + projectId: string; + trigger?: EvalRunTrigger; + scope: string; + window?: EvalRunWindow; + requestedTaskIds?: string[]; + provenance?: EvalProvenance; + metadata?: Record; +} + +export interface EvalRunUpdateInput { + status?: EvalRunStatus; + evaluatedTaskIds?: string[]; + counts?: EvalRunCounts; + aggregateScores?: Record; + summary?: string; + error?: string | null; + provenance?: EvalProvenance; + metadata?: Record; + startedAt?: string | null; + completedAt?: string | null; + cancelledAt?: string | null; +} + +export interface EvalRunListOptions { + projectId?: string; + status?: EvalRunStatus; + trigger?: EvalRunTrigger; + limit?: number; + offset?: number; +} + +export interface EvalTaskResultCreateInput { + taskId: string; + taskSnapshot: EvalTaskSnapshot; + status: "scored" | "skipped" | "error"; + overallScore?: number; + maxScore?: number; + categoryScores?: EvalCategoryScore[]; + rationale?: string; + summary?: string; + evidence?: EvalEvidenceReference[]; + deterministicSignals?: EvalSignal[]; + aiSignals?: EvalSignal[]; + followUps?: EvalFollowUpSuggestion[]; + provenance?: EvalProvenance; + metadata?: Record; +} + +export interface EvalTaskResultUpdateInput { + status?: "scored" | "skipped" | "error"; + overallScore?: number; + maxScore?: number; + categoryScores?: EvalCategoryScore[]; + rationale?: string; + summary?: string; + evidence?: EvalEvidenceReference[]; + deterministicSignals?: EvalSignal[]; + aiSignals?: EvalSignal[]; + followUps?: EvalFollowUpSuggestion[]; + provenance?: EvalProvenance; + metadata?: Record; +} + +export interface EvalTaskResultListOptions { + runId?: string; + taskId?: string; + status?: "scored" | "skipped" | "error"; + limit?: number; + offset?: number; +} + +export interface EvalStoreEvents { + "run:created": [EvalRun]; + "run:updated": [EvalRun]; + "run:deleted": [string]; + "run:event": [{ runId: string; event: EvalRunEvent }]; + "result:created": [EvalTaskResult]; + "result:updated": [EvalTaskResult]; +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 365180f9e..fbe17d0cc 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -695,6 +695,30 @@ export type { ResolvedResearchSettings } from "./research-settings.js"; export { TodoStore } from "./todo-store.js"; export type { TodoStoreEvents } from "./todo-store.js"; +export { EvalLifecycleError, EvalStore } from "./eval-store.js"; +export type { + EvalRun, + EvalRunStatus, + EvalRunTrigger, + EvalRunWindow, + EvalRunCounts, + EvalRunEvent, + EvalRunCreateInput, + EvalRunUpdateInput, + EvalRunListOptions, + EvalTaskSnapshot, + EvalTaskResult, + EvalTaskResultCreateInput, + EvalTaskResultUpdateInput, + EvalTaskResultListOptions, + EvalCategoryScore, + EvalEvidenceReference, + EvalSignal, + EvalFollowUpSuggestion, + EvalProvenance, + EvalStoreEvents, +} from "./eval-types.js"; +export { EVAL_RUN_STATUSES, EVAL_RUN_TRIGGERS, EVAL_SCORE_CATEGORIES } from "./eval-types.js"; // ── Agent Companies Types ────────────────────────────────── diff --git a/packages/core/src/store.ts b/packages/core/src/store.ts index 729244886..17fefd8d4 100644 --- a/packages/core/src/store.ts +++ b/packages/core/src/store.ts @@ -16,6 +16,7 @@ import { RoadmapStore } from "./roadmap-store.js"; import { InsightStore } from "./insight-store.js"; import { ResearchStore } from "./research-store.js"; import { TodoStore } from "./todo-store.js"; +import { EvalStore } from "./eval-store.js"; import { BackwardCompat, ProjectRequiredError } from "./migration.js"; import { CentralCore } from "./central-core.js"; import { getTaskMergeBlocker } from "./task-merge.js"; @@ -512,6 +513,8 @@ export class TaskStore extends EventEmitter { private researchStore: ResearchStore | null = null; /** Cached TodoStore instance */ private todoStore: TodoStore | null = null; + /** Cached EvalStore instance */ + private evalStore: EvalStore | null = null; /** Buffer for batching agent log writes to reduce WAL pressure. */ private agentLogBuffer: Array<{ @@ -6592,6 +6595,17 @@ ${notificationsSection}`; return this.todoStore; } + /** + * Get the EvalStore instance for eval run and task result operations. + * Lazily initializes the EvalStore on first access. + */ + getEvalStore(): EvalStore { + if (!this.evalStore) { + this.evalStore = new EvalStore(this.db); + } + return this.evalStore; + } + // ── Verification Cache ──────────────────────────────────────────────────── /** diff --git a/packages/dashboard/src/chat.ts b/packages/dashboard/src/chat.ts index bffdad0dc..47eabf4bb 100644 --- a/packages/dashboard/src/chat.ts +++ b/packages/dashboard/src/chat.ts @@ -1231,7 +1231,8 @@ export function __setCreateFnAgent(mock: typeof createFnAgent): void { // hit the real engine. Mirror the same fake into the resolved-session slot // so existing test setups that only call `__setCreateFnAgent` continue to // work. - createResolvedAgentSession = (async (options: unknown) => mock(options)) as typeof createResolvedAgentSession; + createResolvedAgentSession = (async (options: Parameters[0]) => + mock(options)) as typeof createResolvedAgentSession; } /** From 03c0348b9e712750a5c03cb4590646e0e0b969f3 Mon Sep 17 00:00:00 2001 From: Fusion Date: Tue, 5 May 2026 12:08:44 -0700 Subject: [PATCH 04/14] feat(FN-3157): add plugin dashboard view registry with nav integration Merged FN-3157 to add a plugin dashboard views system, including a plugin view registry with lazy loading, navigation integration for Header and MobileNavBar, a usePluginDashboardViews hook with cache and refetch support, and tests covering the no-loader path. Also added documentation in `docs/PLUGI Fusion-Task-Id: FN-3157 --- .changeset/fn-3157-plugin-dashboard-views.md | 5 + docs/PLUGIN_AUTHORING.md | 22 +++++ packages/dashboard/app/App.tsx | 6 +- packages/dashboard/app/components/Header.tsx | 4 +- .../dashboard/app/components/MobileNavBar.tsx | 4 +- .../app/components/__tests__/App.test.tsx | 3 +- .../__tests__/usePluginDashboardViews.test.ts | 64 +++++++++++- .../app/hooks/__tests__/useViewState.test.ts | 16 +++ .../app/hooks/usePluginDashboardViews.ts | 19 +++- packages/dashboard/app/hooks/useViewState.ts | 7 +- .../app/plugins/PluginDashboardViewHost.tsx | 22 +---- .../__tests__/pluginViewRegistry.test.tsx | 48 +++++++++ .../app/plugins/pluginViewRegistry.tsx | 98 +++++++++++-------- .../src/__tests__/plugin-routes.test.ts | 10 ++ 14 files changed, 253 insertions(+), 75 deletions(-) create mode 100644 .changeset/fn-3157-plugin-dashboard-views.md create mode 100644 packages/dashboard/app/plugins/__tests__/pluginViewRegistry.test.tsx diff --git a/.changeset/fn-3157-plugin-dashboard-views.md b/.changeset/fn-3157-plugin-dashboard-views.md new file mode 100644 index 000000000..98546898d --- /dev/null +++ b/.changeset/fn-3157-plugin-dashboard-views.md @@ -0,0 +1,5 @@ +--- +"@runfusion/fusion": minor +--- + +Add plugin dashboard view discovery and navigation integration via `GET /api/plugins/dashboard-views`, plugin view ID persistence (`plugin:${pluginId}:${viewId}`), and static host-side plugin view registry rendering. diff --git a/docs/PLUGIN_AUTHORING.md b/docs/PLUGIN_AUTHORING.md index dada3827f..b0238aeff 100644 --- a/docs/PLUGIN_AUTHORING.md +++ b/docs/PLUGIN_AUTHORING.md @@ -555,6 +555,28 @@ Current host constraints: - `componentPath` is stored for authoring symmetry/future expansion, but render resolution is currently done through a host-side static registry (`pluginId + viewId`) - Use stable IDs; runtime view key format is `plugin:${pluginId}:${viewId}` +### Static host registry model + +Dashboard view components are resolved from a host-side registry and must be explicitly registered: + +```ts +import { lazy } from "react"; +import { registerPluginView } from "../app/plugins/pluginViewRegistry"; + +registerPluginView( + "fusion-plugin-dependency-graph", + "graph", + lazy(() => import("@fusion-plugin-examples/dependency-graph/dashboard-view")), +); +``` + +The host then renders plugin views via `PluginDashboardViewHost` using the composite ID. + +Placement guidance: +- `primary`: top-level nav tab (host may limit count on mobile) +- `overflow`: desktop header overflow menu +- `more`: mobile More sheet / secondary nav surfaces + --- ## 9. Registering Agent Runtimes diff --git a/packages/dashboard/app/App.tsx b/packages/dashboard/app/App.tsx index 20f624c58..426128e30 100644 --- a/packages/dashboard/app/App.tsx +++ b/packages/dashboard/app/App.tsx @@ -49,6 +49,7 @@ import { useViewState, type TaskView } from "./hooks/useViewState"; import { useNavigationHistory } from "./hooks/useNavigationHistory"; import { usePluginDashboardViews } from "./hooks/usePluginDashboardViews"; import { PluginDashboardViewHost } from "./plugins/PluginDashboardViewHost"; +import { isPluginViewId } from "./plugins/pluginViewRegistry"; import { useProjectActions } from "./hooks/useProjectActions"; import { useTaskHandlers } from "./hooks/useTaskHandlers"; import { useRemoteNodeData } from "./hooks/useRemoteNodeData"; @@ -448,6 +449,7 @@ function AppInner() { // Redirect to board if feature-gated views are disabled. useEffect(() => { if (!settingsLoaded) return; + if (isPluginViewId(taskView)) return; if (taskView === "skills" && !skillsEnabled) { handleChangeTaskView("board"); } @@ -867,7 +869,7 @@ function AppInner() { } // Project view - if (taskView.startsWith("plugin:")) { + if (isPluginViewId(taskView)) { return ( setShellConnectionManagerOpen(true)} /> ) : undefined} /> - {viewMode === "project" && currentProject && taskView !== "chat" && taskView !== "mailbox" && taskView !== "insights" && taskView !== "devserver" && taskView !== "dev-server" && !taskView.startsWith("plugin:") && ( + {viewMode === "project" && currentProject && taskView !== "chat" && taskView !== "mailbox" && taskView !== "insights" && taskView !== "devserver" && taskView !== "dev-server" && !isPluginViewId(taskView) && ( @@ -757,6 +768,15 @@ export function AgentDetailView({ agentId, projectId, onClose, addToast, onChild )} + setIsImportModalOpen(false)} + onImported={() => { + void handleSavedMutation(); + }} + projectId={projectId} + initialInputMethod="browse" + /> ); } diff --git a/packages/dashboard/app/components/AgentImportModal.tsx b/packages/dashboard/app/components/AgentImportModal.tsx index a4ef17cf0..7d750dda4 100644 --- a/packages/dashboard/app/components/AgentImportModal.tsx +++ b/packages/dashboard/app/components/AgentImportModal.tsx @@ -9,6 +9,7 @@ export interface AgentImportModalProps { onClose: () => void; onImported: () => void; projectId?: string; + initialInputMethod?: InputMethod; } /** Parsed agent preview item for display before import */ @@ -126,10 +127,10 @@ function parseDirectoryAgentManifest(content: string): DirectoryAgentInput { * * Flow: Input → Preview parsed agents → Import → Show results */ -export function AgentImportModal({ isOpen, onClose, onImported, projectId }: AgentImportModalProps) { +export function AgentImportModal({ isOpen, onClose, onImported, projectId, initialInputMethod = "paste" }: AgentImportModalProps) { useMobileScrollLock(isOpen); const [step, setStep] = useState("input"); - const [inputMethod, setInputMethod] = useState("paste"); + const [inputMethod, setInputMethod] = useState(initialInputMethod); const [manifestContent, setManifestContent] = useState(""); const [directoryAgents, setDirectoryAgents] = useState([]); const [companyName, setCompanyName] = useState("Unknown"); @@ -207,7 +208,7 @@ export function AgentImportModal({ isOpen, onClose, onImported, projectId }: Age const reset = useCallback(() => { setStep("input"); - setInputMethod("paste"); + setInputMethod(initialInputMethod); setManifestContent(""); setDirectoryAgents([]); setCompanyName("Unknown"); @@ -226,7 +227,7 @@ export function AgentImportModal({ isOpen, onClose, onImported, projectId }: Age setIsLoadingCompanies(false); setCompaniesError(null); fetchAttemptedRef.current = false; - }, []); + }, [initialInputMethod]); const handleClose = useCallback(() => { reset(); diff --git a/packages/dashboard/app/components/__tests__/AgentDetailView.test.tsx b/packages/dashboard/app/components/__tests__/AgentDetailView.test.tsx index 7b1c7db82..fae49185c 100644 --- a/packages/dashboard/app/components/__tests__/AgentDetailView.test.tsx +++ b/packages/dashboard/app/components/__tests__/AgentDetailView.test.tsx @@ -38,6 +38,7 @@ vi.mock("../../api", () => ({ fetchPluginRuntimes: vi.fn(), upgradeAgentHeartbeatProcedure: vi.fn(), updateGlobalSettings: vi.fn(), + fetchCompanies: vi.fn(), })); vi.mock("../AgentLogViewer", () => ({ @@ -118,7 +119,7 @@ vi.mock("../../hooks/useConfirm", () => ({ useConfirm: () => ({ confirm: mockConfirm }), })); -import { fetchAgent, fetchAgents, updateAgent, updateAgentState, deleteAgent, fetchAgentChildren, fetchAgentRunLogs, fetchAgentRuns, fetchAgentRunDetail, fetchAgentTasks, fetchChainOfCommand, fetchAgentBudgetStatus, resetAgentBudget, updateAgentInstructions, updateAgentSoul, updateAgentMemory, fetchWorkspaceFileContent, saveWorkspaceFileContent, fetchDiscoveredSkills, fetchSkillContent, fetchModels, fetchPluginRuntimes, fetchAgentLogsWithMeta, upgradeAgentHeartbeatProcedure, updateGlobalSettings } from "../../api"; +import { fetchAgent, fetchAgents, updateAgent, updateAgentState, deleteAgent, fetchAgentChildren, fetchAgentRunLogs, fetchAgentRuns, fetchAgentRunDetail, fetchAgentTasks, fetchChainOfCommand, fetchAgentBudgetStatus, resetAgentBudget, updateAgentInstructions, updateAgentSoul, updateAgentMemory, fetchWorkspaceFileContent, saveWorkspaceFileContent, fetchDiscoveredSkills, fetchSkillContent, fetchModels, fetchPluginRuntimes, fetchAgentLogsWithMeta, upgradeAgentHeartbeatProcedure, updateGlobalSettings, fetchCompanies } from "../../api"; import { subscribeSse } from "../../sse-bus"; const mockFetchAgent = vi.mocked(fetchAgent); @@ -146,6 +147,7 @@ const mockFetchPluginRuntimes = vi.mocked(fetchPluginRuntimes); const mockFetchAgentLogsWithMeta = vi.mocked(fetchAgentLogsWithMeta); const mockUpgradeAgentHeartbeatProcedure = vi.mocked(upgradeAgentHeartbeatProcedure); const mockUpdateGlobalSettings = vi.mocked(updateGlobalSettings); +const mockFetchCompanies = vi.mocked(fetchCompanies); const mockSubscribeSse = vi.mocked(subscribeSse); const MOCK_SKILLS = [ @@ -249,6 +251,7 @@ describe("AgentDetailView", () => { procedureFileSeeded: true, }); mockUpdateGlobalSettings.mockResolvedValue({} as any); + mockFetchCompanies.mockResolvedValue({ companies: [] }); }); it("shows loading state initially", () => { @@ -936,11 +939,32 @@ describe("AgentDetailView", () => { const utilityContainer = headerActions?.querySelector(".agent-detail-utility-actions"); expect(utilityContainer).toBeTruthy(); + expect(utilityContainer?.querySelector('[aria-label="Import agents"]')).toBeTruthy(); expect(utilityContainer?.querySelector('[title="Refresh"]')).toBeTruthy(); expect(utilityContainer?.querySelector('[title="Close"]')).toBeTruthy(); }); }); + it("opens the import modal from agent detail in browse mode", async () => { + const user = userEvent.setup(); + mockFetchCompanies.mockResolvedValue({ companies: [{ slug: "acme", name: "Acme AI" }] }); + + render( + , + ); + + await user.click(await screen.findByRole("button", { name: "Import agents" })); + + await waitFor(() => { + expect(screen.getByRole("dialog", { name: "Import agents" })).toBeInTheDocument(); + expect(screen.getByPlaceholderText("Search companies...")).toBeInTheDocument(); + }); + }); + it("keeps mobile inline header controls on the same row as identity", () => { const stylesContent = loadAllAppCss(); diff --git a/packages/dashboard/app/components/__tests__/AgentImportModal.test.tsx b/packages/dashboard/app/components/__tests__/AgentImportModal.test.tsx index c55530f95..9922f5c06 100644 --- a/packages/dashboard/app/components/__tests__/AgentImportModal.test.tsx +++ b/packages/dashboard/app/components/__tests__/AgentImportModal.test.tsx @@ -184,4 +184,11 @@ describe("AgentImportModal", () => { // The browse mode should render the search input (the fetch for companies is async) expect(screen.getByPlaceholderText("Search companies...")).toBeTruthy(); }); + + it("opens directly in browse mode when initialInputMethod is browse", () => { + render(); + + expect(screen.getByPlaceholderText("Search companies...")).toBeTruthy(); + expect(screen.queryByLabelText("Manifest content")).toBeNull(); + }); }); diff --git a/packages/dashboard/app/components/__tests__/agent-modals-mobile.test.tsx b/packages/dashboard/app/components/__tests__/agent-modals-mobile.test.tsx index c11e5be4c..b3d637008 100644 --- a/packages/dashboard/app/components/__tests__/agent-modals-mobile.test.tsx +++ b/packages/dashboard/app/components/__tests__/agent-modals-mobile.test.tsx @@ -42,6 +42,7 @@ vi.mock("../../api", () => ({ fetchAgentBudgetStatus: vi.fn(), resetAgentBudget: vi.fn(), upgradeAgentHeartbeatProcedure: vi.fn(), + fetchCompanies: vi.fn(), })); vi.mock("../AgentLogViewer", () => ({ @@ -80,6 +81,7 @@ const mockGenerateAgentSpec = vi.mocked(api.generateAgentSpec); const mockCancelAgentGeneration = vi.mocked(api.cancelAgentGeneration); const mockFetchAgentBudgetStatus = vi.mocked(api.fetchAgentBudgetStatus); const mockResetAgentBudget = vi.mocked(api.resetAgentBudget); +const mockFetchCompanies = vi.mocked(api.fetchCompanies); const originalFetch = globalThis.fetch; @@ -172,6 +174,7 @@ describe("agent modal mobile CSS structure", () => { mockCancelAgentGeneration.mockResolvedValue({ success: true }); mockFetchAgentBudgetStatus.mockResolvedValue({ agentId: "agent-001", currentUsage: 0, budgetLimit: null, usagePercent: null, thresholdPercent: null, isOverBudget: false, isOverThreshold: false, lastResetAt: null, nextResetAt: null }); mockResetAgentBudget.mockResolvedValue(undefined); + mockFetchCompanies.mockResolvedValue({ companies: [] }); globalThis.fetch = vi.fn(async () => ({ @@ -273,6 +276,12 @@ describe("agent modal mobile CSS structure", () => { expect(document.querySelector(".agent-import-dialog")).toBeTruthy(); }); + it("supports browse-first launch mode", () => { + render(); + + expect(screen.getByPlaceholderText("Search companies...")).toBeInTheDocument(); + }); + it("file upload area has targetable class", () => { render(); From 5083227069b479434d439b126289d6d8cc92958b Mon Sep 17 00:00:00 2001 From: Fusion Date: Tue, 5 May 2026 12:43:40 -0700 Subject: [PATCH 07/14] feat(FN-3388): document scheduled eval batch architecture Added 3 lines documenting the scheduled eval batch architecture in the architecture docs as part of FN-3388 Step 4. Fusion-Task-Id: FN-3388 --- .../src/__tests__/eval-automation.test.ts | 138 ++++++++ packages/core/src/eval-automation.ts | 335 ++++++++++++++++++ 2 files changed, 473 insertions(+) create mode 100644 packages/core/src/__tests__/eval-automation.test.ts create mode 100644 packages/core/src/eval-automation.ts diff --git a/packages/core/src/__tests__/eval-automation.test.ts b/packages/core/src/__tests__/eval-automation.test.ts new file mode 100644 index 000000000..5aefca295 --- /dev/null +++ b/packages/core/src/__tests__/eval-automation.test.ts @@ -0,0 +1,138 @@ +import { describe, expect, it } from "vitest"; +import { createDatabase } from "../db.js"; +import { EvalStore } from "../eval-store.js"; +import { + DEFAULT_TASK_EVALUATION_SCHEDULE, + createScheduledEvalBatchAutomation, + resolveTaskEvaluationSettings, + runScheduledEvalBatch, + syncScheduledEvalBatchAutomation, +} from "../eval-automation.js"; + +function task(id: string, column: "done" | "todo" | "archived", completedAt: string, createdAt = "2026-01-01T00:00:00.000Z") { + return { + id, + column, + createdAt, + updatedAt: createdAt, + executionCompletedAt: completedAt, + title: id, + summary: id, + } as any; +} + +describe("eval-automation", () => { + it("resolves task evaluation settings defaults", () => { + const resolved = resolveTaskEvaluationSettings({}); + expect(resolved.taskEvaluationEnabled).toBe(false); + expect(resolved.taskEvaluationSchedule).toBe(DEFAULT_TASK_EVALUATION_SCHEDULE); + expect(resolved.taskEvaluationFollowUpPolicy).toBe("off"); + }); + + it("creates scheduled eval automation", () => { + const input = createScheduledEvalBatchAutomation({ taskEvaluationSchedule: "0 9 * * *" }); + expect(input.name).toBe("Scheduled Task Evaluation"); + expect(input.cronExpression).toBe("0 9 * * *"); + expect(input.scope).toBe("project"); + }); + + it("syncs schedule create/delete based on enabled flag", async () => { + const schedules: any[] = []; + const automationStore = { + listSchedules: async () => schedules, + createSchedule: async (input: any) => ({ ...input, id: "S-1" }), + deleteSchedule: async () => true, + updateSchedule: async () => undefined, + } as any; + + const created = await syncScheduledEvalBatchAutomation(automationStore, { taskEvaluationEnabled: true }); + expect(created?.name).toBe("Scheduled Task Evaluation"); + + schedules.push({ id: "S-1", name: "Scheduled Task Evaluation" }); + const deleted = await syncScheduledEvalBatchAutomation(automationStore, { taskEvaluationEnabled: false }); + expect(deleted).toBeUndefined(); + }); + + it("selects done tasks on first run and orders deterministically", async () => { + const db = createDatabase("/tmp/fn-eval-automation-1", { inMemory: true }); + db.init(); + const evalStore = new EvalStore(db); + const tasks = [ + task("FN-2", "done", "2026-05-01T01:00:00.000Z", "2026-01-02T00:00:00.000Z"), + task("FN-1", "done", "2026-05-01T01:00:00.000Z", "2026-01-01T00:00:00.000Z"), + task("FN-3", "done", "2026-05-01T02:00:00.000Z"), + task("FN-4", "todo", "2026-05-01T03:00:00.000Z"), + task("FN-5", "archived", "2026-05-01T04:00:00.000Z"), + ]; + + const result = await runScheduledEvalBatch({ + projectId: "proj", + store: { + listTasks: async () => tasks, + getEvalStore: () => evalStore, + } as any, + startedAt: "2026-05-01T05:00:00.000Z", + evaluator: async ({ task }) => ({ status: "scored", categoryScores: [], evidence: [], deterministicSignals: [], followUps: [], summary: task.id }), + }); + + expect(result.status).toBe("completed"); + expect(result.selectedTaskIds).toEqual(["FN-1", "FN-2", "FN-3"]); + + const run = evalStore.getRun(result.runId)!; + expect(run.counts.totalTasks).toBe(3); + expect(run.metadata?.windowEndInclusive).toBe("2026-05-01T05:00:00.000Z"); + const results = evalStore.listTaskResults({ runId: run.id }); + expect(results).toHaveLength(3); + expect(results[0]?.metadata?.windowEndInclusive).toBe("2026-05-01T05:00:00.000Z"); + }); + + it("uses previous windowEndInclusive cursor for incremental selection", async () => { + const db = createDatabase("/tmp/fn-eval-automation-2", { inMemory: true }); + db.init(); + const evalStore = new EvalStore(db); + + evalStore.createRun({ + projectId: "proj", + trigger: "schedule", + scope: "completed-tasks", + window: { until: "2026-05-01T05:00:00.000Z" }, + metadata: { windowEndInclusive: "2026-05-01T05:00:00.000Z" }, + }); + const run = evalStore.listRuns({ projectId: "proj", trigger: "schedule" })[0]!; + evalStore.updateRun(run.id, { status: "completed", completedAt: "2026-05-01T05:05:00.000Z" }); + + const tasks = [ + task("FN-1", "done", "2026-05-01T05:00:00.000Z"), + task("FN-2", "done", "2026-05-01T05:00:00.001Z"), + task("FN-3", "done", "2026-05-01T06:00:00.000Z"), + ]; + + const result = await runScheduledEvalBatch({ + projectId: "proj", + store: { listTasks: async () => tasks, getEvalStore: () => evalStore } as any, + startedAt: "2026-05-01T06:00:00.000Z", + evaluator: async () => ({ status: "skipped", categoryScores: [], evidence: [], deterministicSignals: [], followUps: [] }), + }); + + expect(result.windowStartExclusive).toBe("2026-05-01T05:00:00.000Z"); + expect(result.selectedTaskIds).toEqual(["FN-2", "FN-3"]); + }); + + it("completes no-op batch when no tasks are eligible", async () => { + const db = createDatabase("/tmp/fn-eval-automation-3", { inMemory: true }); + db.init(); + const evalStore = new EvalStore(db); + + const result = await runScheduledEvalBatch({ + projectId: "proj", + store: { listTasks: async () => [task("FN-1", "todo", "2026-05-01T01:00:00.000Z")], getEvalStore: () => evalStore } as any, + startedAt: "2026-05-02T01:00:00.000Z", + evaluator: async () => ({ status: "scored", categoryScores: [], evidence: [], deterministicSignals: [], followUps: [] }), + }); + + expect(result.tasksSelected).toBe(0); + const run = evalStore.getRun(result.runId)!; + expect(run.status).toBe("completed"); + expect(run.counts.totalTasks).toBe(0); + }); +}); diff --git a/packages/core/src/eval-automation.ts b/packages/core/src/eval-automation.ts new file mode 100644 index 000000000..87a4a36e9 --- /dev/null +++ b/packages/core/src/eval-automation.ts @@ -0,0 +1,335 @@ +import type { AutomationStore } from "./automation-store.js"; +import type { ScheduledTask, ScheduledTaskCreateInput } from "./automation.js"; +import type { EvalRun, EvalTaskResultCreateInput } from "./eval-types.js"; +import { EvalLifecycleError } from "./eval-store.js"; +import type { ProjectSettings, Task } from "./types.js"; + +export const TASK_EVALUATION_SCHEDULE_NAME = "Scheduled Task Evaluation"; +export const DEFAULT_TASK_EVALUATION_SCHEDULE = "0 5 * * *"; +export const TASK_EVALUATION_SCHEDULE_COMMAND = "fn eval --scheduled-batch"; + +export interface ResolvedTaskEvaluationSettings { + taskEvaluationEnabled: boolean; + taskEvaluationSchedule: string; + taskEvaluationProvider?: string; + taskEvaluationModelId?: string; + taskEvaluationFollowUpPolicy: "off" | "suggest" | "create"; + taskEvaluationRetention?: number; +} + +export function resolveTaskEvaluationSettings( + settings: Partial, +): ResolvedTaskEvaluationSettings { + return { + taskEvaluationEnabled: settings.taskEvaluationEnabled ?? false, + taskEvaluationSchedule: settings.taskEvaluationSchedule ?? DEFAULT_TASK_EVALUATION_SCHEDULE, + taskEvaluationProvider: settings.taskEvaluationProvider, + taskEvaluationModelId: settings.taskEvaluationModelId, + taskEvaluationFollowUpPolicy: settings.taskEvaluationFollowUpPolicy ?? "off", + taskEvaluationRetention: settings.taskEvaluationRetention, + }; +} + +export function createScheduledEvalBatchAutomation( + settings: Partial, +): ScheduledTaskCreateInput { + const resolved = resolveTaskEvaluationSettings(settings); + return { + name: TASK_EVALUATION_SCHEDULE_NAME, + description: "Evaluates tasks completed since the previous scheduled evaluation batch", + scheduleType: "custom", + cronExpression: resolved.taskEvaluationSchedule, + command: TASK_EVALUATION_SCHEDULE_COMMAND, + enabled: true, + scope: "project", + }; +} + +export async function syncScheduledEvalBatchAutomation( + automationStore: AutomationStore, + settings: Partial, +): Promise { + const { AutomationStore } = await import("./automation-store.js"); + const resolved = resolveTaskEvaluationSettings(settings); + const schedules = await automationStore.listSchedules(); + const existing = schedules.find((s) => s.name === TASK_EVALUATION_SCHEDULE_NAME); + + if (!resolved.taskEvaluationEnabled) { + if (existing) await automationStore.deleteSchedule(existing.id); + return undefined; + } + + if (!AutomationStore.isValidCron(resolved.taskEvaluationSchedule)) { + throw new Error(`Invalid task evaluation schedule: ${resolved.taskEvaluationSchedule}`); + } + + const input = createScheduledEvalBatchAutomation(settings); + if (existing) { + return automationStore.updateSchedule(existing.id, { + scheduleType: "custom", + cronExpression: input.cronExpression, + command: input.command, + enabled: true, + scope: "project", + }); + } + + return automationStore.createSchedule(input); +} + +export interface EvalBatchWindow { + windowStartExclusive?: string; + windowEndInclusive: string; +} + +export interface CompletedTaskEvaluationContext { + run: EvalRun; + task: Task; + taskIndex: number; + totalTasks: number; + window: EvalBatchWindow; +} + +export type CompletedTaskEvaluator = ( + context: CompletedTaskEvaluationContext, +) => Promise>; + +export interface EvalBatchTaskStore { + listTasks(options?: { column?: string }): Promise; + getEvalStore(): import("./eval-store.js").EvalStore; +} + +export interface RunScheduledEvalBatchParams { + store: EvalBatchTaskStore; + projectId: string; + evaluator: CompletedTaskEvaluator; + startedAt?: string; +} + +export interface ScheduledEvalBatchResult { + runId: string; + status: "completed" | "failed"; + windowStartExclusive?: string; + windowEndInclusive: string; + selectedTaskIds: string[]; + tasksSelected: number; +} + +export async function runScheduledEvalBatch( + params: RunScheduledEvalBatchParams, +): Promise { + const startedAt = params.startedAt ?? new Date().toISOString(); + const evalStore = params.store.getEvalStore(); + const priorRuns = evalStore + .listRuns({ projectId: params.projectId, trigger: "schedule" }) + .filter((run) => run.status === "completed") + .sort((a, b) => { + const aWindowEnd = (a.metadata?.windowEndInclusive as string | undefined) ?? a.window.until ?? ""; + const bWindowEnd = (b.metadata?.windowEndInclusive as string | undefined) ?? b.window.until ?? ""; + if (aWindowEnd !== bWindowEnd) return aWindowEnd.localeCompare(bWindowEnd); + return a.id.localeCompare(b.id); + }); + + const previousScheduledBatch = priorRuns.at(-1); + const windowStartExclusive = + (previousScheduledBatch?.metadata?.windowEndInclusive as string | undefined) ?? + previousScheduledBatch?.window.until; + const windowEndInclusive = startedAt; + + let run: EvalRun; + try { + run = evalStore.createRun({ + projectId: params.projectId, + trigger: "schedule", + scope: "completed-tasks", + window: { + since: windowStartExclusive, + until: windowEndInclusive, + windowStartExclusive, + windowEndInclusive, + }, + metadata: { + windowStartExclusive, + windowEndInclusive, + }, + }); + } catch (error) { + if (error instanceof EvalLifecycleError && error.code === "active_run_conflict") { + throw error; + } + throw error; + } + + evalStore.appendRunEvent(run.id, { + type: "info", + message: "Scheduled eval batch started", + status: "pending", + metadata: { windowStartExclusive, windowEndInclusive }, + }); + + evalStore.updateRun(run.id, { status: "running", startedAt }); + + try { + const doneTasks = (await params.store.listTasks({ column: "done" })).filter((task) => + task.column === "done" + && Boolean(task.executionCompletedAt) + && (!windowStartExclusive || task.executionCompletedAt! > windowStartExclusive) + && task.executionCompletedAt! <= windowEndInclusive, + ); + + doneTasks.sort((a, b) => { + const byCompletedAt = (a.executionCompletedAt ?? "").localeCompare(b.executionCompletedAt ?? ""); + if (byCompletedAt !== 0) return byCompletedAt; + const byCreatedAt = a.createdAt.localeCompare(b.createdAt); + if (byCreatedAt !== 0) return byCreatedAt; + return a.id.localeCompare(b.id); + }); + + const selectedTaskIds = doneTasks.map((task) => task.id); + evalStore.updateRun(run.id, { + counts: { totalTasks: selectedTaskIds.length, scoredTasks: 0, skippedTasks: 0, erroredTasks: 0 }, + metadata: { + windowStartExclusive, + windowEndInclusive, + selectedTaskIds, + tasksSelected: selectedTaskIds.length, + }, + }); + + if (doneTasks.length === 0) { + evalStore.appendRunEvent(run.id, { + type: "info", + status: "completed", + message: "Scheduled eval batch completed with no newly done tasks", + metadata: { tasksSelected: 0 }, + }); + evalStore.updateRun(run.id, { + status: "completed", + completedAt: new Date().toISOString(), + summary: "No newly completed tasks found in evaluation window", + }); + return { + runId: run.id, + status: "completed", + windowStartExclusive, + windowEndInclusive, + selectedTaskIds: [], + tasksSelected: 0, + }; + } + + let scoredTasks = 0; + let skippedTasks = 0; + let erroredTasks = 0; + const evaluatedTaskIds: string[] = []; + + for (const [index, task] of doneTasks.entries()) { + try { + const result = await params.evaluator({ + run, + task, + taskIndex: index, + totalTasks: doneTasks.length, + window: { windowStartExclusive, windowEndInclusive }, + }); + + evalStore.createTaskResult(run.id, { + ...result, + taskId: task.id, + taskSnapshot: { + taskId: task.id, + title: task.title, + column: task.column, + createdAt: task.createdAt, + updatedAt: task.updatedAt, + executionCompletedAt: task.executionCompletedAt, + summary: task.summary, + }, + metadata: { + ...(result.metadata ?? {}), + windowEndInclusive, + }, + }); + + evaluatedTaskIds.push(task.id); + if (result.status === "scored") scoredTasks += 1; + else if (result.status === "skipped") skippedTasks += 1; + else erroredTasks += 1; + + evalStore.appendRunEvent(run.id, { + type: "task_evaluated", + message: `Evaluated task ${task.id}`, + taskId: task.id, + metadata: { status: result.status }, + }); + } catch (error) { + erroredTasks += 1; + evalStore.appendRunEvent(run.id, { + type: "error", + message: `Failed evaluating task ${task.id}`, + taskId: task.id, + metadata: { error: error instanceof Error ? error.message : String(error) }, + }); + } + } + + evalStore.updateRun(run.id, { + status: "completed", + evaluatedTaskIds, + counts: { + totalTasks: doneTasks.length, + scoredTasks, + skippedTasks, + erroredTasks, + }, + completedAt: new Date().toISOString(), + summary: `Scheduled eval batch completed for ${doneTasks.length} task(s)`, + metadata: { + windowStartExclusive, + windowEndInclusive, + selectedTaskIds, + tasksSelected: selectedTaskIds.length, + }, + }); + + evalStore.appendRunEvent(run.id, { + type: "status_changed", + status: "completed", + message: `Scheduled eval batch completed (${doneTasks.length} tasks selected)`, + metadata: { scoredTasks, skippedTasks, erroredTasks }, + }); + + return { + runId: run.id, + status: "completed", + windowStartExclusive, + windowEndInclusive, + selectedTaskIds, + tasksSelected: selectedTaskIds.length, + }; + } catch (error) { + evalStore.updateRun(run.id, { + status: "failed", + completedAt: new Date().toISOString(), + error: error instanceof Error ? error.message : String(error), + metadata: { + windowStartExclusive, + windowEndInclusive, + }, + }); + evalStore.appendRunEvent(run.id, { + type: "error", + status: "failed", + message: "Scheduled eval batch failed", + metadata: { error: error instanceof Error ? error.message : String(error) }, + }); + return { + runId: run.id, + status: "failed", + windowStartExclusive, + windowEndInclusive, + selectedTaskIds: [], + tasksSelected: 0, + }; + } +} From 2e8fb88c4471f6227d00658206f19929088bbbc4 Mon Sep 17 00:00:00 2001 From: Fusion Date: Tue, 5 May 2026 13:05:57 -0700 Subject: [PATCH 08/14] feat(FN-3368): tighten research view test coverage and resolve dashboard ty This merge introduces an eval automation domain store with persistence schema and a plugin dashboard view registry (FN-3512/FN-3513), adds scheduled eval batch architecture documentation (FN-3388), and includes substantial test coverage for Research routes and hooks (FN-3368 steps 1-3). The branch a Fusion-Task-Id: FN-3368 --- docs/research/research-hardening-preflight.md | 8 ++- packages/core/src/eval-automation.ts | 2 - packages/core/src/settings-schema.ts | 6 ++ packages/core/src/types.ts | 12 ++++ packages/dashboard/app/App.tsx | 8 ++- .../app/components/__tests__/App.test.tsx | 22 ++++++ .../__tests__/ResearchView.test.tsx | 70 +++++++++++++++++-- .../app/hooks/__tests__/useResearch.test.ts | 29 ++++++++ .../app/plugins/pluginViewRegistry.tsx | 4 +- .../src/__tests__/research-routes.test.ts | 57 ++++++++++++++- 10 files changed, 205 insertions(+), 13 deletions(-) diff --git a/docs/research/research-hardening-preflight.md b/docs/research/research-hardening-preflight.md index a96f7f9e1..415569e42 100644 --- a/docs/research/research-hardening-preflight.md +++ b/docs/research/research-hardening-preflight.md @@ -164,7 +164,13 @@ Key endpoints: - FN-3370 replaces FN-3015's stale insights-backed child scope with the landed research subsystem surfaces in this document (core `ResearchStore` + dashboard `/api/research` + engine orchestrator lifecycle persistence). - Regression coverage work should stay bounded to shipped lifecycle/status/export/task-integration contracts and use follow-up tasks for any unshipped behavior instead of feature expansion. -## 11) Validation references used for this baseline +## 11) Dashboard regression coverage status (FN-3368 refinement) + +- Dashboard interaction tests are anchored to landed standalone research surfaces (`ResearchView`, `ResearchTaskActionModal`, `useResearch`, `App` research route wiring). +- Route regression tests explicitly cover finding-to-task create/enrich provenance metadata, task-document writes, duplicate-attachment skip behavior, archived/missing target guards, and payload validation. +- There is no placeholder/optional assumption that research dashboard files or `/api/research` routes are absent. + +## 12) Validation references used for this baseline - `packages/dashboard/src/__tests__/research-routes.test.ts` - `packages/core/src/__tests__/research-store.test.ts` diff --git a/packages/core/src/eval-automation.ts b/packages/core/src/eval-automation.ts index 87a4a36e9..4dda9f99b 100644 --- a/packages/core/src/eval-automation.ts +++ b/packages/core/src/eval-automation.ts @@ -145,8 +145,6 @@ export async function runScheduledEvalBatch( window: { since: windowStartExclusive, until: windowEndInclusive, - windowStartExclusive, - windowEndInclusive, }, metadata: { windowStartExclusive, diff --git a/packages/core/src/settings-schema.ts b/packages/core/src/settings-schema.ts index 9f0b32262..5de9a7cb6 100644 --- a/packages/core/src/settings-schema.ts +++ b/packages/core/src/settings-schema.ts @@ -242,6 +242,12 @@ export const DEFAULT_PROJECT_SETTINGS = { insightExtractionEnabled: false, insightExtractionSchedule: "0 2 * * *", insightExtractionMinIntervalMs: 86_400_000, + taskEvaluationEnabled: false, + taskEvaluationSchedule: "0 5 * * *", + taskEvaluationProvider: undefined, + taskEvaluationModelId: undefined, + taskEvaluationFollowUpPolicy: "off", + taskEvaluationRetention: undefined, memoryEnabled: true, memoryBackendType: "qmd", memoryAutoSummarizeEnabled: false, diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 1911198ea..d3bd46628 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -1712,6 +1712,18 @@ export interface ProjectSettings { testCommand?: string; /** Custom build command for the project (e.g. "pnpm build") */ buildCommand?: string; + /** Enables automated scheduled evaluation of completed tasks. */ + taskEvaluationEnabled?: boolean; + /** Cron expression for scheduled task evaluation batches. */ + taskEvaluationSchedule?: string; + /** Optional provider override for task evaluation. */ + taskEvaluationProvider?: string; + /** Optional model override for task evaluation. */ + taskEvaluationModelId?: string; + /** Follow-up behavior for evaluation findings. */ + taskEvaluationFollowUpPolicy?: "off" | "suggest" | "create"; + /** Number of days to retain evaluation data. */ + taskEvaluationRetention?: number; /** When true, completed task worktrees are returned to an idle pool instead * of being deleted. New tasks acquire a warm worktree from the pool, * preserving build caches (node_modules, target/, dist/). Default: false. */ diff --git a/packages/dashboard/app/App.tsx b/packages/dashboard/app/App.tsx index 426128e30..433a9823a 100644 --- a/packages/dashboard/app/App.tsx +++ b/packages/dashboard/app/App.tsx @@ -878,8 +878,12 @@ function AppInner() { projectId: currentProject?.id, tasks: isRemote && remoteData.tasks.length > 0 ? remoteData.tasks : tasks, workflowSteps, - openTaskDetail: isMobile ? (task, initialTab) => openDetailTaskWithHistory(task, initialTab) : (task, initialTab) => modalManager.openDetailTask(task, initialTab), - renderTaskCard: (task) => ( + openTaskDetail: isMobile + ? (task: Task | TaskDetail, initialTab?: Parameters[1]) => + openDetailTaskWithHistory(task, initialTab) + : (task: Task | TaskDetail, initialTab?: Parameters[1]) => + modalManager.openDetailTask(task, initialTab), + renderTaskCard: (task: Task) => ( { localStorage.removeItem(taskViewStorageKey()); }); + it("does not expose research navigation when research feature is disabled", async () => { + localStorage.setItem("kb-dashboard-view-mode", "project"); + (fetchSettings as ReturnType).mockResolvedValueOnce({ + ...defaultSettings, + experimentalFeatures: { + ...defaultSettings.experimentalFeatures, + researchView: false, + }, + }); + + render(); + + await waitFor(() => { + expect(screen.getByTestId("view-toggle-overflow-trigger")).toBeInTheDocument(); + }); + + fireEvent.click(screen.getByTestId("view-toggle-overflow-trigger")); + expect(screen.queryByTestId("view-overflow-research")).not.toBeInTheDocument(); + + localStorage.removeItem("kb-dashboard-view-mode"); + }); + it("initializes research view from persisted task-view when feature-enabled", async () => { localStorage.setItem("kb-dashboard-view-mode", "project"); localStorage.setItem(taskViewStorageKey(), "research"); diff --git a/packages/dashboard/app/components/__tests__/ResearchView.test.tsx b/packages/dashboard/app/components/__tests__/ResearchView.test.tsx index 23fd2501c..ac524a1f6 100644 --- a/packages/dashboard/app/components/__tests__/ResearchView.test.tsx +++ b/packages/dashboard/app/components/__tests__/ResearchView.test.tsx @@ -201,12 +201,12 @@ describe("ResearchView", () => { mockUseResearch.mockReturnValue({ ...baseHookValue, - runs: [{ id: "RR-1", title: "t", query: "q", status: "pending" }], + runs: [{ id: "RR-1", title: "t", query: "q", status: "queued" }], selectedRun: { id: "RR-1", title: "t", query: "q", - status: "pending", + status: "queued", events: [{ id: "E-1", message: "queued" }], results: { summary: "Summary", findings: [{ id: "finding-1", heading: "Finding", content: "Impact." }], citations: [] }, }, @@ -239,12 +239,12 @@ describe("ResearchView", () => { const attachRunToTask = vi.fn().mockResolvedValue({}); mockUseResearch.mockReturnValue({ ...baseHookValue, - runs: [{ id: "RR-1", title: "t", query: "q", status: "pending" }], + runs: [{ id: "RR-1", title: "t", query: "q", status: "queued" }], selectedRun: { id: "RR-1", title: "t", query: "q", - status: "pending", + status: "queued", events: [], results: { summary: "Summary", findings: [{ id: "finding-1", heading: "Finding", content: "Impact." }], citations: [] }, }, @@ -298,7 +298,7 @@ describe("ResearchView", () => { setSearchQuery, setSelectedRunId, runs: [ - { id: "RR-1", title: "Alpha", query: "alpha", status: "pending" }, + { id: "RR-1", title: "Alpha", query: "alpha", status: "queued" }, { id: "RR-2", title: "Beta", query: "beta", status: "completed" }, ], }); @@ -460,6 +460,66 @@ describe("ResearchView", () => { expect(await screen.findByTestId("research-state-empty")).toBeInTheDocument(); }); + it("wires create-task modal payload with trimmed fields and attachment toggle", async () => { + const createTaskFromRun = vi.fn().mockResolvedValue({}); + mockUseResearch.mockReturnValue({ + ...baseHookValue, + createTaskFromRun, + runs: [{ id: "RR-1", title: "t", query: "q", status: "completed" }], + selectedRun: { + id: "RR-1", + title: "t", + query: "q", + status: "completed", + events: [], + results: { summary: "Summary", findings: [{ id: "finding-1", heading: "Finding", content: "Impact." }], citations: [] }, + }, + selectedRunId: "RR-1", + }); + mockFetchAuthStatus.mockResolvedValue({ providers: [{ id: "openrouter", type: "api_key", authenticated: true }] }); + + render(); + fireEvent.click((await screen.findAllByText("Create Task"))[0]); + + const dialog = await screen.findByRole("dialog"); + fireEvent.change(within(dialog).getByLabelText("Title"), { target: { value: " Follow up task " } }); + fireEvent.change(within(dialog).getByLabelText("Description"), { target: { value: " Take action now. " } }); + fireEvent.click(within(dialog).getByLabelText("Attach markdown export artifact")); + fireEvent.click(within(dialog).getByRole("button", { name: "Create Task" })); + + await waitFor(() => { + expect(createTaskFromRun).toHaveBeenCalledWith("RR-1", "Follow up task", "finding-1", "Take action now.", "normal", true); + }); + }); + + it("keeps enrich action disabled until a task id is provided", async () => { + mockUseResearch.mockReturnValue({ + ...baseHookValue, + runs: [{ id: "RR-1", title: "t", query: "q", status: "completed" }], + selectedRun: { + id: "RR-1", + title: "t", + query: "q", + status: "completed", + events: [], + results: { summary: "Summary", findings: [{ id: "finding-1", heading: "Finding", content: "Impact." }], citations: [] }, + }, + selectedRunId: "RR-1", + }); + mockFetchAuthStatus.mockResolvedValue({ providers: [{ id: "openrouter", type: "api_key", authenticated: true }] }); + + render(); + fireEvent.click((await screen.findAllByText("Enrich Task"))[0]); + + const dialog = await screen.findByRole("dialog"); + const enrichButton = within(dialog).getByRole("button", { name: "Enrich Task" }); + expect(enrichButton).toBeDisabled(); + + const targetInput = within(dialog).getByRole("combobox", { name: "Target task" }); + fireEvent.change(targetInput, { target: { value: "FN-1" } }); + await waitFor(() => expect(enrichButton).not.toBeDisabled()); + }); + it("includes mobile layout media rule", async () => { const css = await import("../ResearchView.css?inline"); expect(css.default).toContain("@media (max-width: 768px)"); diff --git a/packages/dashboard/app/hooks/__tests__/useResearch.test.ts b/packages/dashboard/app/hooks/__tests__/useResearch.test.ts index ec7e1611c..c64ef1b2c 100644 --- a/packages/dashboard/app/hooks/__tests__/useResearch.test.ts +++ b/packages/dashboard/app/hooks/__tests__/useResearch.test.ts @@ -35,6 +35,7 @@ vi.mock("../../sse-bus", () => ({ describe("useResearch", () => { beforeEach(() => { vi.clearAllMocks(); + vi.useRealTimers(); mockListResearchRuns.mockResolvedValue({ runs: [], availability: { available: true } }); mockGetResearchRun.mockResolvedValue({ run: { id: "RR-2", title: "t" }, availability: { available: true } }); }); @@ -223,4 +224,32 @@ describe("useResearch", () => { ); }); }); + + it("refreshes list and selected run on reconnect", async () => { + let handlers: { onReconnect?: () => void; events?: Record void> } = {}; + mockSubscribeSse.mockImplementationOnce((_url, opts) => { + handlers = opts; + return vi.fn(); + }); + + const { result } = renderHook(() => useResearch({ projectId: "p1" })); + + act(() => { + result.current.setSelectedRunId("RR-2"); + }); + + await waitFor(() => { + expect(mockGetResearchRun).toHaveBeenCalledWith("RR-2", "p1"); + }); + + const listCallsBefore = mockListResearchRuns.mock.calls.length; + + act(() => { + handlers.onReconnect?.(); + }); + + await waitFor(() => { + expect(mockListResearchRuns.mock.calls.length).toBeGreaterThan(listCallsBefore); + }); + }); }); diff --git a/packages/dashboard/app/plugins/pluginViewRegistry.tsx b/packages/dashboard/app/plugins/pluginViewRegistry.tsx index 8b2faaa68..5443e328d 100644 --- a/packages/dashboard/app/plugins/pluginViewRegistry.tsx +++ b/packages/dashboard/app/plugins/pluginViewRegistry.tsx @@ -1,11 +1,11 @@ import { AlertTriangle } from "lucide-react"; -import { lazy, Suspense, type LazyExoticComponent, type ReactNode } from "react"; +import { lazy, Suspense, type LazyExoticComponent, type ReactElement, type ReactNode } from "react"; import { ErrorBoundary } from "../components/ErrorBoundary"; import "./pluginViewRegistry.css"; export type PluginTaskView = `plugin:${string}:${string}`; -type PluginViewComponent = LazyExoticComponent<() => JSX.Element>; +type PluginViewComponent = LazyExoticComponent<() => ReactElement>; const registry = new Map(); diff --git a/packages/dashboard/src/__tests__/research-routes.test.ts b/packages/dashboard/src/__tests__/research-routes.test.ts index aa7bf2323..f8d9f7e81 100644 --- a/packages/dashboard/src/__tests__/research-routes.test.ts +++ b/packages/dashboard/src/__tests__/research-routes.test.ts @@ -85,6 +85,7 @@ function createMockStore(options?: { } return { filename: "RR-1-finding-1.md" }; }), + appendAgentLog: vi.fn(async () => undefined), log: vi.fn(async () => undefined), }; } @@ -197,10 +198,26 @@ describe("research-routes", () => { expect.objectContaining({ source: expect.objectContaining({ sourceType: "research", - sourceMetadata: expect.objectContaining({ runId: "RR-1", findingId: "finding-1" }), + sourceRunId: "RR-1", + sourceMetadata: expect.objectContaining({ runId: "RR-1", findingId: "finding-1", documentKey: "research-RR-1" }), }), }), ); + expect(store.upsertTaskDocument).toHaveBeenCalledWith( + "FN-1", + expect.objectContaining({ + key: "research-RR-1", + author: "research", + metadata: expect.objectContaining({ runId: "RR-1", findingId: "finding-1" }), + }), + ); + expect(store.appendAgentLog).toHaveBeenCalledWith( + "FN-1", + expect.stringContaining("Task created from research finding finding-1 in run RR-1"), + "text", + "research-task-integration", + "executor", + ); }); it("enriches existing task from finding and returns revision", async () => { @@ -221,6 +238,21 @@ describe("research-routes", () => { expect(response.body.taskId).toBe("FN-42"); expect(response.body.documentKey).toBe("research-RR-1"); expect(response.body.revision).toBe(1); + expect(store.upsertTaskDocument).toHaveBeenCalledWith( + "FN-42", + expect.objectContaining({ + key: "research-RR-1", + author: "research", + metadata: expect.objectContaining({ runId: "RR-1", findingId: "finding-1" }), + }), + ); + expect(store.appendAgentLog).toHaveBeenCalledWith( + "FN-42", + expect.stringContaining("Task enriched from research finding finding-1 in run RR-1"), + "text", + "research-task-integration", + "executor", + ); }); it("skips duplicate attachment when original name already exists", async () => { @@ -401,6 +433,29 @@ describe("research-routes", () => { expect(response.body.error).toContain("attachExport must be a boolean"); }); + it("returns 400 when create payload title/description are empty strings", async () => { + const store = createMockStore(); + const app = express(); + app.use(express.json()); + app.use(createResearchRouter(store as any)); + + const response = await performRequest( + app, + "POST", + "/runs/RR-1/findings/finding-1/task", + JSON.stringify({ title: " ", description: " " }), + { "content-type": "application/json" }, + ); + + expect(response.status).toBe(201); + expect(store.createTask).toHaveBeenCalledWith( + expect.objectContaining({ + title: "Research: Finding One", + description: expect.stringContaining("Important actionable result."), + }), + ); + }); + it("returns 400 when attachment exceeds size limit", async () => { const app = express(); app.use(express.json()); From 61bf81d6b84a4c4fa0336025c007440acd8c505f Mon Sep 17 00:00:00 2001 From: Fusion Date: Tue, 5 May 2026 13:19:16 -0700 Subject: [PATCH 09/14] feat(FN-3470): prefer source aliases for runtime plugins and add test artif Merges FN-3470 in two steps: first, runtime plugins now prefer source path aliases over build artifacts, with hardening tests for vitest alias resolution and plugin view registry updates; second, a new `ensure-test-artifacts.mjs` bootstrap script guarantees required test files exist before test runs Fusion-Task-Id: FN-3470 --- docs/contributing.md | 3 +- packages/cli/vitest.config.ts | 12 ++++++ packages/dashboard/app/App.tsx | 6 +-- .../app/plugins/pluginViewRegistry.tsx | 2 +- packages/dashboard/vitest.config.ts | 12 ++++++ .../__tests__/ensure-test-artifacts.test.mjs | 33 +++++++++++++++ .../__tests__/vitest-source-aliases.test.mjs | 41 +++++++++++++++++++ scripts/ci-test-shard.mjs | 2 + scripts/ensure-test-artifacts.mjs | 38 +++++++++++++++++ scripts/test-changed.mjs | 2 + 10 files changed, 146 insertions(+), 5 deletions(-) create mode 100644 scripts/__tests__/ensure-test-artifacts.test.mjs create mode 100644 scripts/__tests__/vitest-source-aliases.test.mjs create mode 100644 scripts/ensure-test-artifacts.mjs diff --git a/docs/contributing.md b/docs/contributing.md index 600dfed6e..5da4f484e 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -58,7 +58,8 @@ Fusion codifies workspace verification as a deterministic contract: - Use `pnpm install --frozen-lockfile` for clean bootstrap and dependency repair paths. - `pnpm test:full` must be runnable in a clean worktree without requiring a prior `pnpm build`. -- This includes clean states where `packages/core/dist`, `packages/engine/dist`, and `packages/dashboard/dist` are absent. +- Root test entrypoints (`pnpm test` via `scripts/test-changed.mjs` and `pnpm test:ci:shard` via `scripts/ci-test-shard.mjs`) call `scripts/ensure-test-artifacts.mjs`, which deterministically builds only missing required workspace dist artifacts (`@fusion/core`, `@fusion/plugin-sdk`, and runtime plugins that export from `dist/*`). +- This includes clean states where those required dist directories are absent. - `pnpm verify:workspace` is the canonical pre-merge gate and runs in strict order: 1. `pnpm lint` 2. `pnpm test:full` diff --git a/packages/cli/vitest.config.ts b/packages/cli/vitest.config.ts index 2d1c8b5c3..7f1b013bb 100644 --- a/packages/cli/vitest.config.ts +++ b/packages/cli/vitest.config.ts @@ -25,6 +25,18 @@ export default defineConfig({ find: /^@fusion-plugin-examples\/droid-runtime$/, replacement: resolve(__dirname, "../../plugins/fusion-plugin-droid-runtime/src/index.ts"), }, + { + find: /^@fusion-plugin-examples\/hermes-runtime$/, + replacement: resolve(__dirname, "../../plugins/fusion-plugin-hermes-runtime/src/index.ts"), + }, + { + find: /^@fusion-plugin-examples\/openclaw-runtime$/, + replacement: resolve(__dirname, "../../plugins/fusion-plugin-openclaw-runtime/src/index.ts"), + }, + { + find: /^@fusion-plugin-examples\/paperclip-runtime$/, + replacement: resolve(__dirname, "../../plugins/fusion-plugin-paperclip-runtime/src/index.ts"), + }, { find: /^@fusion\/test-utils$/, replacement: resolve(__dirname, "../core/src/__test-utils__/workspace.ts") }, ], }, diff --git a/packages/dashboard/app/App.tsx b/packages/dashboard/app/App.tsx index 433a9823a..28a6a36aa 100644 --- a/packages/dashboard/app/App.tsx +++ b/packages/dashboard/app/App.tsx @@ -880,10 +880,10 @@ function AppInner() { workflowSteps, openTaskDetail: isMobile ? (task: Task | TaskDetail, initialTab?: Parameters[1]) => - openDetailTaskWithHistory(task, initialTab) + openDetailTaskWithHistory(task, initialTab) : (task: Task | TaskDetail, initialTab?: Parameters[1]) => - modalManager.openDetailTask(task, initialTab), - renderTaskCard: (task: Task) => ( + modalManager.openDetailTask(task, initialTab), + renderTaskCard: (task: Task | TaskDetail) => ( ReactElement>; +type PluginViewComponent = LazyExoticComponent<() => ReactNode>; const registry = new Map(); diff --git a/packages/dashboard/vitest.config.ts b/packages/dashboard/vitest.config.ts index ca3ef7a80..59f5905eb 100644 --- a/packages/dashboard/vitest.config.ts +++ b/packages/dashboard/vitest.config.ts @@ -21,6 +21,18 @@ export default defineConfig({ __dirname, "../../plugins/fusion-plugin-droid-runtime/src/index.ts", ), + "@fusion-plugin-examples/hermes-runtime": resolve( + __dirname, + "../../plugins/fusion-plugin-hermes-runtime/src/index.ts", + ), + "@fusion-plugin-examples/openclaw-runtime": resolve( + __dirname, + "../../plugins/fusion-plugin-openclaw-runtime/src/index.ts", + ), + "@fusion-plugin-examples/paperclip-runtime": resolve( + __dirname, + "../../plugins/fusion-plugin-paperclip-runtime/src/index.ts", + ), }, }, test: { diff --git a/scripts/__tests__/ensure-test-artifacts.test.mjs b/scripts/__tests__/ensure-test-artifacts.test.mjs new file mode 100644 index 000000000..143b972d8 --- /dev/null +++ b/scripts/__tests__/ensure-test-artifacts.test.mjs @@ -0,0 +1,33 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { detectMissingArtifacts, ensureTestArtifacts } from "../ensure-test-artifacts.mjs"; + +test("detectMissingArtifacts returns missing package list", () => { + const missing = detectMissingArtifacts("/repo", () => false); + assert.ok(missing.length >= 5); + assert.equal(missing[0].name, "@fusion/core"); +}); + +test("ensureTestArtifacts skips build when nothing is missing", () => { + let called = false; + const built = ensureTestArtifacts("/repo", () => { + called = true; + }, () => true); + + assert.equal(called, false); + assert.deepEqual(built, []); +}); + +test("ensureTestArtifacts builds only missing packages", () => { + const calls = []; + const built = ensureTestArtifacts( + "/repo", + (cmd, args, cwd) => calls.push({ cmd, args, cwd }), + (fullPath) => !fullPath.includes("fusion-plugin-openclaw-runtime"), + ); + + assert.deepEqual(built, ["@fusion-plugin-examples/openclaw-runtime"]); + assert.equal(calls.length, 1); + assert.equal(calls[0].cmd, "pnpm"); + assert.deepEqual(calls[0].args, ["--filter", "@fusion-plugin-examples/openclaw-runtime", "build"]); +}); diff --git a/scripts/__tests__/vitest-source-aliases.test.mjs b/scripts/__tests__/vitest-source-aliases.test.mjs new file mode 100644 index 000000000..4408ef904 --- /dev/null +++ b/scripts/__tests__/vitest-source-aliases.test.mjs @@ -0,0 +1,41 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { URL } from "node:url"; + +function read(path) { + return readFileSync(new URL(`../../${path}`, import.meta.url), "utf8"); +} + +test("dashboard vitest config aliases runtime plugins to src", () => { + const content = read("packages/dashboard/vitest.config.ts"); + assert.match(content, /@fusion-plugin-examples\/hermes-runtime/); + assert.match(content, /@fusion-plugin-examples\/openclaw-runtime/); + assert.match(content, /@fusion-plugin-examples\/paperclip-runtime/); + assert.match(content, /plugins\/fusion-plugin-hermes-runtime\/src\/index\.ts/); + assert.match(content, /plugins\/fusion-plugin-openclaw-runtime\/src\/index\.ts/); + assert.match(content, /plugins\/fusion-plugin-paperclip-runtime\/src\/index\.ts/); +}); + +test("cli vitest config aliases runtime plugins to src", () => { + const content = read("packages/cli/vitest.config.ts"); + assert.ok(content.includes("@fusion-plugin-examples\\/droid-runtime")); + assert.ok(content.includes("@fusion-plugin-examples\\/hermes-runtime")); + assert.ok(content.includes("@fusion-plugin-examples\\/openclaw-runtime")); + assert.ok(content.includes("@fusion-plugin-examples\\/paperclip-runtime")); + assert.match(content, /plugins\/fusion-plugin-hermes-runtime\/src\/index\.ts/); + assert.match(content, /plugins\/fusion-plugin-openclaw-runtime\/src\/index\.ts/); + assert.match(content, /plugins\/fusion-plugin-paperclip-runtime\/src\/index\.ts/); +}); + +test("engine and plugin-sdk vitest configs keep source aliases", () => { + const engine = read("packages/engine/vitest.config.ts"); + const sdk = read("packages/plugin-sdk/vitest.config.ts"); + + assert.match(engine, /@fusion\/core/); + assert.match(engine, /\.\.\/core\/src\/index\.ts/); + assert.match(engine, /@fusion\/plugin-sdk/); + + assert.match(sdk, /@fusion\/core/); + assert.match(sdk, /\.\.\/core\/src\/index\.ts/); +}); diff --git a/scripts/ci-test-shard.mjs b/scripts/ci-test-shard.mjs index f441c92eb..dcfbd9359 100644 --- a/scripts/ci-test-shard.mjs +++ b/scripts/ci-test-shard.mjs @@ -3,6 +3,7 @@ import { spawnSync } from "node:child_process"; import path from "node:path"; import { fileURLToPath } from "node:url"; +import { ensureTestArtifacts } from "./ensure-test-artifacts.mjs"; const DEFAULT_TEST_PACKAGES = [ "@fusion/core", @@ -74,6 +75,7 @@ export function main(argv = process.argv.slice(2), env = process.env) { }; run("pnpm", ["sync:fusion-skill:check"], { env: shardEnv }); + ensureTestArtifacts(process.cwd()); const filters = shardPackages.flatMap((pkg) => ["--filter", pkg]); run("pnpm", [...filters, "test"], { env: shardEnv }); } diff --git a/scripts/ensure-test-artifacts.mjs b/scripts/ensure-test-artifacts.mjs new file mode 100644 index 000000000..14b0bd503 --- /dev/null +++ b/scripts/ensure-test-artifacts.mjs @@ -0,0 +1,38 @@ +#!/usr/bin/env node + +import { existsSync } from "node:fs"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; + +export const REQUIRED_BUILD_PACKAGES = [ + { name: "@fusion/core", distEntry: "packages/core/dist/index.js" }, + { name: "@fusion/plugin-sdk", distEntry: "packages/plugin-sdk/dist/index.js" }, + { name: "@fusion-plugin-examples/hermes-runtime", distEntry: "plugins/fusion-plugin-hermes-runtime/dist/index.js" }, + { name: "@fusion-plugin-examples/openclaw-runtime", distEntry: "plugins/fusion-plugin-openclaw-runtime/dist/index.js" }, + { name: "@fusion-plugin-examples/paperclip-runtime", distEntry: "plugins/fusion-plugin-paperclip-runtime/dist/index.js" }, +]; + +export function detectMissingArtifacts(rootDir = process.cwd(), existsFn = existsSync) { + return REQUIRED_BUILD_PACKAGES.filter((pkg) => !existsFn(path.join(rootDir, pkg.distEntry))); +} + +function run(command, args, cwd) { + const result = spawnSync(command, args, { cwd, stdio: "inherit" }); + if (result.status !== 0) { + process.exit(result.status ?? 1); + } +} + +export function ensureTestArtifacts(rootDir = process.cwd(), runFn = run, existsFn = existsSync) { + const missing = detectMissingArtifacts(rootDir, existsFn); + if (missing.length === 0) return []; + + const names = missing.map((pkg) => pkg.name); + console.log(`[test-bootstrap] building missing dist artifacts: ${names.join(", ")}`); + runFn("pnpm", [...names.flatMap((name) => ["--filter", name]), "build"], rootDir); + return names; +} + +if (import.meta.url === `file://${process.argv[1]}`) { + ensureTestArtifacts(); +} diff --git a/scripts/test-changed.mjs b/scripts/test-changed.mjs index 8eaa0d61a..b18a126a2 100644 --- a/scripts/test-changed.mjs +++ b/scripts/test-changed.mjs @@ -5,6 +5,7 @@ import path from "node:path"; import { spawnSync } from "node:child_process"; import { fileURLToPath } from "node:url"; import { createHash } from "node:crypto"; +import { ensureTestArtifacts } from "./ensure-test-artifacts.mjs"; const rootDir = process.env.FUSION_PROJECT_DIR ? path.resolve(process.env.FUSION_PROJECT_DIR) @@ -447,6 +448,7 @@ export function main(argv = process.argv.slice(2)) { const forwardedArgs = argv.filter((arg) => arg !== "--full" && arg !== "--no-cache"); run("pnpm", ["sync:fusion-skill:check"]); + ensureTestArtifacts(rootDir); const baseBranch = getBaseBranch(); const comparisonBase = detectComparisonBase(baseBranch); From 57c67b6bbd61667145c96c62dd28730174885566 Mon Sep 17 00:00:00 2001 From: Fusion Date: Tue, 5 May 2026 13:32:10 -0700 Subject: [PATCH 10/14] feat(FN-3502): add comment-triggered retriage when task is in triage status Merged four commits implementing comment-driven retriage: triage rules now respond to specific comment patterns (Step 1) and surface needs-replan feedback inputs in the UI (Step 2), with documentation for the new behavior and a bug fix restoring workspace typecheck defaults. Changes span the core ta Fusion-Task-Id: FN-3502 --- docs/task-management.md | 14 ++++ packages/core/src/__tests__/store.test.ts | 76 ++++++++++++++++++-- packages/core/src/eval-types.ts | 2 + packages/core/src/settings-schema.ts | 6 ++ packages/core/src/store.ts | 75 ++++++++++++------- packages/core/src/types.ts | 12 ++++ packages/engine/src/__tests__/triage.test.ts | 30 ++++++++ packages/engine/src/triage.ts | 25 +++++-- 8 files changed, 200 insertions(+), 40 deletions(-) diff --git a/docs/task-management.md b/docs/task-management.md index da37903f1..378522d7d 100644 --- a/docs/task-management.md +++ b/docs/task-management.md @@ -231,6 +231,20 @@ This file is the contract for execution and review. Steering comments can be injected mid-run into active executor sessions. +### User comments and triage re-consideration + +User comments can trigger **re-triage** for already-planned but non-executing work: + +- `triage` + `awaiting-approval` → user comment sets `status: "needs-replan"` +- `triage` or `todo` with a real (non-bootstrap-stub) `PROMPT.md` → user comment sets `status: "needs-replan"` +- `triage` or `todo` with only bootstrap-stub/unplanned prompt content → no re-triage transition + +Execution ownership is preserved for active work: + +- User comments on `in-progress` and `in-review` tasks do **not** re-route those tasks back through triage. +- Agent/system comments do **not** trigger comment-driven re-triage. + +This is distinct from steering comments: steering feedback targets the currently running executor session, while comment-driven re-triage requests a fresh specification pass for planned work. ## Refinement Tasks `fn task refine ` creates a new planning task that depends on the original done/in-review task. diff --git a/packages/core/src/__tests__/store.test.ts b/packages/core/src/__tests__/store.test.ts index bf7dc42c3..720696f5a 100644 --- a/packages/core/src/__tests__/store.test.ts +++ b/packages/core/src/__tests__/store.test.ts @@ -5430,7 +5430,7 @@ Task with acceptance criteria expect(updateSpy).toHaveBeenCalled(); const warningCall = warnSpy.mock.calls.find( - (call) => typeof call[0] === "string" && call[0].includes("[task-store] Best-effort post-comment awaiting-approval invalidation failed"), + (call) => typeof call[0] === "string" && call[0].includes("[task-store] Best-effort post-comment re-triage failed"), ); expect(warningCall).toBeDefined(); @@ -5476,7 +5476,7 @@ Task with acceptance criteria expect(logEntrySpy).toHaveBeenCalled(); const warningCall = warnSpy.mock.calls.find( - (call) => typeof call[0] === "string" && call[0].includes("[task-store] Best-effort post-comment awaiting-approval invalidation failed"), + (call) => typeof call[0] === "string" && call[0].includes("[task-store] Best-effort post-comment re-triage failed"), ); expect(warningCall).toBeDefined(); @@ -5654,14 +5654,76 @@ Task with acceptance criteria expect(updated.comments).toHaveLength(1); }); - it("does NOT transition to needs-replan when user comments on non-awaiting-approval triage task", async () => { + it("transitions to needs-replan when user comments on non-awaiting-approval triage task with real spec", async () => { const task = await store.createTask({ description: "Task in triage" }); - // Task is in triage with no status (not awaiting-approval) - expect(task.status).toBeUndefined(); + const promptPath = join(rootDir, ".fusion", "tasks", task.id, "PROMPT.md"); + await writeFile(promptPath, `# Task: ${task.id} - Triage Plan\n\n## Mission\n\nPlanned task.`); - const updated = await store.addComment(task.id, "User feedback", "user"); + await store.addComment(task.id, "User feedback", "user"); + const updated = await store.getTask(task.id); - // Status should remain undefined + expect(updated.status).toBe("needs-replan"); + expect(updated.column).toBe("triage"); + expect(updated.comments?.[0]?.text).toBe("User feedback"); + }); + + it("does NOT transition to needs-replan when user comments on triage task with bootstrap stub prompt", async () => { + const task = await store.createTask({ description: "Task in triage" }); + + await store.addComment(task.id, "User feedback", "user"); + const updated = await store.getTask(task.id); + + expect(updated.status).toBeUndefined(); + }); + + it("transitions todo task to needs-replan when user comments and task has real spec", async () => { + const task = await store.createTask({ description: "Task in todo", column: "todo" }); + const promptPath = join(rootDir, ".fusion", "tasks", task.id, "PROMPT.md"); + await writeFile(promptPath, `# Task: ${task.id} - Todo Plan\n\n## Mission\n\nPlanned task.`); + + await store.addComment(task.id, "Please update approach", "user"); + const updated = await store.getTask(task.id); + + expect(updated.status).toBe("needs-replan"); + expect(updated.column).toBe("todo"); + expect(updated.log.some((entry) => entry.action === "User comment requested re-specification of planned task")).toBe(true); + }); + + it("does NOT transition todo task to needs-replan when prompt matches bootstrap stub", async () => { + const task = await store.createTask({ description: "Task in todo", column: "todo" }); + const promptPath = join(rootDir, ".fusion", "tasks", task.id, "PROMPT.md"); + await writeFile(promptPath, `# ${task.id}\n\nTask in todo\n`); + + await store.addComment(task.id, "Please update approach", "user"); + const updated = await store.getTask(task.id); + + expect(updated.status).toBeUndefined(); + }); + + it("does NOT transition to needs-replan when user comments on in-progress task", async () => { + const task = await store.createTask({ description: "Task in progress", column: "todo" }); + const promptPath = join(rootDir, ".fusion", "tasks", task.id, "PROMPT.md"); + await writeFile(promptPath, `# Task: ${task.id} - Plan\n\n## Mission\n\nPlanned task.`); + await store.moveTask(task.id, "in-progress"); + + await store.addComment(task.id, "Please adjust implementation", "user"); + const updated = await store.getTask(task.id); + + expect(updated.column).toBe("in-progress"); + expect(updated.status).toBeUndefined(); + }); + + it("does NOT transition to needs-replan when user comments on in-review task", async () => { + const task = await store.createTask({ description: "Task in review", column: "todo" }); + const promptPath = join(rootDir, ".fusion", "tasks", task.id, "PROMPT.md"); + await writeFile(promptPath, `# Task: ${task.id} - Plan\n\n## Mission\n\nPlanned task.`); + await store.moveTask(task.id, "in-progress"); + await store.moveTask(task.id, "in-review"); + + await store.addComment(task.id, "Please adjust before merge", "user"); + const updated = await store.getTask(task.id); + + expect(updated.column).toBe("in-review"); expect(updated.status).toBeUndefined(); }); }); diff --git a/packages/core/src/eval-types.ts b/packages/core/src/eval-types.ts index 7da6c22a9..04c6eb721 100644 --- a/packages/core/src/eval-types.ts +++ b/packages/core/src/eval-types.ts @@ -49,6 +49,8 @@ export interface EvalRunWindow { since?: string; until?: string; baselineRunId?: string; + windowStartExclusive?: string; + windowEndInclusive?: string; } export interface EvalProvenance { diff --git a/packages/core/src/settings-schema.ts b/packages/core/src/settings-schema.ts index 5de9a7cb6..70a05efa7 100644 --- a/packages/core/src/settings-schema.ts +++ b/packages/core/src/settings-schema.ts @@ -292,6 +292,12 @@ export const DEFAULT_PROJECT_SETTINGS = { researchDefaultTimeout: 300000, researchMaxSourcesPerRun: 20, researchMaxSynthesisRounds: 2, + taskEvaluationEnabled: false, + taskEvaluationSchedule: "0 5 * * *", + taskEvaluationProvider: undefined, + taskEvaluationModelId: undefined, + taskEvaluationFollowUpPolicy: "off", + taskEvaluationRetention: undefined, } satisfies CompleteSettings; /** diff --git a/packages/core/src/store.ts b/packages/core/src/store.ts index 17fefd8d4..87ff6d774 100644 --- a/packages/core/src/store.ts +++ b/packages/core/src/store.ts @@ -5169,52 +5169,73 @@ export class TaskStore extends EventEmitter { } } - // Phase 3: Invalidate stale spec approval when a user comments on - // a triage task that is awaiting manual approval. The new comment - // means the spec is now stale and must be re-specified/re-reviewed. + // Phase 3: user comments on already-planned, non-executing work should + // trigger triage re-specification. This includes awaiting-approval + // invalidation and todo/triage tasks that have a real non-bootstrap spec. // This remains best-effort: failures are logged for observability but // never fail the comment add operation itself. // Note: The `task` returned above reflects the state BEFORE this // transition. Callers that need the post-transition status should // re-read the task (e.g., via getTask). - if ( - task.column === "triage" - && task.status === "awaiting-approval" - && author === "user" - ) { - let invalidatedStatus = false; + if (author === "user" && (task.column === "todo" || task.column === "triage")) { + let hasRealPrompt = false; try { - await this.updateTask(id, { - status: "needs-replan", - }); - invalidatedStatus = true; + const promptPath = join(this.taskDir(id), "PROMPT.md"); + if (existsSync(promptPath)) { + const prompt = await readFile(promptPath, "utf-8"); + hasRealPrompt = !isBootstrapPromptStub(prompt, task.id, task.title, task.description); + } } catch (err) { - storeLog.warn("Best-effort post-comment awaiting-approval invalidation failed", { + storeLog.warn("Best-effort post-comment re-triage prompt-read failed", { ...commentContextBase, - phase: "addComment:awaiting-approval-invalidation", - stage: "status-update", - nextStatus: "needs-replan", + phase: "addComment:retriage-prompt-read", error: err instanceof Error ? err.message : String(err), }); } - if (invalidatedStatus) { + const shouldInvalidateAwaitingApproval = + task.column === "triage" && task.status === "awaiting-approval"; + const shouldRetriagePlannedTask = hasRealPrompt + && ( + task.column === "todo" + || (task.column === "triage" && task.status !== "awaiting-approval") + ); + + if (shouldInvalidateAwaitingApproval || shouldRetriagePlannedTask) { + const phase = shouldInvalidateAwaitingApproval + ? "addComment:awaiting-approval-invalidation" + : "addComment:planned-task-retriage"; + const action = shouldInvalidateAwaitingApproval + ? "User comment invalidated spec approval — task needs re-specification" + : "User comment requested re-specification of planned task"; + let transitioned = false; + try { - await this.logEntry( - id, - `User comment invalidated spec approval — task needs re-specification`, - undefined, - runContext, - ); + await this.updateTask(id, { status: "needs-replan" }); + transitioned = true; } catch (err) { - storeLog.warn("Best-effort post-comment awaiting-approval invalidation failed", { + storeLog.warn("Best-effort post-comment re-triage failed", { ...commentContextBase, - phase: "addComment:awaiting-approval-invalidation", - stage: "post-invalidation-log-entry", + phase, + stage: "status-update", nextStatus: "needs-replan", error: err instanceof Error ? err.message : String(err), }); } + + if (transitioned) { + try { + await this.logEntry(id, action, text, runContext); + } catch (err) { + storeLog.warn("Best-effort post-comment re-triage failed", { + ...commentContextBase, + phase, + stage: "post-invalidation-log-entry", + nextStatus: "needs-replan", + error: err instanceof Error ? err.message : String(err), + }); + } + } } } diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index d3bd46628..b7e059592 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -1687,6 +1687,18 @@ export interface ProjectSettings { unavailableNodePolicy?: UnavailableNodePolicy; /** Project-level research configuration overrides. */ researchSettings?: ResearchProjectSettings; + /** Enable scheduled evaluation batches for recently completed tasks. */ + taskEvaluationEnabled?: boolean; + /** Cron expression for scheduled task-evaluation batches. */ + taskEvaluationSchedule?: string; + /** Optional provider override for scheduled task evaluation runs. */ + taskEvaluationProvider?: string; + /** Optional model override for scheduled task evaluation runs. */ + taskEvaluationModelId?: string; + /** Follow-up policy for scheduled task evaluation findings. */ + taskEvaluationFollowUpPolicy?: "off" | "suggest" | "create"; + /** Optional retention window (days) for task evaluation history. */ + taskEvaluationRetention?: number; /** Enable or disable the research subsystem for this project. * When undefined, falls back to global settings. * @deprecated Prefer researchSettings.enabled */ diff --git a/packages/engine/src/__tests__/triage.test.ts b/packages/engine/src/__tests__/triage.test.ts index 934de9e38..4707ccb46 100644 --- a/packages/engine/src/__tests__/triage.test.ts +++ b/packages/engine/src/__tests__/triage.test.ts @@ -263,6 +263,7 @@ describe("buildSpecificationPrompt", () => { expect(prompt).toContain(feedback); expect(prompt).not.toContain("Existing Specification"); expect(prompt).toContain("without carrying forward stale assumptions"); + expect(prompt).toContain("Treat the current task title and description as required primary inputs"); }); it("includes attachments when provided", () => { @@ -1170,6 +1171,35 @@ describe("Re-specification flow", () => { expect(revisionLogEntry?.outcome).toBe("Most recent feedback"); }); + + it("prefers latest comment-triggered re-spec feedback log over legacy revision requests", () => { + const taskWithCommentTriggeredFeedback: Task = { + ...taskWithRevisionRequest, + log: [ + { + timestamp: "2026-01-01T00:00:00.000Z", + action: "AI spec revision requested", + outcome: "Older feedback", + }, + { + timestamp: "2026-01-01T00:03:00.000Z", + action: "User comment requested re-specification of planned task", + outcome: "Latest feedback", + }, + ], + }; + + const feedbackLogEntry = [...taskWithCommentTriggeredFeedback.log] + .reverse() + .find((entry) => + entry.action === "User comment requested re-specification of planned task" + || entry.action === "User comment invalidated spec approval — task needs re-specification" + || entry.action === "AI spec revision requested" + ); + + expect(feedbackLogEntry?.outcome).toBe("Latest feedback"); + }); + }); describe("requirePlanApproval setting", () => { diff --git a/packages/engine/src/triage.ts b/packages/engine/src/triage.ts index fad180d83..123c21569 100644 --- a/packages/engine/src/triage.ts +++ b/packages/engine/src/triage.ts @@ -1077,11 +1077,24 @@ export class TriageProcessor { let feedback: string | undefined; if (isReplan) { - // Extract feedback from the most recent "AI spec revision requested" log entry - const revisionLogEntry = [...task.log] + // Prefer explicit re-specification feedback logged by comment-triggered + // and approval-invalidation flows; fall back to legacy revision logs. + const feedbackLogEntry = [...task.log] .reverse() - .find((entry) => entry.action === "AI spec revision requested"); - feedback = revisionLogEntry?.outcome; + .find((entry) => + entry.action === "User comment requested re-specification of planned task" + || entry.action === "User comment invalidated spec approval — task needs re-specification" + || entry.action === "AI spec revision requested" + ); + feedback = feedbackLogEntry?.outcome; + + // Ensure the latest user feedback is always actionable for re-plans. + if (!feedback) { + const latestUserComment = [...(detail.comments || [])] + .reverse() + .find((comment) => comment.author === "user"); + feedback = latestUserComment?.text; + } planLog.log( `${task.id} re-planning with feedback: ${feedback?.slice(0, 100)}...`, @@ -2270,7 +2283,7 @@ Please revise the specification above to address this feedback. Write the comple ## Re-specification Instructions You are creating a fresh replacement specification based on user feedback. -**Important:** Do not reuse stale PROMPT.md content. Start from the current task description, inspect the codebase, and write a complete new specification that addresses the feedback below. +**Important:** Do not reuse stale PROMPT.md content. Treat the current task title and description as required primary inputs, inspect the codebase, and write a complete new specification that addresses the feedback below. ## User Feedback ${feedback} @@ -2340,7 +2353,7 @@ ${task.breakIntoSubtasks ? "- **Break into subtasks:** Yes (user requested)" : " ${task.dependencies.length > 0 ? `- **Dependencies:** ${task.dependencies.join(", ")}` : ""}${revisionSection}${subtaskSection} ## Instructions -${isRevision ? "1. Review the existing specification and user feedback carefully\n2. Revise the PROMPT.md to address the feedback while maintaining the structure\n3. Ensure the specification is detailed enough for an AI agent to execute" : isFreshRespecification ? "1. Read the project structure to understand context (package.json, source files, etc.)\n2. Write a fresh complete PROMPT.md specification to the given path following the format in your system prompt\n3. Address the user feedback without carrying forward stale assumptions from the old spec\n4. Name actual files, functions, and patterns from the codebase — be specific" : "1. Read the project structure to understand context (package.json, source files, etc.)\n2. Write a complete PROMPT.md specification to the given path following the format in your system prompt\n3. The specification must be detailed enough for an autonomous AI agent to implement without asking questions\n4. Name actual files, functions, and patterns from the codebase — be specific"} +${isRevision ? "1. Review the existing specification and user feedback carefully\n2. Revise the PROMPT.md to address the feedback while maintaining the structure\n3. Ensure the specification is detailed enough for an AI agent to execute" : isFreshRespecification ? "1. Read the project structure to understand context (package.json, source files, etc.)\n2. Treat the current task title and description as mandatory primary inputs for a new spec\n3. Write a fresh complete PROMPT.md specification to the given path following the format in your system prompt\n4. Address the user feedback without carrying forward stale assumptions from the old spec\n5. Name actual files, functions, and patterns from the codebase — be specific" : "1. Read the project structure to understand context (package.json, source files, etc.)\n2. Write a complete PROMPT.md specification to the given path following the format in your system prompt\n3. The specification must be detailed enough for an autonomous AI agent to implement without asking questions\n4. Name actual files, functions, and patterns from the codebase — be specific"} Use the write tool to write the specification file.${commandsSection}${completionDocumentationSection}${memorySection}${attachmentsSection}${userCommentsSection}`; } From 36493eca0f681be0834024b4e369c0fe932af8f9 Mon Sep 17 00:00:00 2001 From: Fusion Date: Tue, 5 May 2026 13:42:39 -0700 Subject: [PATCH 11/14] feat(FN-3498): add self-healing and ownership-aware done-task merge reconci The merge completes FN-3498 across three steps: adds ownership-aware done-task reconciliation to the merger, prevents branch-missing head SHA pollution during merge operations, and restores workspace typecheck compatibility. Core changes touch the merger (103 lines) and self-healing module (67 lines Fusion-Task-Id: FN-3498 --- docs/architecture.md | 1 + packages/core/src/eval-automation.ts | 13 ++- packages/core/src/settings-schema.ts | 6 - packages/core/src/types.ts | 12 -- packages/dashboard/app/App.tsx | 8 +- .../PlanningModeModal.planning-flow.test.tsx | 2 +- .../app/plugins/pluginViewRegistry.tsx | 2 +- packages/engine/src/__tests__/merger.test.ts | 60 +++++++--- .../engine/src/__tests__/self-healing.test.ts | 76 +++++++++++++ packages/engine/src/merger.ts | 103 ++++++++++++++---- packages/engine/src/self-healing.ts | 67 +++++++++++- 11 files changed, 277 insertions(+), 73 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 09c856875..4b95bb261 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -405,6 +405,7 @@ See [Memory Plugin Contract](./memory-plugin-contract.md) for the full plan. - `SelfHealingManager` (`self-healing.ts`) — auto-unpause/maintenance recovery actions - `recoverGhostReviewTasks()` is a fallback only for idle, non-terminal `in-review` states. Terminal/actionable states (notably `status: "failed"`) are preserved and **not** auto-kicked back to `todo`. - `recoverMergeableReviewTasks()` only re-enqueues truly eligible tasks; retry-exhausted review tasks are skipped to avoid re-enqueue/no-op loops that keep refreshing `updatedAt`. + - Merge commit attribution is ownership-aware: a `mergeDetails.commitSha` is trusted only when reachable from `HEAD` **and** attributable to the task via `Fusion-Task-Id` trailer or task-ID-bearing subject. Reachable-but-unowned SHAs are rejected to prevent sibling done tasks from sharing misleading merge metadata. - `ProjectEngine` settings lifecycle handlers (`project-engine.ts`) treat `enginePaused` as a soft pause: clearing it dispatches runtime resume and, when `autoMerge` is enabled, performs an `in-review` eligibility sweep to requeue mergeable review tasks. - `UsageLimitPauser` (`usage-limit-detector.ts`) and `withRateLimitRetry` (`rate-limit-retry.ts`) diff --git a/packages/core/src/eval-automation.ts b/packages/core/src/eval-automation.ts index 4dda9f99b..7556b8a79 100644 --- a/packages/core/src/eval-automation.ts +++ b/packages/core/src/eval-automation.ts @@ -20,13 +20,14 @@ export interface ResolvedTaskEvaluationSettings { export function resolveTaskEvaluationSettings( settings: Partial, ): ResolvedTaskEvaluationSettings { + const evalSettings = settings as Partial; return { - taskEvaluationEnabled: settings.taskEvaluationEnabled ?? false, - taskEvaluationSchedule: settings.taskEvaluationSchedule ?? DEFAULT_TASK_EVALUATION_SCHEDULE, - taskEvaluationProvider: settings.taskEvaluationProvider, - taskEvaluationModelId: settings.taskEvaluationModelId, - taskEvaluationFollowUpPolicy: settings.taskEvaluationFollowUpPolicy ?? "off", - taskEvaluationRetention: settings.taskEvaluationRetention, + taskEvaluationEnabled: evalSettings.taskEvaluationEnabled ?? false, + taskEvaluationSchedule: evalSettings.taskEvaluationSchedule ?? DEFAULT_TASK_EVALUATION_SCHEDULE, + taskEvaluationProvider: evalSettings.taskEvaluationProvider, + taskEvaluationModelId: evalSettings.taskEvaluationModelId, + taskEvaluationFollowUpPolicy: evalSettings.taskEvaluationFollowUpPolicy ?? "off", + taskEvaluationRetention: evalSettings.taskEvaluationRetention, }; } diff --git a/packages/core/src/settings-schema.ts b/packages/core/src/settings-schema.ts index 70a05efa7..5de9a7cb6 100644 --- a/packages/core/src/settings-schema.ts +++ b/packages/core/src/settings-schema.ts @@ -292,12 +292,6 @@ export const DEFAULT_PROJECT_SETTINGS = { researchDefaultTimeout: 300000, researchMaxSourcesPerRun: 20, researchMaxSynthesisRounds: 2, - taskEvaluationEnabled: false, - taskEvaluationSchedule: "0 5 * * *", - taskEvaluationProvider: undefined, - taskEvaluationModelId: undefined, - taskEvaluationFollowUpPolicy: "off", - taskEvaluationRetention: undefined, } satisfies CompleteSettings; /** diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index b7e059592..81aee8c94 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -1724,18 +1724,6 @@ export interface ProjectSettings { testCommand?: string; /** Custom build command for the project (e.g. "pnpm build") */ buildCommand?: string; - /** Enables automated scheduled evaluation of completed tasks. */ - taskEvaluationEnabled?: boolean; - /** Cron expression for scheduled task evaluation batches. */ - taskEvaluationSchedule?: string; - /** Optional provider override for task evaluation. */ - taskEvaluationProvider?: string; - /** Optional model override for task evaluation. */ - taskEvaluationModelId?: string; - /** Follow-up behavior for evaluation findings. */ - taskEvaluationFollowUpPolicy?: "off" | "suggest" | "create"; - /** Number of days to retain evaluation data. */ - taskEvaluationRetention?: number; /** When true, completed task worktrees are returned to an idle pool instead * of being deleted. New tasks acquire a warm worktree from the pool, * preserving build caches (node_modules, target/, dist/). Default: false. */ diff --git a/packages/dashboard/app/App.tsx b/packages/dashboard/app/App.tsx index 28a6a36aa..c06f4d1b9 100644 --- a/packages/dashboard/app/App.tsx +++ b/packages/dashboard/app/App.tsx @@ -36,7 +36,7 @@ import { useCurrentProject } from "./hooks/useCurrentProject"; import { ToastProvider, useToast } from "./hooks/useToast"; import { ConfirmDialogProvider } from "./hooks/useConfirm"; import { useTheme } from "./hooks/useTheme"; -import { useModalManager, type DetailTaskOrigin } from "./hooks/useModalManager"; +import { useModalManager, type DetailTaskOrigin, type DetailTaskTab } from "./hooks/useModalManager"; import { useAppSettings } from "./hooks/useAppSettings"; import { useDeepLink } from "./hooks/useDeepLink"; import { useFavorites } from "./hooks/useFavorites"; @@ -879,10 +879,8 @@ function AppInner() { tasks: isRemote && remoteData.tasks.length > 0 ? remoteData.tasks : tasks, workflowSteps, openTaskDetail: isMobile - ? (task: Task | TaskDetail, initialTab?: Parameters[1]) => - openDetailTaskWithHistory(task, initialTab) - : (task: Task | TaskDetail, initialTab?: Parameters[1]) => - modalManager.openDetailTask(task, initialTab), + ? (task: Task | TaskDetail, initialTab?: DetailTaskTab) => openDetailTaskWithHistory(task, initialTab) + : (task: Task | TaskDetail, initialTab?: DetailTaskTab) => modalManager.openDetailTask(task, initialTab), renderTaskCard: (task: Task | TaskDetail) => ( { await waitFor(() => { expect(screen.getByText("What are the key requirements?")).toBeDefined(); - }); + }, { timeout: 5000 }); expect(screen.getByTestId("conversation-history")).toBeDefined(); expect(screen.getByText("What is the scope?")).toBeDefined(); diff --git a/packages/dashboard/app/plugins/pluginViewRegistry.tsx b/packages/dashboard/app/plugins/pluginViewRegistry.tsx index f72fb51f0..5443e328d 100644 --- a/packages/dashboard/app/plugins/pluginViewRegistry.tsx +++ b/packages/dashboard/app/plugins/pluginViewRegistry.tsx @@ -5,7 +5,7 @@ import "./pluginViewRegistry.css"; export type PluginTaskView = `plugin:${string}:${string}`; -type PluginViewComponent = LazyExoticComponent<() => ReactNode>; +type PluginViewComponent = LazyExoticComponent<() => ReactElement>; const registry = new Map(); diff --git a/packages/engine/src/__tests__/merger.test.ts b/packages/engine/src/__tests__/merger.test.ts index 64116878f..0b0e82d53 100644 --- a/packages/engine/src/__tests__/merger.test.ts +++ b/packages/engine/src/__tests__/merger.test.ts @@ -5362,38 +5362,62 @@ describe("aiMergeTask — merge details collection", () => { expect(mergeDetailsCall?.[1].mergeDetails.mergeCommitMessage).toBe("- feat: something"); }); - it("stores partial mergeDetails when branch is not found", async () => { + it("recovers owned landed commit when branch is not found", async () => { const store = createMockStore( - { id: "FN-050", worktree: "/tmp/root/.worktrees/KB-050" }, - [{ id: "FN-050", worktree: "/tmp/root/.worktrees/KB-050", column: "in-review" } as Task], + { + id: "FN-3469", + worktree: "/tmp/root/.worktrees/FN-3469", + baseCommitSha: "base3469", + mergeDetails: { commitSha: "a47b1e5d78d626f8b480f1e90d3d64be2625ff6a" } as any, + }, + [{ id: "FN-3469", worktree: "/tmp/root/.worktrees/FN-3469", column: "in-review" } as Task], ); mockedExecSync.mockImplementation((cmd: any) => { const cmdStr = String(cmd); - // Branch verification fails → branch not found if (cmdStr.includes("rev-parse --verify")) throw new Error("not found"); - // But rev-parse HEAD still works → can capture commitSha (encoding: utf-8 → string) - if (cmdStr === "git rev-parse HEAD" || cmdStr.startsWith("git rev-parse HEAD ")) - return "existingheadsha999"; + if (cmdStr.includes("merge-base --is-ancestor a47b1e5d78d626f8b480f1e90d3d64be2625ff6a HEAD")) return Buffer.from(""); + if (cmdStr.includes("log -1 --format=%H%x1f%s%x1f%b a47b1e5d78d626f8b480f1e90d3d64be2625ff6a")) { + return "a47b1e5d78d626f8b480f1e90d3d64be2625ff6a\u001ffix(FN-3469): title\u001fFusion-Task-Id: FN-3469" as any; + } + if (cmdStr.includes("show --shortstat --format= a47b1e5d78d626f8b480f1e90d3d64be2625ff6a")) { + return "2 files changed, 84 insertions(+), 2 deletions(-)" as any; + } return Buffer.from(""); }); - const result = await aiMergeTask(store, "/tmp/root", "FN-050"); - + const result = await aiMergeTask(store, "/tmp/root", "FN-3469"); expect(result.merged).toBe(false); - expect(result.error).toContain("not found"); - // Find the updateTask call that set mergeDetails - const updateCalls = (store.updateTask as ReturnType).mock.calls; - const mergeDetailsCall = updateCalls.find( + const mergeDetailsCall = (store.updateTask as ReturnType).mock.calls.find( (call: any[]) => call[1]?.mergeDetails !== undefined, ); - expect(mergeDetailsCall).toBeDefined(); + expect(mergeDetailsCall?.[1].mergeDetails).toEqual(expect.objectContaining({ + commitSha: "a47b1e5d78d626f8b480f1e90d3d64be2625ff6a", + mergeCommitMessage: "fix(FN-3469): title", + mergeConfirmed: true, + })); + }); - const mergeDetails = mergeDetailsCall![1].mergeDetails; - expect(mergeDetails.commitSha).toBe("existingheadsha999"); - expect(mergeDetails.mergedAt).toBeDefined(); - expect(mergeDetails.mergeConfirmed).toBe(false); + it("does not persist misleading mergeDetails when branch is not found and no owned commit exists", async () => { + const store = createMockStore( + { id: "FN-3373", worktree: "/tmp/root/.worktrees/FN-3373" }, + [{ id: "FN-3373", worktree: "/tmp/root/.worktrees/FN-3373", column: "in-review" } as Task], + ); + + mockedExecSync.mockImplementation((cmd: any) => { + const cmdStr = String(cmd); + if (cmdStr.includes("rev-parse --verify")) throw new Error("not found"); + return Buffer.from(""); + }); + + const result = await aiMergeTask(store, "/tmp/root", "FN-3373"); + expect(result.merged).toBe(false); + + const mergeDetailsCall = (store.updateTask as ReturnType).mock.calls.find( + (call: any[]) => call[1]?.mergeDetails !== undefined, + ); + expect(mergeDetailsCall).toBeUndefined(); }); it("completes merge even when git commands fail during merge details collection", async () => { diff --git a/packages/engine/src/__tests__/self-healing.test.ts b/packages/engine/src/__tests__/self-healing.test.ts index a127734e7..c878cfd1f 100644 --- a/packages/engine/src/__tests__/self-healing.test.ts +++ b/packages/engine/src/__tests__/self-healing.test.ts @@ -3006,6 +3006,82 @@ describe("stale triage processing eviction before recovery", () => { // ── Maintenance cycle concurrency ────────────────────────────────── +describe("recoverDoneTaskMergeMetadata", () => { + it("upgrades done task metadata to an owned landed commit", async () => { + const store = createMockStore(); + const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" }); + + (store.listTasks as ReturnType).mockResolvedValue([ + { + id: "FN-3469", + column: "done", + paused: false, + baseCommitSha: "base", + mergeDetails: { commitSha: "sharedsha", mergeConfirmed: false }, + modifiedFiles: ["AGENTS.md"], + }, + ]); + + mockedExecSync.mockImplementation((command) => { + const cmd = String(command); + if (cmd.includes("merge-base --is-ancestor sharedsha HEAD")) return "" as any; + if (cmd.includes("log -1 --format=%H%x1f%s%x1f%b sharedsha")) { + return "sharedsha\u001ffix(FN-3468): other\u001fFusion-Task-Id: FN-3468" as any; + } + if (cmd.includes("Fusion-Task-Id: FN-3469")) { + return "a47b1e5\u001ffix(FN-3469): correct lazy-loaded views\n" as any; + } + if (cmd.includes("show --shortstat --format= a47b1e5")) { + return "2 files changed, 84 insertions(+), 2 deletions(-)" as any; + } + return "" as any; + }); + + const repaired = await manager.recoverDoneTaskMergeMetadata(); + + expect(repaired).toBe(1); + expect(store.updateTask).toHaveBeenCalledWith("FN-3469", { + mergeDetails: expect.objectContaining({ + commitSha: "a47b1e5", + mergeConfirmed: true, + }), + }); + + manager.stop(); + }); + + it("clears unowned shared SHA for done task when no owned landed commit exists", async () => { + const store = createMockStore(); + const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" }); + + (store.listTasks as ReturnType).mockResolvedValue([ + { + id: "FN-3373", + column: "done", + paused: false, + mergeDetails: { commitSha: "196adbd", mergeConfirmed: false }, + modifiedFiles: ["packages/cli/src/extension.ts"], + }, + ]); + + mockedExecSync.mockImplementation((command) => { + const cmd = String(command); + if (cmd.includes("merge-base --is-ancestor 196adbd HEAD")) return "" as any; + if (cmd.includes("log -1 --format=%H%x1f%s%x1f%b 196adbd")) { + return "196adbd\u001ffeat(FN-3372): add safety net\u001fFusion-Task-Id: FN-3372" as any; + } + return "" as any; + }); + + const repaired = await manager.recoverDoneTaskMergeMetadata(); + + expect(repaired).toBe(1); + expect(store.updateTask).toHaveBeenCalledWith("FN-3373", { mergeDetails: undefined }); + + manager.stop(); + }); +}); + describe("maintenance cycle concurrency", () => { let store: TaskStore & EventEmitter; let manager: SelfHealingManager; diff --git a/packages/engine/src/merger.ts b/packages/engine/src/merger.ts index 7682a602d..e0781d29a 100644 --- a/packages/engine/src/merger.ts +++ b/packages/engine/src/merger.ts @@ -46,6 +46,7 @@ import { type AgentPromptsConfig, type CanonicalMergeConflictStrategy, type TaskSourceIssue, + type Task, } from "@fusion/core"; import { describeModel, promptWithFallback } from "./pi.js"; import { accumulateSessionTokenUsage } from "./session-token-usage.js"; @@ -250,6 +251,72 @@ interface InferredTestCommand { buildSource?: "explicit" | "inferred"; } +interface OwnedLandedCommit { + sha: string; + subject?: string; + filesChanged?: number; + insertions?: number; + deletions?: number; +} + +function commitOwnedByTask(taskId: string, subject: string, body: string): boolean { + return body.includes(`${FUSION_TASK_ID_TRAILER_KEY}: ${taskId}`) || subject.includes(taskId); +} + +async function findOwnedLandedCommitForTask(rootDir: string, task: Task): Promise { + const tryHydrate = async (sha: string): Promise => { + try { + await execFileAsync("git", ["merge-base", "--is-ancestor", sha, "HEAD"], { cwd: rootDir }); + const { stdout } = await execFileAsync("git", ["log", "-1", "--format=%H%x1f%s%x1f%b", sha], { + cwd: rootDir, + encoding: "utf-8", + }); + const [resolvedSha, subject = "", body = ""] = stdout.trim().split("\x1f"); + if (!resolvedSha || !commitOwnedByTask(task.id, subject, body)) return null; + const owned: OwnedLandedCommit = { sha: resolvedSha, subject }; + try { + const { stdout: statsOut } = await execFileAsync("git", ["show", "--shortstat", "--format=", resolvedSha], { + cwd: rootDir, + encoding: "utf-8", + }); + Object.assign(owned, parseDiffStat(statsOut)); + } catch { + // stats optional + } + return owned; + } catch { + return null; + } + }; + + if (task.mergeDetails?.commitSha) { + const ownedStored = await tryHydrate(task.mergeDetails.commitSha); + if (ownedStored) return ownedStored; + } + + const trailer = `${FUSION_TASK_ID_TRAILER_KEY}: ${task.id}`; + const searches: string[][] = [ + ["log", "--format=%H%x1f%s", "--max-count=20", "--fixed-strings", `--grep=${trailer}`, "HEAD"], + ["log", "--format=%H%x1f%s", "--max-count=20", "--fixed-strings", `--grep=${task.id}`, "HEAD"], + ]; + + for (const args of searches) { + try { + const { stdout } = await execFileAsync("git", args, { cwd: rootDir, encoding: "utf-8" }); + const first = stdout.trim().split("\n").find(Boolean); + if (!first) continue; + const [sha] = first.split("\x1f"); + if (!sha) continue; + const owned = await tryHydrate(sha); + if (owned) return owned; + } catch { + // continue + } + } + + return null; +} + /** * Infer a default test command based on project files. * Returns the command and whether it was explicitly configured or inferred. @@ -2662,25 +2729,23 @@ export async function aiMergeTask( }); } catch { result.error = `Branch '${branch}' not found — moving to done without merge`; - // Best-effort: try to capture current HEAD commitSha even though branch is missing - try { - const commitSha = execSyncText("git rev-parse HEAD", { - cwd: rootDir, - stdio: "pipe", - encoding: "utf-8", - }).trim() || undefined; - if (commitSha) { - await store.updateTask(taskId, { - mergeDetails: { - commitSha, - mergedAt: new Date().toISOString(), - mergeConfirmed: false, - }, - }); - mergerLog.log(`${taskId}: branch not found but captured commitSha ${commitSha.slice(0, 8)}`); - } - } catch { - // No commit SHA available — task will show summary fallback + // Branch is gone; never infer ownership from raw HEAD. Only persist commit + // metadata when we can prove a landed commit belongs to this task. + const ownedCommit = await findOwnedLandedCommitForTask(rootDir, task); + if (ownedCommit) { + await store.updateTask(taskId, { + mergeDetails: { + commitSha: ownedCommit.sha, + filesChanged: ownedCommit.filesChanged, + insertions: ownedCommit.insertions, + deletions: ownedCommit.deletions, + mergeCommitMessage: ownedCommit.subject, + mergedAt: new Date().toISOString(), + mergeConfirmed: true, + prNumber: task.prInfo?.number, + }, + }); + mergerLog.log(`${taskId}: branch missing; recovered owned landed commit ${ownedCommit.sha.slice(0, 8)}`); } // Audit trail: record merge completion (FN-1404) await audit.database({ type: "task:move", target: taskId, metadata: { to: "done", merged: false } }); diff --git a/packages/engine/src/self-healing.ts b/packages/engine/src/self-healing.ts index 8f8827035..d8cc3da81 100644 --- a/packages/engine/src/self-healing.ts +++ b/packages/engine/src/self-healing.ts @@ -117,6 +117,10 @@ interface LandedTaskCommit { deletions?: number; } +function commitOwnedByTask(taskId: string, subject: string, body: string): boolean { + return body.includes(`Fusion-Task-Id: ${taskId}`) || subject.includes(taskId); +} + function shellQuote(value: string): string { return `'${value.replace(/'/g, "'\\''")}'`; } @@ -193,6 +197,7 @@ export class SelfHealingManager { { name: "stale-incomplete-review", fn: () => this.recoverStaleIncompleteReviewTasks().then(() => undefined) }, { name: "failed-pre-merge-steps", fn: () => this.recoverReviewTasksWithFailedPreMergeSteps().then(() => undefined) }, { name: "interrupted-merging", fn: () => this.recoverInterruptedMergingTasks().then(() => undefined) }, + { name: "done-merge-metadata", fn: () => this.recoverDoneTaskMergeMetadata().then(() => undefined) }, { name: "misclassified-failures", fn: () => this.recoverMisclassifiedFailures().then(() => undefined) }, { name: "partial-progress-no-task-done", fn: () => this.recoverPartialProgressNoTaskDoneFailures().then(() => undefined) }, { name: "orphaned-executions", fn: () => this.recoverOrphanedExecutions().then(() => undefined) }, @@ -467,18 +472,16 @@ export class SelfHealingManager { const storedSha = task.mergeDetails?.commitSha; if (storedSha) { try { - // Reachable from HEAD? Use --quiet --exit-code on rev-list. await execAsync( `git merge-base --is-ancestor ${shellQuote(storedSha)} HEAD`, { cwd: this.options.rootDir }, ); - // Yes — fetch its subject + stats. const { stdout } = await execAsync( - `git log -1 --format=%H%x1f%s ${shellQuote(storedSha)}`, + `git log -1 --format=%H%x1f%s%x1f%b ${shellQuote(storedSha)}`, { cwd: this.options.rootDir, maxBuffer: 1024 * 1024 }, ); - const [sha, subject] = stdout.trim().split("\x1f"); - if (sha) { + const [sha, subject = "", body = ""] = stdout.trim().split("\x1f"); + if (sha && commitOwnedByTask(task.id, subject, body)) { const commit: LandedTaskCommit = { sha, subject }; try { const stats = await execAsync(`git show --shortstat --format= ${shellQuote(sha)}`, { @@ -644,6 +647,7 @@ export class SelfHealingManager { { name: "recover-stale-incomplete-review", fn: () => this.recoverStaleIncompleteReviewTasks() }, { name: "recover-failed-pre-merge-steps", fn: () => this.recoverReviewTasksWithFailedPreMergeSteps() }, { name: "recover-interrupted-merging", fn: () => this.recoverInterruptedMergingTasks() }, + { name: "recover-done-merge-metadata", fn: () => this.recoverDoneTaskMergeMetadata() }, { name: "recover-mergeable-review", fn: () => this.recoverMergeableReviewTasks() }, { name: "recover-merged-review", fn: () => this.recoverMergedReviewTasks() }, { name: "recover-misclassified-failures", fn: () => this.recoverMisclassifiedFailures() }, @@ -1218,6 +1222,59 @@ export class SelfHealingManager { } } + async recoverDoneTaskMergeMetadata(): Promise { + try { + const tasks = await this.store.listTasks({ column: "done", slim: true }); + const candidates = tasks.filter((task) => task.column === "done" && !task.paused && Boolean(task.mergeDetails?.commitSha)); + if (candidates.length === 0) return 0; + + let repaired = 0; + for (const task of candidates) { + try { + const landed = await this.findLandedTaskCommit(task); + if (!landed) { + if (task.mergeDetails?.mergeConfirmed === false) { + await this.store.updateTask(task.id, { mergeDetails: undefined }); + await this.store.logEntry(task.id, "Auto-recovered: cleared unowned done-task mergeDetails commitSha"); + repaired++; + } + continue; + } + + const needsRepair = + task.mergeDetails?.commitSha !== landed.sha || + task.mergeDetails?.mergeConfirmed !== true || + task.mergeDetails?.filesChanged === undefined; + + if (!needsRepair) continue; + + await this.store.updateTask(task.id, { + mergeDetails: { + ...task.mergeDetails, + commitSha: landed.sha, + filesChanged: landed.filesChanged, + insertions: landed.insertions, + deletions: landed.deletions, + mergeCommitMessage: landed.subject, + mergedAt: task.mergeDetails?.mergedAt ?? new Date().toISOString(), + mergeConfirmed: true, + prNumber: task.prInfo?.number, + }, + }); + await this.store.logEntry(task.id, `Auto-recovered: reconciled done-task mergeDetails to owned commit ${landed.sha.slice(0, 8)}`); + repaired++; + } catch (err: unknown) { + log.error(`Failed done-task merge metadata recovery for ${task.id}: ${err instanceof Error ? err.message : String(err)}`); + } + } + + return repaired; + } catch (err: unknown) { + log.error(`Done-task merge metadata recovery failed: ${err instanceof Error ? err.message : String(err)}`); + return 0; + } + } + // ── Misclassified failure recovery ─────────────────────────────── /** From 9ba75598c3e5d612542cf78515434b015dbf0db1 Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Tue, 5 May 2026 14:13:07 -0700 Subject: [PATCH 12/14] fix: auto-recover orphaned heartbeat runs from crashed processes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the dashboard crashes mid-heartbeat, the agentRuns row is left in status='active' forever. HeartbeatTriggerScheduler.onTimerTick treats any active run as "still running" and skips every subsequent tick, so agents go silent indefinitely (observed: 6+ hours). The existing in-memory missed-heartbeat watchdog can't help — its trackedAgents map is wiped on process restart. SelfHealingManager.recoverStaleHeartbeatRuns now reconciles these on startup and during periodic maintenance: terminates active runs whose processPid does not match the current process, has no recorded pid, or has been active for more than 6 hours. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../auto-recover-orphan-heartbeat-runs.md | 5 + packages/core/src/agent-store.ts | 19 ++++ .../engine/src/__tests__/self-healing.test.ts | 106 +++++++++++++++++- packages/engine/src/self-healing.ts | 100 +++++++++++++++++ 4 files changed, 229 insertions(+), 1 deletion(-) create mode 100644 .changeset/auto-recover-orphan-heartbeat-runs.md diff --git a/.changeset/auto-recover-orphan-heartbeat-runs.md b/.changeset/auto-recover-orphan-heartbeat-runs.md new file mode 100644 index 000000000..361b39725 --- /dev/null +++ b/.changeset/auto-recover-orphan-heartbeat-runs.md @@ -0,0 +1,5 @@ +--- +"@runfusion/fusion": patch +--- + +Self-heal orphaned `agentRuns` rows left in `status='active'` when the dashboard process crashes mid-heartbeat. The trigger scheduler treats any active run as "still running" and silently skips every subsequent tick, so a single crashed run could leave an agent without heartbeats for hours. SelfHealingManager now reconciles these on startup and during periodic maintenance, terminating runs whose `processPid` does not match the current process or whose age exceeds 6 hours. diff --git a/packages/core/src/agent-store.ts b/packages/core/src/agent-store.ts index 7b53c4eda..1c5588dd7 100644 --- a/packages/core/src/agent-store.ts +++ b/packages/core/src/agent-store.ts @@ -1743,6 +1743,25 @@ export class AgentStore extends EventEmitter { .sort((a, b) => new Date(b.startedAt).getTime() - new Date(a.startedAt).getTime()); } + /** + * List every heartbeat run currently in `status = 'active'` across all + * agents. Used by self-healing to detect orphaned runs from prior process + * incarnations that crashed before calling endHeartbeatRun(). Without this + * sweep an active row blocks all subsequent timer ticks for the agent + * because HeartbeatTriggerScheduler.onTimerTick treats any active run as + * "already running". + */ + async listActiveHeartbeatRuns(): Promise { + const rows = this.db.prepare(` + SELECT data FROM agentRuns + WHERE status = 'active' + ORDER BY startedAt ASC + `).all() as Array<{ data: string }>; + return rows + .map((row) => this.parseJson(row.data, null)) + .filter((run): run is AgentHeartbeatRun => run !== null); + } + // ───────────────────────────────────────────────────────────────────────── // Task Session Management // ───────────────────────────────────────────────────────────────────────── diff --git a/packages/engine/src/__tests__/self-healing.test.ts b/packages/engine/src/__tests__/self-healing.test.ts index c878cfd1f..c74bdf6c7 100644 --- a/packages/engine/src/__tests__/self-healing.test.ts +++ b/packages/engine/src/__tests__/self-healing.test.ts @@ -594,6 +594,108 @@ describe("SelfHealingManager", () => { }); }); + describe("recoverStaleHeartbeatRuns", () => { + function createMockAgentStore(activeRuns: Array<{ id: string; agentId: string; startedAt: string; processPid?: number; status?: string }>): { + store: AgentStore; + ended: Array<{ runId: string; status: string }>; + saved: Array>; + } { + const ended: Array<{ runId: string; status: string }> = []; + const saved: Array> = []; + const detailById = new Map(); + for (const r of activeRuns) { + detailById.set(r.id, { id: r.id, agentId: r.agentId, startedAt: r.startedAt, endedAt: null, status: r.status ?? "active", processPid: r.processPid }); + } + const agentStore = { + listActiveHeartbeatRuns: vi.fn().mockResolvedValue( + activeRuns.map((r) => ({ id: r.id, agentId: r.agentId, startedAt: r.startedAt, endedAt: null, status: "active" as const, processPid: r.processPid })), + ), + getRunDetail: vi.fn().mockImplementation((_agentId: string, runId: string) => Promise.resolve(detailById.get(runId) ?? null)), + saveRun: vi.fn().mockImplementation((run: any) => { + saved.push({ id: run.id, status: run.status, stderrExcerpt: run.stderrExcerpt }); + return Promise.resolve(); + }), + endHeartbeatRun: vi.fn().mockImplementation((runId: string, status: string) => { + ended.push({ runId, status }); + return Promise.resolve(); + }), + } as unknown as AgentStore; + return { store: agentStore, ended, saved }; + } + + it("returns 0 when no agentStore is configured", async () => { + const result = await manager.recoverStaleHeartbeatRuns(); + expect(result).toBe(0); + }); + + it("terminates active runs whose processPid does not match this process", async () => { + const { store: agentStore, ended, saved } = createMockAgentStore([ + { id: "run-orphan", agentId: "agent-a", startedAt: new Date().toISOString(), processPid: 999_999 }, + ]); + const m = new SelfHealingManager(store, { rootDir: "/tmp/test-project", agentStore }); + + const result = await m.recoverStaleHeartbeatRuns(); + + expect(result).toBe(1); + expect(ended).toEqual([{ runId: "run-orphan", status: "terminated" }]); + expect(saved[0]?.status).toBe("terminated"); + expect(saved[0]?.stderrExcerpt).toMatch(/Auto-recovered orphaned heartbeat run/); + m.stop(); + }); + + it("leaves young runs from the current process alone", async () => { + const { store: agentStore, ended } = createMockAgentStore([ + { id: "run-mine", agentId: "agent-b", startedAt: new Date().toISOString(), processPid: process.pid }, + ]); + const m = new SelfHealingManager(store, { rootDir: "/tmp/test-project", agentStore }); + + const result = await m.recoverStaleHeartbeatRuns(); + + expect(result).toBe(0); + expect(ended).toEqual([]); + m.stop(); + }); + + it("terminates legacy active runs that have no recorded processPid", async () => { + const { store: agentStore, ended } = createMockAgentStore([ + { id: "run-legacy", agentId: "agent-c", startedAt: new Date().toISOString() }, + ]); + const m = new SelfHealingManager(store, { rootDir: "/tmp/test-project", agentStore }); + + const result = await m.recoverStaleHeartbeatRuns(); + + expect(result).toBe(1); + expect(ended[0]?.runId).toBe("run-legacy"); + m.stop(); + }); + + it("terminates current-process runs that exceed the max-age threshold", async () => { + const tooOld = new Date(Date.now() - 7 * 60 * 60 * 1000).toISOString(); // 7h ago + const { store: agentStore, ended } = createMockAgentStore([ + { id: "run-stuck", agentId: "agent-d", startedAt: tooOld, processPid: process.pid }, + ]); + const m = new SelfHealingManager(store, { rootDir: "/tmp/test-project", agentStore }); + + const result = await m.recoverStaleHeartbeatRuns(); + + expect(result).toBe(1); + expect(ended[0]?.runId).toBe("run-stuck"); + m.stop(); + }); + + it("runStartupRecovery includes the stale heartbeat runs step", async () => { + vi.mocked(store.getSettings).mockResolvedValue({ + globalPause: false, + enginePaused: false, + } as unknown as Settings); + const spy = vi.spyOn(manager, "recoverStaleHeartbeatRuns").mockResolvedValue(0); + + await manager.runStartupRecovery(); + + expect(spy).toHaveBeenCalledTimes(1); + }); + }); + describe("recoverNoProgressNoTaskDoneFailures", () => { it("requeues clean in-progress no-task_done failures with no step progress", async () => { const managerWithRecovery = new SelfHealingManager(store, { @@ -3254,13 +3356,14 @@ describe("maintenance cycle concurrency", () => { makeSlow("recoverOrphanedPlanningTasks"); makeSlow("recoverGhostReviewTasks"); makeSlow("recoverOrphanedAgents"); + makeSlow("recoverStaleHeartbeatRuns"); await (manager as any).runMaintenance(); // Operations run sequentially (one at a time), not in parallel. expect(maxConcurrent).toBe(1); // All operations should have run (including last one) - expect(executionOrder[executionOrder.length - 1]).toBe("recoverOrphanedAgents"); + expect(executionOrder[executionOrder.length - 1]).toBe("recoverStaleHeartbeatRuns"); }); it("one failing batch 2 operation does not abort the batch", async () => { @@ -3278,6 +3381,7 @@ describe("maintenance cycle concurrency", () => { "recoverOrphanedPlanningTasks", "recoverGhostReviewTasks", "recoverOrphanedAgents", + "recoverStaleHeartbeatRuns", ] as const; // Make one operation fail diff --git a/packages/engine/src/self-healing.ts b/packages/engine/src/self-healing.ts index d8cc3da81..fc608b832 100644 --- a/packages/engine/src/self-healing.ts +++ b/packages/engine/src/self-healing.ts @@ -204,6 +204,7 @@ export class SelfHealingManager { { name: "approved-triage", fn: () => this.recoverApprovedTriageTasks().then(() => undefined) }, { name: "orphaned-planning", fn: () => this.recoverOrphanedPlanningTasks().then(() => undefined) }, { name: "recover-orphaned-agents", fn: () => this.recoverOrphanedAgents().then(() => undefined) }, + { name: "recover-stale-heartbeat-runs", fn: () => this.recoverStaleHeartbeatRuns().then(() => undefined) }, ]; for (const step of steps) { @@ -658,6 +659,7 @@ export class SelfHealingManager { { name: "recover-orphaned-planning", fn: () => this.recoverOrphanedPlanningTasks() }, { name: "recover-ghost-review", fn: () => this.recoverGhostReviewTasks() }, { name: "recover-orphaned-agents", fn: () => this.recoverOrphanedAgents() }, + { name: "recover-stale-heartbeat-runs", fn: () => this.recoverStaleHeartbeatRuns() }, ]; for (const fn of batch2Fns) { try { @@ -1519,6 +1521,104 @@ export class SelfHealingManager { } } + /** + * Default cap (in ms) on how long an active heartbeat run from the current + * process is allowed to remain open before self-healing will terminate it. + * Six hours is well past any legitimate heartbeat tick (default 1 h + * interval, configurable up to a few hours) so reaching this threshold + * means the run record was never closed — typically a process that died + * without our watchdog catching it. + */ + private static readonly STALE_ACTIVE_RUN_MAX_AGE_MS = 6 * 60 * 60 * 1000; + + /** + * Terminate orphaned `agentRuns` rows left in `status = 'active'` by a + * process that crashed before calling endHeartbeatRun(). These rows + * silently break heartbeat scheduling: HeartbeatTriggerScheduler.onTimerTick + * skips every tick that finds an active run, so the agent never gets called + * again until something cleans up. + * + * A run is considered stale when: + * - `processPid` was recorded and does not match the current `process.pid` + * (i.e., the writer process is gone — guaranteed orphan), or + * - `processPid` is missing (legacy data), or + * - the run has been active for longer than STALE_ACTIVE_RUN_MAX_AGE_MS, + * even from the current process (defense in depth against a writer that + * leaks the row without crashing the whole runtime). + * + * The matching `processPid` + young run case is left alone — that is a + * legitimately in-flight heartbeat. + */ + async recoverStaleHeartbeatRuns(): Promise { + const agentStore = this.options.agentStore; + if (!agentStore) { + return 0; + } + + let activeRuns; + try { + activeRuns = await agentStore.listActiveHeartbeatRuns(); + } catch (err: unknown) { + const errorMessage = err instanceof Error ? err.message : String(err); + log.error(`Stale heartbeat run recovery — listing failed: ${errorMessage}`); + return 0; + } + + if (activeRuns.length === 0) { + return 0; + } + + const now = Date.now(); + const currentPid = process.pid; + const maxAgeMs = SelfHealingManager.STALE_ACTIVE_RUN_MAX_AGE_MS; + let recovered = 0; + + for (const run of activeRuns) { + const startedMs = Date.parse(run.startedAt); + const ageMs = Number.isFinite(startedMs) ? Math.max(0, now - startedMs) : Infinity; + const recordedPid = run.processPid; + + const pidMismatch = typeof recordedPid === "number" && recordedPid !== currentPid; + const pidMissing = typeof recordedPid !== "number"; + const tooOld = ageMs >= maxAgeMs; + + if (!pidMismatch && !pidMissing && !tooOld) { + continue; + } + + const reason = pidMismatch + ? `writer pid ${recordedPid} is no longer this process (current pid ${currentPid})` + : pidMissing + ? `no processPid recorded` + : `active for ${Math.round(ageMs / 1000)}s (>= ${Math.round(maxAgeMs / 1000)}s threshold)`; + + try { + const detail = await agentStore.getRunDetail(run.agentId, run.id); + if (detail) { + await agentStore.saveRun({ + ...detail, + endedAt: new Date().toISOString(), + status: "terminated", + stderrExcerpt: `Auto-recovered orphaned heartbeat run: ${reason}`, + }); + } + await agentStore.endHeartbeatRun(run.id, "terminated"); + log.log( + `Auto-recovered: orphan heartbeat run ${run.id} for ${run.agentId} (${reason})`, + ); + recovered++; + } catch (err: unknown) { + const errorMessage = err instanceof Error ? err.message : String(err); + log.error(`Failed to recover stale heartbeat run ${run.id} for ${run.agentId}: ${errorMessage}`); + } + } + + if (recovered > 0) { + log.log(`Recovered ${recovered} stale heartbeat run(s)`); + } + return recovered; + } + /** * Recover `in-progress` tasks that failed only because the agent exited * without calling task_done, and where there is no sign of work to preserve. From 7d23397dca144ffca9e78be6d27a9d07206be8cf Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Tue, 5 May 2026 14:14:56 -0700 Subject: [PATCH 13/14] test: cover concurrent startRun race in heartbeat recovery sweep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Documents the race called out in code review: when recovery samples a stale run id and a fresh run is spawned for the same agent before endHeartbeatRun() lands, only the sampled id is terminated — never the freshly-spawned run. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../engine/src/__tests__/self-healing.test.ts | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/packages/engine/src/__tests__/self-healing.test.ts b/packages/engine/src/__tests__/self-healing.test.ts index c74bdf6c7..96332ffa6 100644 --- a/packages/engine/src/__tests__/self-healing.test.ts +++ b/packages/engine/src/__tests__/self-healing.test.ts @@ -694,6 +694,51 @@ describe("SelfHealingManager", () => { expect(spy).toHaveBeenCalledTimes(1); }); + + // Documents the race between recovery and a concurrent live startRun(). + // Sequence: recovery loads the stale row, then a fresh startRun() saves a + // brand-new run for the same agent, then recovery calls endHeartbeatRun() + // on the stale row. The new run must remain untouched — recovery must + // only terminate the run id it sampled, never the agent's "any active + // run." Otherwise we'd kill the very run we just spawned. + it("only terminates the sampled run id even if a fresh run is started concurrently", async () => { + const oldStarted = new Date(Date.now() - 7 * 60 * 60 * 1000).toISOString(); + const ended: Array<{ runId: string; status: string }> = []; + const saved: Array<{ id: string; status: string }> = []; + + const agentStore = { + listActiveHeartbeatRuns: vi.fn().mockResolvedValue([ + { id: "run-stale", agentId: "agent-x", startedAt: oldStarted, endedAt: null, status: "active", processPid: 999_999 }, + ]), + // Simulate the live process spawning a NEW run after recovery sampled the stale one + // but before it called endHeartbeatRun. getRunDetail still returns the stale row + // because the new run has a different id. + getRunDetail: vi.fn().mockImplementation((_agentId: string, runId: string) => { + if (runId === "run-stale") { + return Promise.resolve({ id: "run-stale", agentId: "agent-x", startedAt: oldStarted, endedAt: null, status: "active", processPid: 999_999 }); + } + return Promise.resolve(null); + }), + saveRun: vi.fn().mockImplementation((run: any) => { + saved.push({ id: run.id, status: run.status }); + return Promise.resolve(); + }), + endHeartbeatRun: vi.fn().mockImplementation((runId: string, status: string) => { + ended.push({ runId, status }); + return Promise.resolve(); + }), + } as unknown as AgentStore; + + const m = new SelfHealingManager(store, { rootDir: "/tmp/test-project", agentStore }); + const result = await m.recoverStaleHeartbeatRuns(); + + expect(result).toBe(1); + expect(ended).toEqual([{ runId: "run-stale", status: "terminated" }]); + // The hypothetical concurrent run-fresh must not have been touched. + expect(ended.some((e) => e.runId === "run-fresh")).toBe(false); + expect(saved.every((s) => s.id === "run-stale")).toBe(true); + m.stop(); + }); }); describe("recoverNoProgressNoTaskDoneFailures", () => { From a77f1b4fca031bfdf8106b7895002195aa6e99e5 Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Tue, 5 May 2026 14:16:30 -0700 Subject: [PATCH 14/14] fix: quiet noisy store poll and skill-resolver info logs Raise checkForChanges slow-poll warn threshold from 100ms to 750ms so warnings only fire when cycles approach the 1s poll interval, and route skill-resolver info diagnostics through log() instead of warn(). Co-Authored-By: Claude Opus 4.7 (1M context) --- .changeset/quiet-noisy-store-and-skill-logs.md | 5 +++++ packages/core/src/store.ts | 4 ++-- packages/engine/src/skill-resolver.ts | 5 ++++- 3 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 .changeset/quiet-noisy-store-and-skill-logs.md diff --git a/.changeset/quiet-noisy-store-and-skill-logs.md b/.changeset/quiet-noisy-store-and-skill-logs.md new file mode 100644 index 000000000..fbba9dc2f --- /dev/null +++ b/.changeset/quiet-noisy-store-and-skill-logs.md @@ -0,0 +1,5 @@ +--- +"@runfusion/fusion": patch +--- + +Reduce log noise: bump `checkForChanges` slow-poll warn threshold from 100ms to 750ms (the 1s poll interval + multiple SQLite queries routinely exceed 100ms without indicating a real problem), and route skill-resolver `info` diagnostics (e.g. "Requested skill: …") through `log()` instead of `warn()` so informational messages no longer surface as warnings. diff --git a/packages/core/src/store.ts b/packages/core/src/store.ts index 87ff6d774..d4b72558a 100644 --- a/packages/core/src/store.ts +++ b/packages/core/src/store.ts @@ -4580,10 +4580,10 @@ export class TaskStore extends EventEmitter { } const elapsed = Date.now() - startTime; - if (elapsed > 100) { + if (elapsed > 750) { storeLog.warn("checkForChanges took longer than expected", { elapsedMs: elapsed, - thresholdMs: 100, + thresholdMs: 750, }); } } catch (err) { diff --git a/packages/engine/src/skill-resolver.ts b/packages/engine/src/skill-resolver.ts index c61850b41..265d302ec 100644 --- a/packages/engine/src/skill-resolver.ts +++ b/packages/engine/src/skill-resolver.ts @@ -471,7 +471,10 @@ export function createSkillsOverrideFromSelection( if (newDiagnostics.length > 0) { const _purpose = sessionPurpose ? `[${sessionPurpose}]` : "skills"; for (const diag of newDiagnostics) { - piLog.warn(`[skills] ${diag.type}: ${diag.message}`); + const msg = `[skills] ${diag.type}: ${diag.message}`; + if (diag.type === "error") piLog.error(msg); + else if (diag.type === "warning") piLog.warn(msg); + else piLog.log(msg); } }