diff --git a/.changeset/auto-recover-orphan-heartbeat-runs.md b/.changeset/auto-recover-orphan-heartbeat-runs.md new file mode 100644 index 000000000..361b39725 --- /dev/null +++ b/.changeset/auto-recover-orphan-heartbeat-runs.md @@ -0,0 +1,5 @@ +--- +"@runfusion/fusion": patch +--- + +Self-heal orphaned `agentRuns` rows left in `status='active'` when the dashboard process crashes mid-heartbeat. The trigger scheduler treats any active run as "still running" and silently skips every subsequent tick, so a single crashed run could leave an agent without heartbeats for hours. SelfHealingManager now reconciles these on startup and during periodic maintenance, terminating runs whose `processPid` does not match the current process or whose age exceeds 6 hours. diff --git a/.changeset/fn-3157-plugin-dashboard-views.md b/.changeset/fn-3157-plugin-dashboard-views.md new file mode 100644 index 000000000..98546898d --- /dev/null +++ b/.changeset/fn-3157-plugin-dashboard-views.md @@ -0,0 +1,5 @@ +--- +"@runfusion/fusion": minor +--- + +Add plugin dashboard view discovery and navigation integration via `GET /api/plugins/dashboard-views`, plugin view ID persistence (`plugin:${pluginId}:${viewId}`), and static host-side plugin view registry rendering. diff --git a/.changeset/fn-3209-planning-refine-fix.md b/.changeset/fn-3209-planning-refine-fix.md new file mode 100644 index 000000000..9abb98766 --- /dev/null +++ b/.changeset/fn-3209-planning-refine-fix.md @@ -0,0 +1,5 @@ +--- +"@runfusion/fusion": patch +--- + +Fix Planning Mode summary refinement so "Refine Further" reliably continues completed/resumed sessions through the backend interview flow instead of showing a blank question screen. diff --git a/.changeset/fn-3513-github-source-metadata.md b/.changeset/fn-3513-github-source-metadata.md new file mode 100644 index 000000000..282e0fb65 --- /dev/null +++ b/.changeset/fn-3513-github-source-metadata.md @@ -0,0 +1,5 @@ +--- +"@runfusion/fusion": patch +--- + +Preserve complete GitHub source metadata for imported issues across CLI and extension import paths, and improve commit reference generation by falling back to `externalIssueId` when `issueNumber` is missing. \ No newline at end of file diff --git a/.changeset/quiet-noisy-store-and-skill-logs.md b/.changeset/quiet-noisy-store-and-skill-logs.md new file mode 100644 index 000000000..fbba9dc2f --- /dev/null +++ b/.changeset/quiet-noisy-store-and-skill-logs.md @@ -0,0 +1,5 @@ +--- +"@runfusion/fusion": patch +--- + +Reduce log noise: bump `checkForChanges` slow-poll warn threshold from 100ms to 750ms (the 1s poll interval + multiple SQLite queries routinely exceed 100ms without indicating a real problem), and route skill-resolver `info` diagnostics (e.g. "Requested skill: …") through `log()` instead of `warn()` so informational messages no longer surface as warnings. diff --git a/docs/PLUGIN_AUTHORING.md b/docs/PLUGIN_AUTHORING.md index dada3827f..b0238aeff 100644 --- a/docs/PLUGIN_AUTHORING.md +++ b/docs/PLUGIN_AUTHORING.md @@ -555,6 +555,28 @@ Current host constraints: - `componentPath` is stored for authoring symmetry/future expansion, but render resolution is currently done through a host-side static registry (`pluginId + viewId`) - Use stable IDs; runtime view key format is `plugin:${pluginId}:${viewId}` +### Static host registry model + +Dashboard view components are resolved from a host-side registry and must be explicitly registered: + +```ts +import { lazy } from "react"; +import { registerPluginView } from "../app/plugins/pluginViewRegistry"; + +registerPluginView( + "fusion-plugin-dependency-graph", + "graph", + lazy(() => import("@fusion-plugin-examples/dependency-graph/dashboard-view")), +); +``` + +The host then renders plugin views via `PluginDashboardViewHost` using the composite ID. + +Placement guidance: +- `primary`: top-level nav tab (host may limit count on mobile) +- `overflow`: desktop header overflow menu +- `more`: mobile More sheet / secondary nav surfaces + --- ## 9. Registering Agent Runtimes diff --git a/docs/agents.md b/docs/agents.md index 922dd1347..0b9e0e9ed 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -97,6 +97,16 @@ Fallback behavior remains unchanged: Execution-ownership sync intentionally avoids assignment-trigger side effects (`agent:assigned` wakeups) that are intended for control-plane delegation. +### Ephemeral agent terminal cleanup + +Runtime-created ephemeral agents are removed immediately after terminal cleanup paths run: + +- Task-worker agents created by `InProcessRuntime` are deleted as soon as they reach `terminated` through completion, error, or `agent:stateChanged` fallback cleanup. +- Spawned child agents created by `TaskExecutor` are deleted immediately inside `terminateChildAgent()` after terminal state update. +- User-managed non-ephemeral agents are never auto-deleted by these pathways. + +Because deletion is immediate, terminated runtime helper agents should not remain visible in the dashboard or `AgentStore` after cleanup completes. + ## Agents View (Dashboard) The agents surface provides: @@ -106,6 +116,7 @@ The agents surface provides: - Org chart nodes intentionally stay compact (role/state/health hierarchy signal only) and do not enumerate per-agent skill badges; detailed skills remain in list/board/detail surfaces - A cross-pane **Overview** strip above the split layout with summary metrics and a disclosure to expand active/running live cards - A compact **Controls** popup for secondary actions (state filter, Show system agents toggle, Import, and global Heartbeat Speed) +- Agent import can also be launched from the selected **Agent Detail** header; this entry opens the import modal directly in the companies.sh browse flow so operators can discover and import packages without leaving the detail context - Detail/config panels - Split-view synchronization: successful saves and lifecycle actions from the right-side Agent Detail pane immediately refresh the left-side list/selection state (no wait for background polling) - A per-agent **Token Usage** panel that summarizes cumulative token consumption for the currently displayed agents diff --git a/docs/architecture.md b/docs/architecture.md index 188c20c32..4b95bb261 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -120,7 +120,7 @@ Concrete references: - **Database adapter**: `packages/core/src/db.ts` - SQLite (`node:sqlite`) with WAL mode + foreign keys - JSON helpers: `toJson`, `toJsonNullable`, `fromJson` - - Core schema tables include: `tasks`, `config`, `workflow_steps`, `activityLog`, `archivedTasks`, `automations`, `agents`, `agentHeartbeats`, `task_documents`, `task_document_revisions`, mission hierarchy tables (`missions`, `milestones`, `slices`, `mission_features`, `mission_events`), plugin/routine tables (`plugins`, `routines`), roadmap tables (`roadmaps`, `roadmap_milestones`, `roadmap_features`), insight tables (`project_insights`, `project_insight_runs`), todo tables (`todo_lists`, `todo_items`), `__meta` + - Core schema tables include: `tasks`, `config`, `workflow_steps`, `activityLog`, `archivedTasks`, `automations`, `agents`, `agentHeartbeats`, `task_documents`, `task_document_revisions`, mission hierarchy tables (`missions`, `milestones`, `slices`, `mission_features`, `mission_events`), plugin/routine tables (`plugins`, `routines`), roadmap tables (`roadmaps`, `roadmap_milestones`, `roadmap_features`), insight tables (`project_insights`, `project_insight_runs`), research tables (`research_runs`, `research_exports`, `research_run_events`), eval tables (`eval_runs`, `eval_task_results`, `eval_run_events`), todo tables (`todo_lists`, `todo_items`), `__meta` - Migration-created tables include: `ai_sessions`, `messages`, `agentRatings`, `chat_sessions`, `chat_messages`, `runAuditEvents`, `mission_contract_assertions`, `mission_feature_assertions`, `mission_validator_runs`, `mission_validator_failures`, `mission_fix_feature_lineage` - `ai_sessions.status` lifecycle includes `draft` (pre-start planning session), then `generating`, `awaiting_input`, terminal `complete` / `error` - **Standalone roadmap model**: `packages/core/src/roadmap-types.ts`, `roadmap-ordering.ts`, `roadmap-store.ts` @@ -144,6 +144,7 @@ Concrete references: - `RoutineStore` (`routine-store.ts`) — recurring routine definitions and run history - `RoadmapStore` (`roadmap-store.ts`) — standalone roadmap CRUD with deterministic ordering and atomic reorder/move operations - `TodoStore` (`todo-store.ts`) — project-scoped todo lists/items with completion, reorder, and composite list+items queries + - `EvalStore` (`eval-store.ts`) — eval run persistence, per-task eval results with durable snapshots, and append-only run event trails ### Chat System @@ -187,6 +188,13 @@ Concrete references: - Provider substitution must remain data-driven: source metadata can carry provider identity, and fetching should resolve providers per source rather than relying on provider ordering. - **Boundary note:** research and insights are parallel subsystems sharing host infrastructure, not one table/store family. +### Task Evaluations + +- `EvalStore` (`eval-store.ts`, `eval-types.ts`) persists eval runs and task-level eval outcomes. +- Backed by `eval_runs`, `eval_task_results`, and `eval_run_events`. +- Data model stores structured scoring/evidence/signal payloads plus durable `taskSnapshot` metadata so historical eval results remain readable even if the live task row later changes or is removed. +- Lifecycle safeguards mirror other core stores: deterministic list ordering, transition guards, terminal immutability for run rows, and active-run conflict protection for scheduled/task-completion triggers. + ### Plugin System - `PluginStore` (`plugin-store.ts`) stores plugin installation state and settings (`plugins` table) @@ -397,6 +405,7 @@ See [Memory Plugin Contract](./memory-plugin-contract.md) for the full plan. - `SelfHealingManager` (`self-healing.ts`) — auto-unpause/maintenance recovery actions - `recoverGhostReviewTasks()` is a fallback only for idle, non-terminal `in-review` states. Terminal/actionable states (notably `status: "failed"`) are preserved and **not** auto-kicked back to `todo`. - `recoverMergeableReviewTasks()` only re-enqueues truly eligible tasks; retry-exhausted review tasks are skipped to avoid re-enqueue/no-op loops that keep refreshing `updatedAt`. + - Merge commit attribution is ownership-aware: a `mergeDetails.commitSha` is trusted only when reachable from `HEAD` **and** attributable to the task via `Fusion-Task-Id` trailer or task-ID-bearing subject. Reachable-but-unowned SHAs are rejected to prevent sibling done tasks from sharing misleading merge metadata. - `ProjectEngine` settings lifecycle handlers (`project-engine.ts`) treat `enginePaused` as a soft pause: clearing it dispatches runtime resume and, when `autoMerge` is enabled, performs an `in-review` eligibility sweep to requeue mergeable review tasks. - `UsageLimitPauser` (`usage-limit-detector.ts`) and `withRateLimitRetry` (`rate-limit-retry.ts`) diff --git a/docs/contributing.md b/docs/contributing.md index 600dfed6e..5da4f484e 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -58,7 +58,8 @@ Fusion codifies workspace verification as a deterministic contract: - Use `pnpm install --frozen-lockfile` for clean bootstrap and dependency repair paths. - `pnpm test:full` must be runnable in a clean worktree without requiring a prior `pnpm build`. -- This includes clean states where `packages/core/dist`, `packages/engine/dist`, and `packages/dashboard/dist` are absent. +- Root test entrypoints (`pnpm test` via `scripts/test-changed.mjs` and `pnpm test:ci:shard` via `scripts/ci-test-shard.mjs`) call `scripts/ensure-test-artifacts.mjs`, which deterministically builds only missing required workspace dist artifacts (`@fusion/core`, `@fusion/plugin-sdk`, and runtime plugins that export from `dist/*`). +- This includes clean states where those required dist directories are absent. - `pnpm verify:workspace` is the canonical pre-merge gate and runs in strict order: 1. `pnpm lint` 2. `pnpm test:full` diff --git a/docs/dashboard-guide.md b/docs/dashboard-guide.md index 687ee10e7..f1a24b1b5 100644 --- a/docs/dashboard-guide.md +++ b/docs/dashboard-guide.md @@ -526,6 +526,12 @@ Possible error codes: The Agent Import feature allows you to import agents from Agent Companies packages. When importing agents from companies.sh or local directories, Fusion now also persists any skill definitions from the package. +### Launch Points + +You can open Agent Import from: +- **Agents view → Controls popup → Import** +- **Agent Detail header → Import** (opens directly to the companies.sh browse catalog) + ### How It Works 1. **Select Source**: Choose to import from: diff --git a/docs/research/research-hardening-preflight.md b/docs/research/research-hardening-preflight.md index a96f7f9e1..415569e42 100644 --- a/docs/research/research-hardening-preflight.md +++ b/docs/research/research-hardening-preflight.md @@ -164,7 +164,13 @@ Key endpoints: - FN-3370 replaces FN-3015's stale insights-backed child scope with the landed research subsystem surfaces in this document (core `ResearchStore` + dashboard `/api/research` + engine orchestrator lifecycle persistence). - Regression coverage work should stay bounded to shipped lifecycle/status/export/task-integration contracts and use follow-up tasks for any unshipped behavior instead of feature expansion. -## 11) Validation references used for this baseline +## 11) Dashboard regression coverage status (FN-3368 refinement) + +- Dashboard interaction tests are anchored to landed standalone research surfaces (`ResearchView`, `ResearchTaskActionModal`, `useResearch`, `App` research route wiring). +- Route regression tests explicitly cover finding-to-task create/enrich provenance metadata, task-document writes, duplicate-attachment skip behavior, archived/missing target guards, and payload validation. +- There is no placeholder/optional assumption that research dashboard files or `/api/research` routes are absent. + +## 12) Validation references used for this baseline - `packages/dashboard/src/__tests__/research-routes.test.ts` - `packages/core/src/__tests__/research-store.test.ts` diff --git a/docs/storage.md b/docs/storage.md index 576a88e5a..11af37265 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -6,7 +6,7 @@ - **Backend settings keys defined in `@fusion/core`:** **78** total - **Global settings:** 17 (`GlobalSettings`) - **Project settings:** 61 (`ProjectSettings`) -- **SQLite tables in project DB schema (`packages/core/src/db.ts`):** **36** (including migration-created tables) +- **SQLite tables in project DB schema (`packages/core/src/db.ts`):** **39** (including migration-created tables) - **Issues identified:** **9** - High: 2 - Medium: 5 @@ -209,6 +209,9 @@ Additional backend notes: | `research_runs` | Research run state (query, topic, status, lifecycle, sources, results, citations, events, exports, token usage). Supports project-scoped active-run uniqueness via `(projectId, trigger, status)` index. Terminal runs are immutable. | | `research_exports` | Persisted export records for research runs (`runId` FK cascade). Stores format, content, and optional file path. | | `research_run_events` | Append-only event log for research run lifecycle tracking (`runId` FK cascade, ordered by `seq`). Records status transitions, phase changes, step lifecycle, and failure classifications. | +| `eval_runs` | Eval run lifecycle state (status, trigger, scope, evaluation window boundaries, evaluated task IDs/counts, aggregate scores, provenance). | +| `eval_task_results` | Per-task eval outcomes linked to runs (`runId` FK cascade), including durable task snapshots, category scores, evidence references, deterministic/AI signal payloads, rationale, and follow-up suggestions. | +| `eval_run_events` | Append-only eval run event trail (`runId` FK cascade, ordered by `seq`) for orchestration/debug auditing and downstream API/UI drill-down. | --- diff --git a/docs/task-management.md b/docs/task-management.md index ceecbfb65..378522d7d 100644 --- a/docs/task-management.md +++ b/docs/task-management.md @@ -25,6 +25,7 @@ Use the 💡 button to open planning mode: - Break-into-tasks descriptions are structured with subtask-specific guidance first, then a separate larger-plan context section (plus `## Planning Interview Context` when interview history exists) - Sessions persist when the modal is closed — resume from the sidebar list at any time; reasoning context is restored automatically - Back navigation rewinds the server-side planning session to the previous answered question so you can revise earlier answers and continue from the corrected turn +- On the summary screen, **Refine Further** continues through the backend planning session (including resumed completed sessions) and waits for a real follow-up question or updated summary; it does not switch to an empty question view ### 3) Todo item → Plan Mode @@ -230,6 +231,20 @@ This file is the contract for execution and review. Steering comments can be injected mid-run into active executor sessions. +### User comments and triage re-consideration + +User comments can trigger **re-triage** for already-planned but non-executing work: + +- `triage` + `awaiting-approval` → user comment sets `status: "needs-replan"` +- `triage` or `todo` with a real (non-bootstrap-stub) `PROMPT.md` → user comment sets `status: "needs-replan"` +- `triage` or `todo` with only bootstrap-stub/unplanned prompt content → no re-triage transition + +Execution ownership is preserved for active work: + +- User comments on `in-progress` and `in-review` tasks do **not** re-route those tasks back through triage. +- Agent/system comments do **not** trigger comment-driven re-triage. + +This is distinct from steering comments: steering feedback targets the currently running executor session, while comment-driven re-triage requests a fresh specification pass for planned work. ## Refinement Tasks `fn task refine ` creates a new planning task that depends on the original done/in-review task. @@ -289,6 +304,8 @@ Archive entries preserve key metadata needed for restoration, including: Import issues: +- GitHub-imported tasks retain typed source issue metadata (`sourceIssue.provider/repository/externalIssueId/issueNumber/url`), which executor and merger flows use to include `Ref: owner/repo#N` in commit bodies. + ```bash fn task import owner/repo --labels bug --limit 20 fn task import owner/repo --interactive diff --git a/packages/cli/src/__tests__/extension.test.ts b/packages/cli/src/__tests__/extension.test.ts index d907f3658..db2f1c957 100644 --- a/packages/cli/src/__tests__/extension.test.ts +++ b/packages/cli/src/__tests__/extension.test.ts @@ -1185,6 +1185,10 @@ describe.skipIf(!SHOULD_RUN_EXTENSION_INTEGRATION)("fn pi extension", () => { issueNumber: 1, url: "https://github.com/acme/demo/issues/1", }); + expect(issueOneTask?.source?.sourceMetadata).toEqual({ + issueUrl: "https://github.com/acme/demo/issues/1", + issueNumber: 1, + }); }); it("fn_task_browse_github_issues lists issues via gh api", async () => { diff --git a/packages/cli/src/commands/__tests__/task.test.ts b/packages/cli/src/commands/__tests__/task.test.ts index a79a2fa16..b1850a7ba 100644 --- a/packages/cli/src/commands/__tests__/task.test.ts +++ b/packages/cli/src/commands/__tests__/task.test.ts @@ -170,7 +170,7 @@ describe("runTaskShow", () => { [{ sourceType: "task_refine", sourceParentTaskId: "FN-2904" }, "Source: Refinement of FN-2904"], [{ sourceType: "task_duplicate", sourceParentTaskId: "FN-2905" }, "Source: Duplicate of FN-2905"], [ - { sourceType: "github_import", sourceMetadata: { issueUrl: "https://github.com/owner/repo/issues/42" } }, + { sourceType: "github_import", sourceMetadata: { issueUrl: "https://github.com/owner/repo/issues/42", issueNumber: 42 } }, "Source: GitHub Import (https://github.com/owner/repo/issues/42)", ], [ @@ -1115,7 +1115,7 @@ describe("runTaskImportGitHubInteractive", () => { issueNumber: 1, url: "https://github.com/owner/repo/issues/1", }, - source: { sourceType: "github_import", sourceMetadata: { issueUrl: "https://github.com/owner/repo/issues/1" } }, + source: { sourceType: "github_import", sourceMetadata: { issueUrl: "https://github.com/owner/repo/issues/1", issueNumber: 1 } }, }); expect(mockCreateTask).toHaveBeenCalledWith({ title: "Third Issue", @@ -1129,7 +1129,7 @@ describe("runTaskImportGitHubInteractive", () => { issueNumber: 3, url: "https://github.com/owner/repo/issues/3", }, - source: { sourceType: "github_import", sourceMetadata: { issueUrl: "https://github.com/owner/repo/issues/3" } }, + source: { sourceType: "github_import", sourceMetadata: { issueUrl: "https://github.com/owner/repo/issues/3", issueNumber: 3 } }, }); }); @@ -1187,7 +1187,7 @@ describe("runTaskImportGitHubInteractive", () => { issueNumber: 2, url: "https://github.com/owner/repo/issues/2", }, - source: { sourceType: "github_import", sourceMetadata: { issueUrl: "https://github.com/owner/repo/issues/2" } }, + source: { sourceType: "github_import", sourceMetadata: { issueUrl: "https://github.com/owner/repo/issues/2", issueNumber: 2 } }, }); const skipLine = logSpy.mock.calls.find( @@ -1441,7 +1441,7 @@ describe("runTaskImportFromGitHub", () => { issueNumber: 1, url: "https://github.com/owner/repo/issues/1", }, - source: { sourceType: "github_import", sourceMetadata: { issueUrl: "https://github.com/owner/repo/issues/1" } }, + source: { sourceType: "github_import", sourceMetadata: { issueUrl: "https://github.com/owner/repo/issues/1", issueNumber: 1 } }, }); const successLine = logSpy.mock.calls.find( @@ -1524,7 +1524,7 @@ describe("runTaskImportFromGitHub", () => { issueNumber: 1, url: "https://github.com/owner/repo/issues/1", }, - source: { sourceType: "github_import", sourceMetadata: { issueUrl: "https://github.com/owner/repo/issues/1" } }, + source: { sourceType: "github_import", sourceMetadata: { issueUrl: "https://github.com/owner/repo/issues/1", issueNumber: 1 } }, }); }); @@ -1546,7 +1546,7 @@ describe("runTaskImportFromGitHub", () => { issueNumber: 1, url: "https://github.com/owner/repo/issues/1", }, - source: { sourceType: "github_import", sourceMetadata: { issueUrl: "https://github.com/owner/repo/issues/1" } }, + source: { sourceType: "github_import", sourceMetadata: { issueUrl: "https://github.com/owner/repo/issues/1", issueNumber: 1 } }, }); }); }); diff --git a/packages/cli/src/commands/task.ts b/packages/cli/src/commands/task.ts index 78647de33..23827321a 100644 --- a/packages/cli/src/commands/task.ts +++ b/packages/cli/src/commands/task.ts @@ -984,21 +984,16 @@ export async function runTaskImportGitHubInteractive( const description = `${body}\n\nSource: ${issue.html_url}`; // Create the task + const source = buildGitHubIssueSource(owner, repo, issue); const task = await store.createTask({ title: title || undefined, description, column: "triage", dependencies: [], - sourceIssue: { - provider: "github", - repository: `${owner}/${repo}`, - externalIssueId: String(issue.number), - issueNumber: issue.number, - url: issue.html_url, - }, + sourceIssue: source.sourceIssue, source: { sourceType: "github_import", - sourceMetadata: { issueUrl: issue.html_url }, + sourceMetadata: source.sourceMetadata, }, }); @@ -1067,6 +1062,19 @@ export interface TaskImportOptions { labels?: string[]; } +function buildGitHubIssueSource(owner: string, repo: string, issue: { number: number; html_url: string }) { + return { + sourceIssue: { + provider: "github" as const, + repository: `${owner}/${repo}`, + externalIssueId: String(issue.number), + issueNumber: issue.number, + url: issue.html_url, + }, + sourceMetadata: { issueUrl: issue.html_url, issueNumber: issue.number }, + }; +} + export async function runTaskImportFromGitHub( ownerRepo: string, options: TaskImportOptions = {}, @@ -1131,21 +1139,16 @@ export async function runTaskImportFromGitHub( const description = `${body}\n\nSource: ${issue.html_url}`; // Create the task + const source = buildGitHubIssueSource(owner, repo, issue); const task = await store.createTask({ title: title || undefined, description, column: "triage", dependencies: [], - sourceIssue: { - provider: "github", - repository: `${owner}/${repo}`, - externalIssueId: String(issue.number), - issueNumber: issue.number, - url: issue.html_url, - }, + sourceIssue: source.sourceIssue, source: { sourceType: "github_import", - sourceMetadata: { issueUrl: issue.html_url }, + sourceMetadata: source.sourceMetadata, }, }); diff --git a/packages/cli/src/extension.ts b/packages/cli/src/extension.ts index 8557000e5..c775a7a05 100644 --- a/packages/cli/src/extension.ts +++ b/packages/cli/src/extension.ts @@ -251,6 +251,19 @@ async function fetchGitHubIssuesViaGh( } } +function buildGitHubIssueSource(owner: string, repo: string, issue: { number: number; html_url: string }) { + return { + sourceIssue: { + provider: "github" as const, + repository: `${owner}/${repo}`, + externalIssueId: String(issue.number), + issueNumber: issue.number, + url: issue.html_url, + }, + sourceMetadata: { issueUrl: issue.html_url, issueNumber: issue.number }, + }; +} + async function fetchGitHubIssueViaGh( owner: string, repo: string, @@ -988,21 +1001,16 @@ export default function kbExtension(pi: ExtensionAPI) { const body = issue.body?.trim() || "(no description)"; const description = `${body}\n\nSource: ${sourceUrl}`; + const source = buildGitHubIssueSource(owner, repo, issue); const task = await store.createTask({ title: title || undefined, description, column: "triage", dependencies: [], - sourceIssue: { - provider: "github", - repository: `${owner}/${repo}`, - externalIssueId: String(issue.number), - issueNumber: issue.number, - url: issue.html_url, - }, + sourceIssue: source.sourceIssue, source: { sourceType: "github_import", - sourceMetadata: { issueUrl: issue.html_url }, + sourceMetadata: source.sourceMetadata, }, }); @@ -1085,21 +1093,16 @@ export default function kbExtension(pi: ExtensionAPI) { const body = issue.body?.trim() || "(no description)"; const description = `${body}\n\nSource: ${sourceUrl}`; + const source = buildGitHubIssueSource(owner, repo, issue); const task = await store.createTask({ title: title || undefined, description, column: "triage", dependencies: [], - sourceIssue: { - provider: "github", - repository: `${owner}/${repo}`, - externalIssueId: String(issue.number), - issueNumber: issue.number, - url: issue.html_url, - }, + sourceIssue: source.sourceIssue, source: { sourceType: "github_import", - sourceMetadata: { issueUrl: issue.html_url }, + sourceMetadata: source.sourceMetadata, }, }); diff --git a/packages/cli/vitest.config.ts b/packages/cli/vitest.config.ts index 2d1c8b5c3..7f1b013bb 100644 --- a/packages/cli/vitest.config.ts +++ b/packages/cli/vitest.config.ts @@ -25,6 +25,18 @@ export default defineConfig({ find: /^@fusion-plugin-examples\/droid-runtime$/, replacement: resolve(__dirname, "../../plugins/fusion-plugin-droid-runtime/src/index.ts"), }, + { + find: /^@fusion-plugin-examples\/hermes-runtime$/, + replacement: resolve(__dirname, "../../plugins/fusion-plugin-hermes-runtime/src/index.ts"), + }, + { + find: /^@fusion-plugin-examples\/openclaw-runtime$/, + replacement: resolve(__dirname, "../../plugins/fusion-plugin-openclaw-runtime/src/index.ts"), + }, + { + find: /^@fusion-plugin-examples\/paperclip-runtime$/, + replacement: resolve(__dirname, "../../plugins/fusion-plugin-paperclip-runtime/src/index.ts"), + }, { find: /^@fusion\/test-utils$/, replacement: resolve(__dirname, "../core/src/__test-utils__/workspace.ts") }, ], }, diff --git a/packages/core/src/__tests__/db.test.ts b/packages/core/src/__tests__/db.test.ts index 62458e9ff..734db6fed 100644 --- a/packages/core/src/__tests__/db.test.ts +++ b/packages/core/src/__tests__/db.test.ts @@ -160,7 +160,7 @@ describe("Database", () => { }); it("seeds schema version", () => { - expect(db.getSchemaVersion()).toBe(61); + expect(db.getSchemaVersion()).toBe(62); }); it("seeds lastModified", () => { @@ -183,7 +183,7 @@ describe("Database", () => { it("is idempotent - calling init() twice does not fail", () => { expect(() => db.init()).not.toThrow(); - expect(db.getSchemaVersion()).toBe(61); + expect(db.getSchemaVersion()).toBe(62); }); it("does not overwrite existing config on re-init", () => { @@ -957,7 +957,7 @@ describe("schema migrations", () => { db.init(); // Verify version bumped to 29 (includes v1→v2 through v26→v29) - expect(db.getSchemaVersion()).toBe(61); + expect(db.getSchemaVersion()).toBe(62); // Verify new columns exist and existing data is intact const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>; @@ -982,11 +982,11 @@ describe("schema migrations", () => { const db = new Database(fusionDir); db.init(); - expect(db.getSchemaVersion()).toBe(61); + expect(db.getSchemaVersion()).toBe(62); // Re-init should not fail db.init(); - expect(db.getSchemaVersion()).toBe(61); + expect(db.getSchemaVersion()).toBe(62); db.close(); }); @@ -1021,7 +1021,7 @@ describe("schema migrations", () => { db.init(); - expect(db.getSchemaVersion()).toBe(61); + expect(db.getSchemaVersion()).toBe(62); const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>; expect(cols.map((col) => col.name)).toContain("priority"); @@ -1062,7 +1062,7 @@ describe("schema migrations", () => { db.init(); - expect(db.getSchemaVersion()).toBe(61); + expect(db.getSchemaVersion()).toBe(62); const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>; const colNames = cols.map((col) => col.name); @@ -1131,7 +1131,7 @@ describe("schema migrations", () => { db.init(); - expect(db.getSchemaVersion()).toBe(61); + expect(db.getSchemaVersion()).toBe(62); const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>; const colNames = cols.map((col) => col.name); @@ -1234,7 +1234,7 @@ describe("schema migrations", () => { db.init(); - expect(db.getSchemaVersion()).toBe(61); + expect(db.getSchemaVersion()).toBe(62); const cols = db.prepare("PRAGMA table_info(chat_messages)").all() as Array<{ name: string }>; expect(cols.map((col) => col.name)).toContain("attachments"); @@ -1308,7 +1308,7 @@ describe("schema migrations", () => { db.init(); - expect(db.getSchemaVersion()).toBe(61); + expect(db.getSchemaVersion()).toBe(62); const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name = 'agentRatings'").all() as Array<{ name: string }>; expect(tables).toEqual([{ name: "agentRatings" }]); @@ -1332,7 +1332,7 @@ describe("schema migrations", () => { db.init(); - expect(db.getSchemaVersion()).toBe(61); + expect(db.getSchemaVersion()).toBe(62); const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name = 'mission_events'").all() as Array<{ name: string }>; expect(tables).toEqual([{ name: "mission_events" }]); @@ -1436,7 +1436,7 @@ describe("schema migrations", () => { db.init(); // Verify version bumped to 29 - expect(db.getSchemaVersion()).toBe(61); + expect(db.getSchemaVersion()).toBe(62); // Verify new columns exist and existing data is intact const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>; @@ -1905,7 +1905,7 @@ describe("createDatabase factory", () => { const db = createDatabase(fusionDir); db.init(); - expect(db.getSchemaVersion()).toBe(61); + expect(db.getSchemaVersion()).toBe(62); expect(db.getLastModified()).toBeGreaterThan(0); db.close(); diff --git a/packages/core/src/__tests__/eval-automation.test.ts b/packages/core/src/__tests__/eval-automation.test.ts new file mode 100644 index 000000000..5aefca295 --- /dev/null +++ b/packages/core/src/__tests__/eval-automation.test.ts @@ -0,0 +1,138 @@ +import { describe, expect, it } from "vitest"; +import { createDatabase } from "../db.js"; +import { EvalStore } from "../eval-store.js"; +import { + DEFAULT_TASK_EVALUATION_SCHEDULE, + createScheduledEvalBatchAutomation, + resolveTaskEvaluationSettings, + runScheduledEvalBatch, + syncScheduledEvalBatchAutomation, +} from "../eval-automation.js"; + +function task(id: string, column: "done" | "todo" | "archived", completedAt: string, createdAt = "2026-01-01T00:00:00.000Z") { + return { + id, + column, + createdAt, + updatedAt: createdAt, + executionCompletedAt: completedAt, + title: id, + summary: id, + } as any; +} + +describe("eval-automation", () => { + it("resolves task evaluation settings defaults", () => { + const resolved = resolveTaskEvaluationSettings({}); + expect(resolved.taskEvaluationEnabled).toBe(false); + expect(resolved.taskEvaluationSchedule).toBe(DEFAULT_TASK_EVALUATION_SCHEDULE); + expect(resolved.taskEvaluationFollowUpPolicy).toBe("off"); + }); + + it("creates scheduled eval automation", () => { + const input = createScheduledEvalBatchAutomation({ taskEvaluationSchedule: "0 9 * * *" }); + expect(input.name).toBe("Scheduled Task Evaluation"); + expect(input.cronExpression).toBe("0 9 * * *"); + expect(input.scope).toBe("project"); + }); + + it("syncs schedule create/delete based on enabled flag", async () => { + const schedules: any[] = []; + const automationStore = { + listSchedules: async () => schedules, + createSchedule: async (input: any) => ({ ...input, id: "S-1" }), + deleteSchedule: async () => true, + updateSchedule: async () => undefined, + } as any; + + const created = await syncScheduledEvalBatchAutomation(automationStore, { taskEvaluationEnabled: true }); + expect(created?.name).toBe("Scheduled Task Evaluation"); + + schedules.push({ id: "S-1", name: "Scheduled Task Evaluation" }); + const deleted = await syncScheduledEvalBatchAutomation(automationStore, { taskEvaluationEnabled: false }); + expect(deleted).toBeUndefined(); + }); + + it("selects done tasks on first run and orders deterministically", async () => { + const db = createDatabase("/tmp/fn-eval-automation-1", { inMemory: true }); + db.init(); + const evalStore = new EvalStore(db); + const tasks = [ + task("FN-2", "done", "2026-05-01T01:00:00.000Z", "2026-01-02T00:00:00.000Z"), + task("FN-1", "done", "2026-05-01T01:00:00.000Z", "2026-01-01T00:00:00.000Z"), + task("FN-3", "done", "2026-05-01T02:00:00.000Z"), + task("FN-4", "todo", "2026-05-01T03:00:00.000Z"), + task("FN-5", "archived", "2026-05-01T04:00:00.000Z"), + ]; + + const result = await runScheduledEvalBatch({ + projectId: "proj", + store: { + listTasks: async () => tasks, + getEvalStore: () => evalStore, + } as any, + startedAt: "2026-05-01T05:00:00.000Z", + evaluator: async ({ task }) => ({ status: "scored", categoryScores: [], evidence: [], deterministicSignals: [], followUps: [], summary: task.id }), + }); + + expect(result.status).toBe("completed"); + expect(result.selectedTaskIds).toEqual(["FN-1", "FN-2", "FN-3"]); + + const run = evalStore.getRun(result.runId)!; + expect(run.counts.totalTasks).toBe(3); + expect(run.metadata?.windowEndInclusive).toBe("2026-05-01T05:00:00.000Z"); + const results = evalStore.listTaskResults({ runId: run.id }); + expect(results).toHaveLength(3); + expect(results[0]?.metadata?.windowEndInclusive).toBe("2026-05-01T05:00:00.000Z"); + }); + + it("uses previous windowEndInclusive cursor for incremental selection", async () => { + const db = createDatabase("/tmp/fn-eval-automation-2", { inMemory: true }); + db.init(); + const evalStore = new EvalStore(db); + + evalStore.createRun({ + projectId: "proj", + trigger: "schedule", + scope: "completed-tasks", + window: { until: "2026-05-01T05:00:00.000Z" }, + metadata: { windowEndInclusive: "2026-05-01T05:00:00.000Z" }, + }); + const run = evalStore.listRuns({ projectId: "proj", trigger: "schedule" })[0]!; + evalStore.updateRun(run.id, { status: "completed", completedAt: "2026-05-01T05:05:00.000Z" }); + + const tasks = [ + task("FN-1", "done", "2026-05-01T05:00:00.000Z"), + task("FN-2", "done", "2026-05-01T05:00:00.001Z"), + task("FN-3", "done", "2026-05-01T06:00:00.000Z"), + ]; + + const result = await runScheduledEvalBatch({ + projectId: "proj", + store: { listTasks: async () => tasks, getEvalStore: () => evalStore } as any, + startedAt: "2026-05-01T06:00:00.000Z", + evaluator: async () => ({ status: "skipped", categoryScores: [], evidence: [], deterministicSignals: [], followUps: [] }), + }); + + expect(result.windowStartExclusive).toBe("2026-05-01T05:00:00.000Z"); + expect(result.selectedTaskIds).toEqual(["FN-2", "FN-3"]); + }); + + it("completes no-op batch when no tasks are eligible", async () => { + const db = createDatabase("/tmp/fn-eval-automation-3", { inMemory: true }); + db.init(); + const evalStore = new EvalStore(db); + + const result = await runScheduledEvalBatch({ + projectId: "proj", + store: { listTasks: async () => [task("FN-1", "todo", "2026-05-01T01:00:00.000Z")], getEvalStore: () => evalStore } as any, + startedAt: "2026-05-02T01:00:00.000Z", + evaluator: async () => ({ status: "scored", categoryScores: [], evidence: [], deterministicSignals: [], followUps: [] }), + }); + + expect(result.tasksSelected).toBe(0); + const run = evalStore.getRun(result.runId)!; + expect(run.status).toBe("completed"); + expect(run.counts.totalTasks).toBe(0); + }); +}); diff --git a/packages/core/src/__tests__/eval-store.test.ts b/packages/core/src/__tests__/eval-store.test.ts new file mode 100644 index 000000000..64b0006e8 --- /dev/null +++ b/packages/core/src/__tests__/eval-store.test.ts @@ -0,0 +1,82 @@ +import { beforeEach, describe, expect, it } from "vitest"; +import { createDatabase, type Database } from "../db.js"; +import { EvalLifecycleError, EvalStore } from "../eval-store.js"; + +let db: Database; +let store: EvalStore; + +beforeEach(() => { + db = createDatabase("/tmp/fn-eval-store-test", { inMemory: true }); + db.init(); + store = new EvalStore(db); +}); + +describe("EvalStore", () => { + it("creates and lists runs with deterministic ordering", () => { + const runA = store.createRun({ projectId: "p1", scope: "completed-since-last", requestedTaskIds: ["FN-1"] }); + const runB = store.createRun({ projectId: "p1", scope: "completed-since-last", requestedTaskIds: ["FN-2"] }); + + const runs = store.listRuns({ projectId: "p1" }); + expect(runs.map((run) => run.id)).toEqual([runA.id, runB.id].sort()); + }); + + it("enforces active run conflict for scheduled trigger", () => { + store.createRun({ projectId: "p1", scope: "window", trigger: "schedule" }); + expect(() => store.createRun({ projectId: "p1", scope: "window", trigger: "schedule" })).toThrow(EvalLifecycleError); + }); + + it("enforces terminal immutability", () => { + const run = store.createRun({ projectId: "p1", scope: "window" }); + store.updateRun(run.id, { status: "completed" }); + expect(() => store.updateRun(run.id, { summary: "late change" })).toThrow(EvalLifecycleError); + }); + + it("creates results and preserves task snapshot after tasks row deletion", () => { + const run = store.createRun({ projectId: "p1", scope: "window" }); + const result = store.createTaskResult(run.id, { + taskId: "FN-123", + taskSnapshot: { taskId: "FN-123", title: "Snapshot title", status: "done", summary: "task summary" }, + status: "scored", + overallScore: 0.8, + categoryScores: [{ category: "quality", score: 0.8 }], + evidence: [{ type: "task_log", ref: "log:1" }], + deterministicSignals: [{ signalId: "s1", kind: "test", name: "tests-pass", passed: true }], + }); + + db.prepare("DELETE FROM tasks WHERE id = ?").run("FN-123"); + + const fetched = store.getTaskResult(result.id); + expect(fetched?.taskSnapshot.title).toBe("Snapshot title"); + expect(fetched?.taskId).toBe("FN-123"); + }); + + it("persists run window boundaries and evaluated task rollups", () => { + const run = store.createRun({ + projectId: "p1", + trigger: "schedule", + scope: "completed-since-last", + window: { since: "2026-05-01T00:00:00.000Z", until: "2026-05-02T00:00:00.000Z", baselineRunId: "ER-BASE" }, + requestedTaskIds: ["FN-1", "FN-2"], + }); + + const updated = store.updateRun(run.id, { + status: "running", + evaluatedTaskIds: ["FN-1", "FN-2"], + counts: { totalTasks: 2, scoredTasks: 1, skippedTasks: 1, erroredTasks: 0 }, + }); + + expect(updated?.window.since).toBe("2026-05-01T00:00:00.000Z"); + expect(updated?.evaluatedTaskIds).toEqual(["FN-1", "FN-2"]); + expect(updated?.counts.scoredTasks).toBe(1); + }); + + it("appends run events with sequential ordering", () => { + const run = store.createRun({ projectId: "p1", scope: "window" }); + const evt1 = store.appendRunEvent(run.id, { type: "info", message: "started" }); + const evt2 = store.appendRunEvent(run.id, { type: "task_evaluated", message: "scored", taskId: "FN-1" }); + + const events = store.listRunEvents(run.id); + expect(events.map((event) => event.id)).toEqual([evt1.id, evt2.id]); + expect(events.map((event) => event.seq)).toEqual([1, 2]); + }); +}); diff --git a/packages/core/src/__tests__/insight-store.test.ts b/packages/core/src/__tests__/insight-store.test.ts index aec09ec10..c8f7b95b0 100644 --- a/packages/core/src/__tests__/insight-store.test.ts +++ b/packages/core/src/__tests__/insight-store.test.ts @@ -886,7 +886,7 @@ describe("Migration: pre-33 DB upgrade", () => { // Step 1: Create a fresh database at v33 (runs all migrations up to 33) const db1 = createDatabase(legacyDir); db1.init(); - expect(db1.getSchemaVersion()).toBe(61); + expect(db1.getSchemaVersion()).toBe(62); db1.close(); // Step 2: Manually downgrade to version 32 and drop insight tables @@ -921,7 +921,7 @@ describe("Migration: pre-33 DB upgrade", () => { expect(tableNamesBefore).not.toContain("project_insight_runs"); // Now run init — this triggers the v32→v33 migration db3.init(); - expect(db3.getSchemaVersion()).toBe(61); + expect(db3.getSchemaVersion()).toBe(62); // Step 4: Verify insight tables exist after migration const tablesAfter = db3.prepare( @@ -952,12 +952,12 @@ describe("Migration: pre-33 DB upgrade", () => { try { const db1 = createDatabase(testDir); db1.init(); - expect(db1.getSchemaVersion()).toBe(61); + expect(db1.getSchemaVersion()).toBe(62); db1.close(); const db2 = createDatabase(testDir); expect(() => db2.init()).not.toThrow(); - expect(db2.getSchemaVersion()).toBe(61); + expect(db2.getSchemaVersion()).toBe(62); db2.close(); } finally { rmSync(testDir, { recursive: true, force: true }); diff --git a/packages/core/src/__tests__/mission-store.test.ts b/packages/core/src/__tests__/mission-store.test.ts index 176145cb2..7ee0ee6ab 100644 --- a/packages/core/src/__tests__/mission-store.test.ts +++ b/packages/core/src/__tests__/mission-store.test.ts @@ -2629,7 +2629,7 @@ describe("MissionStore", () => { describe("Loop State & Validator Run Schema (v31)", () => { it("schema version is 40 after migration", () => { - expect(db.getSchemaVersion()).toBe(61); + expect(db.getSchemaVersion()).toBe(62); }); it("mission_features table has loop state columns", () => { diff --git a/packages/core/src/__tests__/roadmap-store.test.ts b/packages/core/src/__tests__/roadmap-store.test.ts index 1c35afe96..6170c83d4 100644 --- a/packages/core/src/__tests__/roadmap-store.test.ts +++ b/packages/core/src/__tests__/roadmap-store.test.ts @@ -742,7 +742,7 @@ describe("RoadmapStore", () => { describe("schema version", () => { it("schema version is 40 after init", () => { - expect(db.getSchemaVersion()).toBe(61); + expect(db.getSchemaVersion()).toBe(62); }); }); diff --git a/packages/core/src/__tests__/run-audit.test.ts b/packages/core/src/__tests__/run-audit.test.ts index 1b860ec7e..3c0e72a4e 100644 --- a/packages/core/src/__tests__/run-audit.test.ts +++ b/packages/core/src/__tests__/run-audit.test.ts @@ -465,7 +465,7 @@ describe("Run Audit", () => { }); it("schema version is bumped to 40", () => { - expect(db.getSchemaVersion()).toBe(61); + expect(db.getSchemaVersion()).toBe(62); }); }); }); diff --git a/packages/core/src/__tests__/store.test.ts b/packages/core/src/__tests__/store.test.ts index bf7dc42c3..720696f5a 100644 --- a/packages/core/src/__tests__/store.test.ts +++ b/packages/core/src/__tests__/store.test.ts @@ -5430,7 +5430,7 @@ Task with acceptance criteria expect(updateSpy).toHaveBeenCalled(); const warningCall = warnSpy.mock.calls.find( - (call) => typeof call[0] === "string" && call[0].includes("[task-store] Best-effort post-comment awaiting-approval invalidation failed"), + (call) => typeof call[0] === "string" && call[0].includes("[task-store] Best-effort post-comment re-triage failed"), ); expect(warningCall).toBeDefined(); @@ -5476,7 +5476,7 @@ Task with acceptance criteria expect(logEntrySpy).toHaveBeenCalled(); const warningCall = warnSpy.mock.calls.find( - (call) => typeof call[0] === "string" && call[0].includes("[task-store] Best-effort post-comment awaiting-approval invalidation failed"), + (call) => typeof call[0] === "string" && call[0].includes("[task-store] Best-effort post-comment re-triage failed"), ); expect(warningCall).toBeDefined(); @@ -5654,14 +5654,76 @@ Task with acceptance criteria expect(updated.comments).toHaveLength(1); }); - it("does NOT transition to needs-replan when user comments on non-awaiting-approval triage task", async () => { + it("transitions to needs-replan when user comments on non-awaiting-approval triage task with real spec", async () => { const task = await store.createTask({ description: "Task in triage" }); - // Task is in triage with no status (not awaiting-approval) - expect(task.status).toBeUndefined(); + const promptPath = join(rootDir, ".fusion", "tasks", task.id, "PROMPT.md"); + await writeFile(promptPath, `# Task: ${task.id} - Triage Plan\n\n## Mission\n\nPlanned task.`); - const updated = await store.addComment(task.id, "User feedback", "user"); + await store.addComment(task.id, "User feedback", "user"); + const updated = await store.getTask(task.id); - // Status should remain undefined + expect(updated.status).toBe("needs-replan"); + expect(updated.column).toBe("triage"); + expect(updated.comments?.[0]?.text).toBe("User feedback"); + }); + + it("does NOT transition to needs-replan when user comments on triage task with bootstrap stub prompt", async () => { + const task = await store.createTask({ description: "Task in triage" }); + + await store.addComment(task.id, "User feedback", "user"); + const updated = await store.getTask(task.id); + + expect(updated.status).toBeUndefined(); + }); + + it("transitions todo task to needs-replan when user comments and task has real spec", async () => { + const task = await store.createTask({ description: "Task in todo", column: "todo" }); + const promptPath = join(rootDir, ".fusion", "tasks", task.id, "PROMPT.md"); + await writeFile(promptPath, `# Task: ${task.id} - Todo Plan\n\n## Mission\n\nPlanned task.`); + + await store.addComment(task.id, "Please update approach", "user"); + const updated = await store.getTask(task.id); + + expect(updated.status).toBe("needs-replan"); + expect(updated.column).toBe("todo"); + expect(updated.log.some((entry) => entry.action === "User comment requested re-specification of planned task")).toBe(true); + }); + + it("does NOT transition todo task to needs-replan when prompt matches bootstrap stub", async () => { + const task = await store.createTask({ description: "Task in todo", column: "todo" }); + const promptPath = join(rootDir, ".fusion", "tasks", task.id, "PROMPT.md"); + await writeFile(promptPath, `# ${task.id}\n\nTask in todo\n`); + + await store.addComment(task.id, "Please update approach", "user"); + const updated = await store.getTask(task.id); + + expect(updated.status).toBeUndefined(); + }); + + it("does NOT transition to needs-replan when user comments on in-progress task", async () => { + const task = await store.createTask({ description: "Task in progress", column: "todo" }); + const promptPath = join(rootDir, ".fusion", "tasks", task.id, "PROMPT.md"); + await writeFile(promptPath, `# Task: ${task.id} - Plan\n\n## Mission\n\nPlanned task.`); + await store.moveTask(task.id, "in-progress"); + + await store.addComment(task.id, "Please adjust implementation", "user"); + const updated = await store.getTask(task.id); + + expect(updated.column).toBe("in-progress"); + expect(updated.status).toBeUndefined(); + }); + + it("does NOT transition to needs-replan when user comments on in-review task", async () => { + const task = await store.createTask({ description: "Task in review", column: "todo" }); + const promptPath = join(rootDir, ".fusion", "tasks", task.id, "PROMPT.md"); + await writeFile(promptPath, `# Task: ${task.id} - Plan\n\n## Mission\n\nPlanned task.`); + await store.moveTask(task.id, "in-progress"); + await store.moveTask(task.id, "in-review"); + + await store.addComment(task.id, "Please adjust before merge", "user"); + const updated = await store.getTask(task.id); + + expect(updated.column).toBe("in-review"); expect(updated.status).toBeUndefined(); }); }); diff --git a/packages/core/src/__tests__/task-documents.test.ts b/packages/core/src/__tests__/task-documents.test.ts index c80d00b55..b64173607 100644 --- a/packages/core/src/__tests__/task-documents.test.ts +++ b/packages/core/src/__tests__/task-documents.test.ts @@ -51,7 +51,7 @@ describe("TaskStore task documents", () => { expect(tableNames.has("task_documents")).toBe(true); expect(tableNames.has("task_document_revisions")).toBe(true); - expect(db.getSchemaVersion()).toBe(61); + expect(db.getSchemaVersion()).toBe(62); const index = db .prepare( diff --git a/packages/core/src/agent-store.ts b/packages/core/src/agent-store.ts index 7b53c4eda..1c5588dd7 100644 --- a/packages/core/src/agent-store.ts +++ b/packages/core/src/agent-store.ts @@ -1743,6 +1743,25 @@ export class AgentStore extends EventEmitter { .sort((a, b) => new Date(b.startedAt).getTime() - new Date(a.startedAt).getTime()); } + /** + * List every heartbeat run currently in `status = 'active'` across all + * agents. Used by self-healing to detect orphaned runs from prior process + * incarnations that crashed before calling endHeartbeatRun(). Without this + * sweep an active row blocks all subsequent timer ticks for the agent + * because HeartbeatTriggerScheduler.onTimerTick treats any active run as + * "already running". + */ + async listActiveHeartbeatRuns(): Promise { + const rows = this.db.prepare(` + SELECT data FROM agentRuns + WHERE status = 'active' + ORDER BY startedAt ASC + `).all() as Array<{ data: string }>; + return rows + .map((row) => this.parseJson(row.data, null)) + .filter((run): run is AgentHeartbeatRun => run !== null); + } + // ───────────────────────────────────────────────────────────────────────── // Task Session Management // ───────────────────────────────────────────────────────────────────────── diff --git a/packages/core/src/db.ts b/packages/core/src/db.ts index aa3f9d781..2be2f85c7 100644 --- a/packages/core/src/db.ts +++ b/packages/core/src/db.ts @@ -88,7 +88,7 @@ export function probeFts5(db: DatabaseSync): boolean { // ── Schema Definition ──────────────────────────────────────────────── -const SCHEMA_VERSION = 61; +const SCHEMA_VERSION = 62; function normalizeTaskComments( steeringComments: SteeringComment[] | undefined, @@ -466,6 +466,71 @@ CREATE TABLE IF NOT EXISTS research_run_events ( ); CREATE INDEX IF NOT EXISTS idxResearchRunEventsRunIdSeq ON research_run_events(runId, seq); +-- Eval run persistence (FN-3387) +CREATE TABLE IF NOT EXISTS eval_runs ( + id TEXT PRIMARY KEY, + projectId TEXT NOT NULL, + status TEXT NOT NULL, + trigger TEXT NOT NULL, + scope TEXT NOT NULL, + window TEXT NOT NULL DEFAULT '{}', + requestedTaskIds TEXT NOT NULL DEFAULT '[]', + evaluatedTaskIds TEXT NOT NULL DEFAULT '[]', + counts TEXT NOT NULL DEFAULT '{"totalTasks":0,"scoredTasks":0,"skippedTasks":0,"erroredTasks":0}', + aggregateScores TEXT, + summary TEXT, + error TEXT, + provenance TEXT, + metadata TEXT, + createdAt TEXT NOT NULL, + updatedAt TEXT NOT NULL, + startedAt TEXT, + completedAt TEXT, + cancelledAt TEXT +); +CREATE INDEX IF NOT EXISTS idxEvalRunsProjectIdCreatedAt ON eval_runs(projectId, createdAt); +CREATE INDEX IF NOT EXISTS idxEvalRunsProjectTriggerStatus ON eval_runs(projectId, trigger, status); +CREATE INDEX IF NOT EXISTS idxEvalRunsStatusCreatedAt ON eval_runs(status, createdAt); + +CREATE TABLE IF NOT EXISTS eval_task_results ( + id TEXT PRIMARY KEY, + runId TEXT NOT NULL, + taskId TEXT NOT NULL, + taskSnapshot TEXT NOT NULL, + status TEXT NOT NULL, + overallScore REAL, + maxScore REAL, + categoryScores TEXT NOT NULL DEFAULT '[]', + rationale TEXT, + summary TEXT, + evidence TEXT NOT NULL DEFAULT '[]', + deterministicSignals TEXT NOT NULL DEFAULT '[]', + aiSignals TEXT, + followUps TEXT NOT NULL DEFAULT '[]', + provenance TEXT, + metadata TEXT, + createdAt TEXT NOT NULL, + updatedAt TEXT NOT NULL, + FOREIGN KEY (runId) REFERENCES eval_runs(id) ON DELETE CASCADE +); +CREATE INDEX IF NOT EXISTS idxEvalTaskResultsRunIdCreatedAt ON eval_task_results(runId, createdAt); +CREATE INDEX IF NOT EXISTS idxEvalTaskResultsTaskIdCreatedAt ON eval_task_results(taskId, createdAt); +CREATE INDEX IF NOT EXISTS idxEvalTaskResultsStatusRunId ON eval_task_results(status, runId); + +CREATE TABLE IF NOT EXISTS eval_run_events ( + id TEXT PRIMARY KEY, + runId TEXT NOT NULL, + seq INTEGER NOT NULL, + type TEXT NOT NULL, + message TEXT NOT NULL, + status TEXT, + taskId TEXT, + metadata TEXT, + createdAt TEXT NOT NULL, + FOREIGN KEY (runId) REFERENCES eval_runs(id) ON DELETE CASCADE +); +CREATE INDEX IF NOT EXISTS idxEvalRunEventsRunIdSeq ON eval_run_events(runId, seq); + -- Schema version tracking CREATE TABLE IF NOT EXISTS __meta ( key TEXT PRIMARY KEY, @@ -2433,6 +2498,80 @@ export class Database { }); } + if (version < 62) { + this.applyMigration(62, () => { + this.db.exec(` + CREATE TABLE IF NOT EXISTS eval_runs ( + id TEXT PRIMARY KEY, + projectId TEXT NOT NULL, + status TEXT NOT NULL, + trigger TEXT NOT NULL, + scope TEXT NOT NULL, + window TEXT NOT NULL DEFAULT '{}', + requestedTaskIds TEXT NOT NULL DEFAULT '[]', + evaluatedTaskIds TEXT NOT NULL DEFAULT '[]', + counts TEXT NOT NULL DEFAULT '{"totalTasks":0,"scoredTasks":0,"skippedTasks":0,"erroredTasks":0}', + aggregateScores TEXT, + summary TEXT, + error TEXT, + provenance TEXT, + metadata TEXT, + createdAt TEXT NOT NULL, + updatedAt TEXT NOT NULL, + startedAt TEXT, + completedAt TEXT, + cancelledAt TEXT + ) + `); + this.db.exec(`CREATE INDEX IF NOT EXISTS idxEvalRunsProjectIdCreatedAt ON eval_runs(projectId, createdAt)`); + this.db.exec(`CREATE INDEX IF NOT EXISTS idxEvalRunsProjectTriggerStatus ON eval_runs(projectId, trigger, status)`); + this.db.exec(`CREATE INDEX IF NOT EXISTS idxEvalRunsStatusCreatedAt ON eval_runs(status, createdAt)`); + + this.db.exec(` + CREATE TABLE IF NOT EXISTS eval_task_results ( + id TEXT PRIMARY KEY, + runId TEXT NOT NULL, + taskId TEXT NOT NULL, + taskSnapshot TEXT NOT NULL, + status TEXT NOT NULL, + overallScore REAL, + maxScore REAL, + categoryScores TEXT NOT NULL DEFAULT '[]', + rationale TEXT, + summary TEXT, + evidence TEXT NOT NULL DEFAULT '[]', + deterministicSignals TEXT NOT NULL DEFAULT '[]', + aiSignals TEXT, + followUps TEXT NOT NULL DEFAULT '[]', + provenance TEXT, + metadata TEXT, + createdAt TEXT NOT NULL, + updatedAt TEXT NOT NULL, + FOREIGN KEY (runId) REFERENCES eval_runs(id) ON DELETE CASCADE + ) + `); + this.db.exec(`CREATE INDEX IF NOT EXISTS idxEvalTaskResultsRunIdCreatedAt ON eval_task_results(runId, createdAt)`); + this.db.exec(`CREATE INDEX IF NOT EXISTS idxEvalTaskResultsTaskIdCreatedAt ON eval_task_results(taskId, createdAt)`); + this.db.exec(`CREATE INDEX IF NOT EXISTS idxEvalTaskResultsStatusRunId ON eval_task_results(status, runId)`); + + this.db.exec(` + CREATE TABLE IF NOT EXISTS eval_run_events ( + id TEXT PRIMARY KEY, + runId TEXT NOT NULL, + seq INTEGER NOT NULL, + type TEXT NOT NULL, + message TEXT NOT NULL, + status TEXT, + taskId TEXT, + metadata TEXT, + createdAt TEXT NOT NULL, + FOREIGN KEY (runId) REFERENCES eval_runs(id) ON DELETE CASCADE + ) + `); + this.db.exec(`CREATE INDEX IF NOT EXISTS idxEvalRunEventsRunIdSeq ON eval_run_events(runId, seq)`); + }); + } + } /** diff --git a/packages/core/src/eval-automation.ts b/packages/core/src/eval-automation.ts new file mode 100644 index 000000000..7556b8a79 --- /dev/null +++ b/packages/core/src/eval-automation.ts @@ -0,0 +1,334 @@ +import type { AutomationStore } from "./automation-store.js"; +import type { ScheduledTask, ScheduledTaskCreateInput } from "./automation.js"; +import type { EvalRun, EvalTaskResultCreateInput } from "./eval-types.js"; +import { EvalLifecycleError } from "./eval-store.js"; +import type { ProjectSettings, Task } from "./types.js"; + +export const TASK_EVALUATION_SCHEDULE_NAME = "Scheduled Task Evaluation"; +export const DEFAULT_TASK_EVALUATION_SCHEDULE = "0 5 * * *"; +export const TASK_EVALUATION_SCHEDULE_COMMAND = "fn eval --scheduled-batch"; + +export interface ResolvedTaskEvaluationSettings { + taskEvaluationEnabled: boolean; + taskEvaluationSchedule: string; + taskEvaluationProvider?: string; + taskEvaluationModelId?: string; + taskEvaluationFollowUpPolicy: "off" | "suggest" | "create"; + taskEvaluationRetention?: number; +} + +export function resolveTaskEvaluationSettings( + settings: Partial, +): ResolvedTaskEvaluationSettings { + const evalSettings = settings as Partial; + return { + taskEvaluationEnabled: evalSettings.taskEvaluationEnabled ?? false, + taskEvaluationSchedule: evalSettings.taskEvaluationSchedule ?? DEFAULT_TASK_EVALUATION_SCHEDULE, + taskEvaluationProvider: evalSettings.taskEvaluationProvider, + taskEvaluationModelId: evalSettings.taskEvaluationModelId, + taskEvaluationFollowUpPolicy: evalSettings.taskEvaluationFollowUpPolicy ?? "off", + taskEvaluationRetention: evalSettings.taskEvaluationRetention, + }; +} + +export function createScheduledEvalBatchAutomation( + settings: Partial, +): ScheduledTaskCreateInput { + const resolved = resolveTaskEvaluationSettings(settings); + return { + name: TASK_EVALUATION_SCHEDULE_NAME, + description: "Evaluates tasks completed since the previous scheduled evaluation batch", + scheduleType: "custom", + cronExpression: resolved.taskEvaluationSchedule, + command: TASK_EVALUATION_SCHEDULE_COMMAND, + enabled: true, + scope: "project", + }; +} + +export async function syncScheduledEvalBatchAutomation( + automationStore: AutomationStore, + settings: Partial, +): Promise { + const { AutomationStore } = await import("./automation-store.js"); + const resolved = resolveTaskEvaluationSettings(settings); + const schedules = await automationStore.listSchedules(); + const existing = schedules.find((s) => s.name === TASK_EVALUATION_SCHEDULE_NAME); + + if (!resolved.taskEvaluationEnabled) { + if (existing) await automationStore.deleteSchedule(existing.id); + return undefined; + } + + if (!AutomationStore.isValidCron(resolved.taskEvaluationSchedule)) { + throw new Error(`Invalid task evaluation schedule: ${resolved.taskEvaluationSchedule}`); + } + + const input = createScheduledEvalBatchAutomation(settings); + if (existing) { + return automationStore.updateSchedule(existing.id, { + scheduleType: "custom", + cronExpression: input.cronExpression, + command: input.command, + enabled: true, + scope: "project", + }); + } + + return automationStore.createSchedule(input); +} + +export interface EvalBatchWindow { + windowStartExclusive?: string; + windowEndInclusive: string; +} + +export interface CompletedTaskEvaluationContext { + run: EvalRun; + task: Task; + taskIndex: number; + totalTasks: number; + window: EvalBatchWindow; +} + +export type CompletedTaskEvaluator = ( + context: CompletedTaskEvaluationContext, +) => Promise>; + +export interface EvalBatchTaskStore { + listTasks(options?: { column?: string }): Promise; + getEvalStore(): import("./eval-store.js").EvalStore; +} + +export interface RunScheduledEvalBatchParams { + store: EvalBatchTaskStore; + projectId: string; + evaluator: CompletedTaskEvaluator; + startedAt?: string; +} + +export interface ScheduledEvalBatchResult { + runId: string; + status: "completed" | "failed"; + windowStartExclusive?: string; + windowEndInclusive: string; + selectedTaskIds: string[]; + tasksSelected: number; +} + +export async function runScheduledEvalBatch( + params: RunScheduledEvalBatchParams, +): Promise { + const startedAt = params.startedAt ?? new Date().toISOString(); + const evalStore = params.store.getEvalStore(); + const priorRuns = evalStore + .listRuns({ projectId: params.projectId, trigger: "schedule" }) + .filter((run) => run.status === "completed") + .sort((a, b) => { + const aWindowEnd = (a.metadata?.windowEndInclusive as string | undefined) ?? a.window.until ?? ""; + const bWindowEnd = (b.metadata?.windowEndInclusive as string | undefined) ?? b.window.until ?? ""; + if (aWindowEnd !== bWindowEnd) return aWindowEnd.localeCompare(bWindowEnd); + return a.id.localeCompare(b.id); + }); + + const previousScheduledBatch = priorRuns.at(-1); + const windowStartExclusive = + (previousScheduledBatch?.metadata?.windowEndInclusive as string | undefined) ?? + previousScheduledBatch?.window.until; + const windowEndInclusive = startedAt; + + let run: EvalRun; + try { + run = evalStore.createRun({ + projectId: params.projectId, + trigger: "schedule", + scope: "completed-tasks", + window: { + since: windowStartExclusive, + until: windowEndInclusive, + }, + metadata: { + windowStartExclusive, + windowEndInclusive, + }, + }); + } catch (error) { + if (error instanceof EvalLifecycleError && error.code === "active_run_conflict") { + throw error; + } + throw error; + } + + evalStore.appendRunEvent(run.id, { + type: "info", + message: "Scheduled eval batch started", + status: "pending", + metadata: { windowStartExclusive, windowEndInclusive }, + }); + + evalStore.updateRun(run.id, { status: "running", startedAt }); + + try { + const doneTasks = (await params.store.listTasks({ column: "done" })).filter((task) => + task.column === "done" + && Boolean(task.executionCompletedAt) + && (!windowStartExclusive || task.executionCompletedAt! > windowStartExclusive) + && task.executionCompletedAt! <= windowEndInclusive, + ); + + doneTasks.sort((a, b) => { + const byCompletedAt = (a.executionCompletedAt ?? "").localeCompare(b.executionCompletedAt ?? ""); + if (byCompletedAt !== 0) return byCompletedAt; + const byCreatedAt = a.createdAt.localeCompare(b.createdAt); + if (byCreatedAt !== 0) return byCreatedAt; + return a.id.localeCompare(b.id); + }); + + const selectedTaskIds = doneTasks.map((task) => task.id); + evalStore.updateRun(run.id, { + counts: { totalTasks: selectedTaskIds.length, scoredTasks: 0, skippedTasks: 0, erroredTasks: 0 }, + metadata: { + windowStartExclusive, + windowEndInclusive, + selectedTaskIds, + tasksSelected: selectedTaskIds.length, + }, + }); + + if (doneTasks.length === 0) { + evalStore.appendRunEvent(run.id, { + type: "info", + status: "completed", + message: "Scheduled eval batch completed with no newly done tasks", + metadata: { tasksSelected: 0 }, + }); + evalStore.updateRun(run.id, { + status: "completed", + completedAt: new Date().toISOString(), + summary: "No newly completed tasks found in evaluation window", + }); + return { + runId: run.id, + status: "completed", + windowStartExclusive, + windowEndInclusive, + selectedTaskIds: [], + tasksSelected: 0, + }; + } + + let scoredTasks = 0; + let skippedTasks = 0; + let erroredTasks = 0; + const evaluatedTaskIds: string[] = []; + + for (const [index, task] of doneTasks.entries()) { + try { + const result = await params.evaluator({ + run, + task, + taskIndex: index, + totalTasks: doneTasks.length, + window: { windowStartExclusive, windowEndInclusive }, + }); + + evalStore.createTaskResult(run.id, { + ...result, + taskId: task.id, + taskSnapshot: { + taskId: task.id, + title: task.title, + column: task.column, + createdAt: task.createdAt, + updatedAt: task.updatedAt, + executionCompletedAt: task.executionCompletedAt, + summary: task.summary, + }, + metadata: { + ...(result.metadata ?? {}), + windowEndInclusive, + }, + }); + + evaluatedTaskIds.push(task.id); + if (result.status === "scored") scoredTasks += 1; + else if (result.status === "skipped") skippedTasks += 1; + else erroredTasks += 1; + + evalStore.appendRunEvent(run.id, { + type: "task_evaluated", + message: `Evaluated task ${task.id}`, + taskId: task.id, + metadata: { status: result.status }, + }); + } catch (error) { + erroredTasks += 1; + evalStore.appendRunEvent(run.id, { + type: "error", + message: `Failed evaluating task ${task.id}`, + taskId: task.id, + metadata: { error: error instanceof Error ? error.message : String(error) }, + }); + } + } + + evalStore.updateRun(run.id, { + status: "completed", + evaluatedTaskIds, + counts: { + totalTasks: doneTasks.length, + scoredTasks, + skippedTasks, + erroredTasks, + }, + completedAt: new Date().toISOString(), + summary: `Scheduled eval batch completed for ${doneTasks.length} task(s)`, + metadata: { + windowStartExclusive, + windowEndInclusive, + selectedTaskIds, + tasksSelected: selectedTaskIds.length, + }, + }); + + evalStore.appendRunEvent(run.id, { + type: "status_changed", + status: "completed", + message: `Scheduled eval batch completed (${doneTasks.length} tasks selected)`, + metadata: { scoredTasks, skippedTasks, erroredTasks }, + }); + + return { + runId: run.id, + status: "completed", + windowStartExclusive, + windowEndInclusive, + selectedTaskIds, + tasksSelected: selectedTaskIds.length, + }; + } catch (error) { + evalStore.updateRun(run.id, { + status: "failed", + completedAt: new Date().toISOString(), + error: error instanceof Error ? error.message : String(error), + metadata: { + windowStartExclusive, + windowEndInclusive, + }, + }); + evalStore.appendRunEvent(run.id, { + type: "error", + status: "failed", + message: "Scheduled eval batch failed", + metadata: { error: error instanceof Error ? error.message : String(error) }, + }); + return { + runId: run.id, + status: "failed", + windowStartExclusive, + windowEndInclusive, + selectedTaskIds: [], + tasksSelected: 0, + }; + } +} diff --git a/packages/core/src/eval-store.ts b/packages/core/src/eval-store.ts new file mode 100644 index 000000000..fec60a4f2 --- /dev/null +++ b/packages/core/src/eval-store.ts @@ -0,0 +1,454 @@ +import { EventEmitter } from "node:events"; +import { randomUUID } from "node:crypto"; +import type { Database } from "./db.js"; +import { fromJson, toJson, toJsonNullable } from "./db.js"; +import type { + EvalRun, + EvalRunCreateInput, + EvalRunEvent, + EvalRunListOptions, + EvalRunStatus, + EvalRunUpdateInput, + EvalStoreEvents, + EvalTaskResult, + EvalTaskResultCreateInput, + EvalTaskResultListOptions, + EvalTaskResultUpdateInput, +} from "./eval-types.js"; + +const TERMINAL_STATUSES = new Set(["completed", "failed", "cancelled"]); +const ACTIVE_STATUSES = new Set(["pending", "running"]); +const VALID_TRANSITIONS: Record = { + pending: ["running", "completed", "failed", "cancelled"], + running: ["completed", "failed", "cancelled"], + completed: [], + failed: [], + cancelled: [], +}; + +export class EvalLifecycleError extends Error { + constructor(message: string, readonly code: "invalid_transition" | "terminal_immutable" | "active_run_conflict") { + super(message); + this.name = "EvalLifecycleError"; + } +} + +function generateRunId(): string { + return `ER-${Date.now().toString(36).toUpperCase()}-${Math.random().toString(36).slice(2, 7).toUpperCase()}`; +} + +function generateResultId(): string { + return `ETR-${randomUUID()}`; +} + +function generateEventId(): string { + return `ERE-${randomUUID()}`; +} + +export class EvalStore extends EventEmitter { + constructor(private readonly db: Database) { + super(); + this.setMaxListeners(50); + } + + createRun(input: EvalRunCreateInput): EvalRun { + const now = new Date().toISOString(); + if ((input.trigger === "schedule" || input.trigger === "task_completion") && this.hasActiveRun(input.projectId, input.trigger)) { + throw new EvalLifecycleError(`Active eval run already exists for project ${input.projectId} trigger ${input.trigger}`, "active_run_conflict"); + } + + const run: EvalRun = { + id: generateRunId(), + projectId: input.projectId, + status: "pending", + trigger: input.trigger ?? "manual", + scope: input.scope, + window: input.window ?? {}, + requestedTaskIds: input.requestedTaskIds ?? [], + evaluatedTaskIds: [], + counts: { totalTasks: input.requestedTaskIds?.length ?? 0, scoredTasks: 0, skippedTasks: 0, erroredTasks: 0 }, + provenance: input.provenance, + metadata: input.metadata, + createdAt: now, + updatedAt: now, + }; + + this.db.prepare(` + INSERT INTO eval_runs ( + id, projectId, status, trigger, scope, window, requestedTaskIds, evaluatedTaskIds, + counts, aggregateScores, summary, error, provenance, metadata, + createdAt, updatedAt, startedAt, completedAt, cancelledAt + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `).run( + run.id, + run.projectId, + run.status, + run.trigger, + run.scope, + toJson(run.window), + toJson(run.requestedTaskIds), + toJson(run.evaluatedTaskIds), + toJson(run.counts), + null, + null, + null, + toJsonNullable(run.provenance), + toJsonNullable(run.metadata), + run.createdAt, + run.updatedAt, + null, + null, + null, + ); + + this.db.bumpLastModified(); + this.emit("run:created", run); + return run; + } + + getRun(id: string): EvalRun | undefined { + const row = this.db.prepare("SELECT * FROM eval_runs WHERE id = ?").get(id) as Record | undefined; + return row ? this.rowToRun(row) : undefined; + } + + listRuns(options: EvalRunListOptions = {}): EvalRun[] { + const clauses: string[] = []; + const params: Array = []; + if (options.projectId) { + clauses.push("projectId = ?"); + params.push(options.projectId); + } + if (options.status) { + clauses.push("status = ?"); + params.push(options.status); + } + if (options.trigger) { + clauses.push("trigger = ?"); + params.push(options.trigger); + } + const where = clauses.length ? `WHERE ${clauses.join(" AND ")}` : ""; + const limit = options.limit !== undefined ? `LIMIT ${options.limit}` : ""; + const offset = options.offset !== undefined ? `OFFSET ${options.offset}` : ""; + + const rows = this.db.prepare(` + SELECT * FROM eval_runs + ${where} + ORDER BY createdAt ASC, id ASC + ${limit} + ${offset} + `).all(...params) as Record[]; + + return rows.map((row) => this.rowToRun(row)); + } + + updateRun(id: string, input: EvalRunUpdateInput): EvalRun | undefined { + const existing = this.getRun(id); + if (!existing) return undefined; + + if (TERMINAL_STATUSES.has(existing.status) && Object.keys(input).some((k) => k !== "status")) { + throw new EvalLifecycleError(`Eval run ${id} is terminal and immutable`, "terminal_immutable"); + } + + if (input.status && input.status !== existing.status) { + if (!VALID_TRANSITIONS[existing.status].includes(input.status)) { + throw new EvalLifecycleError(`Invalid eval run status transition: ${existing.status} -> ${input.status}`, "invalid_transition"); + } + } + + const now = new Date().toISOString(); + const updated: EvalRun = { + ...existing, + ...input, + error: input.error === null ? undefined : (input.error ?? existing.error), + metadata: input.metadata ? { ...(existing.metadata ?? {}), ...input.metadata } : existing.metadata, + provenance: input.provenance ? { ...(existing.provenance ?? {}), ...input.provenance } : existing.provenance, + updatedAt: now, + startedAt: input.startedAt === null ? undefined : (input.startedAt ?? existing.startedAt), + completedAt: input.completedAt === null ? undefined : (input.completedAt ?? existing.completedAt), + cancelledAt: input.cancelledAt === null ? undefined : (input.cancelledAt ?? existing.cancelledAt), + }; + + this.persistRun(updated); + this.emit("run:updated", updated); + return updated; + } + + deleteRun(id: string): boolean { + const result = this.db.prepare("DELETE FROM eval_runs WHERE id = ?").run(id) as { changes?: number }; + const deleted = (result.changes ?? 0) > 0; + if (deleted) { + this.db.bumpLastModified(); + this.emit("run:deleted", id); + } + return deleted; + } + + createTaskResult(runId: string, input: EvalTaskResultCreateInput): EvalTaskResult { + const run = this.getRun(runId); + if (!run) throw new Error(`Eval run not found: ${runId}`); + + const now = new Date().toISOString(); + const result: EvalTaskResult = { + id: generateResultId(), + runId, + taskId: input.taskId, + taskSnapshot: input.taskSnapshot, + status: input.status, + overallScore: input.overallScore, + maxScore: input.maxScore, + categoryScores: input.categoryScores ?? [], + rationale: input.rationale, + summary: input.summary, + evidence: input.evidence ?? [], + deterministicSignals: input.deterministicSignals ?? [], + aiSignals: input.aiSignals, + followUps: input.followUps ?? [], + provenance: input.provenance, + metadata: input.metadata, + createdAt: now, + updatedAt: now, + }; + + this.db.prepare(` + INSERT INTO eval_task_results ( + id, runId, taskId, taskSnapshot, status, overallScore, maxScore, + categoryScores, rationale, summary, evidence, deterministicSignals, aiSignals, + followUps, provenance, metadata, createdAt, updatedAt + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `).run( + result.id, + result.runId, + result.taskId, + toJson(result.taskSnapshot), + result.status, + result.overallScore ?? null, + result.maxScore ?? null, + toJson(result.categoryScores), + result.rationale ?? null, + result.summary ?? null, + toJson(result.evidence), + toJson(result.deterministicSignals), + toJsonNullable(result.aiSignals), + toJson(result.followUps), + toJsonNullable(result.provenance), + toJsonNullable(result.metadata), + result.createdAt, + result.updatedAt, + ); + + this.db.bumpLastModified(); + this.emit("result:created", result); + return result; + } + + getTaskResult(id: string): EvalTaskResult | undefined { + const row = this.db.prepare("SELECT * FROM eval_task_results WHERE id = ?").get(id) as Record | undefined; + return row ? this.rowToResult(row) : undefined; + } + + listTaskResults(options: EvalTaskResultListOptions = {}): EvalTaskResult[] { + const clauses: string[] = []; + const params: Array = []; + if (options.runId) { + clauses.push("runId = ?"); + params.push(options.runId); + } + if (options.taskId) { + clauses.push("taskId = ?"); + params.push(options.taskId); + } + if (options.status) { + clauses.push("status = ?"); + params.push(options.status); + } + const where = clauses.length ? `WHERE ${clauses.join(" AND ")}` : ""; + const limit = options.limit !== undefined ? `LIMIT ${options.limit}` : ""; + const offset = options.offset !== undefined ? `OFFSET ${options.offset}` : ""; + + const rows = this.db.prepare(` + SELECT * FROM eval_task_results + ${where} + ORDER BY createdAt ASC, id ASC + ${limit} + ${offset} + `).all(...params) as Record[]; + return rows.map((row) => this.rowToResult(row)); + } + + updateTaskResult(id: string, input: EvalTaskResultUpdateInput): EvalTaskResult | undefined { + const existing = this.getTaskResult(id); + if (!existing) return undefined; + + const now = new Date().toISOString(); + const updated: EvalTaskResult = { + ...existing, + ...input, + metadata: input.metadata ? { ...(existing.metadata ?? {}), ...input.metadata } : existing.metadata, + provenance: input.provenance ? { ...(existing.provenance ?? {}), ...input.provenance } : existing.provenance, + updatedAt: now, + }; + + this.db.prepare(` + UPDATE eval_task_results SET + status = ?, overallScore = ?, maxScore = ?, categoryScores = ?, rationale = ?, summary = ?, + evidence = ?, deterministicSignals = ?, aiSignals = ?, followUps = ?, provenance = ?, metadata = ?, updatedAt = ? + WHERE id = ? + `).run( + updated.status, + updated.overallScore ?? null, + updated.maxScore ?? null, + toJson(updated.categoryScores), + updated.rationale ?? null, + updated.summary ?? null, + toJson(updated.evidence), + toJson(updated.deterministicSignals), + toJsonNullable(updated.aiSignals), + toJson(updated.followUps), + toJsonNullable(updated.provenance), + toJsonNullable(updated.metadata), + updated.updatedAt, + id, + ); + + this.db.bumpLastModified(); + this.emit("result:updated", updated); + return updated; + } + + appendRunEvent(runId: string, event: Omit): EvalRunEvent { + const run = this.getRun(runId); + if (!run) throw new Error(`Eval run not found: ${runId}`); + + const maxSeq = this.db.prepare("SELECT COALESCE(MAX(seq), 0) as maxSeq FROM eval_run_events WHERE runId = ?").get(runId) as { maxSeq: number }; + const created: EvalRunEvent = { + id: generateEventId(), + runId, + seq: (maxSeq?.maxSeq ?? 0) + 1, + type: event.type, + message: event.message, + status: event.status, + taskId: event.taskId, + metadata: event.metadata, + createdAt: new Date().toISOString(), + }; + + this.db.prepare(` + INSERT INTO eval_run_events (id, runId, seq, type, message, status, taskId, metadata, createdAt) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + `).run( + created.id, + created.runId, + created.seq, + created.type, + created.message, + created.status ?? null, + created.taskId ?? null, + toJsonNullable(created.metadata), + created.createdAt, + ); + + this.db.bumpLastModified(); + this.emit("run:event", { runId, event: created }); + return created; + } + + listRunEvents(runId: string): EvalRunEvent[] { + const rows = this.db.prepare("SELECT * FROM eval_run_events WHERE runId = ? ORDER BY seq ASC, id ASC").all(runId) as Record[]; + return rows.map((row) => this.rowToEvent(row)); + } + + private hasActiveRun(projectId: string, trigger: string): boolean { + const placeholders = Array.from(ACTIVE_STATUSES).map(() => "?").join(", "); + const row = this.db.prepare(`SELECT id FROM eval_runs WHERE projectId = ? AND trigger = ? AND status IN (${placeholders}) LIMIT 1`) + .get(projectId, trigger, ...Array.from(ACTIVE_STATUSES)) as { id?: string } | undefined; + return Boolean(row?.id); + } + + private persistRun(run: EvalRun): void { + this.db.prepare(` + UPDATE eval_runs SET + status = ?, scope = ?, window = ?, requestedTaskIds = ?, evaluatedTaskIds = ?, counts = ?, aggregateScores = ?, + summary = ?, error = ?, provenance = ?, metadata = ?, updatedAt = ?, startedAt = ?, completedAt = ?, cancelledAt = ? + WHERE id = ? + `).run( + run.status, + run.scope, + toJson(run.window), + toJson(run.requestedTaskIds), + toJson(run.evaluatedTaskIds), + toJson(run.counts), + toJsonNullable(run.aggregateScores), + run.summary ?? null, + run.error ?? null, + toJsonNullable(run.provenance), + toJsonNullable(run.metadata), + run.updatedAt, + run.startedAt ?? null, + run.completedAt ?? null, + run.cancelledAt ?? null, + run.id, + ); + this.db.bumpLastModified(); + } + + private rowToRun(row: Record): EvalRun { + return { + id: String(row.id), + projectId: String(row.projectId), + status: row.status as EvalRunStatus, + trigger: row.trigger as EvalRun["trigger"], + scope: String(row.scope), + window: fromJson(row.window as string) ?? {}, + requestedTaskIds: fromJson(row.requestedTaskIds as string) ?? [], + evaluatedTaskIds: fromJson(row.evaluatedTaskIds as string) ?? [], + counts: fromJson(row.counts as string) ?? { totalTasks: 0, scoredTasks: 0, skippedTasks: 0, erroredTasks: 0 }, + aggregateScores: fromJson>(row.aggregateScores as string), + summary: (row.summary as string | null) ?? undefined, + error: (row.error as string | null) ?? undefined, + provenance: fromJson(row.provenance as string), + metadata: fromJson(row.metadata as string), + createdAt: String(row.createdAt), + updatedAt: String(row.updatedAt), + startedAt: (row.startedAt as string | null) ?? undefined, + completedAt: (row.completedAt as string | null) ?? undefined, + cancelledAt: (row.cancelledAt as string | null) ?? undefined, + }; + } + + private rowToResult(row: Record): EvalTaskResult { + return { + id: String(row.id), + runId: String(row.runId), + taskId: String(row.taskId), + taskSnapshot: fromJson(row.taskSnapshot as string) ?? { taskId: String(row.taskId) }, + status: row.status as EvalTaskResult["status"], + overallScore: row.overallScore == null ? undefined : Number(row.overallScore), + maxScore: row.maxScore == null ? undefined : Number(row.maxScore), + categoryScores: fromJson(row.categoryScores as string) ?? [], + rationale: (row.rationale as string | null) ?? undefined, + summary: (row.summary as string | null) ?? undefined, + evidence: fromJson(row.evidence as string) ?? [], + deterministicSignals: fromJson(row.deterministicSignals as string) ?? [], + aiSignals: fromJson(row.aiSignals as string), + followUps: fromJson(row.followUps as string) ?? [], + provenance: fromJson(row.provenance as string), + metadata: fromJson(row.metadata as string), + createdAt: String(row.createdAt), + updatedAt: String(row.updatedAt), + }; + } + + private rowToEvent(row: Record): EvalRunEvent { + return { + id: String(row.id), + runId: String(row.runId), + seq: Number(row.seq), + type: row.type as EvalRunEvent["type"], + message: String(row.message), + status: (row.status as EvalRunStatus | null) ?? undefined, + taskId: (row.taskId as string | null) ?? undefined, + metadata: fromJson(row.metadata as string), + createdAt: String(row.createdAt), + }; + } +} diff --git a/packages/core/src/eval-types.ts b/packages/core/src/eval-types.ts new file mode 100644 index 000000000..04c6eb721 --- /dev/null +++ b/packages/core/src/eval-types.ts @@ -0,0 +1,241 @@ +/** + * Eval Domain Types + * + * Contracts for eval run persistence and per-task evaluation results. + */ + +export const EVAL_RUN_STATUSES = [ + "pending", + "running", + "completed", + "failed", + "cancelled", +] as const; + +export type EvalRunStatus = typeof EVAL_RUN_STATUSES[number]; + +export const EVAL_RUN_TRIGGERS = ["manual", "schedule", "api", "task_completion"] as const; + +export type EvalRunTrigger = typeof EVAL_RUN_TRIGGERS[number]; + +export const EVAL_SCORE_CATEGORIES = [ + "correctness", + "completeness", + "quality", + "reliability", + "tests", + "documentation", +] as const; + +export type EvalScoreCategory = typeof EVAL_SCORE_CATEGORIES[number]; + +export interface EvalTaskSnapshot { + taskId: string; + title?: string; + column?: string; + status?: string; + priority?: string; + size?: string; + reviewLevel?: number; + createdAt?: string; + updatedAt?: string; + executionCompletedAt?: string; + summary?: string; + labels?: string[]; + metadata?: Record; +} + +export interface EvalRunWindow { + since?: string; + until?: string; + baselineRunId?: string; + windowStartExclusive?: string; + windowEndInclusive?: string; +} + +export interface EvalProvenance { + evaluatorProvider?: string; + evaluatorModelId?: string; + evaluatorVersion?: string; + promptVersion?: string; + runConfig?: Record; + metadata?: Record; +} + +export interface EvalSignal { + signalId: string; + kind: string; + name: string; + passed?: boolean; + score?: number; + value?: number | string | boolean | null; + threshold?: number; + unit?: string; + summary?: string; + details?: Record; +} + +export interface EvalEvidenceReference { + type: "task_log" | "task_document" | "file" | "command" | "test" | "other"; + ref: string; + excerpt?: string; + metadata?: Record; +} + +export interface EvalCategoryScore { + category: EvalScoreCategory | string; + score: number; + maxScore?: number; + rationale?: string; +} + +export interface EvalFollowUpSuggestion { + title: string; + description: string; + priority?: "low" | "normal" | "high" | "urgent"; + tags?: string[]; + metadata?: Record; +} + +export interface EvalTaskResult { + id: string; + runId: string; + taskId: string; + taskSnapshot: EvalTaskSnapshot; + status: "scored" | "skipped" | "error"; + overallScore?: number; + maxScore?: number; + categoryScores: EvalCategoryScore[]; + rationale?: string; + summary?: string; + evidence: EvalEvidenceReference[]; + deterministicSignals: EvalSignal[]; + aiSignals?: EvalSignal[]; + followUps: EvalFollowUpSuggestion[]; + provenance?: EvalProvenance; + metadata?: Record; + createdAt: string; + updatedAt: string; +} + +export interface EvalRunCounts { + totalTasks: number; + scoredTasks: number; + skippedTasks: number; + erroredTasks: number; +} + +export interface EvalRun { + id: string; + projectId: string; + status: EvalRunStatus; + trigger: EvalRunTrigger; + scope: string; + window: EvalRunWindow; + requestedTaskIds: string[]; + evaluatedTaskIds: string[]; + counts: EvalRunCounts; + aggregateScores?: Record; + summary?: string; + error?: string; + provenance?: EvalProvenance; + metadata?: Record; + createdAt: string; + updatedAt: string; + startedAt?: string; + completedAt?: string; + cancelledAt?: string; +} + +export interface EvalRunEvent { + id: string; + runId: string; + seq: number; + type: "status_changed" | "task_evaluated" | "info" | "warning" | "error"; + message: string; + status?: EvalRunStatus; + taskId?: string; + metadata?: Record; + createdAt: string; +} + +export interface EvalRunCreateInput { + projectId: string; + trigger?: EvalRunTrigger; + scope: string; + window?: EvalRunWindow; + requestedTaskIds?: string[]; + provenance?: EvalProvenance; + metadata?: Record; +} + +export interface EvalRunUpdateInput { + status?: EvalRunStatus; + evaluatedTaskIds?: string[]; + counts?: EvalRunCounts; + aggregateScores?: Record; + summary?: string; + error?: string | null; + provenance?: EvalProvenance; + metadata?: Record; + startedAt?: string | null; + completedAt?: string | null; + cancelledAt?: string | null; +} + +export interface EvalRunListOptions { + projectId?: string; + status?: EvalRunStatus; + trigger?: EvalRunTrigger; + limit?: number; + offset?: number; +} + +export interface EvalTaskResultCreateInput { + taskId: string; + taskSnapshot: EvalTaskSnapshot; + status: "scored" | "skipped" | "error"; + overallScore?: number; + maxScore?: number; + categoryScores?: EvalCategoryScore[]; + rationale?: string; + summary?: string; + evidence?: EvalEvidenceReference[]; + deterministicSignals?: EvalSignal[]; + aiSignals?: EvalSignal[]; + followUps?: EvalFollowUpSuggestion[]; + provenance?: EvalProvenance; + metadata?: Record; +} + +export interface EvalTaskResultUpdateInput { + status?: "scored" | "skipped" | "error"; + overallScore?: number; + maxScore?: number; + categoryScores?: EvalCategoryScore[]; + rationale?: string; + summary?: string; + evidence?: EvalEvidenceReference[]; + deterministicSignals?: EvalSignal[]; + aiSignals?: EvalSignal[]; + followUps?: EvalFollowUpSuggestion[]; + provenance?: EvalProvenance; + metadata?: Record; +} + +export interface EvalTaskResultListOptions { + runId?: string; + taskId?: string; + status?: "scored" | "skipped" | "error"; + limit?: number; + offset?: number; +} + +export interface EvalStoreEvents { + "run:created": [EvalRun]; + "run:updated": [EvalRun]; + "run:deleted": [string]; + "run:event": [{ runId: string; event: EvalRunEvent }]; + "result:created": [EvalTaskResult]; + "result:updated": [EvalTaskResult]; +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 365180f9e..fbe17d0cc 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -695,6 +695,30 @@ export type { ResolvedResearchSettings } from "./research-settings.js"; export { TodoStore } from "./todo-store.js"; export type { TodoStoreEvents } from "./todo-store.js"; +export { EvalLifecycleError, EvalStore } from "./eval-store.js"; +export type { + EvalRun, + EvalRunStatus, + EvalRunTrigger, + EvalRunWindow, + EvalRunCounts, + EvalRunEvent, + EvalRunCreateInput, + EvalRunUpdateInput, + EvalRunListOptions, + EvalTaskSnapshot, + EvalTaskResult, + EvalTaskResultCreateInput, + EvalTaskResultUpdateInput, + EvalTaskResultListOptions, + EvalCategoryScore, + EvalEvidenceReference, + EvalSignal, + EvalFollowUpSuggestion, + EvalProvenance, + EvalStoreEvents, +} from "./eval-types.js"; +export { EVAL_RUN_STATUSES, EVAL_RUN_TRIGGERS, EVAL_SCORE_CATEGORIES } from "./eval-types.js"; // ── Agent Companies Types ────────────────────────────────── diff --git a/packages/core/src/settings-schema.ts b/packages/core/src/settings-schema.ts index 9f0b32262..5de9a7cb6 100644 --- a/packages/core/src/settings-schema.ts +++ b/packages/core/src/settings-schema.ts @@ -242,6 +242,12 @@ export const DEFAULT_PROJECT_SETTINGS = { insightExtractionEnabled: false, insightExtractionSchedule: "0 2 * * *", insightExtractionMinIntervalMs: 86_400_000, + taskEvaluationEnabled: false, + taskEvaluationSchedule: "0 5 * * *", + taskEvaluationProvider: undefined, + taskEvaluationModelId: undefined, + taskEvaluationFollowUpPolicy: "off", + taskEvaluationRetention: undefined, memoryEnabled: true, memoryBackendType: "qmd", memoryAutoSummarizeEnabled: false, diff --git a/packages/core/src/store.ts b/packages/core/src/store.ts index 729244886..d4b72558a 100644 --- a/packages/core/src/store.ts +++ b/packages/core/src/store.ts @@ -16,6 +16,7 @@ import { RoadmapStore } from "./roadmap-store.js"; import { InsightStore } from "./insight-store.js"; import { ResearchStore } from "./research-store.js"; import { TodoStore } from "./todo-store.js"; +import { EvalStore } from "./eval-store.js"; import { BackwardCompat, ProjectRequiredError } from "./migration.js"; import { CentralCore } from "./central-core.js"; import { getTaskMergeBlocker } from "./task-merge.js"; @@ -512,6 +513,8 @@ export class TaskStore extends EventEmitter { private researchStore: ResearchStore | null = null; /** Cached TodoStore instance */ private todoStore: TodoStore | null = null; + /** Cached EvalStore instance */ + private evalStore: EvalStore | null = null; /** Buffer for batching agent log writes to reduce WAL pressure. */ private agentLogBuffer: Array<{ @@ -4577,10 +4580,10 @@ export class TaskStore extends EventEmitter { } const elapsed = Date.now() - startTime; - if (elapsed > 100) { + if (elapsed > 750) { storeLog.warn("checkForChanges took longer than expected", { elapsedMs: elapsed, - thresholdMs: 100, + thresholdMs: 750, }); } } catch (err) { @@ -5166,52 +5169,73 @@ export class TaskStore extends EventEmitter { } } - // Phase 3: Invalidate stale spec approval when a user comments on - // a triage task that is awaiting manual approval. The new comment - // means the spec is now stale and must be re-specified/re-reviewed. + // Phase 3: user comments on already-planned, non-executing work should + // trigger triage re-specification. This includes awaiting-approval + // invalidation and todo/triage tasks that have a real non-bootstrap spec. // This remains best-effort: failures are logged for observability but // never fail the comment add operation itself. // Note: The `task` returned above reflects the state BEFORE this // transition. Callers that need the post-transition status should // re-read the task (e.g., via getTask). - if ( - task.column === "triage" - && task.status === "awaiting-approval" - && author === "user" - ) { - let invalidatedStatus = false; + if (author === "user" && (task.column === "todo" || task.column === "triage")) { + let hasRealPrompt = false; try { - await this.updateTask(id, { - status: "needs-replan", - }); - invalidatedStatus = true; + const promptPath = join(this.taskDir(id), "PROMPT.md"); + if (existsSync(promptPath)) { + const prompt = await readFile(promptPath, "utf-8"); + hasRealPrompt = !isBootstrapPromptStub(prompt, task.id, task.title, task.description); + } } catch (err) { - storeLog.warn("Best-effort post-comment awaiting-approval invalidation failed", { + storeLog.warn("Best-effort post-comment re-triage prompt-read failed", { ...commentContextBase, - phase: "addComment:awaiting-approval-invalidation", - stage: "status-update", - nextStatus: "needs-replan", + phase: "addComment:retriage-prompt-read", error: err instanceof Error ? err.message : String(err), }); } - if (invalidatedStatus) { + const shouldInvalidateAwaitingApproval = + task.column === "triage" && task.status === "awaiting-approval"; + const shouldRetriagePlannedTask = hasRealPrompt + && ( + task.column === "todo" + || (task.column === "triage" && task.status !== "awaiting-approval") + ); + + if (shouldInvalidateAwaitingApproval || shouldRetriagePlannedTask) { + const phase = shouldInvalidateAwaitingApproval + ? "addComment:awaiting-approval-invalidation" + : "addComment:planned-task-retriage"; + const action = shouldInvalidateAwaitingApproval + ? "User comment invalidated spec approval — task needs re-specification" + : "User comment requested re-specification of planned task"; + let transitioned = false; + try { - await this.logEntry( - id, - `User comment invalidated spec approval — task needs re-specification`, - undefined, - runContext, - ); + await this.updateTask(id, { status: "needs-replan" }); + transitioned = true; } catch (err) { - storeLog.warn("Best-effort post-comment awaiting-approval invalidation failed", { + storeLog.warn("Best-effort post-comment re-triage failed", { ...commentContextBase, - phase: "addComment:awaiting-approval-invalidation", - stage: "post-invalidation-log-entry", + phase, + stage: "status-update", nextStatus: "needs-replan", error: err instanceof Error ? err.message : String(err), }); } + + if (transitioned) { + try { + await this.logEntry(id, action, text, runContext); + } catch (err) { + storeLog.warn("Best-effort post-comment re-triage failed", { + ...commentContextBase, + phase, + stage: "post-invalidation-log-entry", + nextStatus: "needs-replan", + error: err instanceof Error ? err.message : String(err), + }); + } + } } } @@ -6592,6 +6616,17 @@ ${notificationsSection}`; return this.todoStore; } + /** + * Get the EvalStore instance for eval run and task result operations. + * Lazily initializes the EvalStore on first access. + */ + getEvalStore(): EvalStore { + if (!this.evalStore) { + this.evalStore = new EvalStore(this.db); + } + return this.evalStore; + } + // ── Verification Cache ──────────────────────────────────────────────────── /** diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 1911198ea..81aee8c94 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -1687,6 +1687,18 @@ export interface ProjectSettings { unavailableNodePolicy?: UnavailableNodePolicy; /** Project-level research configuration overrides. */ researchSettings?: ResearchProjectSettings; + /** Enable scheduled evaluation batches for recently completed tasks. */ + taskEvaluationEnabled?: boolean; + /** Cron expression for scheduled task-evaluation batches. */ + taskEvaluationSchedule?: string; + /** Optional provider override for scheduled task evaluation runs. */ + taskEvaluationProvider?: string; + /** Optional model override for scheduled task evaluation runs. */ + taskEvaluationModelId?: string; + /** Follow-up policy for scheduled task evaluation findings. */ + taskEvaluationFollowUpPolicy?: "off" | "suggest" | "create"; + /** Optional retention window (days) for task evaluation history. */ + taskEvaluationRetention?: number; /** Enable or disable the research subsystem for this project. * When undefined, falls back to global settings. * @deprecated Prefer researchSettings.enabled */ diff --git a/packages/dashboard/app/App.tsx b/packages/dashboard/app/App.tsx index 20f624c58..c06f4d1b9 100644 --- a/packages/dashboard/app/App.tsx +++ b/packages/dashboard/app/App.tsx @@ -36,7 +36,7 @@ import { useCurrentProject } from "./hooks/useCurrentProject"; import { ToastProvider, useToast } from "./hooks/useToast"; import { ConfirmDialogProvider } from "./hooks/useConfirm"; import { useTheme } from "./hooks/useTheme"; -import { useModalManager, type DetailTaskOrigin } from "./hooks/useModalManager"; +import { useModalManager, type DetailTaskOrigin, type DetailTaskTab } from "./hooks/useModalManager"; import { useAppSettings } from "./hooks/useAppSettings"; import { useDeepLink } from "./hooks/useDeepLink"; import { useFavorites } from "./hooks/useFavorites"; @@ -49,6 +49,7 @@ import { useViewState, type TaskView } from "./hooks/useViewState"; import { useNavigationHistory } from "./hooks/useNavigationHistory"; import { usePluginDashboardViews } from "./hooks/usePluginDashboardViews"; import { PluginDashboardViewHost } from "./plugins/PluginDashboardViewHost"; +import { isPluginViewId } from "./plugins/pluginViewRegistry"; import { useProjectActions } from "./hooks/useProjectActions"; import { useTaskHandlers } from "./hooks/useTaskHandlers"; import { useRemoteNodeData } from "./hooks/useRemoteNodeData"; @@ -448,6 +449,7 @@ function AppInner() { // Redirect to board if feature-gated views are disabled. useEffect(() => { if (!settingsLoaded) return; + if (isPluginViewId(taskView)) return; if (taskView === "skills" && !skillsEnabled) { handleChangeTaskView("board"); } @@ -867,7 +869,7 @@ function AppInner() { } // Project view - if (taskView.startsWith("plugin:")) { + if (isPluginViewId(taskView)) { return ( 0 ? remoteData.tasks : tasks, workflowSteps, - openTaskDetail: isMobile ? (task, initialTab) => openDetailTaskWithHistory(task, initialTab) : (task, initialTab) => modalManager.openDetailTask(task, initialTab), - renderTaskCard: (task) => ( + openTaskDetail: isMobile + ? (task: Task | TaskDetail, initialTab?: DetailTaskTab) => openDetailTaskWithHistory(task, initialTab) + : (task: Task | TaskDetail, initialTab?: DetailTaskTab) => modalManager.openDetailTask(task, initialTab), + renderTaskCard: (task: Task | TaskDetail) => ( setShellConnectionManagerOpen(true)} /> ) : undefined} /> - {viewMode === "project" && currentProject && taskView !== "chat" && taskView !== "mailbox" && taskView !== "insights" && taskView !== "devserver" && taskView !== "dev-server" && !taskView.startsWith("plugin:") && ( + {viewMode === "project" && currentProject && taskView !== "chat" && taskView !== "mailbox" && taskView !== "insights" && taskView !== "devserver" && taskView !== "dev-server" && !isPluginViewId(taskView) && ( (null); const { confirm } = useConfirm(); const [logs, setLogs] = useState([]); + const [isImportModalOpen, setIsImportModalOpen] = useState(false); const [isLoading, setIsLoading] = useState(true); const [activeTab, setActiveTab] = useState(initialTab ?? "dashboard"); const [isStreaming, setIsStreaming] = useState(false); @@ -613,6 +615,15 @@ export function AgentDetailView({ agentId, projectId, onClose, addToast, onChild {/* Utility actions: refresh + close */}
+ @@ -757,6 +768,15 @@ export function AgentDetailView({ agentId, projectId, onClose, addToast, onChild
)} + setIsImportModalOpen(false)} + onImported={() => { + void handleSavedMutation(); + }} + projectId={projectId} + initialInputMethod="browse" + /> ); } diff --git a/packages/dashboard/app/components/AgentImportModal.tsx b/packages/dashboard/app/components/AgentImportModal.tsx index a4ef17cf0..7d750dda4 100644 --- a/packages/dashboard/app/components/AgentImportModal.tsx +++ b/packages/dashboard/app/components/AgentImportModal.tsx @@ -9,6 +9,7 @@ export interface AgentImportModalProps { onClose: () => void; onImported: () => void; projectId?: string; + initialInputMethod?: InputMethod; } /** Parsed agent preview item for display before import */ @@ -126,10 +127,10 @@ function parseDirectoryAgentManifest(content: string): DirectoryAgentInput { * * Flow: Input → Preview parsed agents → Import → Show results */ -export function AgentImportModal({ isOpen, onClose, onImported, projectId }: AgentImportModalProps) { +export function AgentImportModal({ isOpen, onClose, onImported, projectId, initialInputMethod = "paste" }: AgentImportModalProps) { useMobileScrollLock(isOpen); const [step, setStep] = useState("input"); - const [inputMethod, setInputMethod] = useState("paste"); + const [inputMethod, setInputMethod] = useState(initialInputMethod); const [manifestContent, setManifestContent] = useState(""); const [directoryAgents, setDirectoryAgents] = useState([]); const [companyName, setCompanyName] = useState("Unknown"); @@ -207,7 +208,7 @@ export function AgentImportModal({ isOpen, onClose, onImported, projectId }: Age const reset = useCallback(() => { setStep("input"); - setInputMethod("paste"); + setInputMethod(initialInputMethod); setManifestContent(""); setDirectoryAgents([]); setCompanyName("Unknown"); @@ -226,7 +227,7 @@ export function AgentImportModal({ isOpen, onClose, onImported, projectId }: Age setIsLoadingCompanies(false); setCompaniesError(null); fetchAttemptedRef.current = false; - }, []); + }, [initialInputMethod]); const handleClose = useCallback(() => { reset(); diff --git a/packages/dashboard/app/components/Header.tsx b/packages/dashboard/app/components/Header.tsx index 800a0a185..f85f432ba 100644 --- a/packages/dashboard/app/components/Header.tsx +++ b/packages/dashboard/app/components/Header.tsx @@ -13,7 +13,7 @@ import { useViewportMode, type ViewportMode } from "../hooks/useViewportMode"; import { getTrailingPath } from "../utils/pathDisplay"; import type { TaskView } from "../hooks/useViewState"; import type { PluginDashboardViewEntry } from "../api"; -import { buildPluginTaskViewId } from "../plugins/pluginViewRegistry"; +import { buildPluginTaskViewId, isPluginViewId } from "../plugins/pluginViewRegistry"; import { getPluginNavIcon } from "./pluginNavIcon"; export { useViewportMode }; @@ -1152,7 +1152,7 @@ export function Header({ <>