feat(FN-5284): surface db corruption via health checks, self-healing, and d

- fix(FN-5284): add db corruption audit mutation type
- feat(FN-5284): complete Step 7 — document corruption surfacing
- feat(FN-5284): complete Step 6 — add db corruption banner
- feat(FN-5284): complete Step 5 — extend health payload
- feat(FN-5284): complete Step 3 — surface db corruption in self-healing
- feat(FN-5284): complete Step 2 — add db corruption notification event
- feat(FN-5284): complete Step 1 — expose integrity check errors

Fusion-Task-Id: FN-5284
This commit is contained in:
Fusion (runfusion.ai)
2026-05-20 04:13:43 -07:00
committed by gsxdsm
parent 12a6ee9af1
commit 1b49cbc94d
21 changed files with 712 additions and 11 deletions

View File

@@ -465,6 +465,8 @@ describe("Database", () => {
expect(dbB.integrityCheckLastRunAt).toBeTruthy();
expect(dbA.corruptionDetected).toBe(false);
expect(dbB.corruptionDetected).toBe(false);
expect(dbA.integrityCheckErrors).toEqual([]);
expect(dbB.integrityCheckErrors).toEqual([]);
} finally {
dbA.close();
dbB.close();
@@ -478,7 +480,7 @@ describe("Database", () => {
vi.useFakeTimers();
const integritySpy = vi.spyOn(Database.prototype, "integrityCheck").mockReturnValue({
ok: false,
errors: ["malformed database"],
errors: ["malformed database", "broken index"],
});
const freshDir = makeTmpDir();
const freshFusionDir = join(freshDir, ".fusion");
@@ -498,6 +500,8 @@ describe("Database", () => {
expect(dbB.integrityCheckLastRunAt).toBeTruthy();
expect(dbA.corruptionDetected).toBe(true);
expect(dbB.corruptionDetected).toBe(true);
expect(dbA.integrityCheckErrors).toEqual(["malformed database", "broken index"]);
expect(dbB.integrityCheckErrors).toEqual(["malformed database", "broken index"]);
} finally {
dbA.close();
dbB.close();
@@ -1036,6 +1040,7 @@ describe("Database", () => {
it("returns ok for healthy databases and leaves corruption flag false", () => {
expect(db.corruptionDetected).toBe(false);
expect(db.integrityCheck()).toEqual({ ok: true });
expect(db.integrityCheckErrors).toEqual([]);
});
it("keeps corruptionDetected false after init for healthy database", () => {

View File

@@ -16,6 +16,8 @@ describe("TaskStore.getDatabaseHealth", () => {
const health = store.getDatabaseHealth();
expect(health.healthy).toBe(true);
expect(health.corruptionDetected).toBe(false);
expect(health.corruptionErrors).toEqual([]);
expect(health.isRunning).toBe(false);
expect(health.lastCheckedAt).toBeNull();
});
@@ -27,19 +29,34 @@ describe("TaskStore.getDatabaseHealth", () => {
const health = store.getDatabaseHealth();
expect(health.healthy).toBe(true);
expect(health.corruptionDetected).toBe(false);
expect(health.corruptionErrors).toEqual([]);
expect(health.isRunning).toBe(true);
});
it("reports unhealthy when corruption has been detected", () => {
const db = store.getDatabase();
db.corruptionDetected = true;
db.integrityCheckErrors = ["bad row", "bad index"];
db.integrityCheckPending = false;
db.integrityCheckLastRunAt = "2026-05-11T12:34:56.000Z";
const health = store.getDatabaseHealth();
expect(health.healthy).toBe(false);
expect(health.corruptionDetected).toBe(true);
expect(health.corruptionErrors).toEqual(["bad row", "bad index"]);
expect(health.isRunning).toBe(false);
expect(health.lastCheckedAt?.toISOString()).toBe("2026-05-11T12:34:56.000Z");
});
it("caps corruption errors to the first five entries", () => {
const db = store.getDatabase();
db.corruptionDetected = true;
db.integrityCheckErrors = ["one", "two", "three", "four", "five", "six"];
const health = store.getDatabaseHealth();
expect(health.corruptionErrors).toEqual(["one", "two", "three", "four", "five"]);
});
});

View File

@@ -1251,6 +1251,7 @@ export class Database {
/** Returns the database file path (or ":memory:" for in-memory databases). */
get path(): string { return this.dbPath; }
corruptionDetected = false;
integrityCheckErrors: string[] = [];
integrityCheckPending = false;
integrityCheckLastRunAt: string | null = null;
/** Tracks transaction nesting depth for savepoint-based nested transactions. */
@@ -3684,6 +3685,7 @@ export class Database {
participant.integrityCheckPending = false;
participant.integrityCheckLastRunAt = startedAt;
participant.corruptionDetected = !integrity.ok;
participant.integrityCheckErrors = integrity.ok ? [] : [...integrity.errors];
}
if (!integrity.ok) {

View File

@@ -9329,11 +9329,16 @@ ${stepsSection}`;
getDatabaseHealth(): {
healthy: boolean;
corruptionDetected: boolean;
corruptionErrors: string[];
lastCheckedAt: Date | null;
isRunning: boolean;
} {
const corruptionDetected = this.db.corruptionDetected;
return {
healthy: !this.db.corruptionDetected,
healthy: !corruptionDetected,
corruptionDetected,
corruptionErrors: this.db.integrityCheckErrors.slice(0, 5),
lastCheckedAt: this.db.integrityCheckLastRunAt ? new Date(this.db.integrityCheckLastRunAt) : null,
isRunning: this.db.integrityCheckPending,
};

View File

@@ -404,6 +404,7 @@ export type NtfyNotificationEvent =
| "planning-awaiting-input"
| "gridlock"
| "board-stall-unrecovered"
| "db-corruption-detected"
| "fallback-used"
| "memory-dreams-processed"
| "token-budget"
@@ -422,6 +423,7 @@ export const NOTIFICATION_EVENTS = [
"planning-awaiting-input",
"gridlock",
"board-stall-unrecovered",
"db-corruption-detected",
"fallback-used",
"memory-dreams-processed",
"token-budget",

View File

@@ -25,6 +25,7 @@ import { CliBinaryInstallBanner } from "./components/CliBinaryInstallBanner";
import { SetupWarningBanner } from "./components/SetupWarningBanner";
import { CapacityRiskBanner } from "./components/CapacityRiskBanner";
import { TaskIdIntegrityBanner } from "./components/TaskIdIntegrityBanner";
import { DbCorruptionBanner } from "./components/DbCorruptionBanner";
import { UpdateAvailableBanner } from "./components/UpdateAvailableBanner";
import { ApprovalNotificationBanner } from "./components/ApprovalNotificationBanner";
import { OnboardingResumeCard } from "./components/OnboardingResumeCard";
@@ -78,7 +79,7 @@ import { NativeShellConnectionManager } from "./components/NativeShellConnection
import { ShellConnectionStatus } from "./components/ShellConnectionStatus";
import { getShellConnectionNativeResult, type ShellConnectionNativeResult } from "./shell-native";
import type { AiSessionSummary, DashboardHealthResponse } from "./api";
import { api, fetchDashboardHealth, fetchUnreadCount, fetchTaskDetail, fetchWorkflowSteps } from "./api";
import { api, fetchDashboardHealth, fetchUnreadCount, fetchTaskDetail, fetchWorkflowSteps, refreshDashboardHealth } from "./api";
import { getScopedItem, removeScopedItem, setScopedItem } from "./utils/projectStorage";
import { subscribeSse } from "./sse-bus";
import { AUTH_TOKEN_RECOVERY_REQUIRED_EVENT } from "./auth";
@@ -690,6 +691,8 @@ function AppInner() {
const [quickChatOpen, setQuickChatOpen] = useState(false);
const [authTokenRecoveryOpen, setAuthTokenRecoveryOpen] = useState(false);
const [dashboardHealth, setDashboardHealth] = useState<DashboardHealthResponse | null>(null);
const [dbCorruptionRefreshing, setDbCorruptionRefreshing] = useState(false);
const [dbCorruptionRefreshError, setDbCorruptionRefreshError] = useState<string | null>(null);
const [setupWarningDismissed, setSetupWarningDismissed] = useState(
() => getScopedItem(SETUP_WARNING_DISMISSED_KEY, currentProject?.id) === "true",
);
@@ -709,6 +712,19 @@ function AppInner() {
);
}, [currentProject?.id]);
const refreshDbCorruptionHealth = useCallback(async () => {
setDbCorruptionRefreshing(true);
setDbCorruptionRefreshError(null);
try {
const health = await refreshDashboardHealth();
setDashboardHealth(health);
} catch (error) {
setDbCorruptionRefreshError(error instanceof Error ? error.message : "Failed to refresh database health.");
} finally {
setDbCorruptionRefreshing(false);
}
}, []);
useEffect(() => {
let cancelled = false;
@@ -1800,7 +1816,12 @@ function AppInner() {
}
return {
...current,
status: report.status === "anomaly" || !current.database.healthy ? "degraded" : "ok",
status:
report.status === "anomaly"
|| !current.database.healthy
|| current.database.corruptionDetected
? "degraded"
: "ok",
taskIdIntegrity: {
...report,
recommendedAction,
@@ -1810,6 +1831,15 @@ function AppInner() {
}}
/>
)}
{viewMode === "project" && currentProject && dashboardHealth?.database?.corruptionDetected === true && (
<DbCorruptionBanner
errors={dashboardHealth.database.corruptionErrors}
lastCheckedAt={dashboardHealth.database.lastCheckedAt}
onRefresh={refreshDbCorruptionHealth}
refreshing={dbCorruptionRefreshing}
refreshError={dbCorruptionRefreshError}
/>
)}
{viewMode === "project" && currentProject && !setupReadinessLoading && hasWarnings && !setupWarningDismissed && (
<SetupWarningBanner
hasAiProvider={hasAiProvider}

View File

@@ -211,6 +211,8 @@ export interface DashboardHealthResponse {
uptime: number;
database: {
healthy: boolean;
corruptionDetected: boolean;
corruptionErrors: string[];
lastCheckedAt: string | null;
isRunning: boolean;
};

View File

@@ -0,0 +1,101 @@
/* === DbCorruptionBanner === */
.db-corruption-banner {
display: flex;
flex-direction: column;
gap: var(--space-md);
padding: var(--space-lg);
margin-bottom: var(--space-md);
border-radius: var(--radius-md);
border: var(--btn-border-width) solid color-mix(in srgb, var(--color-error) 24%, transparent);
border-inline-start: var(--space-xs) solid var(--color-error);
background: color-mix(in srgb, var(--color-error) 10%, transparent);
box-shadow: var(--shadow-md);
}
.db-corruption-banner__header {
display: flex;
align-items: flex-start;
justify-content: space-between;
gap: var(--space-md);
}
.db-corruption-banner__headline-wrap {
display: flex;
align-items: center;
gap: var(--space-sm);
min-width: 0;
}
.db-corruption-banner__headline-copy,
.db-corruption-banner__item {
display: flex;
flex-direction: column;
gap: var(--space-xs);
}
.db-corruption-banner__headline,
.db-corruption-banner__footer-label {
color: var(--text);
}
.db-corruption-banner__headline {
margin: 0;
font-size: 1rem;
}
.db-corruption-banner__meta,
.db-corruption-banner__body,
.db-corruption-banner__footer,
.db-corruption-banner__error {
margin: 0;
color: var(--text-muted);
}
.db-corruption-banner__list {
display: flex;
flex-direction: column;
gap: var(--space-sm);
margin: 0;
padding-inline-start: calc(var(--space-lg) + var(--space-xs));
}
.db-corruption-banner__error-code,
.db-corruption-banner__inline-code {
font-family: var(--font-mono);
color: var(--text);
white-space: normal;
overflow-wrap: anywhere;
}
.db-corruption-banner__refresh,
.db-corruption-banner__refresh-icon {
flex-shrink: 0;
}
.db-corruption-banner__refresh-icon--spinning {
animation: status-dot-pulse var(--transition-slow) ease-in-out infinite;
}
.db-corruption-banner__link {
color: var(--text);
}
.db-corruption-banner__error {
color: var(--color-error);
}
@media (max-width: 768px) {
.db-corruption-banner {
padding: var(--space-md);
}
.db-corruption-banner__header {
flex-direction: column;
align-items: stretch;
}
.db-corruption-banner__refresh {
width: 100%;
justify-content: center;
}
}

View File

@@ -0,0 +1,74 @@
import { AlertTriangle, RefreshCw } from "lucide-react";
import "./DbCorruptionBanner.css";
interface DbCorruptionBannerProps {
errors: string[];
lastCheckedAt: string | null;
onRefresh: () => void | Promise<void>;
refreshing: boolean;
refreshError: string | null;
}
export function DbCorruptionBanner({
errors,
lastCheckedAt,
onRefresh,
refreshing,
refreshError,
}: DbCorruptionBannerProps) {
if (errors.length === 0) {
return null;
}
const visibleErrors = errors.slice(0, 5);
const checkedAtLabel = lastCheckedAt ? new Date(lastCheckedAt).toLocaleString() : null;
return (
<section className="db-corruption-banner" role="alert" aria-live="assertive">
<div className="db-corruption-banner__header">
<div className="db-corruption-banner__headline-wrap">
<span className="status-dot status-dot--error" aria-hidden="true" />
<AlertTriangle aria-hidden="true" />
<div className="db-corruption-banner__headline-copy">
<h2 className="db-corruption-banner__headline">Database corruption detected</h2>
{checkedAtLabel ? (
<p className="db-corruption-banner__meta">Last checked: {checkedAtLabel}</p>
) : null}
</div>
</div>
<button
type="button"
className="btn btn-sm db-corruption-banner__refresh"
onClick={() => {
void onRefresh();
}}
disabled={refreshing}
>
<RefreshCw className={refreshing ? "db-corruption-banner__refresh-icon db-corruption-banner__refresh-icon--spinning" : "db-corruption-banner__refresh-icon"} aria-hidden="true" />
{refreshing ? "Refreshing…" : "Refresh health"}
</button>
</div>
<p className="db-corruption-banner__body">
Fusion&apos;s background SQLite integrity check reported corruption. Review the failing objects below before continuing critical operations.
</p>
<ul className="db-corruption-banner__list">
{visibleErrors.map((error, index) => (
<li key={`${index}:${error}`} className="db-corruption-banner__item">
<code className="db-corruption-banner__error-code">{error}</code>
</li>
))}
</ul>
<p className="db-corruption-banner__footer">
<strong className="db-corruption-banner__footer-label">What to do:</strong>{" "}
Back up the project, try <code className="db-corruption-banner__inline-code">fn db --vacuum</code> if the database still opens cleanly, and restore from a known-good backup if corruption persists. See{" "}
<a href="docs/storage.md" target="_blank" rel="noreferrer" className="db-corruption-banner__link">docs/storage.md</a>
{" "}for the storage layout and recovery guidance.
</p>
{refreshError ? <p className="db-corruption-banner__error">{refreshError}</p> : null}
</section>
);
}

View File

@@ -71,7 +71,13 @@ vi.mock("../../api", async (importOriginal) => {
status: "ok",
version: "1.0.0",
uptime: 1,
database: { healthy: true, lastCheckedAt: null, isRunning: false },
database: {
healthy: true,
corruptionDetected: false,
corruptionErrors: [],
lastCheckedAt: null,
isRunning: false,
},
taskIdIntegrity: { status: "ok", checkedAt: "2026-05-12T00:00:00.000Z", anomalies: [], recommendedAction: null },
})),
fetchPluginDashboardViews: vi.fn(() => Promise.resolve([])),

View File

@@ -0,0 +1,56 @@
import { describe, expect, it, vi } from "vitest";
import { fireEvent, render, screen } from "@testing-library/react";
import { DbCorruptionBanner } from "../DbCorruptionBanner";
describe("DbCorruptionBanner", () => {
it("renders errors and last-checked timestamp", () => {
render(
<DbCorruptionBanner
errors={["bad row", "bad index"]}
lastCheckedAt="2026-05-20T00:05:00.000Z"
onRefresh={() => undefined}
refreshing={false}
refreshError={null}
/>,
);
expect(screen.getByRole("alert")).toBeInTheDocument();
expect(screen.getByText("Database corruption detected")).toBeInTheDocument();
expect(screen.getByText(/Last checked:/)).toBeInTheDocument();
expect(screen.getByText("bad row")).toBeInTheDocument();
expect(screen.getByText("bad index")).toBeInTheDocument();
expect(screen.getByRole("link", { name: "docs/storage.md" })).toHaveAttribute("href", "docs/storage.md");
});
it("calls onRefresh when the button is clicked", () => {
const onRefresh = vi.fn();
render(
<DbCorruptionBanner
errors={["bad row"]}
lastCheckedAt={null}
onRefresh={onRefresh}
refreshing={false}
refreshError={null}
/>,
);
fireEvent.click(screen.getByRole("button", { name: "Refresh health" }));
expect(onRefresh).toHaveBeenCalledTimes(1);
});
it("renders nothing when there are no errors", () => {
const { container } = render(
<DbCorruptionBanner
errors={[]}
lastCheckedAt={null}
onRefresh={() => undefined}
refreshing={false}
refreshError={null}
/>,
);
expect(container).toBeEmptyDOMElement();
});
});

View File

@@ -62,7 +62,13 @@ describe("TaskIdIntegrityBanner", () => {
status: "ok",
version: "1.0.0",
uptime: 1,
database: { healthy: true, lastCheckedAt: null, isRunning: false },
database: {
healthy: true,
corruptionDetected: false,
corruptionErrors: [],
lastCheckedAt: null,
isRunning: false,
},
taskIdIntegrity: {
status: "ok",
checkedAt: "2026-05-12T10:05:00.000Z",

View File

@@ -67,6 +67,8 @@ class MockStore extends EventEmitter {
getDatabaseHealth() {
return {
healthy: true,
corruptionDetected: false,
corruptionErrors: [],
isRunning: false,
lastCheckedAt: null,
};

View File

@@ -93,6 +93,8 @@ function createMockStore(overrides: Partial<TaskStore> = {}): TaskStore {
}),
getDatabaseHealth: vi.fn().mockReturnValue({
healthy: true,
corruptionDetected: false,
corruptionErrors: [],
isRunning: false,
lastCheckedAt: null,
}),
@@ -356,6 +358,8 @@ describe("createServer health and headless mode", () => {
uptime: expect.any(Number),
database: {
healthy: true,
corruptionDetected: false,
corruptionErrors: [],
isRunning: false,
lastCheckedAt: null,
},
@@ -372,6 +376,8 @@ describe("createServer health and headless mode", () => {
const store = createMockStore({
getDatabaseHealth: vi.fn().mockReturnValue({
healthy: false,
corruptionDetected: true,
corruptionErrors: ["bad row", "bad index"],
isRunning: false,
lastCheckedAt: new Date("2026-05-11T10:00:00.000Z"),
}),
@@ -387,6 +393,8 @@ describe("createServer health and headless mode", () => {
uptime: expect.any(Number),
database: {
healthy: false,
corruptionDetected: true,
corruptionErrors: ["bad row", "bad index"],
isRunning: false,
lastCheckedAt: "2026-05-11T10:00:00.000Z",
},
@@ -425,6 +433,8 @@ describe("createServer health and headless mode", () => {
uptime: expect.any(Number),
database: {
healthy: true,
corruptionDetected: false,
corruptionErrors: [],
isRunning: false,
lastCheckedAt: null,
},
@@ -472,6 +482,8 @@ describe("createServer health and headless mode", () => {
uptime: expect.any(Number),
database: {
healthy: true,
corruptionDetected: false,
corruptionErrors: [],
isRunning: false,
lastCheckedAt: null,
},
@@ -492,6 +504,41 @@ describe("createServer health and headless mode", () => {
});
});
it("reports degraded status from /api/health/refresh when corruption is detected", async () => {
const store = createMockStore({
getDatabaseHealth: vi.fn().mockReturnValue({
healthy: false,
corruptionDetected: true,
corruptionErrors: ["bad row"],
isRunning: false,
lastCheckedAt: new Date("2026-05-12T12:00:00.000Z"),
}),
});
const app = createServer(store);
const res = await REQUEST(app, "POST", "/api/health/refresh");
expect(res.status).toBe(200);
expect(res.body).toEqual({
status: "degraded",
version: CLI_PACKAGE_VERSION,
uptime: expect.any(Number),
database: {
healthy: false,
corruptionDetected: true,
corruptionErrors: ["bad row"],
isRunning: false,
lastCheckedAt: "2026-05-12T12:00:00.000Z",
},
taskIdIntegrity: {
status: "ok",
checkedAt: "2026-05-12T00:00:00.000Z",
anomalies: [],
recommendedAction: null,
},
});
});
it("returns reliability metrics payload from /api/health/reliability", async () => {
vi.useFakeTimers();
vi.setSystemTime(new Date("2026-05-13T12:00:00.000Z"));

View File

@@ -98,7 +98,7 @@ function buildHealthPayload(store: TaskStore, cliPackageVersion: string) {
const database = store.getDatabaseHealth();
const taskIdIntegrity = buildTaskIdIntegrityHealth(store.getTaskIdIntegrityReport());
return {
status: !database.healthy || taskIdIntegrity.status === "anomaly" ? "degraded" : "ok",
status: !database.healthy || database.corruptionDetected || taskIdIntegrity.status === "anomaly" ? "degraded" : "ok",
version: cliPackageVersion,
uptime: Math.floor(process.uptime()),
database,
@@ -1325,7 +1325,7 @@ export function createServer(store: TaskStore, options?: ServerOptions): ReturnT
const report = store.refreshTaskIdIntegrityReport();
const database = store.getDatabaseHealth();
res.json({
status: !database.healthy || report.status === "anomaly" ? "degraded" : "ok",
status: !database.healthy || database.corruptionDetected || report.status === "anomaly" ? "degraded" : "ok",
version: cliPackageVersion,
uptime: Math.floor(process.uptime()),
database,

View File

@@ -0,0 +1,260 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { EventEmitter } from "node:events";
import type { Settings, TaskStore } from "@fusion/core";
import { SelfHealingManager } from "../self-healing.js";
import type { NotificationService } from "../notification/notification-service.js";
import * as notifierModule from "../notifier.js";
function createMockStore(overrides: Record<string, unknown> = {}): TaskStore & EventEmitter {
const emitter = new EventEmitter();
return Object.assign(emitter, {
getSettings: vi.fn().mockResolvedValue({
maintenanceIntervalMs: 0,
globalPause: false,
enginePaused: false,
ntfyEnabled: true,
ntfyTopic: "fusion-alerts",
ntfyEvents: ["db-corruption-detected"],
} as unknown as Settings),
getDatabaseHealth: vi.fn().mockReturnValue({
healthy: true,
corruptionDetected: false,
corruptionErrors: [],
lastCheckedAt: null,
isRunning: false,
}),
recordRunAuditEvent: vi.fn().mockResolvedValue(undefined),
...overrides,
}) as unknown as TaskStore & EventEmitter;
}
const BATCH1_METHODS = [
"pruneWorktrees",
"cleanupOrphans",
"cleanupOrphanedBranches",
"enforceWorktreeCap",
] as const;
const BATCH2_METHODS = [
"recoverCompletedTasks",
"recoverStrandedCompletedTodoTasks",
"recoverStaleIncompleteReviewTasks",
"recoverReviewTasksWithFailedPreMergeSteps",
"recoverInterruptedMergingTasks",
"recoverDoneTaskMergeMetadata",
"recoverStaleMergingStatus",
"finalizeNoOpReviewTasks",
"reconcileDoneTaskIntegrity",
"reconcileStaleMergerStatus",
"recoverMergeableReviewTasks",
"recoverMergedReviewTasks",
"recoverAlreadyMergedReviewTasks",
"recoverCompletionHandoffLimbo",
"recoverBranchMisboundInReviewTasks",
"recoverForeignOnlyContaminatedInReviewTasks",
"recoverOrphanOnlyScopeViolations",
"recoverStuckMergeDeadlocks",
"recoverMisclassifiedFailures",
"recoverMissingWorktreeReviewFailures",
"recoverNoProgressNoTaskDoneFailures",
"recoverPartialProgressNoTaskDoneFailures",
"recoverOrphanedExecutions",
"recoverApprovedTriageTasks",
"recoverStarvedRefinementTriageTasks",
"recoverOrphanedPlanningTasks",
"recoverGhostReviewTasks",
"recoverOrphanedAgents",
"recoverStaleHeartbeatRuns",
"recoverAgentsRunningOnInactiveTasks",
"recoverDriftedAgentTaskLinks",
"clearStaleBlockedBy",
"autoReboundPausedScopeDecay",
"autoArchiveResolvedMetaTasks",
"autoArchiveStalledMetaTasks",
"runBoardStallAutoRecoverySweep",
"reconcileSelfDefeatingDependencies",
"reclaimPrConflicts",
"reclaimSelfOwnedBranchConflicts",
"reconcileTaskWorktreeMetadata",
"reconcileInReviewBranchRebind",
"reclaimStaleActiveBranches",
"surfaceInReviewStalls",
"surfaceInReviewStalled",
"surfaceStalePausedReviews",
"surfaceStalePausedTodos",
"auditNoCommitsExpectedCandidates",
] as const;
function stubMaintenance(manager: SelfHealingManager) {
for (const method of BATCH1_METHODS) {
vi.spyOn(manager as never, method).mockResolvedValue(0 as never);
}
vi.spyOn(manager as never, "checkpointWal").mockReturnValue(undefined as never);
for (const method of BATCH2_METHODS) {
vi.spyOn(manager as never, method).mockResolvedValue(0 as never);
}
vi.spyOn(manager, "archiveStaleDoneTasks").mockResolvedValue(0);
}
describe("FN-5284: self-healing DB corruption surfacing", () => {
beforeEach(() => {
vi.useFakeTimers();
vi.setSystemTime(new Date("2026-05-20T00:00:00.000Z"));
vi.spyOn(notifierModule, "getActiveNotificationService").mockReturnValue(undefined);
vi.spyOn(notifierModule, "sendNtfyNotification").mockResolvedValue(undefined);
});
afterEach(() => {
vi.restoreAllMocks();
vi.useRealTimers();
});
it("does not dispatch or audit when the database is healthy", async () => {
const store = createMockStore();
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
stubMaintenance(manager);
await (manager as any).runMaintenance();
expect(notifierModule.getActiveNotificationService).not.toHaveBeenCalled();
expect(notifierModule.sendNtfyNotification).not.toHaveBeenCalled();
expect(store.recordRunAuditEvent).not.toHaveBeenCalled();
});
it("dispatches a notification and records an audit event on first corruption detection", async () => {
const dispatch = vi.fn().mockResolvedValue(undefined);
vi.spyOn(notifierModule, "getActiveNotificationService").mockReturnValue({ dispatch } as unknown as NotificationService);
const store = createMockStore({
getDatabaseHealth: vi.fn().mockReturnValue({
healthy: false,
corruptionDetected: true,
corruptionErrors: ["bad row", "bad index"],
lastCheckedAt: new Date("2026-05-20T00:05:00.000Z"),
isRunning: false,
}),
});
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
stubMaintenance(manager);
await (manager as any).runMaintenance();
expect(dispatch).toHaveBeenCalledWith("db-corruption-detected", {
event: "db-corruption-detected",
timestamp: "2026-05-20T00:00:00.000Z",
metadata: {
errors: ["bad row", "bad index"],
lastCheckedAt: "2026-05-20T00:05:00.000Z",
},
});
expect(store.recordRunAuditEvent).toHaveBeenCalledWith(
expect.objectContaining({
domain: "database",
mutationType: "task:auto-db-corruption-detected",
target: "database",
metadata: expect.objectContaining({
errors: ["bad row", "bad index"],
notificationDispatched: true,
}),
}),
);
});
it("respects the cooldown and avoids duplicate dispatches and audits", async () => {
const dispatch = vi.fn().mockResolvedValue(undefined);
vi.spyOn(notifierModule, "getActiveNotificationService").mockReturnValue({ dispatch } as unknown as NotificationService);
const store = createMockStore({
getDatabaseHealth: vi.fn().mockReturnValue({
healthy: false,
corruptionDetected: true,
corruptionErrors: ["bad row"],
lastCheckedAt: new Date("2026-05-20T00:05:00.000Z"),
isRunning: false,
}),
});
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
stubMaintenance(manager);
await (manager as any).runMaintenance();
await (manager as any).runMaintenance();
expect(dispatch).toHaveBeenCalledTimes(1);
expect(store.recordRunAuditEvent).toHaveBeenCalledTimes(1);
});
it("re-notifies after corruption clears and is detected again", async () => {
const dispatch = vi.fn().mockResolvedValue(undefined);
vi.spyOn(notifierModule, "getActiveNotificationService").mockReturnValue({ dispatch } as unknown as NotificationService);
const getDatabaseHealth = vi.fn()
.mockReturnValueOnce({
healthy: false,
corruptionDetected: true,
corruptionErrors: ["first error"],
lastCheckedAt: new Date("2026-05-20T00:05:00.000Z"),
isRunning: false,
})
.mockReturnValueOnce({
healthy: true,
corruptionDetected: false,
corruptionErrors: [],
lastCheckedAt: new Date("2026-05-20T00:15:00.000Z"),
isRunning: false,
})
.mockReturnValueOnce({
healthy: false,
corruptionDetected: true,
corruptionErrors: ["second error"],
lastCheckedAt: new Date("2026-05-20T00:25:00.000Z"),
isRunning: false,
});
const store = createMockStore({ getDatabaseHealth });
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
stubMaintenance(manager);
await (manager as any).runMaintenance();
await (manager as any).runMaintenance();
await (manager as any).runMaintenance();
expect(dispatch).toHaveBeenNthCalledWith(1, "db-corruption-detected", expect.objectContaining({
metadata: expect.objectContaining({ errors: ["first error"] }),
}));
expect(dispatch).toHaveBeenNthCalledWith(2, "db-corruption-detected", expect.objectContaining({
metadata: expect.objectContaining({ errors: ["second error"] }),
}));
expect(dispatch).toHaveBeenCalledTimes(2);
expect(store.recordRunAuditEvent).toHaveBeenCalledTimes(2);
});
it("records an audit even when no notification channel is active", async () => {
const store = createMockStore({
getSettings: vi.fn().mockResolvedValue({
maintenanceIntervalMs: 0,
globalPause: false,
enginePaused: false,
ntfyEnabled: false,
ntfyTopic: "fusion-alerts",
ntfyEvents: ["db-corruption-detected"],
} as unknown as Settings),
getDatabaseHealth: vi.fn().mockReturnValue({
healthy: false,
corruptionDetected: true,
corruptionErrors: ["bad row"],
lastCheckedAt: new Date("2026-05-20T00:05:00.000Z"),
isRunning: false,
}),
});
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
stubMaintenance(manager);
await (manager as any).runMaintenance();
expect(notifierModule.sendNtfyNotification).not.toHaveBeenCalled();
expect(store.recordRunAuditEvent).toHaveBeenCalledWith(
expect.objectContaining({
mutationType: "task:auto-db-corruption-detected",
metadata: expect.objectContaining({ notificationDispatched: false }),
}),
);
});
});

View File

@@ -29,6 +29,7 @@ export const DEFAULT_NTFY_EVENTS: readonly NtfyNotificationEvent[] = [
"planning-awaiting-input",
"gridlock",
"board-stall-unrecovered",
"db-corruption-detected",
"fallback-used",
"token-budget",
"message:agent-to-user",

View File

@@ -254,6 +254,8 @@ export type DatabaseMutationType =
| "task:auto-board-stall-broken"
/** Metadata: { holderIds: string[], followerCount: number, windowMs: number, ntfyDispatched: boolean } */
| "task:auto-board-stall-unrecovered"
/** Metadata: { errors: string[], lastCheckedAt: string | null, notificationDispatched: boolean } */
| "task:auto-db-corruption-detected"
| "task:in-review-stall-deadlock-disposed"
| "task:finalize-unproven-blocked"
| "task:integrity-reconcile-modified-files"

View File

@@ -30,7 +30,7 @@ import { existsSync, mkdirSync, readdirSync, rmSync, statSync, writeFileSync } f
import { isAbsolute, join, relative, resolve } from "node:path";
import { IN_REVIEW_STALL_DEADLOCK_LOG_PREFIX, IN_REVIEW_STALL_LOG_PREFIX, countRecentIdenticalStallEntries, detectSelfDefeatingDependency, getInReviewStalledSignal, getInReviewStallReason, getPrimaryPrInfo, getStalePausedReviewSignal, getStalePausedTodoSignal, getTaskHardMergeBlocker, getTaskMergeBlocker, isEphemeralAgent, type AgentStore, type ChatStore, type MessageStore, type TaskStore, type Settings, type Task, type MergeDetails, type TaskPriority, type MergeResult } from "@fusion/core";
import type { MeshLeaseManager } from "./mesh-lease-manager.js";
import { createLogger } from "./logger.js";
import { createLogger, schedulerLog } from "./logger.js";
import { RemovalReason, getRegisteredWorktreeBranchMap, getRegisteredWorktreePaths, isUsableTaskWorktree, removeWorktree, resolveWorktreeBackend, scanIdleWorktrees, scanOrphanedBranches } from "./worktree-pool.js";
import {
classifyMissingWorktreeSessionStartFailure,
@@ -51,6 +51,7 @@ import type { OwnedLandedClassification } from "./merger.js";
import { recoverForeignOnlyContamination } from "./recovery/foreign-only-contamination.js";
import {
buildNtfyClickUrl,
getActiveNotificationService,
isNtfyEventEnabled,
resolveNtfyEvents,
sendNtfyNotification,
@@ -64,6 +65,7 @@ const worktreeMetadataReconcileLog = createLogger("worktree-metadata-reconcile")
const execAsync = promisify(exec);
const DONE_TASK_INTEGRITY_SWEEP_LIMIT = 50;
const BOARD_STALL_NOTIFICATION_COOLDOWN_MS = 60 * 60_000;
const DB_CORRUPTION_NOTIFICATION_COOLDOWN_MS = 60 * 60 * 1000;
export const STALE_ACTIVE_BRANCH_EXECUTION_GRACE_MS = 10 * 60_000;
export const COMPLETION_HANDOFF_LIMBO_GRACE_MS = 5 * 60_000;
export const MAX_COMPLETION_HANDOFF_LIMBO_RECOVERIES = 3;
@@ -534,6 +536,7 @@ export class SelfHealingManager {
private maintenanceTickCounter = 0;
private readonly processBootStartedAt = Date.now();
private dependencyBlockedTodoReporter: DependencyBlockedTodoReporter | null = null;
private lastDbCorruptionNotifiedAt: number | null = null;
private boardStallWindow: {
windowStartMs: number;
@@ -1317,6 +1320,7 @@ export class SelfHealingManager {
{ name: "surface-in-review-stalled", fn: () => this.surfaceInReviewStalled() },
{ name: "surface-stale-paused-reviews", fn: () => this.surfaceStalePausedReviews() },
{ name: "surface-stale-paused-todos", fn: () => this.surfaceStalePausedTodos() },
{ name: "surface-db-corruption", fn: () => this.surfaceDbCorruption() },
{ name: "audit-no-commits-expected-candidates", fn: () => this.auditNoCommitsExpectedCandidates() },
];
for (const fn of batch2Fns) {
@@ -3199,6 +3203,78 @@ export class SelfHealingManager {
return { holders: [], recovered: 0, unrecovered: false };
}
private async surfaceDbCorruption(): Promise<void> {
const health = this.store.getDatabaseHealth();
if (!health.corruptionDetected) {
this.lastDbCorruptionNotifiedAt = null;
return;
}
const now = Date.now();
if (
this.lastDbCorruptionNotifiedAt !== null
&& now - this.lastDbCorruptionNotifiedAt < DB_CORRUPTION_NOTIFICATION_COOLDOWN_MS
) {
return;
}
const settings = await this.store.getSettings();
const errors = health.corruptionErrors.slice(0, 5);
let notificationDispatched = false;
try {
const notificationService = getActiveNotificationService();
if (notificationService) {
await notificationService.dispatch("db-corruption-detected", {
event: "db-corruption-detected",
timestamp: new Date().toISOString(),
metadata: {
errors,
lastCheckedAt: health.lastCheckedAt?.toISOString() ?? null,
},
});
notificationDispatched = true;
} else {
const enabled = Boolean(settings.ntfyEnabled && settings.ntfyTopic);
const events = resolveNtfyEvents(settings.ntfyEvents);
if (enabled && isNtfyEventEnabled(events, "db-corruption-detected")) {
const clickUrl = buildNtfyClickUrl({ dashboardHost: settings.ntfyDashboardHost });
await sendNtfyNotification({
ntfyBaseUrl: settings.ntfyBaseUrl,
ntfyAccessToken: settings.ntfyAccessToken,
topic: settings.ntfyTopic!,
title: "Database corruption detected",
message: `Background SQLite integrity check detected corruption. Errors: ${errors.join(" | ") || "unknown"}.`,
priority: "urgent",
clickUrl,
});
notificationDispatched = true;
}
}
} catch (error: unknown) {
schedulerLog.log(
`Failed to dispatch db-corruption-detected notification: ${error instanceof Error ? error.message : String(error)}`,
);
}
const auditor = createRunAuditor(this.store, {
runId: generateSyntheticRunId("fn5284-db-corruption", "global"),
agentId: "self-healing",
phase: "db-corruption-detected",
});
await auditor.database({
type: "task:auto-db-corruption-detected",
target: "database",
metadata: {
errors,
lastCheckedAt: health.lastCheckedAt?.toISOString() ?? null,
notificationDispatched,
},
});
this.lastDbCorruptionNotifiedAt = now;
}
async clearStaleBlockedBy(): Promise<number> {
try {
const settings = await this.store.getSettings();