feat(FN-5284): surface db corruption via health checks, self-healing, and d
- fix(FN-5284): add db corruption audit mutation type - feat(FN-5284): complete Step 7 — document corruption surfacing - feat(FN-5284): complete Step 6 — add db corruption banner - feat(FN-5284): complete Step 5 — extend health payload - feat(FN-5284): complete Step 3 — surface db corruption in self-healing - feat(FN-5284): complete Step 2 — add db corruption notification event - feat(FN-5284): complete Step 1 — expose integrity check errors Fusion-Task-Id: FN-5284
This commit is contained in:
committed by
gsxdsm
parent
12a6ee9af1
commit
1b49cbc94d
@@ -465,6 +465,8 @@ describe("Database", () => {
|
||||
expect(dbB.integrityCheckLastRunAt).toBeTruthy();
|
||||
expect(dbA.corruptionDetected).toBe(false);
|
||||
expect(dbB.corruptionDetected).toBe(false);
|
||||
expect(dbA.integrityCheckErrors).toEqual([]);
|
||||
expect(dbB.integrityCheckErrors).toEqual([]);
|
||||
} finally {
|
||||
dbA.close();
|
||||
dbB.close();
|
||||
@@ -478,7 +480,7 @@ describe("Database", () => {
|
||||
vi.useFakeTimers();
|
||||
const integritySpy = vi.spyOn(Database.prototype, "integrityCheck").mockReturnValue({
|
||||
ok: false,
|
||||
errors: ["malformed database"],
|
||||
errors: ["malformed database", "broken index"],
|
||||
});
|
||||
const freshDir = makeTmpDir();
|
||||
const freshFusionDir = join(freshDir, ".fusion");
|
||||
@@ -498,6 +500,8 @@ describe("Database", () => {
|
||||
expect(dbB.integrityCheckLastRunAt).toBeTruthy();
|
||||
expect(dbA.corruptionDetected).toBe(true);
|
||||
expect(dbB.corruptionDetected).toBe(true);
|
||||
expect(dbA.integrityCheckErrors).toEqual(["malformed database", "broken index"]);
|
||||
expect(dbB.integrityCheckErrors).toEqual(["malformed database", "broken index"]);
|
||||
} finally {
|
||||
dbA.close();
|
||||
dbB.close();
|
||||
@@ -1036,6 +1040,7 @@ describe("Database", () => {
|
||||
it("returns ok for healthy databases and leaves corruption flag false", () => {
|
||||
expect(db.corruptionDetected).toBe(false);
|
||||
expect(db.integrityCheck()).toEqual({ ok: true });
|
||||
expect(db.integrityCheckErrors).toEqual([]);
|
||||
});
|
||||
|
||||
it("keeps corruptionDetected false after init for healthy database", () => {
|
||||
|
||||
@@ -16,6 +16,8 @@ describe("TaskStore.getDatabaseHealth", () => {
|
||||
const health = store.getDatabaseHealth();
|
||||
|
||||
expect(health.healthy).toBe(true);
|
||||
expect(health.corruptionDetected).toBe(false);
|
||||
expect(health.corruptionErrors).toEqual([]);
|
||||
expect(health.isRunning).toBe(false);
|
||||
expect(health.lastCheckedAt).toBeNull();
|
||||
});
|
||||
@@ -27,19 +29,34 @@ describe("TaskStore.getDatabaseHealth", () => {
|
||||
const health = store.getDatabaseHealth();
|
||||
|
||||
expect(health.healthy).toBe(true);
|
||||
expect(health.corruptionDetected).toBe(false);
|
||||
expect(health.corruptionErrors).toEqual([]);
|
||||
expect(health.isRunning).toBe(true);
|
||||
});
|
||||
|
||||
it("reports unhealthy when corruption has been detected", () => {
|
||||
const db = store.getDatabase();
|
||||
db.corruptionDetected = true;
|
||||
db.integrityCheckErrors = ["bad row", "bad index"];
|
||||
db.integrityCheckPending = false;
|
||||
db.integrityCheckLastRunAt = "2026-05-11T12:34:56.000Z";
|
||||
|
||||
const health = store.getDatabaseHealth();
|
||||
|
||||
expect(health.healthy).toBe(false);
|
||||
expect(health.corruptionDetected).toBe(true);
|
||||
expect(health.corruptionErrors).toEqual(["bad row", "bad index"]);
|
||||
expect(health.isRunning).toBe(false);
|
||||
expect(health.lastCheckedAt?.toISOString()).toBe("2026-05-11T12:34:56.000Z");
|
||||
});
|
||||
|
||||
it("caps corruption errors to the first five entries", () => {
|
||||
const db = store.getDatabase();
|
||||
db.corruptionDetected = true;
|
||||
db.integrityCheckErrors = ["one", "two", "three", "four", "five", "six"];
|
||||
|
||||
const health = store.getDatabaseHealth();
|
||||
|
||||
expect(health.corruptionErrors).toEqual(["one", "two", "three", "four", "five"]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1251,6 +1251,7 @@ export class Database {
|
||||
/** Returns the database file path (or ":memory:" for in-memory databases). */
|
||||
get path(): string { return this.dbPath; }
|
||||
corruptionDetected = false;
|
||||
integrityCheckErrors: string[] = [];
|
||||
integrityCheckPending = false;
|
||||
integrityCheckLastRunAt: string | null = null;
|
||||
/** Tracks transaction nesting depth for savepoint-based nested transactions. */
|
||||
@@ -3684,6 +3685,7 @@ export class Database {
|
||||
participant.integrityCheckPending = false;
|
||||
participant.integrityCheckLastRunAt = startedAt;
|
||||
participant.corruptionDetected = !integrity.ok;
|
||||
participant.integrityCheckErrors = integrity.ok ? [] : [...integrity.errors];
|
||||
}
|
||||
|
||||
if (!integrity.ok) {
|
||||
|
||||
@@ -9329,11 +9329,16 @@ ${stepsSection}`;
|
||||
|
||||
getDatabaseHealth(): {
|
||||
healthy: boolean;
|
||||
corruptionDetected: boolean;
|
||||
corruptionErrors: string[];
|
||||
lastCheckedAt: Date | null;
|
||||
isRunning: boolean;
|
||||
} {
|
||||
const corruptionDetected = this.db.corruptionDetected;
|
||||
return {
|
||||
healthy: !this.db.corruptionDetected,
|
||||
healthy: !corruptionDetected,
|
||||
corruptionDetected,
|
||||
corruptionErrors: this.db.integrityCheckErrors.slice(0, 5),
|
||||
lastCheckedAt: this.db.integrityCheckLastRunAt ? new Date(this.db.integrityCheckLastRunAt) : null,
|
||||
isRunning: this.db.integrityCheckPending,
|
||||
};
|
||||
|
||||
@@ -404,6 +404,7 @@ export type NtfyNotificationEvent =
|
||||
| "planning-awaiting-input"
|
||||
| "gridlock"
|
||||
| "board-stall-unrecovered"
|
||||
| "db-corruption-detected"
|
||||
| "fallback-used"
|
||||
| "memory-dreams-processed"
|
||||
| "token-budget"
|
||||
@@ -422,6 +423,7 @@ export const NOTIFICATION_EVENTS = [
|
||||
"planning-awaiting-input",
|
||||
"gridlock",
|
||||
"board-stall-unrecovered",
|
||||
"db-corruption-detected",
|
||||
"fallback-used",
|
||||
"memory-dreams-processed",
|
||||
"token-budget",
|
||||
|
||||
@@ -25,6 +25,7 @@ import { CliBinaryInstallBanner } from "./components/CliBinaryInstallBanner";
|
||||
import { SetupWarningBanner } from "./components/SetupWarningBanner";
|
||||
import { CapacityRiskBanner } from "./components/CapacityRiskBanner";
|
||||
import { TaskIdIntegrityBanner } from "./components/TaskIdIntegrityBanner";
|
||||
import { DbCorruptionBanner } from "./components/DbCorruptionBanner";
|
||||
import { UpdateAvailableBanner } from "./components/UpdateAvailableBanner";
|
||||
import { ApprovalNotificationBanner } from "./components/ApprovalNotificationBanner";
|
||||
import { OnboardingResumeCard } from "./components/OnboardingResumeCard";
|
||||
@@ -78,7 +79,7 @@ import { NativeShellConnectionManager } from "./components/NativeShellConnection
|
||||
import { ShellConnectionStatus } from "./components/ShellConnectionStatus";
|
||||
import { getShellConnectionNativeResult, type ShellConnectionNativeResult } from "./shell-native";
|
||||
import type { AiSessionSummary, DashboardHealthResponse } from "./api";
|
||||
import { api, fetchDashboardHealth, fetchUnreadCount, fetchTaskDetail, fetchWorkflowSteps } from "./api";
|
||||
import { api, fetchDashboardHealth, fetchUnreadCount, fetchTaskDetail, fetchWorkflowSteps, refreshDashboardHealth } from "./api";
|
||||
import { getScopedItem, removeScopedItem, setScopedItem } from "./utils/projectStorage";
|
||||
import { subscribeSse } from "./sse-bus";
|
||||
import { AUTH_TOKEN_RECOVERY_REQUIRED_EVENT } from "./auth";
|
||||
@@ -690,6 +691,8 @@ function AppInner() {
|
||||
const [quickChatOpen, setQuickChatOpen] = useState(false);
|
||||
const [authTokenRecoveryOpen, setAuthTokenRecoveryOpen] = useState(false);
|
||||
const [dashboardHealth, setDashboardHealth] = useState<DashboardHealthResponse | null>(null);
|
||||
const [dbCorruptionRefreshing, setDbCorruptionRefreshing] = useState(false);
|
||||
const [dbCorruptionRefreshError, setDbCorruptionRefreshError] = useState<string | null>(null);
|
||||
const [setupWarningDismissed, setSetupWarningDismissed] = useState(
|
||||
() => getScopedItem(SETUP_WARNING_DISMISSED_KEY, currentProject?.id) === "true",
|
||||
);
|
||||
@@ -709,6 +712,19 @@ function AppInner() {
|
||||
);
|
||||
}, [currentProject?.id]);
|
||||
|
||||
const refreshDbCorruptionHealth = useCallback(async () => {
|
||||
setDbCorruptionRefreshing(true);
|
||||
setDbCorruptionRefreshError(null);
|
||||
try {
|
||||
const health = await refreshDashboardHealth();
|
||||
setDashboardHealth(health);
|
||||
} catch (error) {
|
||||
setDbCorruptionRefreshError(error instanceof Error ? error.message : "Failed to refresh database health.");
|
||||
} finally {
|
||||
setDbCorruptionRefreshing(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
|
||||
@@ -1800,7 +1816,12 @@ function AppInner() {
|
||||
}
|
||||
return {
|
||||
...current,
|
||||
status: report.status === "anomaly" || !current.database.healthy ? "degraded" : "ok",
|
||||
status:
|
||||
report.status === "anomaly"
|
||||
|| !current.database.healthy
|
||||
|| current.database.corruptionDetected
|
||||
? "degraded"
|
||||
: "ok",
|
||||
taskIdIntegrity: {
|
||||
...report,
|
||||
recommendedAction,
|
||||
@@ -1810,6 +1831,15 @@ function AppInner() {
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
{viewMode === "project" && currentProject && dashboardHealth?.database?.corruptionDetected === true && (
|
||||
<DbCorruptionBanner
|
||||
errors={dashboardHealth.database.corruptionErrors}
|
||||
lastCheckedAt={dashboardHealth.database.lastCheckedAt}
|
||||
onRefresh={refreshDbCorruptionHealth}
|
||||
refreshing={dbCorruptionRefreshing}
|
||||
refreshError={dbCorruptionRefreshError}
|
||||
/>
|
||||
)}
|
||||
{viewMode === "project" && currentProject && !setupReadinessLoading && hasWarnings && !setupWarningDismissed && (
|
||||
<SetupWarningBanner
|
||||
hasAiProvider={hasAiProvider}
|
||||
|
||||
@@ -211,6 +211,8 @@ export interface DashboardHealthResponse {
|
||||
uptime: number;
|
||||
database: {
|
||||
healthy: boolean;
|
||||
corruptionDetected: boolean;
|
||||
corruptionErrors: string[];
|
||||
lastCheckedAt: string | null;
|
||||
isRunning: boolean;
|
||||
};
|
||||
|
||||
101
packages/dashboard/app/components/DbCorruptionBanner.css
Normal file
101
packages/dashboard/app/components/DbCorruptionBanner.css
Normal file
@@ -0,0 +1,101 @@
|
||||
/* === DbCorruptionBanner === */
|
||||
.db-corruption-banner {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: var(--space-md);
|
||||
padding: var(--space-lg);
|
||||
margin-bottom: var(--space-md);
|
||||
border-radius: var(--radius-md);
|
||||
border: var(--btn-border-width) solid color-mix(in srgb, var(--color-error) 24%, transparent);
|
||||
border-inline-start: var(--space-xs) solid var(--color-error);
|
||||
background: color-mix(in srgb, var(--color-error) 10%, transparent);
|
||||
box-shadow: var(--shadow-md);
|
||||
}
|
||||
|
||||
.db-corruption-banner__header {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
justify-content: space-between;
|
||||
gap: var(--space-md);
|
||||
}
|
||||
|
||||
.db-corruption-banner__headline-wrap {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: var(--space-sm);
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.db-corruption-banner__headline-copy,
|
||||
.db-corruption-banner__item {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: var(--space-xs);
|
||||
}
|
||||
|
||||
.db-corruption-banner__headline,
|
||||
.db-corruption-banner__footer-label {
|
||||
color: var(--text);
|
||||
}
|
||||
|
||||
.db-corruption-banner__headline {
|
||||
margin: 0;
|
||||
font-size: 1rem;
|
||||
}
|
||||
|
||||
.db-corruption-banner__meta,
|
||||
.db-corruption-banner__body,
|
||||
.db-corruption-banner__footer,
|
||||
.db-corruption-banner__error {
|
||||
margin: 0;
|
||||
color: var(--text-muted);
|
||||
}
|
||||
|
||||
.db-corruption-banner__list {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: var(--space-sm);
|
||||
margin: 0;
|
||||
padding-inline-start: calc(var(--space-lg) + var(--space-xs));
|
||||
}
|
||||
|
||||
.db-corruption-banner__error-code,
|
||||
.db-corruption-banner__inline-code {
|
||||
font-family: var(--font-mono);
|
||||
color: var(--text);
|
||||
white-space: normal;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
.db-corruption-banner__refresh,
|
||||
.db-corruption-banner__refresh-icon {
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.db-corruption-banner__refresh-icon--spinning {
|
||||
animation: status-dot-pulse var(--transition-slow) ease-in-out infinite;
|
||||
}
|
||||
|
||||
.db-corruption-banner__link {
|
||||
color: var(--text);
|
||||
}
|
||||
|
||||
.db-corruption-banner__error {
|
||||
color: var(--color-error);
|
||||
}
|
||||
|
||||
@media (max-width: 768px) {
|
||||
.db-corruption-banner {
|
||||
padding: var(--space-md);
|
||||
}
|
||||
|
||||
.db-corruption-banner__header {
|
||||
flex-direction: column;
|
||||
align-items: stretch;
|
||||
}
|
||||
|
||||
.db-corruption-banner__refresh {
|
||||
width: 100%;
|
||||
justify-content: center;
|
||||
}
|
||||
}
|
||||
74
packages/dashboard/app/components/DbCorruptionBanner.tsx
Normal file
74
packages/dashboard/app/components/DbCorruptionBanner.tsx
Normal file
@@ -0,0 +1,74 @@
|
||||
import { AlertTriangle, RefreshCw } from "lucide-react";
|
||||
|
||||
import "./DbCorruptionBanner.css";
|
||||
|
||||
interface DbCorruptionBannerProps {
|
||||
errors: string[];
|
||||
lastCheckedAt: string | null;
|
||||
onRefresh: () => void | Promise<void>;
|
||||
refreshing: boolean;
|
||||
refreshError: string | null;
|
||||
}
|
||||
|
||||
export function DbCorruptionBanner({
|
||||
errors,
|
||||
lastCheckedAt,
|
||||
onRefresh,
|
||||
refreshing,
|
||||
refreshError,
|
||||
}: DbCorruptionBannerProps) {
|
||||
if (errors.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const visibleErrors = errors.slice(0, 5);
|
||||
const checkedAtLabel = lastCheckedAt ? new Date(lastCheckedAt).toLocaleString() : null;
|
||||
|
||||
return (
|
||||
<section className="db-corruption-banner" role="alert" aria-live="assertive">
|
||||
<div className="db-corruption-banner__header">
|
||||
<div className="db-corruption-banner__headline-wrap">
|
||||
<span className="status-dot status-dot--error" aria-hidden="true" />
|
||||
<AlertTriangle aria-hidden="true" />
|
||||
<div className="db-corruption-banner__headline-copy">
|
||||
<h2 className="db-corruption-banner__headline">Database corruption detected</h2>
|
||||
{checkedAtLabel ? (
|
||||
<p className="db-corruption-banner__meta">Last checked: {checkedAtLabel}</p>
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-sm db-corruption-banner__refresh"
|
||||
onClick={() => {
|
||||
void onRefresh();
|
||||
}}
|
||||
disabled={refreshing}
|
||||
>
|
||||
<RefreshCw className={refreshing ? "db-corruption-banner__refresh-icon db-corruption-banner__refresh-icon--spinning" : "db-corruption-banner__refresh-icon"} aria-hidden="true" />
|
||||
{refreshing ? "Refreshing…" : "Refresh health"}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<p className="db-corruption-banner__body">
|
||||
Fusion's background SQLite integrity check reported corruption. Review the failing objects below before continuing critical operations.
|
||||
</p>
|
||||
|
||||
<ul className="db-corruption-banner__list">
|
||||
{visibleErrors.map((error, index) => (
|
||||
<li key={`${index}:${error}`} className="db-corruption-banner__item">
|
||||
<code className="db-corruption-banner__error-code">{error}</code>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
|
||||
<p className="db-corruption-banner__footer">
|
||||
<strong className="db-corruption-banner__footer-label">What to do:</strong>{" "}
|
||||
Back up the project, try <code className="db-corruption-banner__inline-code">fn db --vacuum</code> if the database still opens cleanly, and restore from a known-good backup if corruption persists. See{" "}
|
||||
<a href="docs/storage.md" target="_blank" rel="noreferrer" className="db-corruption-banner__link">docs/storage.md</a>
|
||||
{" "}for the storage layout and recovery guidance.
|
||||
</p>
|
||||
{refreshError ? <p className="db-corruption-banner__error">{refreshError}</p> : null}
|
||||
</section>
|
||||
);
|
||||
}
|
||||
@@ -71,7 +71,13 @@ vi.mock("../../api", async (importOriginal) => {
|
||||
status: "ok",
|
||||
version: "1.0.0",
|
||||
uptime: 1,
|
||||
database: { healthy: true, lastCheckedAt: null, isRunning: false },
|
||||
database: {
|
||||
healthy: true,
|
||||
corruptionDetected: false,
|
||||
corruptionErrors: [],
|
||||
lastCheckedAt: null,
|
||||
isRunning: false,
|
||||
},
|
||||
taskIdIntegrity: { status: "ok", checkedAt: "2026-05-12T00:00:00.000Z", anomalies: [], recommendedAction: null },
|
||||
})),
|
||||
fetchPluginDashboardViews: vi.fn(() => Promise.resolve([])),
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { fireEvent, render, screen } from "@testing-library/react";
|
||||
|
||||
import { DbCorruptionBanner } from "../DbCorruptionBanner";
|
||||
|
||||
describe("DbCorruptionBanner", () => {
|
||||
it("renders errors and last-checked timestamp", () => {
|
||||
render(
|
||||
<DbCorruptionBanner
|
||||
errors={["bad row", "bad index"]}
|
||||
lastCheckedAt="2026-05-20T00:05:00.000Z"
|
||||
onRefresh={() => undefined}
|
||||
refreshing={false}
|
||||
refreshError={null}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(screen.getByRole("alert")).toBeInTheDocument();
|
||||
expect(screen.getByText("Database corruption detected")).toBeInTheDocument();
|
||||
expect(screen.getByText(/Last checked:/)).toBeInTheDocument();
|
||||
expect(screen.getByText("bad row")).toBeInTheDocument();
|
||||
expect(screen.getByText("bad index")).toBeInTheDocument();
|
||||
expect(screen.getByRole("link", { name: "docs/storage.md" })).toHaveAttribute("href", "docs/storage.md");
|
||||
});
|
||||
|
||||
it("calls onRefresh when the button is clicked", () => {
|
||||
const onRefresh = vi.fn();
|
||||
render(
|
||||
<DbCorruptionBanner
|
||||
errors={["bad row"]}
|
||||
lastCheckedAt={null}
|
||||
onRefresh={onRefresh}
|
||||
refreshing={false}
|
||||
refreshError={null}
|
||||
/>,
|
||||
);
|
||||
|
||||
fireEvent.click(screen.getByRole("button", { name: "Refresh health" }));
|
||||
|
||||
expect(onRefresh).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("renders nothing when there are no errors", () => {
|
||||
const { container } = render(
|
||||
<DbCorruptionBanner
|
||||
errors={[]}
|
||||
lastCheckedAt={null}
|
||||
onRefresh={() => undefined}
|
||||
refreshing={false}
|
||||
refreshError={null}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(container).toBeEmptyDOMElement();
|
||||
});
|
||||
});
|
||||
@@ -62,7 +62,13 @@ describe("TaskIdIntegrityBanner", () => {
|
||||
status: "ok",
|
||||
version: "1.0.0",
|
||||
uptime: 1,
|
||||
database: { healthy: true, lastCheckedAt: null, isRunning: false },
|
||||
database: {
|
||||
healthy: true,
|
||||
corruptionDetected: false,
|
||||
corruptionErrors: [],
|
||||
lastCheckedAt: null,
|
||||
isRunning: false,
|
||||
},
|
||||
taskIdIntegrity: {
|
||||
status: "ok",
|
||||
checkedAt: "2026-05-12T10:05:00.000Z",
|
||||
|
||||
@@ -67,6 +67,8 @@ class MockStore extends EventEmitter {
|
||||
getDatabaseHealth() {
|
||||
return {
|
||||
healthy: true,
|
||||
corruptionDetected: false,
|
||||
corruptionErrors: [],
|
||||
isRunning: false,
|
||||
lastCheckedAt: null,
|
||||
};
|
||||
|
||||
@@ -93,6 +93,8 @@ function createMockStore(overrides: Partial<TaskStore> = {}): TaskStore {
|
||||
}),
|
||||
getDatabaseHealth: vi.fn().mockReturnValue({
|
||||
healthy: true,
|
||||
corruptionDetected: false,
|
||||
corruptionErrors: [],
|
||||
isRunning: false,
|
||||
lastCheckedAt: null,
|
||||
}),
|
||||
@@ -356,6 +358,8 @@ describe("createServer health and headless mode", () => {
|
||||
uptime: expect.any(Number),
|
||||
database: {
|
||||
healthy: true,
|
||||
corruptionDetected: false,
|
||||
corruptionErrors: [],
|
||||
isRunning: false,
|
||||
lastCheckedAt: null,
|
||||
},
|
||||
@@ -372,6 +376,8 @@ describe("createServer health and headless mode", () => {
|
||||
const store = createMockStore({
|
||||
getDatabaseHealth: vi.fn().mockReturnValue({
|
||||
healthy: false,
|
||||
corruptionDetected: true,
|
||||
corruptionErrors: ["bad row", "bad index"],
|
||||
isRunning: false,
|
||||
lastCheckedAt: new Date("2026-05-11T10:00:00.000Z"),
|
||||
}),
|
||||
@@ -387,6 +393,8 @@ describe("createServer health and headless mode", () => {
|
||||
uptime: expect.any(Number),
|
||||
database: {
|
||||
healthy: false,
|
||||
corruptionDetected: true,
|
||||
corruptionErrors: ["bad row", "bad index"],
|
||||
isRunning: false,
|
||||
lastCheckedAt: "2026-05-11T10:00:00.000Z",
|
||||
},
|
||||
@@ -425,6 +433,8 @@ describe("createServer health and headless mode", () => {
|
||||
uptime: expect.any(Number),
|
||||
database: {
|
||||
healthy: true,
|
||||
corruptionDetected: false,
|
||||
corruptionErrors: [],
|
||||
isRunning: false,
|
||||
lastCheckedAt: null,
|
||||
},
|
||||
@@ -472,6 +482,8 @@ describe("createServer health and headless mode", () => {
|
||||
uptime: expect.any(Number),
|
||||
database: {
|
||||
healthy: true,
|
||||
corruptionDetected: false,
|
||||
corruptionErrors: [],
|
||||
isRunning: false,
|
||||
lastCheckedAt: null,
|
||||
},
|
||||
@@ -492,6 +504,41 @@ describe("createServer health and headless mode", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("reports degraded status from /api/health/refresh when corruption is detected", async () => {
|
||||
const store = createMockStore({
|
||||
getDatabaseHealth: vi.fn().mockReturnValue({
|
||||
healthy: false,
|
||||
corruptionDetected: true,
|
||||
corruptionErrors: ["bad row"],
|
||||
isRunning: false,
|
||||
lastCheckedAt: new Date("2026-05-12T12:00:00.000Z"),
|
||||
}),
|
||||
});
|
||||
const app = createServer(store);
|
||||
|
||||
const res = await REQUEST(app, "POST", "/api/health/refresh");
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body).toEqual({
|
||||
status: "degraded",
|
||||
version: CLI_PACKAGE_VERSION,
|
||||
uptime: expect.any(Number),
|
||||
database: {
|
||||
healthy: false,
|
||||
corruptionDetected: true,
|
||||
corruptionErrors: ["bad row"],
|
||||
isRunning: false,
|
||||
lastCheckedAt: "2026-05-12T12:00:00.000Z",
|
||||
},
|
||||
taskIdIntegrity: {
|
||||
status: "ok",
|
||||
checkedAt: "2026-05-12T00:00:00.000Z",
|
||||
anomalies: [],
|
||||
recommendedAction: null,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("returns reliability metrics payload from /api/health/reliability", async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2026-05-13T12:00:00.000Z"));
|
||||
|
||||
@@ -98,7 +98,7 @@ function buildHealthPayload(store: TaskStore, cliPackageVersion: string) {
|
||||
const database = store.getDatabaseHealth();
|
||||
const taskIdIntegrity = buildTaskIdIntegrityHealth(store.getTaskIdIntegrityReport());
|
||||
return {
|
||||
status: !database.healthy || taskIdIntegrity.status === "anomaly" ? "degraded" : "ok",
|
||||
status: !database.healthy || database.corruptionDetected || taskIdIntegrity.status === "anomaly" ? "degraded" : "ok",
|
||||
version: cliPackageVersion,
|
||||
uptime: Math.floor(process.uptime()),
|
||||
database,
|
||||
@@ -1325,7 +1325,7 @@ export function createServer(store: TaskStore, options?: ServerOptions): ReturnT
|
||||
const report = store.refreshTaskIdIntegrityReport();
|
||||
const database = store.getDatabaseHealth();
|
||||
res.json({
|
||||
status: !database.healthy || report.status === "anomaly" ? "degraded" : "ok",
|
||||
status: !database.healthy || database.corruptionDetected || report.status === "anomaly" ? "degraded" : "ok",
|
||||
version: cliPackageVersion,
|
||||
uptime: Math.floor(process.uptime()),
|
||||
database,
|
||||
|
||||
260
packages/engine/src/__tests__/self-healing-db-corruption.test.ts
Normal file
260
packages/engine/src/__tests__/self-healing-db-corruption.test.ts
Normal file
@@ -0,0 +1,260 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { EventEmitter } from "node:events";
|
||||
|
||||
import type { Settings, TaskStore } from "@fusion/core";
|
||||
|
||||
import { SelfHealingManager } from "../self-healing.js";
|
||||
import type { NotificationService } from "../notification/notification-service.js";
|
||||
import * as notifierModule from "../notifier.js";
|
||||
|
||||
function createMockStore(overrides: Record<string, unknown> = {}): TaskStore & EventEmitter {
|
||||
const emitter = new EventEmitter();
|
||||
return Object.assign(emitter, {
|
||||
getSettings: vi.fn().mockResolvedValue({
|
||||
maintenanceIntervalMs: 0,
|
||||
globalPause: false,
|
||||
enginePaused: false,
|
||||
ntfyEnabled: true,
|
||||
ntfyTopic: "fusion-alerts",
|
||||
ntfyEvents: ["db-corruption-detected"],
|
||||
} as unknown as Settings),
|
||||
getDatabaseHealth: vi.fn().mockReturnValue({
|
||||
healthy: true,
|
||||
corruptionDetected: false,
|
||||
corruptionErrors: [],
|
||||
lastCheckedAt: null,
|
||||
isRunning: false,
|
||||
}),
|
||||
recordRunAuditEvent: vi.fn().mockResolvedValue(undefined),
|
||||
...overrides,
|
||||
}) as unknown as TaskStore & EventEmitter;
|
||||
}
|
||||
|
||||
const BATCH1_METHODS = [
|
||||
"pruneWorktrees",
|
||||
"cleanupOrphans",
|
||||
"cleanupOrphanedBranches",
|
||||
"enforceWorktreeCap",
|
||||
] as const;
|
||||
|
||||
const BATCH2_METHODS = [
|
||||
"recoverCompletedTasks",
|
||||
"recoverStrandedCompletedTodoTasks",
|
||||
"recoverStaleIncompleteReviewTasks",
|
||||
"recoverReviewTasksWithFailedPreMergeSteps",
|
||||
"recoverInterruptedMergingTasks",
|
||||
"recoverDoneTaskMergeMetadata",
|
||||
"recoverStaleMergingStatus",
|
||||
"finalizeNoOpReviewTasks",
|
||||
"reconcileDoneTaskIntegrity",
|
||||
"reconcileStaleMergerStatus",
|
||||
"recoverMergeableReviewTasks",
|
||||
"recoverMergedReviewTasks",
|
||||
"recoverAlreadyMergedReviewTasks",
|
||||
"recoverCompletionHandoffLimbo",
|
||||
"recoverBranchMisboundInReviewTasks",
|
||||
"recoverForeignOnlyContaminatedInReviewTasks",
|
||||
"recoverOrphanOnlyScopeViolations",
|
||||
"recoverStuckMergeDeadlocks",
|
||||
"recoverMisclassifiedFailures",
|
||||
"recoverMissingWorktreeReviewFailures",
|
||||
"recoverNoProgressNoTaskDoneFailures",
|
||||
"recoverPartialProgressNoTaskDoneFailures",
|
||||
"recoverOrphanedExecutions",
|
||||
"recoverApprovedTriageTasks",
|
||||
"recoverStarvedRefinementTriageTasks",
|
||||
"recoverOrphanedPlanningTasks",
|
||||
"recoverGhostReviewTasks",
|
||||
"recoverOrphanedAgents",
|
||||
"recoverStaleHeartbeatRuns",
|
||||
"recoverAgentsRunningOnInactiveTasks",
|
||||
"recoverDriftedAgentTaskLinks",
|
||||
"clearStaleBlockedBy",
|
||||
"autoReboundPausedScopeDecay",
|
||||
"autoArchiveResolvedMetaTasks",
|
||||
"autoArchiveStalledMetaTasks",
|
||||
"runBoardStallAutoRecoverySweep",
|
||||
"reconcileSelfDefeatingDependencies",
|
||||
"reclaimPrConflicts",
|
||||
"reclaimSelfOwnedBranchConflicts",
|
||||
"reconcileTaskWorktreeMetadata",
|
||||
"reconcileInReviewBranchRebind",
|
||||
"reclaimStaleActiveBranches",
|
||||
"surfaceInReviewStalls",
|
||||
"surfaceInReviewStalled",
|
||||
"surfaceStalePausedReviews",
|
||||
"surfaceStalePausedTodos",
|
||||
"auditNoCommitsExpectedCandidates",
|
||||
] as const;
|
||||
|
||||
function stubMaintenance(manager: SelfHealingManager) {
|
||||
for (const method of BATCH1_METHODS) {
|
||||
vi.spyOn(manager as never, method).mockResolvedValue(0 as never);
|
||||
}
|
||||
vi.spyOn(manager as never, "checkpointWal").mockReturnValue(undefined as never);
|
||||
for (const method of BATCH2_METHODS) {
|
||||
vi.spyOn(manager as never, method).mockResolvedValue(0 as never);
|
||||
}
|
||||
vi.spyOn(manager, "archiveStaleDoneTasks").mockResolvedValue(0);
|
||||
}
|
||||
|
||||
describe("FN-5284: self-healing DB corruption surfacing", () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2026-05-20T00:00:00.000Z"));
|
||||
vi.spyOn(notifierModule, "getActiveNotificationService").mockReturnValue(undefined);
|
||||
vi.spyOn(notifierModule, "sendNtfyNotification").mockResolvedValue(undefined);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("does not dispatch or audit when the database is healthy", async () => {
|
||||
const store = createMockStore();
|
||||
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
|
||||
stubMaintenance(manager);
|
||||
|
||||
await (manager as any).runMaintenance();
|
||||
|
||||
expect(notifierModule.getActiveNotificationService).not.toHaveBeenCalled();
|
||||
expect(notifierModule.sendNtfyNotification).not.toHaveBeenCalled();
|
||||
expect(store.recordRunAuditEvent).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("dispatches a notification and records an audit event on first corruption detection", async () => {
|
||||
const dispatch = vi.fn().mockResolvedValue(undefined);
|
||||
vi.spyOn(notifierModule, "getActiveNotificationService").mockReturnValue({ dispatch } as unknown as NotificationService);
|
||||
const store = createMockStore({
|
||||
getDatabaseHealth: vi.fn().mockReturnValue({
|
||||
healthy: false,
|
||||
corruptionDetected: true,
|
||||
corruptionErrors: ["bad row", "bad index"],
|
||||
lastCheckedAt: new Date("2026-05-20T00:05:00.000Z"),
|
||||
isRunning: false,
|
||||
}),
|
||||
});
|
||||
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
|
||||
stubMaintenance(manager);
|
||||
|
||||
await (manager as any).runMaintenance();
|
||||
|
||||
expect(dispatch).toHaveBeenCalledWith("db-corruption-detected", {
|
||||
event: "db-corruption-detected",
|
||||
timestamp: "2026-05-20T00:00:00.000Z",
|
||||
metadata: {
|
||||
errors: ["bad row", "bad index"],
|
||||
lastCheckedAt: "2026-05-20T00:05:00.000Z",
|
||||
},
|
||||
});
|
||||
expect(store.recordRunAuditEvent).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
domain: "database",
|
||||
mutationType: "task:auto-db-corruption-detected",
|
||||
target: "database",
|
||||
metadata: expect.objectContaining({
|
||||
errors: ["bad row", "bad index"],
|
||||
notificationDispatched: true,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("respects the cooldown and avoids duplicate dispatches and audits", async () => {
|
||||
const dispatch = vi.fn().mockResolvedValue(undefined);
|
||||
vi.spyOn(notifierModule, "getActiveNotificationService").mockReturnValue({ dispatch } as unknown as NotificationService);
|
||||
const store = createMockStore({
|
||||
getDatabaseHealth: vi.fn().mockReturnValue({
|
||||
healthy: false,
|
||||
corruptionDetected: true,
|
||||
corruptionErrors: ["bad row"],
|
||||
lastCheckedAt: new Date("2026-05-20T00:05:00.000Z"),
|
||||
isRunning: false,
|
||||
}),
|
||||
});
|
||||
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
|
||||
stubMaintenance(manager);
|
||||
|
||||
await (manager as any).runMaintenance();
|
||||
await (manager as any).runMaintenance();
|
||||
|
||||
expect(dispatch).toHaveBeenCalledTimes(1);
|
||||
expect(store.recordRunAuditEvent).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("re-notifies after corruption clears and is detected again", async () => {
|
||||
const dispatch = vi.fn().mockResolvedValue(undefined);
|
||||
vi.spyOn(notifierModule, "getActiveNotificationService").mockReturnValue({ dispatch } as unknown as NotificationService);
|
||||
const getDatabaseHealth = vi.fn()
|
||||
.mockReturnValueOnce({
|
||||
healthy: false,
|
||||
corruptionDetected: true,
|
||||
corruptionErrors: ["first error"],
|
||||
lastCheckedAt: new Date("2026-05-20T00:05:00.000Z"),
|
||||
isRunning: false,
|
||||
})
|
||||
.mockReturnValueOnce({
|
||||
healthy: true,
|
||||
corruptionDetected: false,
|
||||
corruptionErrors: [],
|
||||
lastCheckedAt: new Date("2026-05-20T00:15:00.000Z"),
|
||||
isRunning: false,
|
||||
})
|
||||
.mockReturnValueOnce({
|
||||
healthy: false,
|
||||
corruptionDetected: true,
|
||||
corruptionErrors: ["second error"],
|
||||
lastCheckedAt: new Date("2026-05-20T00:25:00.000Z"),
|
||||
isRunning: false,
|
||||
});
|
||||
const store = createMockStore({ getDatabaseHealth });
|
||||
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
|
||||
stubMaintenance(manager);
|
||||
|
||||
await (manager as any).runMaintenance();
|
||||
await (manager as any).runMaintenance();
|
||||
await (manager as any).runMaintenance();
|
||||
|
||||
expect(dispatch).toHaveBeenNthCalledWith(1, "db-corruption-detected", expect.objectContaining({
|
||||
metadata: expect.objectContaining({ errors: ["first error"] }),
|
||||
}));
|
||||
expect(dispatch).toHaveBeenNthCalledWith(2, "db-corruption-detected", expect.objectContaining({
|
||||
metadata: expect.objectContaining({ errors: ["second error"] }),
|
||||
}));
|
||||
expect(dispatch).toHaveBeenCalledTimes(2);
|
||||
expect(store.recordRunAuditEvent).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("records an audit even when no notification channel is active", async () => {
|
||||
const store = createMockStore({
|
||||
getSettings: vi.fn().mockResolvedValue({
|
||||
maintenanceIntervalMs: 0,
|
||||
globalPause: false,
|
||||
enginePaused: false,
|
||||
ntfyEnabled: false,
|
||||
ntfyTopic: "fusion-alerts",
|
||||
ntfyEvents: ["db-corruption-detected"],
|
||||
} as unknown as Settings),
|
||||
getDatabaseHealth: vi.fn().mockReturnValue({
|
||||
healthy: false,
|
||||
corruptionDetected: true,
|
||||
corruptionErrors: ["bad row"],
|
||||
lastCheckedAt: new Date("2026-05-20T00:05:00.000Z"),
|
||||
isRunning: false,
|
||||
}),
|
||||
});
|
||||
const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" });
|
||||
stubMaintenance(manager);
|
||||
|
||||
await (manager as any).runMaintenance();
|
||||
|
||||
expect(notifierModule.sendNtfyNotification).not.toHaveBeenCalled();
|
||||
expect(store.recordRunAuditEvent).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
mutationType: "task:auto-db-corruption-detected",
|
||||
metadata: expect.objectContaining({ notificationDispatched: false }),
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -29,6 +29,7 @@ export const DEFAULT_NTFY_EVENTS: readonly NtfyNotificationEvent[] = [
|
||||
"planning-awaiting-input",
|
||||
"gridlock",
|
||||
"board-stall-unrecovered",
|
||||
"db-corruption-detected",
|
||||
"fallback-used",
|
||||
"token-budget",
|
||||
"message:agent-to-user",
|
||||
|
||||
@@ -254,6 +254,8 @@ export type DatabaseMutationType =
|
||||
| "task:auto-board-stall-broken"
|
||||
/** Metadata: { holderIds: string[], followerCount: number, windowMs: number, ntfyDispatched: boolean } */
|
||||
| "task:auto-board-stall-unrecovered"
|
||||
/** Metadata: { errors: string[], lastCheckedAt: string | null, notificationDispatched: boolean } */
|
||||
| "task:auto-db-corruption-detected"
|
||||
| "task:in-review-stall-deadlock-disposed"
|
||||
| "task:finalize-unproven-blocked"
|
||||
| "task:integrity-reconcile-modified-files"
|
||||
|
||||
@@ -30,7 +30,7 @@ import { existsSync, mkdirSync, readdirSync, rmSync, statSync, writeFileSync } f
|
||||
import { isAbsolute, join, relative, resolve } from "node:path";
|
||||
import { IN_REVIEW_STALL_DEADLOCK_LOG_PREFIX, IN_REVIEW_STALL_LOG_PREFIX, countRecentIdenticalStallEntries, detectSelfDefeatingDependency, getInReviewStalledSignal, getInReviewStallReason, getPrimaryPrInfo, getStalePausedReviewSignal, getStalePausedTodoSignal, getTaskHardMergeBlocker, getTaskMergeBlocker, isEphemeralAgent, type AgentStore, type ChatStore, type MessageStore, type TaskStore, type Settings, type Task, type MergeDetails, type TaskPriority, type MergeResult } from "@fusion/core";
|
||||
import type { MeshLeaseManager } from "./mesh-lease-manager.js";
|
||||
import { createLogger } from "./logger.js";
|
||||
import { createLogger, schedulerLog } from "./logger.js";
|
||||
import { RemovalReason, getRegisteredWorktreeBranchMap, getRegisteredWorktreePaths, isUsableTaskWorktree, removeWorktree, resolveWorktreeBackend, scanIdleWorktrees, scanOrphanedBranches } from "./worktree-pool.js";
|
||||
import {
|
||||
classifyMissingWorktreeSessionStartFailure,
|
||||
@@ -51,6 +51,7 @@ import type { OwnedLandedClassification } from "./merger.js";
|
||||
import { recoverForeignOnlyContamination } from "./recovery/foreign-only-contamination.js";
|
||||
import {
|
||||
buildNtfyClickUrl,
|
||||
getActiveNotificationService,
|
||||
isNtfyEventEnabled,
|
||||
resolveNtfyEvents,
|
||||
sendNtfyNotification,
|
||||
@@ -64,6 +65,7 @@ const worktreeMetadataReconcileLog = createLogger("worktree-metadata-reconcile")
|
||||
const execAsync = promisify(exec);
|
||||
const DONE_TASK_INTEGRITY_SWEEP_LIMIT = 50;
|
||||
const BOARD_STALL_NOTIFICATION_COOLDOWN_MS = 60 * 60_000;
|
||||
const DB_CORRUPTION_NOTIFICATION_COOLDOWN_MS = 60 * 60 * 1000;
|
||||
export const STALE_ACTIVE_BRANCH_EXECUTION_GRACE_MS = 10 * 60_000;
|
||||
export const COMPLETION_HANDOFF_LIMBO_GRACE_MS = 5 * 60_000;
|
||||
export const MAX_COMPLETION_HANDOFF_LIMBO_RECOVERIES = 3;
|
||||
@@ -534,6 +536,7 @@ export class SelfHealingManager {
|
||||
private maintenanceTickCounter = 0;
|
||||
private readonly processBootStartedAt = Date.now();
|
||||
private dependencyBlockedTodoReporter: DependencyBlockedTodoReporter | null = null;
|
||||
private lastDbCorruptionNotifiedAt: number | null = null;
|
||||
|
||||
private boardStallWindow: {
|
||||
windowStartMs: number;
|
||||
@@ -1317,6 +1320,7 @@ export class SelfHealingManager {
|
||||
{ name: "surface-in-review-stalled", fn: () => this.surfaceInReviewStalled() },
|
||||
{ name: "surface-stale-paused-reviews", fn: () => this.surfaceStalePausedReviews() },
|
||||
{ name: "surface-stale-paused-todos", fn: () => this.surfaceStalePausedTodos() },
|
||||
{ name: "surface-db-corruption", fn: () => this.surfaceDbCorruption() },
|
||||
{ name: "audit-no-commits-expected-candidates", fn: () => this.auditNoCommitsExpectedCandidates() },
|
||||
];
|
||||
for (const fn of batch2Fns) {
|
||||
@@ -3199,6 +3203,78 @@ export class SelfHealingManager {
|
||||
return { holders: [], recovered: 0, unrecovered: false };
|
||||
}
|
||||
|
||||
private async surfaceDbCorruption(): Promise<void> {
|
||||
const health = this.store.getDatabaseHealth();
|
||||
if (!health.corruptionDetected) {
|
||||
this.lastDbCorruptionNotifiedAt = null;
|
||||
return;
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
if (
|
||||
this.lastDbCorruptionNotifiedAt !== null
|
||||
&& now - this.lastDbCorruptionNotifiedAt < DB_CORRUPTION_NOTIFICATION_COOLDOWN_MS
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
const settings = await this.store.getSettings();
|
||||
const errors = health.corruptionErrors.slice(0, 5);
|
||||
let notificationDispatched = false;
|
||||
|
||||
try {
|
||||
const notificationService = getActiveNotificationService();
|
||||
if (notificationService) {
|
||||
await notificationService.dispatch("db-corruption-detected", {
|
||||
event: "db-corruption-detected",
|
||||
timestamp: new Date().toISOString(),
|
||||
metadata: {
|
||||
errors,
|
||||
lastCheckedAt: health.lastCheckedAt?.toISOString() ?? null,
|
||||
},
|
||||
});
|
||||
notificationDispatched = true;
|
||||
} else {
|
||||
const enabled = Boolean(settings.ntfyEnabled && settings.ntfyTopic);
|
||||
const events = resolveNtfyEvents(settings.ntfyEvents);
|
||||
if (enabled && isNtfyEventEnabled(events, "db-corruption-detected")) {
|
||||
const clickUrl = buildNtfyClickUrl({ dashboardHost: settings.ntfyDashboardHost });
|
||||
await sendNtfyNotification({
|
||||
ntfyBaseUrl: settings.ntfyBaseUrl,
|
||||
ntfyAccessToken: settings.ntfyAccessToken,
|
||||
topic: settings.ntfyTopic!,
|
||||
title: "Database corruption detected",
|
||||
message: `Background SQLite integrity check detected corruption. Errors: ${errors.join(" | ") || "unknown"}.`,
|
||||
priority: "urgent",
|
||||
clickUrl,
|
||||
});
|
||||
notificationDispatched = true;
|
||||
}
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
schedulerLog.log(
|
||||
`Failed to dispatch db-corruption-detected notification: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
}
|
||||
|
||||
const auditor = createRunAuditor(this.store, {
|
||||
runId: generateSyntheticRunId("fn5284-db-corruption", "global"),
|
||||
agentId: "self-healing",
|
||||
phase: "db-corruption-detected",
|
||||
});
|
||||
await auditor.database({
|
||||
type: "task:auto-db-corruption-detected",
|
||||
target: "database",
|
||||
metadata: {
|
||||
errors,
|
||||
lastCheckedAt: health.lastCheckedAt?.toISOString() ?? null,
|
||||
notificationDispatched,
|
||||
},
|
||||
});
|
||||
|
||||
this.lastDbCorruptionNotifiedAt = now;
|
||||
}
|
||||
|
||||
async clearStaleBlockedBy(): Promise<number> {
|
||||
try {
|
||||
const settings = await this.store.getSettings();
|
||||
|
||||
Reference in New Issue
Block a user