FN-8902: bound and cache model registry refreshes

Bound model-catalog refreshes so requests retain available models when providers stall.

- Add timeout-bounded, single-flight engine registry refreshes with fallback model retention.
- Cache refresh outcomes per registry and invalidate them after all credential mutations, including default-instance changes.
- Cover refresh cache and route behavior, and document the resilience contract.

Files changed:
 .changeset/fn-8902-models-refresh-bounding.md      |   7 +
 docs/dashboard-guide.md                            |   6 +
 .../__tests__/model-registry-refresh-cache.test.ts |  77 +++++++
 .../register-model-routes-refresh-bounding.test.ts | 246 +++++++++++++++++++++
 .../dashboard/src/model-registry-refresh-cache.ts  | 153 +++++++++++++
 packages/dashboard/src/routes.ts                   |   5 +
 .../dashboard/src/routes/register-auth-routes.ts   |  24 +-
 .../dashboard/src/routes/register-model-routes.ts  |  12 +-
 .../src/__tests__/model-registry-refresh.test.ts   |  33 +++
 packages/engine/src/auth/model-registry-refresh.ts | 125 +++++++----
 packages/engine/src/index.ts                       |   3 +
 11 files changed, 642 insertions(+), 49 deletions(-)

Fusion-Task-Id: FN-8902

Fusion-Task-Lineage: 9f0eaf34-6d28-49aa-b065-274be14ea500

Co-authored-by: Fusion (runfusion.ai) <noreply@runfusion.ai>
This commit is contained in:
gsxdsm
2026-08-11 19:00:44 -07:00
parent 7b4b2b547b
commit 5ce3a973e6
11 changed files with 645 additions and 52 deletions

View File

@@ -0,0 +1,7 @@
---
"@runfusion/fusion": patch
---
summary: Model list no longer hangs when a provider catalog stalls.
category: fix
dev: Adds bounded engine refresh seams, a generation-keyed per-registry request cache, and credential-mutation invalidation.

View File

@@ -2438,3 +2438,9 @@ The shared Task Detail Definition view shows the persisted spec alignment, lates
### Promote release-gate enrichment
`GET /api/tasks` may attach a transient `releaseGate` verdict to hold-lane cards. It includes the resolved release target, pre-release Plan Review facts, and capacity-boundary state, so Promote visibility exactly matches the server while the verdict is fresh. SSE does not carry this field: `useTasks` retains it only while its visible-evidence fingerprint and task row clock match, and for at most `RELEASE_GATE_VERDICT_MAX_AGE_MS` (30 seconds). Otherwise the card uses the conservative client fallback because workflow IR, continuations, and prompt content are not browser-visible.
### Model catalog refresh resilience
`GET /api/models` bounds each catalog refresh to 15 seconds and continues serving the registry's retained `getAvailable()` rows when a provider stalls or fails. Refreshes are single-flight per registry instance: a timed-out operation can continue in the provider runtime, but Fusion never starts another concurrently. A successful refresh is fresh for 60 seconds from its successful settlement; a failed refresh uses a separate 60-second retry window measured from its attempt start, so a failure is never reported as fresh. After a failed refresh settles, the next attempt starts only after both settlement and that retry interval.
Saving or removing API keys, completing OAuth login/manual-code flows, logging out, and removing credential instances invalidate that registry's generation and clear both windows. If a credential change happens while an uncancellable refresh is already running, the model list temporarily serves its retained rows rather than overlapping the refresh. Once that old refresh settles, the first following request starts a current-credential refresh with no additional cache-window wait.

View File

@@ -0,0 +1,77 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import {
__resetModelRegistryRefreshCacheForTests,
invalidateModelRegistryRefreshCache,
refreshModelRegistryForRequest,
} from "../model-registry-refresh-cache.js";
const options = { timeoutMs: 20, successTtlMs: 60, failureRetryMs: 60 };
describe("model registry request refresh cache", () => {
beforeEach(() => __resetModelRegistryRefreshCacheForTests());
afterEach(() => vi.useRealTimers());
it("caches successful refreshes per registry instance", async () => {
const refreshA = vi.fn(async () => undefined);
const refreshB = vi.fn(async () => undefined);
const a = { refresh: refreshA };
const b = { refresh: refreshB };
expect(await refreshModelRegistryForRequest(a, options)).toBe("completed");
expect(await refreshModelRegistryForRequest(a, options)).toBe("cached");
expect(await refreshModelRegistryForRequest(b, options)).toBe("completed");
expect(refreshA).toHaveBeenCalledOnce();
expect(refreshB).toHaveBeenCalledOnce();
});
it("does not overlap a hung refresh and returns expired requests immediately", async () => {
vi.useFakeTimers();
const refresh = vi.fn(() => new Promise<void>(() => {}));
const registry = { refresh };
const first = refreshModelRegistryForRequest(registry, options);
await vi.advanceTimersByTimeAsync(20);
await expect(first).resolves.toBe("timed_out");
await expect(refreshModelRegistryForRequest(registry, options)).resolves.toBe("timed_out");
await Promise.all(Array.from({ length: 10 }, () => refreshModelRegistryForRequest(registry, options)));
expect(refresh).toHaveBeenCalledOnce();
});
it("suppresses settled failures from attempt start without calling them fresh", async () => {
vi.useFakeTimers();
const refresh = vi.fn(async () => { throw new Error("provider failed"); });
const registry = { refresh };
expect(await refreshModelRegistryForRequest(registry, options)).toBe("failed");
await vi.runAllTicks();
expect(await refreshModelRegistryForRequest(registry, options)).toBe("negative_cached");
expect(refresh).toHaveBeenCalledOnce();
await vi.advanceTimersByTimeAsync(60);
expect(await refreshModelRegistryForRequest(registry, options)).toBe("failed");
expect(refresh).toHaveBeenCalledTimes(2);
});
it("preserves a stale-generation flight and requires a current refresh after it settles", async () => {
let resolve!: () => void;
const first = new Promise<void>((done) => { resolve = done; });
const refresh = vi.fn()
.mockImplementationOnce(() => first)
.mockResolvedValueOnce(undefined);
const registry = { refresh };
const pending = refreshModelRegistryForRequest(registry, options);
invalidateModelRegistryRefreshCache(registry);
await expect(refreshModelRegistryForRequest(registry, options)).resolves.toBe("stale_in_flight");
expect(refresh).toHaveBeenCalledOnce();
resolve();
await expect(pending).resolves.toBe("completed");
await Promise.resolve();
expect(await refreshModelRegistryForRequest(registry, options)).toBe("completed");
expect(refresh).toHaveBeenCalledTimes(2);
});
it("invalidation bypasses a prior successful window", async () => {
const refresh = vi.fn(async () => undefined);
const registry = { refresh };
await refreshModelRegistryForRequest(registry, options);
invalidateModelRegistryRefreshCache(registry);
await refreshModelRegistryForRequest(registry, options);
expect(refresh).toHaveBeenCalledTimes(2);
});
});

View File

@@ -0,0 +1,246 @@
/*
FNXC:ModelCatalog 2026-08-12-01:27:
FN-8902 requires production-shaped route coverage, not cache-only assertions: a credential save
must invalidate the same registry while a refresh is hung without losing its single-flight slot.
Changing the active credential instance is also a credential mutation, so it must invalidate the
same cache before `/api/models` can reuse its successful window. These direct handler fixtures
preserve the API boundary while keeping the 300-second regression
reproduction deterministic with fake timers. Cached requests must also run the supplemental-model
registration branch so their live rows match a completed-refresh response.
*/
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { Router } from "express";
import {
MODEL_REGISTRY_REFRESH_FAILURE_RETRY_MS,
__resetModelRegistryRefreshCacheForTests,
} from "../model-registry-refresh-cache.js";
import { registerAuthRoutes } from "../routes/register-auth-routes.js";
import { registerModelRoutes } from "../routes/register-model-routes.js";
const rows = [{ provider: "openai", id: "gpt-test", name: "Retained", reasoning: true, contextWindow: 8_192 }];
type Handler = (req: { body?: Record<string, unknown>; params?: Record<string, string> }, res: { json: (body: unknown) => void }) => Promise<void>;
type Registry = { refresh: () => Promise<void>; getAvailable: () => typeof rows };
function deferred<T>() {
let resolve!: (value: T | PromiseLike<T>) => void;
let reject!: (reason?: unknown) => void;
const promise = new Promise<T>((resolvePromise, rejectPromise) => {
resolve = resolvePromise;
reject = rejectPromise;
});
return { promise, resolve, reject };
}
function register(registry?: Registry, configuredOAuthProviders: string[] = []) {
const getHandlers = new Map<string, Handler>();
const postHandlers = new Map<string, Handler>();
const router = {
get: vi.fn((path: string, handler: Handler) => getHandlers.set(path, handler)),
post: vi.fn((path: string, handler: Handler) => postHandlers.set(path, handler)),
delete: vi.fn(),
put: vi.fn(),
} as unknown as Router;
const warn = vi.fn();
const authStorage = {
reload: vi.fn(),
getOAuthProviders: () => configuredOAuthProviders.map((id) => ({ id })),
hasAuth: (provider: string) => configuredOAuthProviders.includes(provider),
hasApiKey: (provider: string) => provider === "openai",
getApiKeyProviders: () => [{ id: "openai", name: "OpenAI" }],
setApiKey: vi.fn().mockResolvedValue(undefined),
getInstance: vi.fn(() => ({ type: "api_key", key: "sk-test" })),
setDefaultInstance: vi.fn().mockResolvedValue(undefined),
};
const context = {
router,
store: {
getGlobalSettingsStore: () => ({ getSettings: vi.fn().mockResolvedValue({}) }),
getSettingsFast: vi.fn().mockResolvedValue({}),
},
runtimeLogger: { child: vi.fn(() => ({ warn })) },
options: registry ? {
modelRegistry: registry,
authStorage: {
...authStorage,
hasAuth: (provider: string) => configuredOAuthProviders.includes(provider),
},
} : { authStorage },
getScopedStore: vi.fn(),
rethrowAsApiError: (error: unknown) => { throw error; },
};
registerModelRoutes(context as never);
registerAuthRoutes(context as never);
return {
handler: getHandlers.get("/models")!,
saveApiKey: postHandlers.get("/auth/api-key")!,
setDefaultInstance: postHandlers.get("/auth/providers/:provider/default-instance")!,
warn,
authStorage,
};
}
async function request(handler: Handler): Promise<{ models: typeof rows }> {
const json = vi.fn();
await handler({}, { json });
return json.mock.calls[0]?.[0] as { models: typeof rows };
}
async function saveApiKey(handler: Handler) {
const json = vi.fn();
await handler({ body: { provider: "openai", apiKey: "sk-test" } }, { json });
expect(json).toHaveBeenCalledWith(expect.objectContaining({ success: true }));
}
async function setDefaultInstance(handler: Handler) {
const json = vi.fn();
await handler({ params: { provider: "openai" }, body: { instance: "secondary" } }, { json });
expect(json).toHaveBeenCalledWith(expect.objectContaining({ success: true }));
}
async function flushSettlements() {
await Promise.resolve();
await Promise.resolve();
}
describe("registerModelRoutes refresh bounding", () => {
beforeEach(() => __resetModelRegistryRefreshCacheForTests());
afterEach(() => vi.useRealTimers());
it("returns retained rows when a registry refresh never settles", async () => {
vi.useFakeTimers();
const refresh = vi.fn(() => new Promise<void>(() => {}));
const { handler } = register({ refresh, getAvailable: () => rows });
const pending = request(handler);
await vi.advanceTimersByTimeAsync(15_000);
await expect(pending).resolves.toMatchObject({ models: rows });
expect(refresh).toHaveBeenCalledOnce();
});
it("never overlaps hung registry refreshes across sequential and concurrent requests", async () => {
vi.useFakeTimers();
const refresh = vi.fn(() => new Promise<void>(() => {}));
const { handler } = register({ refresh, getAvailable: () => rows });
const first = request(handler);
await vi.advanceTimersByTimeAsync(15_000);
await first;
await Promise.all(Array.from({ length: 10 }, () => request(handler)));
expect(refresh).toHaveBeenCalledOnce();
});
it("invalidates through POST /auth/api-key without overlapping an old hung refresh", async () => {
vi.useFakeTimers();
const first = deferred<void>();
const refresh = vi.fn()
.mockImplementationOnce(() => first.promise)
.mockResolvedValueOnce(undefined);
const { handler, saveApiKey: save, warn } = register({ refresh, getAvailable: () => rows });
const initial = request(handler);
await saveApiKey(save);
// A new credential generation cannot start alongside the uncancellable old refresh.
await expect(request(handler)).resolves.toMatchObject({ models: rows });
expect(refresh).toHaveBeenCalledOnce();
expect(warn).toHaveBeenCalledWith(expect.stringContaining("stale_in_flight"));
first.resolve();
await flushSettlements();
await vi.advanceTimersByTimeAsync(15_000);
await expect(initial).resolves.toMatchObject({ models: rows });
// The old generation's late success is discarded, so this is a real current-generation refresh.
await expect(request(handler)).resolves.toMatchObject({ models: rows });
expect(refresh).toHaveBeenCalledTimes(2);
});
it("uses the failure retry window at the route boundary and refreshes after it expires", async () => {
vi.useFakeTimers();
const refresh = vi.fn().mockRejectedValue(new Error("catalog unavailable"));
const { handler } = register({ refresh, getAvailable: () => rows });
await expect(request(handler)).resolves.toMatchObject({ models: rows });
await flushSettlements();
await expect(request(handler)).resolves.toMatchObject({ models: rows });
expect(refresh).toHaveBeenCalledOnce();
await vi.advanceTimersByTimeAsync(MODEL_REGISTRY_REFRESH_FAILURE_RETRY_MS);
await expect(request(handler)).resolves.toMatchObject({ models: rows });
expect(refresh).toHaveBeenCalledTimes(2);
});
it("keeps registry freshness instance-scoped when a credential mutation invalidates one router", async () => {
const refreshA = vi.fn().mockResolvedValue(undefined);
const refreshB = vi.fn().mockResolvedValue(undefined);
const first = register({ refresh: refreshA, getAvailable: () => rows });
const second = register({ refresh: refreshB, getAvailable: () => rows });
await request(first.handler);
await request(second.handler);
await saveApiKey(first.saveApiKey);
await request(first.handler);
await request(second.handler);
expect(refreshA).toHaveBeenCalledTimes(2);
expect(refreshB).toHaveBeenCalledOnce();
});
it("invalidates the successful catalog window when the default credential instance changes", async () => {
const refresh = vi.fn().mockResolvedValue(undefined);
const { handler, setDefaultInstance: setDefault } = register({ refresh, getAvailable: () => rows });
await request(handler);
await setDefaultInstance(setDefault);
await request(handler);
expect(refresh).toHaveBeenCalledTimes(2);
});
it("runs supplemental registrations for cached requests as it does after a refresh", async () => {
const registeredProviders = new Map<string, { models: Array<Record<string, unknown>> }>([
["openai-codex", { models: [] }],
]);
const registry = {
refresh: vi.fn().mockResolvedValue(undefined),
registeredProviders,
registerProvider: vi.fn((provider: string, config: { models: Array<Record<string, unknown>> }) => {
registeredProviders.set(provider, { models: config.models });
}),
getAll: () => [...registeredProviders.entries()].flatMap(([provider, config]) => config.models.map((model) => ({
...model,
provider,
}))),
getAvailable: () => [
...rows,
...registeredProviders.get("openai-codex")!.models.map((model) => ({
provider: "openai-codex",
id: String(model.id),
name: String(model.name),
reasoning: Boolean(model.reasoning),
contextWindow: Number(model.contextWindow),
})),
],
};
const { handler } = register(registry as never, ["openai-codex"]);
const fresh = await request(handler);
expect(fresh.models.filter((model) => model.provider === "openai-codex").map((model) => model.id)).toEqual(expect.arrayContaining([
"gpt-5.6-luna", "gpt-5.6-sol", "gpt-5.6-terra",
]));
// Simulate a catalog replacement between polls; the cached path must reapply supplements.
registeredProviders.set("openai-codex", { models: [] });
const cached = await request(handler);
expect(registry.refresh).toHaveBeenCalledOnce();
expect(cached.models).toEqual(fresh.models);
});
it("preserves route dedupe and the absent-registry empty-list branch", async () => {
const duplicateRows = [...rows, { ...rows[0] }];
const { handler } = register({
refresh: vi.fn().mockResolvedValue(undefined),
getAvailable: () => duplicateRows,
} as never);
await expect(request(handler)).resolves.toMatchObject({ models: rows });
const absent = register();
await expect(request(absent.handler)).resolves.toMatchObject({ models: [] });
});
});

View File

@@ -0,0 +1,153 @@
import {
DEFAULT_MODEL_REGISTRY_REFRESH_TIMEOUT_MS,
boundExistingModelRegistryRefresh,
startFusionModelRegistryRefresh,
type RefreshableModelRegistry,
} from "@fusion/engine";
/*
FNXC:ModelCatalog 2026-08-12-01:00:
FN-8902 measured a ModelRegistry.refresh() stall of about 300 seconds. `/api/models` therefore
uses a per-registry WeakMap single flight: timing out a request never releases the underlying,
uncancellable refresh slot or permits concurrent provider reloads. The engine seam keeps that
promise observable while preserving its runtime-aware timeout implementation.
Success freshness (settled success) and failure retry (attempt start) deliberately use different
fields and anchors. A failed catalog reload must never claim the live getAvailable() catalog is
fresh, and a settle-anchored retry interval cannot bound an operation that may hang for minutes.
Credential changes bump a generation rather than deleting an entry, preserving in-flight tracking.
A mutation during an uncancellable old-generation refresh temporarily serves live but stale rows;
once it settles, the next request starts a current-generation refresh without an extra window wait.
*/
export const MODEL_REGISTRY_REQUEST_REFRESH_TIMEOUT_MS = DEFAULT_MODEL_REGISTRY_REFRESH_TIMEOUT_MS;
export const MODEL_REGISTRY_REFRESH_SUCCESS_TTL_MS = 60_000;
export const MODEL_REGISTRY_REFRESH_FAILURE_RETRY_MS = 60_000;
export type ModelRegistryRequestRefreshOutcome =
| "completed"
| "cached"
| "negative_cached"
| "stale_in_flight"
| "timed_out"
| "failed";
type RefreshCacheEntry = {
generation: number;
freshAsOfGeneration?: number;
succeededAt?: number;
lastAttemptStartedAt?: number;
lastAttemptGeneration?: number;
lastOutcome?: "completed" | "timed_out" | "failed";
lastOutcomeAt?: number;
inFlight?: { promise: Promise<unknown>; generation: number; startedAt: number };
};
let entries = new WeakMap<object, RefreshCacheEntry>();
export type RefreshModelRegistryForRequestOptions = {
/** Injectable clock and windows are test seams; production uses the exported defaults. */
now?: () => number;
timeoutMs?: number;
successTtlMs?: number;
failureRetryMs?: number;
};
function entryFor(registry: object): RefreshCacheEntry {
let entry = entries.get(registry);
if (!entry) {
entry = { generation: 0 };
entries.set(registry, entry);
}
return entry;
}
/** Bump only this registry's generation without ever dropping an in-flight refresh. */
export function invalidateModelRegistryRefreshCache(registry: object): void {
const entry = entryFor(registry);
entry.generation += 1;
entry.freshAsOfGeneration = undefined;
entry.succeededAt = undefined;
entry.lastAttemptStartedAt = undefined;
entry.lastAttemptGeneration = undefined;
entry.lastOutcome = undefined;
entry.lastOutcomeAt = undefined;
}
/** Test-only full reset, including any tracked underlying refresh promises. */
export function __resetModelRegistryRefreshCacheForTests(): void {
entries = new WeakMap<object, RefreshCacheEntry>();
}
/**
* Bound and single-flight a live registry refresh. This never throws: callers
* always retain `getAvailable()` as their model-row source of truth.
*/
export async function refreshModelRegistryForRequest(
registry: RefreshableModelRegistry,
options: RefreshModelRegistryForRequestOptions = {},
): Promise<ModelRegistryRequestRefreshOutcome> {
const now = options.now ?? Date.now;
const timeoutMs = options.timeoutMs ?? MODEL_REGISTRY_REQUEST_REFRESH_TIMEOUT_MS;
const successTtlMs = options.successTtlMs ?? MODEL_REGISTRY_REFRESH_SUCCESS_TTL_MS;
const failureRetryMs = options.failureRetryMs ?? MODEL_REGISTRY_REFRESH_FAILURE_RETRY_MS;
const entry = entryFor(registry);
const nowMs = now();
if (entry.inFlight) {
if (entry.inFlight.generation !== entry.generation) return "stale_in_flight";
const elapsed = nowMs - entry.inFlight.startedAt;
if (elapsed >= timeoutMs) return "timed_out";
return boundExistingModelRegistryRefresh(entry.inFlight.promise, { timeoutMs: timeoutMs - elapsed });
}
if (
entry.freshAsOfGeneration === entry.generation
&& entry.succeededAt !== undefined
&& nowMs - entry.succeededAt < successTtlMs
) return "cached";
if (
entry.lastAttemptGeneration === entry.generation
&& (entry.lastOutcome === "timed_out" || entry.lastOutcome === "failed")
&& entry.lastAttemptStartedAt !== undefined
&& nowMs - entry.lastAttemptStartedAt < failureRetryMs
) return "negative_cached";
entry.lastAttemptStartedAt = nowMs;
entry.lastAttemptGeneration = entry.generation;
entry.lastOutcome = undefined;
entry.lastOutcomeAt = undefined;
try {
const started = startFusionModelRegistryRefresh(registry, { timeoutMs });
const inFlight = { promise: started.underlying, generation: entry.generation, startedAt: nowMs };
entry.inFlight = inFlight;
// The underlying promise is intentionally retained after a request timeout.
void inFlight.promise.then(
() => {
if (entry.inFlight?.promise === inFlight.promise) entry.inFlight = undefined;
if (inFlight.generation !== entry.generation) return;
entry.lastOutcome = "completed";
entry.lastOutcomeAt = now();
entry.freshAsOfGeneration = entry.generation;
entry.succeededAt = now();
},
() => {
if (entry.inFlight?.promise === inFlight.promise) entry.inFlight = undefined;
if (inFlight.generation !== entry.generation) return;
entry.lastOutcome = "failed";
entry.lastOutcomeAt = now();
},
).catch(() => {});
// Guarding here makes a late underlying rejection harmless even if callers
// only observe the bounded outcome.
void inFlight.promise.catch(() => {});
return await started.bounded;
} catch {
// start is designed not to throw, but this route cache must remain fail-soft.
entry.lastOutcome = "failed";
entry.lastOutcomeAt = now();
return "failed";
}
}

View File

@@ -110,6 +110,7 @@ const TASK_DETAIL_ACTIVITY_LOG_LIMIT = 500;
* intentionally remain exported from this file for existing tests/importers.
*/
export { __resetBatchImportRateLimiter } from "./routes/register-git-github.js";
export { __resetModelRegistryRefreshCacheForTests } from "./model-registry-refresh-cache.js";
/**
* Minimal interface matching pi 0.80.8+ ModelRuntime's ModelRegistry
@@ -122,6 +123,10 @@ export interface ModelRegistryLike {
* before reading getAvailable() and surface any refresh failure to the caller.
*/
refresh(): Promise<void>;
/** Optional runtime passthrough lets request refreshes use the engine abort-aware path. */
modelRuntime?: {
refresh: (options?: { allowNetwork?: boolean; signal?: AbortSignal; force?: boolean }) => Promise<unknown>;
};
/** Get models that have auth configured. */
getAvailable(): Array<{ id: string; name: string; provider: string; reasoning: boolean; contextWindow: number }>;
/** Optional pi ModelRegistry surface used for supplemental model registration. */

View File

@@ -13,6 +13,7 @@ import { probeLlamaCpp } from "../llama-cpp-probe.js";
import { ApiError, badRequest, conflict } from "../api-error.js";
import { clearUsageCache } from "../usage.js";
import { invalidateAllGlobalSettingsCaches } from "../project-store-resolver.js";
import { invalidateModelRegistryRefreshCache } from "../model-registry-refresh-cache.js";
import type { AuthStorageLike } from "../routes.js";
import type { ApiRouteRegistrar } from "./types.js";
import {
@@ -35,6 +36,17 @@ export const registerAuthRoutes: ApiRouteRegistrar = (ctx) => {
const { router, options, store, getScopedStore, rethrowAsApiError } = ctx;
const authStorage = options?.authStorage;
/*
FNXC:ModelCatalog 2026-08-12-01:00:
FN-8902 makes catalog freshness safe across credential changes by bumping only
this registry's generation. The bump clears success and failure windows but
preserves an uncancellable in-flight refresh: a mutation during that flight
deliberately accepts bounded temporary staleness rather than overlapping it.
*/
const invalidateModelsAfterCredentialMutation = () => {
if (options?.modelRegistry) invalidateModelRegistryRefreshCache(options.modelRegistry);
};
/*
FNXC:ProviderAuth 2026-07-14-14:22:
CLI-backed providers own their credentials and have dedicated status rows below. Runtime model registration can also expose those ids through getApiKeyProviders(); exclude them from the generic API-key union so Grok cannot render twice as both "missing API key" and ready via its authenticated CLI.
@@ -1717,7 +1729,8 @@ export const registerAuthRoutes: ApiRouteRegistrar = (ctx) => {
loginPromise
.then(() => {
// Login completed (user finished OAuth in browser)
// Login completed (user finished OAuth in browser).
invalidateModelsAfterCredentialMutation();
})
.catch((err: unknown) => {
// Login failed — also reject auth URL if not yet received
@@ -1842,6 +1855,7 @@ export const registerAuthRoutes: ApiRouteRegistrar = (ctx) => {
activeLogin.inputSubmitted = true;
await deliverManualOAuthCallbackToLocalListener(storageProvider, code);
activeLogin.resolveInput(normalizeManualOAuthInputForProvider(storageProvider, code));
invalidateModelsAfterCredentialMutation();
res.json({ success: true, submitted: true });
} catch (err: unknown) {
if (err instanceof ApiError) {
@@ -1933,6 +1947,7 @@ export const registerAuthRoutes: ApiRouteRegistrar = (ctx) => {
await storage.logout(toOauthCredentialProviderId(provider));
}
clearUsageCache();
invalidateModelsAfterCredentialMutation();
res.json({ success: true });
} catch (err: unknown) {
if (err instanceof ApiError) {
@@ -1997,7 +2012,7 @@ export const registerAuthRoutes: ApiRouteRegistrar = (ctx) => {
refreshError = error instanceof Error ? error.message : String(error);
}
options?.modelRegistry?.refresh?.();
invalidateModelsAfterCredentialMutation();
clearUsageCache();
res.json({
success: true,
@@ -2049,7 +2064,7 @@ export const registerAuthRoutes: ApiRouteRegistrar = (ctx) => {
} else {
await storage.clearApiKey(provider);
}
// No model refresh needed on delete: removing the key leaves nothing to sync.
invalidateModelsAfterCredentialMutation();
clearUsageCache();
res.json({ success: true });
} catch (err: unknown) {
@@ -2102,6 +2117,8 @@ export const registerAuthRoutes: ApiRouteRegistrar = (ctx) => {
const storage = getAuthStorage();
if (!storage.getInstance?.(ref) || !storage.setDefaultInstance) throw new ApiError(404, "Credential instance not found");
await storage.setDefaultInstance(ref);
invalidateModelsAfterCredentialMutation();
clearUsageCache();
res.json({ success: true });
} catch (err: unknown) { if (err instanceof ApiError) throw err; rethrowAsApiError(err); }
});
@@ -2112,6 +2129,7 @@ export const registerAuthRoutes: ApiRouteRegistrar = (ctx) => {
const storage = getAuthStorage();
if (!storage.getInstance?.(ref) || !storage.removeInstance) throw new ApiError(404, "Credential instance not found");
await storage.removeInstance(ref);
invalidateModelsAfterCredentialMutation();
clearUsageCache();
res.json({ success: true });
} catch (err: unknown) { if (err instanceof ApiError) throw err; rethrowAsApiError(err); }

View File

@@ -9,6 +9,7 @@ import { getGrokPickerModels, GROK_PICKER_PROVIDER_ID } from "../grok-model-cach
import { getClaudePickerModels, CLAUDE_PICKER_PROVIDER_ID } from "../claude-model-cache.js";
import { getOmpPickerModels, OMP_PICKER_PROVIDER_ID } from "../omp-model-cache.js";
import { getHermesPickerModels, HERMES_PICKER_PROVIDER_ID } from "../hermes-model-cache.js";
import { refreshModelRegistryForRequest } from "../model-registry-refresh-cache.js";
import type { AuthStorageLike } from "../routes.js";
import type { ApiRouteRegistrar } from "./types.js";
@@ -330,7 +331,16 @@ export const registerModelRoutes: ApiRouteRegistrar = (ctx) => {
}
try {
await options.modelRegistry.refresh();
const refreshOutcome = await refreshModelRegistryForRequest(options.modelRegistry);
if (["timed_out", "failed", "stale_in_flight", "negative_cached"].includes(refreshOutcome)) {
runtimeLogger.child("models").warn(`Model registry refresh outcome: ${refreshOutcome}; serving retained catalog`);
}
/*
FNXC:ModelCatalog 2026-08-12-01:00:
FN-8902 bounds and caches only the refresh operation. Supplemental merges and
dedupe remain unconditional per request because refresh can replace provider
rows; cached, failed, or timed-out paths must return the same live catalog shape.
*/
if (options.modelRegistry.registerProvider) {
mergeSupplementalAnthropicModels(options.modelRegistry as Parameters<typeof mergeSupplementalAnthropicModels>[0], (message) => runtimeLogger.child("models").warn(message));
/*

View File

@@ -1,7 +1,9 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import {
DEFAULT_MODEL_REGISTRY_REFRESH_TIMEOUT_MS,
boundExistingModelRegistryRefresh,
refreshFusionModelRegistry,
startFusionModelRegistryRefresh,
} from "../auth/model-registry-refresh.js";
describe("refreshFusionModelRegistry", () => {
@@ -51,4 +53,35 @@ describe("refreshFusionModelRegistry", () => {
it("defaults timeout to the create-path bound", () => {
expect(DEFAULT_MODEL_REGISTRY_REFRESH_TIMEOUT_MS).toBe(15_000);
});
it("exposes one faithful underlying refresh alongside its bounded outcome", async () => {
const refresh = vi.fn(async () => "catalog");
const started = startFusionModelRegistryRefresh({ refresh });
await expect(started.underlying).resolves.toBe("catalog");
await expect(started.bounded).resolves.toBe("completed");
expect(refresh).toHaveBeenCalledOnce();
});
it("bounds an existing promise without starting another refresh", async () => {
vi.useFakeTimers();
const refresh = vi.fn();
let resolve!: () => void;
const underlying = new Promise<void>((done) => { resolve = done; });
const pending = boundExistingModelRegistryRefresh(underlying, { timeoutMs: 20 });
await vi.advanceTimersByTimeAsync(20);
await expect(pending).resolves.toBe("timed_out");
expect(refresh).not.toHaveBeenCalled();
const completed = boundExistingModelRegistryRefresh(Promise.resolve(), { timeoutMs: 20 });
await expect(completed).resolves.toBe("completed");
resolve();
});
it("maps a late existing rejection to failed without an unhandled rejection", async () => {
let reject!: (reason: Error) => void;
const underlying = new Promise<void>((_, fail) => { reject = fail; });
const started = startFusionModelRegistryRefresh({ refresh: () => underlying });
void started.bounded;
reject(new Error("late provider failure"));
await expect(started.bounded).resolves.toBe("failed");
});
});

View File

@@ -1,13 +1,10 @@
/*
FNXC:ModelRegistry 2026-07-21-17:15:
pi 0.80.8+ ModelRegistry.refresh() delegates to ModelRuntime.reloadConfig() /
refresh(), which performs remote model-catalog fetches and availability checks
with no timeout on the post-create path. A hung provider catalog (observed as a
stuck HTTPS connection to Cloudflare while the TUI stayed on "Loading
extensions…") blocked fn dashboard / serve / daemon forever after extensions
had already finished loading. Bound every Fusion-owned await so startup always
progresses; createFusionModelRegistry already ran a 15s network refresh and
cached models remain usable.
FNXC:ModelRegistry 2026-08-12-01:00:
ModelRegistry.refresh() can leave an uncancellable provider-catalog operation running after its
bounded await expires. FN-8902 exposes that one underlying promise so request-path callers can
retain single-flight ownership and apply later waits only for their remaining budget, rather than
starting concurrent catalog reloads. On the ModelRuntime path the first bound abort can reject the
underlying promise; consumers must account for that as a failed late settlement, never a refresh.
*/
/** Default bound for Fusion-owned model-registry refresh awaits (matches ModelRuntime.create). */
@@ -33,6 +30,82 @@ export type RefreshFusionModelRegistryOptions = {
log?: (message: string) => void;
};
/** Options for bounding a refresh that has already been started. */
export type BoundExistingModelRegistryRefreshOptions = Pick<RefreshFusionModelRegistryOptions, "timeoutMs" | "log">;
function refreshTimeoutError(timeoutMs: number): Error {
return new Error(`Model registry refresh timed out after ${timeoutMs}ms`);
}
async function boundModelRegistryRefresh(
underlying: Promise<unknown>,
options: BoundExistingModelRegistryRefreshOptions,
controller?: AbortController,
): Promise<ModelRegistryRefreshOutcome> {
const timeoutMs = options.timeoutMs ?? DEFAULT_MODEL_REGISTRY_REFRESH_TIMEOUT_MS;
let timedOut = false;
let timer: ReturnType<typeof setTimeout> | undefined;
const timeout = new Promise<never>((_, reject) => {
timer = setTimeout(() => {
timedOut = true;
controller?.abort();
reject(refreshTimeoutError(timeoutMs));
}, timeoutMs);
});
try {
await Promise.race([underlying, timeout]);
return "completed";
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
if (timedOut || controller?.signal.aborted || /timed out/i.test(message)) {
options.log?.(`Model registry refresh timed out after ${timeoutMs}ms; continuing with cached models`);
return "timed_out";
}
options.log?.(`Model registry refresh failed: ${message}`);
return "failed";
} finally {
if (timer) clearTimeout(timer);
}
}
/**
* Apply Fusion's wall-clock outcome mapping to an already-running refresh.
* This intentionally creates no AbortSignal: only the original starter can
* signal a ModelRuntime operation, and registry.refresh() cannot be cancelled.
*/
export function boundExistingModelRegistryRefresh(
underlying: Promise<unknown>,
options: BoundExistingModelRegistryRefreshOptions = {},
): Promise<ModelRegistryRefreshOutcome> {
return boundModelRegistryRefresh(underlying, options);
}
/**
* Start exactly one registry refresh and expose both its faithful underlying
* promise and its bounded outcome. A retained catch prevents a late rejection
* from becoming unhandled when callers only await `bounded`.
*/
export function startFusionModelRegistryRefresh(
modelRegistry: RefreshableModelRegistry,
options: RefreshFusionModelRegistryOptions = {},
): { underlying: Promise<unknown>; bounded: Promise<ModelRegistryRefreshOutcome> } {
const controller = new AbortController();
const allowNetwork = options.allowNetwork ?? true;
const runtime = modelRegistry.modelRuntime;
const underlying = typeof runtime?.refresh === "function"
? Promise.resolve().then(() => runtime.refresh({ allowNetwork, signal: controller.signal }))
: Promise.resolve().then(() => modelRegistry.refresh());
// Keep a rejection observed independently of the bounded race without changing
// the promise returned to callers that need its original settlement.
void underlying.catch(() => {});
return {
underlying,
bounded: boundModelRegistryRefresh(underlying, options, controller),
};
}
/**
* Await a model-registry refresh with a hard wall-clock bound.
* Prefers ModelRuntime.refresh({ signal }) when present so in-flight catalog
@@ -43,43 +116,5 @@ export async function refreshFusionModelRegistry(
modelRegistry: RefreshableModelRegistry,
options: RefreshFusionModelRegistryOptions = {},
): Promise<ModelRegistryRefreshOutcome> {
const timeoutMs = options.timeoutMs ?? DEFAULT_MODEL_REGISTRY_REFRESH_TIMEOUT_MS;
const allowNetwork = options.allowNetwork ?? true;
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), timeoutMs);
try {
const runtime = modelRegistry.modelRuntime;
const work = runtime
? runtime.refresh({ allowNetwork, signal: controller.signal })
: Promise.resolve(modelRegistry.refresh());
await Promise.race([
work,
new Promise<never>((_, reject) => {
const onAbort = () => {
reject(new Error(`Model registry refresh timed out after ${timeoutMs}ms`));
};
if (controller.signal.aborted) {
onAbort();
return;
}
controller.signal.addEventListener("abort", onAbort, { once: true });
}),
]);
return "completed";
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
const timedOut = controller.signal.aborted || /timed out/i.test(message);
if (timedOut) {
options.log?.(
`Model registry refresh timed out after ${timeoutMs}ms; continuing with cached models`,
);
return "timed_out";
}
options.log?.(`Model registry refresh failed: ${message}`);
return "failed";
} finally {
clearTimeout(timer);
}
return startFusionModelRegistryRefresh(modelRegistry, options).bounded;
}

View File

@@ -10,7 +10,10 @@ export type { AgentActionGateContext, AgentActionGateDecision } from "./agents/a
export { createFusionAuthStorage, createFusionModelRegistry } from "./auth/auth-storage.js";
export {
DEFAULT_MODEL_REGISTRY_REFRESH_TIMEOUT_MS,
boundExistingModelRegistryRefresh,
refreshFusionModelRegistry,
startFusionModelRegistryRefresh,
type BoundExistingModelRegistryRefreshOptions,
type ModelRegistryRefreshOutcome,
type RefreshableModelRegistry,
type RefreshFusionModelRegistryOptions,