From 13f7266b6797f3e0c5ba9f2e793da4fe46806cd8 Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Sat, 1 Aug 2026 01:34:43 -0700 Subject: [PATCH] FN-8670: reuse model runtime fixtures in engine tests Share a warmed Pi model runtime across engine catalog tests. - Add an isolated in-memory model registry fixture backed by a per-file shared runtime. - Warm the runtime before catalog tests and verify custom-provider registry isolation. - Document the required fixture pattern for real Pi SDK catalog tests. Files changed: docs/testing.md | 1 + .../engine/src/__tests__/_model-runtime-fixture.ts | 40 ++++++++++++++++++++++ .../custom-providers-openai-completions.test.ts | 15 +++----- .../custom-providers-openai-responses.test.ts | 15 +++----- .../src/__tests__/provider-registration.test.ts | 33 ++++++++++++------ 5 files changed, 73 insertions(+), 31 deletions(-) Fusion-Task-Id: FN-8670 Fusion-Task-Lineage: 7410d114-1853-44cc-a09b-a997fbc7f119 Co-authored-by: Fusion (runfusion.ai) --- docs/testing.md | 1 + .../src/__tests__/_model-runtime-fixture.ts | 40 +++++++++++++++++++ ...ustom-providers-openai-completions.test.ts | 15 +++---- .../custom-providers-openai-responses.test.ts | 15 +++---- .../__tests__/provider-registration.test.ts | 33 ++++++++++----- 5 files changed, 73 insertions(+), 31 deletions(-) create mode 100644 packages/engine/src/__tests__/_model-runtime-fixture.ts diff --git a/docs/testing.md b/docs/testing.md index f71a21585f..9a0f531692 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -631,6 +631,7 @@ When a test owns a process-wide singleton that has asynchronous owners (timers, - Prefer fake timers over real polling/time waits (FN-2707 pattern: advance timers inside `act(...)`, restore with `afterEach(() => vi.useRealTimers())`). - Do **not** mask slowness by raising worker/concurrency knobs (`FUSION_TEST_TOTAL_WORKERS`, `FUSION_TEST_CONCURRENCY`, `VITEST_MAX_WORKERS`, workspace concurrency settings). - Do **not** add net-new real-network calls, real-`setTimeout` polling loops, or mock-the-world component shells when a narrower seam exists. +- Real Pi SDK catalog tests in the engine package must use `src/__tests__/_model-runtime-fixture.ts`: warm its shared runtime in `beforeAll` and request a fresh registry rather than constructing `ModelRuntime` inside timed test bodies. - Use the canonical taxonomy in **What NOT to write** and **What TO keep unconditionally** when deciding trim vs keep. - See `docs/test-speed-audit-FN-5048.md` for the measured baseline offender list and optimization priorities. diff --git a/packages/engine/src/__tests__/_model-runtime-fixture.ts b/packages/engine/src/__tests__/_model-runtime-fixture.ts new file mode 100644 index 0000000000..653c36f68d --- /dev/null +++ b/packages/engine/src/__tests__/_model-runtime-fixture.ts @@ -0,0 +1,40 @@ +import { ModelRegistry, ModelRuntime } from "@earendil-works/pi-coding-agent"; + +/* +FNXC:ModelCatalog 2026-08-01-08:14: +FN-8670 moves real Pi SDK catalog construction out of timed test bodies into beforeAll. The runtime is +memoized per test file because Vitest file-level isolation gives each file its own module graph. +ModelRegistry.registerProvider writes into its shared ModelRuntime, so each handed-out registry clears +prior extension providers before use. The SDK remains real, not stubbed, because this coverage catches +catalog regressions such as FN-8564's native Kimi K3; widening timeouts or adding retries is forbidden here. +*/ +let sharedModelRuntime: Promise | undefined; + +export function getSharedModelRuntime(): Promise { + sharedModelRuntime ??= ModelRuntime.create({ + credentials: { + read: async () => undefined, + list: async () => [], + modify: async (_id, fn) => fn(undefined), + delete: async () => undefined, + }, + modelsPath: null, + allowModelNetwork: false, + }); + return sharedModelRuntime; +} + +export async function warmSharedModelRuntime(): Promise { + await getSharedModelRuntime(); +} + +export async function createInMemoryModelRegistry(): Promise { + const modelRegistry = new ModelRegistry(await getSharedModelRuntime()); + + for (const providerId of modelRegistry.getRegisteredProviderIds()) { + modelRegistry.unregisterProvider(providerId); + } + await modelRegistry.refresh(); + + return modelRegistry; +} diff --git a/packages/engine/src/__tests__/custom-providers-openai-completions.test.ts b/packages/engine/src/__tests__/custom-providers-openai-completions.test.ts index 6c75f9ec72..32f5c1f107 100644 --- a/packages/engine/src/__tests__/custom-providers-openai-completions.test.ts +++ b/packages/engine/src/__tests__/custom-providers-openai-completions.test.ts @@ -1,5 +1,4 @@ -import { afterEach, describe, expect, it, vi } from "vitest"; -import { ModelRegistry, ModelRuntime } from "@earendil-works/pi-coding-agent"; +import { afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { completeSimple } from "@earendil-works/pi-ai/compat"; /* FNXC:Dependencies 2026-07-01-08:16: @@ -7,6 +6,7 @@ The pi 0.80 SDK keeps compatibility helpers under ./compat and exposes provider */ import { convertMessages } from "@earendil-works/pi-ai/api/openai-completions"; import { customProviderRegistryKey, type CustomProvider } from "@fusion/core"; +import { createInMemoryModelRegistry, warmSharedModelRuntime } from "./_model-runtime-fixture.js"; function createSseResponse(): Response { const stream = new ReadableStream({ @@ -21,14 +21,9 @@ function createSseResponse(): Response { } -async function createInMemoryModelRegistry(): Promise { - const runtime = await ModelRuntime.create({ - credentials: { read: async () => undefined, list: async () => [], modify: async (_id, fn) => fn(undefined), delete: async () => undefined }, - modelsPath: null, - allowModelNetwork: false, - }); - return new ModelRegistry(runtime); -} +beforeAll(async () => { + await warmSharedModelRuntime(); +}); describe("custom providers openai-completions regression", () => { afterEach(() => { diff --git a/packages/engine/src/__tests__/custom-providers-openai-responses.test.ts b/packages/engine/src/__tests__/custom-providers-openai-responses.test.ts index b7438bf7aa..89bb8b507b 100644 --- a/packages/engine/src/__tests__/custom-providers-openai-responses.test.ts +++ b/packages/engine/src/__tests__/custom-providers-openai-responses.test.ts @@ -1,19 +1,14 @@ import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { ModelRegistry, ModelRuntime } from "@earendil-works/pi-coding-agent"; +import { afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; import { customProviderRegistryKey, type CustomProvider } from "@fusion/core"; import { readCustomProviders } from "../custom-providers.js"; +import { createInMemoryModelRegistry, warmSharedModelRuntime } from "./_model-runtime-fixture.js"; -async function createInMemoryModelRegistry(): Promise { - const runtime = await ModelRuntime.create({ - credentials: { read: async () => undefined, list: async () => [], modify: async (_id, fn) => fn(undefined), delete: async () => undefined }, - modelsPath: null, - allowModelNetwork: false, - }); - return new ModelRegistry(runtime); -} +beforeAll(async () => { + await warmSharedModelRuntime(); +}); describe("custom providers openai-responses regression", () => { let homeDir: string; diff --git a/packages/engine/src/__tests__/provider-registration.test.ts b/packages/engine/src/__tests__/provider-registration.test.ts index 525098a56e..083749b1ea 100644 --- a/packages/engine/src/__tests__/provider-registration.test.ts +++ b/packages/engine/src/__tests__/provider-registration.test.ts @@ -1,9 +1,9 @@ -import { afterEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { customProviderRegistryKey, type CustomProvider } from "@fusion/core"; -import { ModelRegistry, ModelRuntime } from "@earendil-works/pi-coding-agent"; import { completeSimple } from "@earendil-works/pi-ai/compat"; import { seedDashboardProviders } from "../provider-registration.js"; import { registerCustomProviders } from "../custom-provider-registry.js"; +import { createInMemoryModelRegistry, warmSharedModelRuntime } from "./_model-runtime-fixture.js"; /* FNXC:ProviderRegistration 2026-07-07-00:00: @@ -36,15 +36,6 @@ function makeAuthStorage() { } as any; } -async function createInMemoryModelRegistry(): Promise { - const runtime = await ModelRuntime.create({ - credentials: { read: async () => undefined, list: async () => [], modify: async (_id, fn) => fn(undefined), delete: async () => undefined }, - modelsPath: null, - allowModelNetwork: false, - }); - return new ModelRegistry(runtime); -} - function makeModelRegistry() { const registeredProviders = new Map }>(); return { @@ -94,6 +85,10 @@ function makeStore(initialCustomProviders?: CustomProvider[]) { }; } +beforeAll(async () => { + await warmSharedModelRuntime(); +}); + const customProvider = (overrides: Partial = {}): CustomProvider => ({ id: "550e8400-e29b-41d4-a716-446655440000", name: "Acme AI", @@ -144,6 +139,22 @@ describe("seedDashboardProviders", () => { }); }); + it("hands each real registry an isolated custom-provider catalog", async () => { + const provider = customProvider({ id: "isolation-provider-id", name: "Fixture Isolation Provider" }); + const registryA = await createInMemoryModelRegistry(); + registryA.registerProvider(customProviderRegistryKey(provider, [provider]), { + baseUrl: provider.baseUrl, + api: "openai-completions", + apiKey: provider.apiKey, + models: [{ id: "acme-1", name: "Acme Model 1", reasoning: false, input: ["text"], cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, contextWindow: 128000, maxTokens: 16384 }], + }); + + const registryB = await createInMemoryModelRegistry(); + + expect(registryB.find("fixture-isolation-provider", "acme-1")).toBeUndefined(); + expect(registryB.find("kimi-coding", "k3")).toMatchObject({ provider: "kimi-coding", id: "k3" }); + }); + it("registers one custom provider alongside built-ins", async () => { const store = makeStore([customProvider()]); const authStorage = makeAuthStorage();