Merge pull request #63 from magreenberg1/feat/cross-session-prompt-caching

feat(engine): cross-session prompt caching to reduce per-task token consumption
This commit is contained in:
gsxdsm
2026-05-10 15:00:17 -07:00
committed by GitHub
16 changed files with 1002 additions and 61 deletions

View File

@@ -0,0 +1,31 @@
import { describe, it, expect } from "vitest";
import type { AgentRuntimeOptions } from "../agent-runtime.js";
import type { SystemPromptLayers } from "../prompt-layers.js";
describe("AgentRuntimeOptions.systemPromptLayers", () => {
it("accepts systemPromptLayers alongside systemPrompt", () => {
const layers: SystemPromptLayers = {
stable: "You are a reviewer.",
dynamic: "Check for bugs.",
};
const options: AgentRuntimeOptions = {
cwd: "/tmp/test",
systemPrompt: "You are a reviewer.\n\nCheck for bugs.",
systemPromptLayers: layers,
};
expect(options.systemPromptLayers).toBeDefined();
expect(options.systemPromptLayers!.stable).toBe("You are a reviewer.");
expect(options.systemPromptLayers!.dynamic).toBe("Check for bugs.");
});
it("works without systemPromptLayers (backward compatible)", () => {
const options: AgentRuntimeOptions = {
cwd: "/tmp/test",
systemPrompt: "You are a reviewer.",
};
expect(options.systemPromptLayers).toBeUndefined();
});
});

View File

@@ -0,0 +1,318 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { PathLike } from "node:fs";
/**
* Tests that createFnAgent correctly wires prompt layers to
* DefaultResourceLoader and that tools are sorted deterministically.
*
* These tests verify the actual subsystem wiring rather than testing
* the layer logic in isolation.
*/
const createAgentSessionMock = vi.fn();
const createCodingToolsMock = vi.fn(() => []);
const createReadOnlyToolsMock = vi.fn(() => []);
const createExtensionRuntimeMock = vi.fn();
const discoverAndLoadExtensionsMock = vi.fn().mockResolvedValue({
runtime: { pendingProviderRegistrations: [] },
errors: [],
});
const packageManagerResolveMock = vi.fn().mockResolvedValue({ extensions: [] });
const findMock = vi.fn();
const getAllMock = vi.fn(() => [] as any[]);
const registerProviderMock = vi.fn();
const refreshMock = vi.fn();
const settingsManagerInMemoryMock = vi.fn(() => ({ kind: "settings-manager" }));
const setFallbackResolverMock = vi.fn();
const reloadMock = vi.fn(async () => {});
const execSyncMock = vi.fn((_cmd?: any, _opts?: any) => "");
const existsSyncMock = vi.fn((_path: PathLike) => false);
const readFileSyncMock = vi.fn((_path?: any) => "{}");
const readCustomProvidersMock = vi.fn(() => []);
// Capture DefaultResourceLoader constructor args
let capturedResourceLoaderOptions: any = null;
vi.mock("node:child_process", () => {
const execSyncFn = execSyncMock;
const kPromisifyCustom = Symbol.for("nodejs.util.promisify.custom");
const execFn: any = vi.fn((cmd: string, opts: any, cb: any) => {
const callback = typeof opts === "function" ? opts : cb;
const options = typeof opts === "function" ? {} : (opts ?? {});
try {
const out = execSyncFn(cmd, { ...options, stdio: ["pipe", "pipe", "pipe"] });
const stdout = out === undefined ? "" : out.toString();
if (typeof callback === "function") callback(null, stdout, "");
} catch (err) {
if (typeof callback === "function") {
const error = err as { stdout?: string; stderr?: string };
callback(err, error?.stdout?.toString?.() ?? "", error?.stderr?.toString?.() ?? "");
}
}
});
execFn[kPromisifyCustom] = (cmd: string, opts?: any) =>
new Promise((resolve, reject) => {
execFn(cmd, opts, (err: any, stdout: string, stderr: string) => {
if (err) {
(err as Record<string, unknown>).stdout = stdout;
(err as Record<string, unknown>).stderr = stderr;
reject(err);
} else {
resolve({ stdout, stderr });
}
});
});
return { execSync: execSyncFn, exec: execFn };
});
vi.mock("node:fs", async () => {
const actual = await vi.importActual<typeof import("node:fs")>("node:fs");
return {
...actual,
existsSync: existsSyncMock,
readFileSync: readFileSyncMock,
};
});
vi.mock("../custom-providers.js", () => ({
readCustomProviders: readCustomProvidersMock,
}));
vi.mock("@mariozechner/pi-coding-agent", () => ({
AuthStorage: {
create: () => ({
setFallbackResolver: setFallbackResolverMock,
}),
},
createAgentSession: createAgentSessionMock,
createBashTool: () => ({ name: "bash" }),
createCodingTools: createCodingToolsMock,
createEditTool: () => ({ name: "edit" }),
createExtensionRuntime: createExtensionRuntimeMock,
createFindTool: () => ({ name: "find" }),
createGrepTool: () => ({ name: "grep" }),
createLsTool: () => ({ name: "ls" }),
createReadOnlyTools: createReadOnlyToolsMock,
createReadTool: () => ({ name: "read" }),
createWriteTool: () => ({ name: "write" }),
DefaultResourceLoader: class {
constructor(options: any) {
capturedResourceLoaderOptions = options;
}
async reload() {
await reloadMock();
}
},
DefaultPackageManager: class {
async resolve() {
return packageManagerResolveMock();
}
},
discoverAndLoadExtensions: discoverAndLoadExtensionsMock,
getAgentDir: () => "/mock-agent-dir",
ModelRegistry: class {
static create(..._args: unknown[]) {
return new (this as unknown as new () => unknown)();
}
find(provider: string, modelId: string) {
return findMock(provider, modelId);
}
getAll() {
return getAllMock();
}
registerProvider(name: string, config: unknown) {
return registerProviderMock(name, config);
}
refresh() {
return refreshMock();
}
},
SessionManager: {
inMemory: () => ({ kind: "session-manager" }),
},
SettingsManager: {
create: vi.fn(),
inMemory: settingsManagerInMemoryMock,
},
}));
describe("createFnAgent prompt layer wiring", () => {
beforeEach(() => {
vi.clearAllMocks();
capturedResourceLoaderOptions = null;
execSyncMock.mockReturnValue("");
existsSyncMock.mockReturnValue(false);
readFileSyncMock.mockReturnValue("{}");
readCustomProvidersMock.mockReturnValue([]);
findMock.mockImplementation((provider: string, modelId: string) => ({ provider, id: modelId }));
createAgentSessionMock.mockResolvedValue({
session: {
prompt: vi.fn(),
subscribe: vi.fn(),
dispose: vi.fn(),
setThinkingLevel: vi.fn(),
},
});
});
it("passes stable layer as systemPromptOverride when layers provided", async () => {
const { createFnAgent } = await import("../pi.js");
await createFnAgent({
cwd: "/tmp/test-project",
systemPrompt: "Stable prefix.\n\nDynamic suffix.",
systemPromptLayers: {
stable: "Stable prefix.",
dynamic: "Dynamic suffix.",
},
});
expect(capturedResourceLoaderOptions).toBeDefined();
const override = capturedResourceLoaderOptions.systemPromptOverride();
expect(override).toBe("Stable prefix.");
});
it("passes dynamic layer via appendSystemPromptOverride when layers provided", async () => {
const { createFnAgent } = await import("../pi.js");
await createFnAgent({
cwd: "/tmp/test-project",
systemPrompt: "Stable.\n\nDynamic content.",
systemPromptLayers: {
stable: "Stable.",
dynamic: "Dynamic content.",
},
});
expect(capturedResourceLoaderOptions).toBeDefined();
const appended = capturedResourceLoaderOptions.appendSystemPromptOverride();
expect(appended).toEqual(["Dynamic content."]);
});
it("falls back to full systemPrompt when no layers provided", async () => {
const { createFnAgent } = await import("../pi.js");
await createFnAgent({
cwd: "/tmp/test-project",
systemPrompt: "Full system prompt.",
});
expect(capturedResourceLoaderOptions).toBeDefined();
const override = capturedResourceLoaderOptions.systemPromptOverride();
expect(override).toBe("Full system prompt.");
});
it("returns empty array from appendSystemPromptOverride when no layers", async () => {
const { createFnAgent } = await import("../pi.js");
await createFnAgent({
cwd: "/tmp/test-project",
systemPrompt: "Full prompt.",
});
const appended = capturedResourceLoaderOptions.appendSystemPromptOverride();
expect(appended).toEqual([]);
});
it("returns empty array from appendSystemPromptOverride when dynamic is empty", async () => {
const { createFnAgent } = await import("../pi.js");
await createFnAgent({
cwd: "/tmp/test-project",
systemPrompt: "Stable only.",
systemPromptLayers: {
stable: "Stable only.",
dynamic: "",
},
});
const appended = capturedResourceLoaderOptions.appendSystemPromptOverride();
expect(appended).toEqual([]);
});
});
describe("createFnAgent deterministic tool ordering", () => {
beforeEach(() => {
vi.clearAllMocks();
capturedResourceLoaderOptions = null;
execSyncMock.mockReturnValue("");
existsSyncMock.mockReturnValue(false);
readFileSyncMock.mockReturnValue("{}");
readCustomProvidersMock.mockReturnValue([]);
findMock.mockImplementation((provider: string, modelId: string) => ({ provider, id: modelId }));
createAgentSessionMock.mockResolvedValue({
session: {
prompt: vi.fn(),
subscribe: vi.fn(),
dispose: vi.fn(),
setThinkingLevel: vi.fn(),
},
});
});
it("passes tools to createAgentSession in alphabetical order", async () => {
const { createFnAgent } = await import("../pi.js");
await createFnAgent({
cwd: "/tmp/test-project",
systemPrompt: "Test.",
tools: "coding",
});
expect(createAgentSessionMock).toHaveBeenCalled();
const callArgs = createAgentSessionMock.mock.calls[0][0];
const toolNames = (callArgs.customTools ?? []).map((t: any) => t.name);
// Tools should be in alphabetical order
const sorted = [...toolNames].sort();
expect(toolNames).toEqual(sorted);
});
it("sorts custom tools mixed with built-in tools", async () => {
const { createFnAgent } = await import("../pi.js");
await createFnAgent({
cwd: "/tmp/test-project",
systemPrompt: "Test.",
tools: "coding",
customTools: [
{ name: "zzz_custom", execute: vi.fn() } as any,
{ name: "aaa_custom", execute: vi.fn() } as any,
],
});
const callArgs = createAgentSessionMock.mock.calls[0][0];
const toolNames = (callArgs.customTools ?? []).map((t: any) => t.name);
const sorted = [...toolNames].sort();
expect(toolNames).toEqual(sorted);
});
it("sorts readonly tools with custom tools", async () => {
createReadOnlyToolsMock.mockReturnValueOnce([
{ name: "read" },
{ name: "grep" },
{ name: "find" },
] as any);
const { createFnAgent } = await import("../pi.js");
await createFnAgent({
cwd: "/tmp/test-project",
systemPrompt: "Test.",
tools: "readonly",
customTools: [
{ name: "fn_task_list", execute: vi.fn() } as any,
],
});
const callArgs = createAgentSessionMock.mock.calls[0][0];
const toolNames = (callArgs.customTools ?? []).map((t: any) => t.name);
const sorted = [...toolNames].sort();
expect(toolNames).toEqual(sorted);
});
});

View File

@@ -0,0 +1,49 @@
import { describe, it, expect } from "vitest";
import { buildPromptLayers, collapsePromptLayers, type SystemPromptLayers } from "../prompt-layers.js";
import { REVIEWER_SYSTEM_PROMPT } from "../reviewer.js";
describe("cross-session prompt cache integration", () => {
const MEMORY_INSTRUCTIONS = "\n## Memory\n\nUse fn_memory_search to look up relevant context.";
function simulateReviewerSession(sessionIndex: number): SystemPromptLayers {
return buildPromptLayers({
basePrompt: REVIEWER_SYSTEM_PROMPT,
agentInstructions: `Session ${sessionIndex}: custom instructions that vary per agent.`,
memorySection: MEMORY_INSTRUCTIONS,
pluginContributions: sessionIndex % 2 === 0
? "## Plugin: lint\n\nCheck lint rules."
: "",
});
}
it("produces byte-identical stable prefixes across 10 reviewer sessions", () => {
const sessions = Array.from({ length: 10 }, (_, i) => simulateReviewerSession(i));
const stablePrefix = sessions[0].stable;
for (let i = 1; i < sessions.length; i++) {
expect(sessions[i].stable).toBe(stablePrefix);
}
});
it("dynamic layers vary across sessions as expected", () => {
const sessions = Array.from({ length: 5 }, (_, i) => simulateReviewerSession(i));
const uniqueDynamics = new Set(sessions.map((s) => s.dynamic));
expect(uniqueDynamics.size).toBeGreaterThan(1);
});
it("collapsed layers produce valid non-empty strings", () => {
const sessions = Array.from({ length: 5 }, (_, i) => simulateReviewerSession(i));
for (const session of sessions) {
const collapsed = collapsePromptLayers(session);
expect(collapsed.length).toBeGreaterThan(0);
expect(collapsed).toContain("independent code and plan reviewer");
}
});
it("stable prefix starts with REVIEWER_SYSTEM_PROMPT", () => {
const layers = simulateReviewerSession(0);
expect(layers.stable.startsWith(REVIEWER_SYSTEM_PROMPT)).toBe(true);
});
});

View File

@@ -0,0 +1,196 @@
import { describe, it, expect } from "vitest";
import { buildSystemPromptWithInstructions } from "../agent-instructions.js";
import { buildPromptLayers, collapsePromptLayers } from "../prompt-layers.js";
/**
* Backward compatibility tests: verify that the new layered prompt approach
* produces byte-identical output to the legacy buildSystemPromptWithInstructions
* + manual concatenation pattern used by each subsystem.
*/
describe("prompt layers backward compatibility", () => {
describe("collapsed layers match buildSystemPromptWithInstructions", () => {
it("matches when only base prompt is provided", () => {
const basePrompt = "You are a reviewer.";
const oldResult = buildSystemPromptWithInstructions(basePrompt, "");
const layers = buildPromptLayers({ basePrompt });
const newResult = collapsePromptLayers(layers);
expect(newResult).toBe(oldResult);
});
it("matches with base prompt and instructions", () => {
const basePrompt = "You are a reviewer.";
const instructions = "Check for SQL injection.";
const oldResult = buildSystemPromptWithInstructions(basePrompt, instructions);
const layers = buildPromptLayers({
basePrompt,
agentInstructions: instructions,
});
const newResult = collapsePromptLayers(layers);
expect(newResult).toBe(oldResult);
});
it("matches with base prompt, instructions, and plugin contributions", () => {
const basePrompt = "You are an executor.";
const instructions = "Follow TDD.";
const plugins = "## Plugin: security\n\nScan for vulnerabilities.";
// Old approach: buildSystemPromptWithInstructions + manual concatenation
const oldSystemPrompt = buildSystemPromptWithInstructions(basePrompt, instructions);
const oldResult = `${oldSystemPrompt}\n\n${plugins}`;
// New approach: buildPromptLayers + collapsePromptLayers
const layers = buildPromptLayers({
basePrompt,
agentInstructions: instructions,
pluginContributions: plugins,
});
const newResult = collapsePromptLayers(layers);
expect(newResult).toBe(oldResult);
});
it("matches with empty instructions and plugin contributions", () => {
const basePrompt = "You are a triage agent.";
const plugins = "## Plugin: research\n\nUse web search.";
// Old approach: buildSystemPromptWithInstructions returns base (empty instructions)
// then plugins appended
const oldSystemPrompt = buildSystemPromptWithInstructions(basePrompt, "");
const oldResult = `${oldSystemPrompt}\n\n${plugins}`;
const layers = buildPromptLayers({
basePrompt,
pluginContributions: plugins,
});
const newResult = collapsePromptLayers(layers);
expect(newResult).toBe(oldResult);
});
it("matches with empty instructions and no plugins", () => {
const basePrompt = "You are a heartbeat agent.";
const oldResult = buildSystemPromptWithInstructions(basePrompt, "");
const layers = buildPromptLayers({ basePrompt });
const newResult = collapsePromptLayers(layers);
expect(newResult).toBe(oldResult);
});
});
describe("reviewer assembly pattern", () => {
it("preserves memory-before-instructions ordering in dynamic layer", () => {
const basePrompt = "You are an independent code and plan reviewer.";
const memoryInstructions = "## Memory\n\nUse fn_memory_search.";
const agentInstructions = "Focus on security.";
const plugins = "## Plugin: lint\n\nCheck eslint.";
// The old reviewer pattern was:
// buildSystemPromptWithInstructions(base + memory, instructions) + plugins
// which produced: base + memory → instructions → plugins
//
// The new pattern moves memory from stable to dynamic (so stable prefix
// is byte-identical even if memory changes mid-task), but preserves the
// relative ordering: memory → instructions → plugins in the dynamic layer.
const layers = buildPromptLayers({
basePrompt,
agentInstructions,
memorySection: memoryInstructions,
pluginContributions: plugins,
});
const result = collapsePromptLayers(layers);
// Base prompt is the stable layer (no memory)
expect(layers.stable).toBe(basePrompt);
// Dynamic layer preserves: memory → instructions → plugins ordering
const memoryIdx = result.indexOf("## Memory");
const instructionsIdx = result.indexOf("## Custom Instructions");
const pluginsIdx = result.indexOf("## Plugin:");
expect(memoryIdx).toBeGreaterThan(0);
expect(instructionsIdx).toBeGreaterThan(memoryIdx);
expect(pluginsIdx).toBeGreaterThan(instructionsIdx);
});
});
describe("executor assembly pattern", () => {
it("reproduces the executor prompt assembly", () => {
const basePrompt = "You are a task execution agent.";
const agentInstructions = "Follow the spec precisely.";
const plugins = "## Plugin: deploy\n\nCheck CI status.";
// Old executor pattern:
// 1. buildSystemPromptWithInstructions(base, instructions)
// 2. if plugins: concatenate
const oldSystemPrompt = buildSystemPromptWithInstructions(
basePrompt,
agentInstructions,
);
const oldResult = `${oldSystemPrompt}\n\n${plugins}`;
const layers = buildPromptLayers({
basePrompt,
agentInstructions,
pluginContributions: plugins,
});
const newResult = collapsePromptLayers(layers);
expect(newResult).toBe(oldResult);
});
});
describe("heartbeat assembly pattern", () => {
it("reproduces the heartbeat prompt assembly with multi-part instructions", () => {
const basePrompt = "You are a heartbeat agent.";
const identitySection = "## Identity\n\nYou are Agent-1.";
const memoryInstructions = "## Memory\n\nUse memory tools.";
const selfImprovePrompt = "## Self-Improvement\n\nReview your performance.";
// Old heartbeat pattern:
// 1. Join identity + memory + selfImprove with \n\n
// 2. buildSystemPromptWithInstructions(base, joined)
// 3. if plugins: concatenate
const joinedInstructions = [identitySection, memoryInstructions, selfImprovePrompt]
.filter((part) => part.trim())
.join("\n\n");
const oldResult = buildSystemPromptWithInstructions(basePrompt, joinedInstructions);
const layers = buildPromptLayers({
basePrompt,
agentInstructions: joinedInstructions,
});
const newResult = collapsePromptLayers(layers);
expect(newResult).toBe(oldResult);
});
});
describe("triage assembly pattern", () => {
it("reproduces the triage prompt assembly with multi-part instructions", () => {
const basePrompt = "You are a task specification agent.";
const identitySection = "## Identity\n\nYou are TriageBot.";
const triageInstructions = "Be thorough.";
const researchGuidance = "## Research\n\nUse web search when needed.";
// Old triage pattern:
// 1. Join identity + instructions + research with \n\n
// 2. buildSystemPromptWithInstructions(base, joined)
const joinedInstructions = [identitySection, triageInstructions, researchGuidance]
.filter((section) => section.trim())
.join("\n\n");
const oldResult = buildSystemPromptWithInstructions(basePrompt, joinedInstructions);
const layers = buildPromptLayers({
basePrompt,
agentInstructions: joinedInstructions,
});
const newResult = collapsePromptLayers(layers);
expect(newResult).toBe(oldResult);
});
});
});

View File

@@ -0,0 +1,125 @@
import { describe, it, expect } from "vitest";
import {
type SystemPromptLayers,
buildPromptLayers,
collapsePromptLayers,
} from "../prompt-layers.js";
describe("buildPromptLayers", () => {
it("separates base prompt into stable layer", () => {
const layers = buildPromptLayers({
basePrompt: "You are a reviewer.",
});
expect(layers.stable).toBe("You are a reviewer.");
expect(layers.dynamic).toBe("");
});
it("puts agent instructions into dynamic layer", () => {
const layers = buildPromptLayers({
basePrompt: "You are a reviewer.",
agentInstructions: "Always check for SQL injection.",
});
expect(layers.stable).toBe("You are a reviewer.");
expect(layers.dynamic).toContain("Always check for SQL injection.");
});
it("puts memory section into dynamic layer", () => {
const layers = buildPromptLayers({
basePrompt: "You are a reviewer.",
memorySection: "## Agent Memory\n\nRemember to check tests.",
});
expect(layers.stable).toBe("You are a reviewer.");
expect(layers.dynamic).toContain("Agent Memory");
});
it("puts plugin contributions into dynamic layer", () => {
const layers = buildPromptLayers({
basePrompt: "You are a reviewer.",
pluginContributions: "## Plugin: security\n\nScan for CVEs.",
});
expect(layers.stable).toBe("You are a reviewer.");
expect(layers.dynamic).toContain("security");
});
it("puts performance feedback into dynamic layer", () => {
const layers = buildPromptLayers({
basePrompt: "You are a reviewer.",
performanceFeedback: "## Performance Feedback\n\n- Average score: 8.5",
});
expect(layers.stable).toBe("You are a reviewer.");
expect(layers.dynamic).toContain("Performance Feedback");
});
it("combines multiple dynamic sections with double newlines", () => {
const layers = buildPromptLayers({
basePrompt: "Base.",
agentInstructions: "Instructions.",
memorySection: "Memory.",
pluginContributions: "Plugins.",
});
expect(layers.dynamic).toBe(
"Memory.\n\n## Custom Instructions\n\nInstructions.\n\nPlugins."
);
});
it("omits empty dynamic sections", () => {
const layers = buildPromptLayers({
basePrompt: "Base.",
agentInstructions: "",
memorySection: "",
pluginContributions: "Plugins.",
});
expect(layers.dynamic).toBe("Plugins.");
expect(layers.dynamic).not.toContain("Custom Instructions");
});
it("produces deterministic output for identical inputs", () => {
const input = {
basePrompt: "Base.",
agentInstructions: "Inst.",
memorySection: "Mem.",
pluginContributions: "Plug.",
performanceFeedback: "Perf.",
};
const layers1 = buildPromptLayers(input);
const layers2 = buildPromptLayers(input);
expect(layers1.stable).toBe(layers2.stable);
expect(layers1.dynamic).toBe(layers2.dynamic);
});
});
describe("collapsePromptLayers", () => {
it("returns stable when dynamic is empty", () => {
const result = collapsePromptLayers({ stable: "Base.", dynamic: "" });
expect(result).toBe("Base.");
});
it("joins stable and dynamic with double newline", () => {
const result = collapsePromptLayers({
stable: "Base.",
dynamic: "Dynamic.",
});
expect(result).toBe("Base.\n\nDynamic.");
});
it("matches legacy buildSystemPromptWithInstructions output", () => {
const layers = buildPromptLayers({
basePrompt: "You are a reviewer.",
agentInstructions: "Check for bugs.",
});
const collapsed = collapsePromptLayers(layers);
expect(collapsed).toBe(
"You are a reviewer.\n\n## Custom Instructions\n\nCheck for bugs."
);
});
});

View File

@@ -0,0 +1,49 @@
import { describe, it, expect } from "vitest";
import { buildPromptLayers, collapsePromptLayers } from "../prompt-layers.js";
describe("reviewer prompt layering", () => {
const REVIEWER_BASE = "You are an independent code and plan reviewer.";
const MEMORY_INSTRUCTIONS = "\n## Memory\n\nUse fn_memory_search to look up context.";
it("puts base prompt in stable layer and memory in dynamic layer", () => {
const layers = buildPromptLayers({
basePrompt: REVIEWER_BASE,
agentInstructions: "Custom reviewer guidance.",
memorySection: MEMORY_INSTRUCTIONS,
pluginContributions: "## Plugin: lint\n\nCheck lint.",
});
expect(layers.stable).toBe(REVIEWER_BASE);
expect(layers.stable).not.toContain("Memory");
expect(layers.dynamic).toContain("Memory");
expect(layers.dynamic).toContain("Custom reviewer guidance");
});
it("produces identical stable layer across simulated sessions", () => {
const layers1 = buildPromptLayers({
basePrompt: REVIEWER_BASE,
agentInstructions: "Session 1 instructions.",
memorySection: MEMORY_INSTRUCTIONS,
});
const layers2 = buildPromptLayers({
basePrompt: REVIEWER_BASE,
agentInstructions: "Session 2 instructions.",
memorySection: MEMORY_INSTRUCTIONS,
});
expect(layers1.stable).toBe(layers2.stable);
});
it("collapsed layers match legacy concatenation", () => {
const layers = buildPromptLayers({
basePrompt: REVIEWER_BASE,
agentInstructions: "Check for bugs.",
pluginContributions: "## Plugin: sec\n\nScan.",
});
const collapsed = collapsePromptLayers(layers);
expect(collapsed).toBe(
`${REVIEWER_BASE}\n\n## Custom Instructions\n\nCheck for bugs.\n\n## Plugin: sec\n\nScan.`
);
});
});

View File

@@ -0,0 +1,20 @@
import { describe, it, expect } from "vitest";
import { computeCacheHitRatio } from "../session-token-usage.js";
describe("computeCacheHitRatio", () => {
it("returns 0 when no tokens used", () => {
expect(computeCacheHitRatio(0, 0)).toBe(0);
});
it("returns 0 when no cached tokens", () => {
expect(computeCacheHitRatio(1000, 0)).toBe(0);
});
it("returns ratio of cached to total input", () => {
expect(computeCacheHitRatio(500, 500)).toBeCloseTo(0.5);
});
it("returns close to 1 when mostly cached", () => {
expect(computeCacheHitRatio(100, 9900)).toBeCloseTo(0.99);
});
});

View File

@@ -26,10 +26,10 @@ import { createTaskCreateTool, createTaskLogToolWithContext, createTaskDocumentW
import { AgentLogger } from "./agent-logger.js";
import {
resolveAgentInstructionsWithRatings,
buildSystemPromptWithInstructions,
buildPluginPromptSection,
resolveAgentHeartbeatProcedure,
} from "./agent-instructions.js";
import { buildPromptLayers, collapsePromptLayers } from "./prompt-layers.js";
import { heartbeatLog, formatError } from "./logger.js";
import { createRunAuditor, type EngineRunContext } from "./run-audit.js";
import { promptWithFallback } from "./pi.js";
@@ -1838,23 +1838,22 @@ export class HeartbeatMonitor {
}
}
const systemPrompt = buildSystemPromptWithInstructions(
baseHeartbeatSystemPrompt,
[resolvedInstructionsForIdentity, memoryInstructions, selfImprovePrompt].filter((part) => part.trim()).join("\n\n"),
);
const heartbeatContributions = this.pluginRunner
?.getPromptContributionsForSurface("heartbeat")
?? [];
if (heartbeatContributions.length > 0) {
heartbeatLog.log(`applied ${heartbeatContributions.length} plugin prompt contributions for heartbeat surface`);
}
// Build structured layers for cross-session prompt caching.
const heartbeatPluginContributions = buildPluginPromptSection(
"heartbeat",
this.pluginRunner,
);
const systemPromptFinal = heartbeatPluginContributions
? `${systemPrompt}\n\n${heartbeatPluginContributions}`
: systemPrompt;
if (heartbeatPluginContributions) {
heartbeatLog.log(`applied plugin prompt contributions for heartbeat surface`);
}
const heartbeatLayers = buildPromptLayers({
basePrompt: baseHeartbeatSystemPrompt,
agentInstructions: [resolvedInstructionsForIdentity, memoryInstructions, selfImprovePrompt].filter((part) => part.trim()).join("\n\n"),
pluginContributions: heartbeatPluginContributions,
});
const systemPromptFinal = collapsePromptLayers(heartbeatLayers);
// fn_heartbeat_done must be the last tool in the array (stable terminal signal)
heartbeatTools.push(heartbeatDoneTool);
@@ -1948,6 +1947,7 @@ export class HeartbeatMonitor {
pluginRunner: this.pluginRunner,
cwd: rootDir,
systemPrompt: systemPromptFinal,
systemPromptLayers: heartbeatLayers,
tools: "coding",
customTools: heartbeatTools,
defaultProvider: heartbeatSessionModels.defaultProvider,

View File

@@ -19,6 +19,7 @@ import type { PermanentAgentGatingContext } from "@fusion/core";
import type { SkillSelectionContext } from "./skill-resolver.js";
import type { FallbackModelUsedPayload } from "./pi.js";
import type { AgentActionGateContext } from "./agent-action-gate.js";
import type { SystemPromptLayers } from "./prompt-layers.js";
/**
* Options for creating an agent session.
@@ -36,6 +37,17 @@ export interface AgentRuntimeOptions {
cwd: string;
/** System prompt for the agent */
systemPrompt: string;
/**
* Optional structured prompt layers for cross-session caching.
* When present, runtimes that support prompt caching use the `stable`
* layer as a cacheable prefix and the `dynamic` layer as the per-session
* suffix. Runtimes that don't support caching ignore this and use
* `systemPrompt` (the collapsed string) instead.
*
* Callers MUST also provide `systemPrompt` as the collapsed equivalent
* for backward compatibility.
*/
systemPromptLayers?: SystemPromptLayers;
/** Tool set to use: "coding" for full tools, "readonly" for read-only access */
tools?: "coding" | "readonly";
/** Additional custom tools to merge with the base toolset */

View File

@@ -53,6 +53,7 @@ import {
buildSystemPromptWithInstructions,
buildPluginPromptSection,
} from "./agent-instructions.js";
import { buildPromptLayers, collapsePromptLayers } from "./prompt-layers.js";
import type { AgentReflectionService } from "./agent-reflection.js";
import { createRunAuditor, generateSyntheticRunId, type EngineRunContext } from "./run-audit.js";
import { evaluateSpecStaleness, getPromptPath } from "./spec-staleness.js";
@@ -3077,21 +3078,23 @@ export class TaskExecutor {
// Resolve per-agent custom instructions for the executor role
const executorInstructions = await this.resolveInstructionsForRole("executor");
const executorSystemPrompt = buildSystemPromptWithInstructions(
getExecutorSystemPrompt(settings),
executorInstructions,
);
const executorSystemContributions = this.options.pluginRunner?.getPromptContributionsForSurface("executor-system") ?? [];
if (executorSystemContributions.length > 0) {
executorLog.log(`${task.id}: applied ${executorSystemContributions.length} plugin prompt contributions for executor-system surface`);
}
// Build structured layers for cross-session prompt caching.
const executorPluginContributions = buildPluginPromptSection(
"executor-system",
this.options.pluginRunner,
);
const executorSystemPromptFinal = executorPluginContributions
? `${executorSystemPrompt}\n\n${executorPluginContributions}`
: executorSystemPrompt;
if (executorPluginContributions) {
executorLog.log(`${task.id}: applied plugin prompt contributions for executor-system surface`);
}
const executorLayers = buildPromptLayers({
basePrompt: getExecutorSystemPrompt(settings),
agentInstructions: executorInstructions,
pluginContributions: executorPluginContributions,
});
const executorSystemPromptFinal = collapsePromptLayers(executorLayers);
// sessionFile must be let because it's destructured alongside session which is reassigned
// eslint-disable-next-line prefer-const
@@ -3101,6 +3104,7 @@ export class TaskExecutor {
pluginRunner: this.options.pluginRunner,
cwd: worktreePath,
systemPrompt: executorSystemPromptFinal,
systemPromptLayers: executorLayers,
tools: "coding",
customTools,
onText: agentLogger.onText,
@@ -3430,6 +3434,7 @@ export class TaskExecutor {
pluginRunner: this.options.pluginRunner,
cwd: worktreePath,
systemPrompt: executorSystemPromptFinal,
systemPromptLayers: executorLayers,
tools: "coding",
customTools,
onText: agentLogger.onText,

View File

@@ -56,6 +56,7 @@ import {
type AgentActionGateContext,
} from "./agent-action-gate.js";
import { resolvePermanentAgentToolDecision } from "./permanent-agent-gating.js";
import type { SystemPromptLayers } from "./prompt-layers.js";
export interface AgentResult {
session: AgentSession;
@@ -711,6 +712,11 @@ export type BuiltinWebToolName = "WebSearch" | "WebFetch";
export interface AgentOptions {
cwd: string;
systemPrompt: string;
/** Structured prompt layers for cross-session caching. When provided,
* the stable layer is used as systemPromptOverride and the dynamic
* layer as appendSystemPromptOverride. Falls back to systemPrompt
* when not provided. */
systemPromptLayers?: SystemPromptLayers;
tools?: "coding" | "readonly";
customTools?: ToolDefinition[];
/** Optional allowlist of builtin runtime web tools to keep enabled. */
@@ -1715,8 +1721,11 @@ export async function createFnAgent(options: AgentOptions): Promise<AgentResult>
cwd: resolvedProjectRoot,
agentDir: getFusionAgentDir(),
settingsManager,
systemPromptOverride: () => options.systemPrompt,
appendSystemPromptOverride: () => [],
systemPromptOverride: () => options.systemPromptLayers?.stable ?? options.systemPrompt,
appendSystemPromptOverride: () =>
options.systemPromptLayers?.dynamic
? [options.systemPromptLayers.dynamic]
: [],
...(effectiveExtensionPaths.length > 0 ? { additionalExtensionPaths: [...effectiveExtensionPaths] } : {}),
...(skillsOverrideFn ? { skillsOverride: skillsOverrideFn } : {}),
});
@@ -1748,6 +1757,17 @@ export async function createFnAgent(options: AgentOptions): Promise<AgentResult>
boundaryContext.worktreePath,
boundaryContext.worktreeProjectRoot,
);
// Sort tools alphabetically by name for deterministic ordering.
// Prompt caching requires the tool list to be byte-identical across
// sessions — reordering breaks cache prefix matching.
// Exception: fn_heartbeat_done must remain last (stable terminal signal
// required by the heartbeat executor — see agent-heartbeat.ts).
customToolList.sort((a, b) => a.name.localeCompare(b.name));
const heartbeatDoneIdx = customToolList.findIndex((t) => t.name === "fn_heartbeat_done");
if (heartbeatDoneIdx >= 0 && heartbeatDoneIdx < customToolList.length - 1) {
const [doneTool] = customToolList.splice(heartbeatDoneIdx, 1);
customToolList.push(doneTool);
}
// Last-chance abort hook. Fires *here* — after every awaited setup step
// in createFnAgent (provider registration, worktree validation, resource
// loader reload) and immediately before the actual LLM session spawn.
@@ -1774,7 +1794,7 @@ export async function createFnAgent(options: AgentOptions): Promise<AgentResult>
...customToolList.map((tool) => tool.name),
...options.builtinToolsAllowlist,
]),
];
].sort();
}
return createAgentSession(createSessionOptions);

View File

@@ -0,0 +1,85 @@
/**
* Structured system prompt layers for cross-session caching.
*
* The `stable` layer contains content that is identical across sessions of
* the same role (base role prompt). The `dynamic` layer holds per-session
* content (agent instructions, memory, performance feedback, plugins).
*
* When the stable layer is byte-identical across consecutive API calls,
* Anthropic's prompt cache gives a 90% read discount. OpenAI caches
* matching prefixes automatically at 50% discount.
*/
export interface SystemPromptLayers {
/** Role-specific base prompt — identical across all sessions of this role. */
stable: string;
/** Per-session content: agent instructions, memory, feedback, plugins. */
dynamic: string;
}
export interface PromptLayerInput {
/** The base role system prompt (e.g. REVIEWER_SYSTEM_PROMPT). */
basePrompt: string;
/** Resolved agent instructions (instructionsText + instructionsPath + soul). */
agentInstructions?: string;
/** Formatted memory section (agent memory + workspace memory). */
memorySection?: string;
/** Formatted plugin prompt contributions. */
pluginContributions?: string;
/** Formatted performance feedback section. */
performanceFeedback?: string;
}
/**
* Build structured prompt layers from the components that currently get
* concatenated into a single system prompt string.
*
* The stable layer is ONLY the base role prompt. Everything else goes into
* the dynamic layer so that the stable prefix is byte-identical across
* sessions of the same role, enabling cross-session prompt caching.
*/
export function buildPromptLayers(input: PromptLayerInput): SystemPromptLayers {
const { basePrompt, agentInstructions, memorySection, pluginContributions, performanceFeedback } = input;
const dynamicParts: string[] = [];
// Memory section comes before instructions to preserve the relative
// ordering from the legacy buildSystemPromptWithInstructions approach,
// where memory was concatenated onto basePrompt before instructions
// were appended.
const trimmedMemory = memorySection?.trim() ?? "";
if (trimmedMemory) {
dynamicParts.push(trimmedMemory);
}
const trimmedInstructions = agentInstructions?.trim() ?? "";
if (trimmedInstructions) {
dynamicParts.push(`## Custom Instructions\n\n${trimmedInstructions}`);
}
const trimmedPlugins = pluginContributions?.trim() ?? "";
if (trimmedPlugins) {
dynamicParts.push(trimmedPlugins);
}
const trimmedFeedback = performanceFeedback?.trim() ?? "";
if (trimmedFeedback) {
dynamicParts.push(trimmedFeedback);
}
return {
stable: basePrompt,
dynamic: dynamicParts.join("\n\n"),
};
}
/**
* Collapse layers back into a single string for backward compatibility.
* Runtimes that don't support structured caching use this to get the same
* concatenated prompt as before.
*/
export function collapsePromptLayers(layers: SystemPromptLayers): string {
if (!layers.dynamic) {
return layers.stable;
}
return `${layers.stable}\n\n${layers.dynamic}`;
}

View File

@@ -18,9 +18,9 @@ import { reviewerLog } from "./logger.js";
import { checkSessionError } from "./usage-limit-detector.js";
import {
resolveAgentInstructions,
buildSystemPromptWithInstructions,
buildPluginPromptSection,
} from "./agent-instructions.js";
import { buildPromptLayers, collapsePromptLayers } from "./prompt-layers.js";
import { createFallbackModelObserver } from "./fallback-model-observer.js";
import { createMemoryGetTool, createMemorySearchTool, createWebFetchTool } from "./agent-tools.js";
@@ -405,26 +405,36 @@ export async function reviewStep(
}
}
const reviewerBasePrompt = resolveAgentPrompt("reviewer", options.agentPrompts) || REVIEWER_SYSTEM_PROMPT;
// Memory goes in the dynamic layer (not concatenated onto basePrompt) so the
// stable prefix is byte-identical across sessions even if memory changes.
// The leading "\n" separator is no longer needed — buildPromptLayers handles
// section joining with "\n\n".
const memorySection = options.rootDir && options.settings?.memoryEnabled !== false
? "\n" + buildReviewerMemoryInstructions(options.rootDir, options.settings)
? buildReviewerMemoryInstructions(options.rootDir, options.settings)
: "";
const reviewerSystemPrompt = buildSystemPromptWithInstructions(
reviewerBasePrompt + memorySection,
reviewerInstructions,
);
const reviewerContributions = options.pluginRunner
?.getPromptContributionsForSurface("reviewer")
?? [];
if (reviewerContributions.length > 0) {
reviewerLog.log(`applied ${reviewerContributions.length} plugin prompt contributions for reviewer surface`);
}
// Build structured layers for cross-session prompt caching.
// The stable layer (base prompt only) is byte-identical across all
// reviewer sessions in this task, enabling cache hits. Memory goes
// into the dynamic layer because it can change between sessions.
const reviewerPluginContributions = buildPluginPromptSection(
"reviewer",
options.pluginRunner,
);
const reviewerSystemPromptFinal = reviewerPluginContributions
? `${reviewerSystemPrompt}\n\n${reviewerPluginContributions}`
: reviewerSystemPrompt;
if (reviewerPluginContributions) {
reviewerLog.log(`applied plugin prompt contributions for reviewer surface`);
}
const layers = buildPromptLayers({
basePrompt: reviewerBasePrompt,
agentInstructions: reviewerInstructions,
memorySection,
pluginContributions: reviewerPluginContributions,
});
// Collapsed string for backward compatibility with runtimes that don't
// support layers (plugin runtimes, older pi versions).
const reviewerSystemPromptFinal = collapsePromptLayers(layers);
// Build skill selection context (assigned agent skills take precedence over role fallback)
let skillContext = undefined;
@@ -495,6 +505,7 @@ export async function reviewStep(
pluginRunner: options.pluginRunner,
cwd,
systemPrompt: reviewerSystemPromptFinal,
systemPromptLayers: layers,
tools: "readonly",
customTools: [createWebFetchTool(), ...(memoryTools ?? [])],
onText: agentLogger ? agentLogger.onText : (delta) => options.onText?.(delta),

View File

@@ -90,3 +90,21 @@ export async function accumulateSessionTokenUsage(
log.warn(`${taskId}: session token usage accumulate failed: ${message}`);
}
}
/**
* Compute the cache hit ratio: `cachedTokens / (inputTokens + cachedTokens)`.
* Returns a number in [0, 1], or 0 when both arguments are 0.
*
* Compatible with stored `task.tokenUsage` fields: pass `inputTokens` (which
* includes cache-write tokens per `accumulateSessionTokenUsage`) and
* `cachedTokens` (cache-read tokens). Note this differs slightly from the
* Anthropic console metric, which excludes cache-write from the denominator.
*/
export function computeCacheHitRatio(
inputTokens: number,
cachedTokens: number,
): number {
const total = inputTokens + cachedTokens;
if (total === 0) return 0;
return cachedTokens / total;
}

View File

@@ -484,3 +484,4 @@ export function createSkillsOverrideFromSelection(
};
};
}

View File

@@ -30,9 +30,9 @@ import { AgentLogger } from "./agent-logger.js";
import {
resolveAgentInstructions,
resolveAgentInstructionsWithRatings,
buildSystemPromptWithInstructions,
buildPluginPromptSection,
} from "./agent-instructions.js";
import { buildPromptLayers, collapsePromptLayers } from "./prompt-layers.js";
import { createFallbackModelObserver } from "./fallback-model-observer.js";
import { planLog, reviewerLog, formatError } from "./logger.js";
import {
@@ -1014,30 +1014,29 @@ export class TriageProcessor {
const triageIdentitySection = assignedAgent
? `## Identity\n\nYou are ${assignedAgent.name}${assignedAgent.title?.trim() ? `, ${assignedAgent.title.trim()}` : ""} (agent ID: ${assignedAgent.id}, role: ${assignedAgent.role}).`
: "";
const triageSystemPrompt = buildSystemPromptWithInstructions(
resolveAgentPrompt("triage", settings.agentPrompts)
// Build structured layers for cross-session prompt caching.
const triagePluginContributions = buildPluginPromptSection(
"triage",
this.options.pluginRunner,
);
if (triagePluginContributions) {
planLog.log(`${task.id}: applied plugin prompt contributions for triage surface`);
}
const triageLayers = buildPromptLayers({
basePrompt: resolveAgentPrompt("triage", settings.agentPrompts)
|| (isFast ? FAST_TRIAGE_SYSTEM_PROMPT : TRIAGE_SYSTEM_PROMPT),
[
agentInstructions: [
triageIdentitySection,
triageInstructions,
isResearchToolSurfaceEnabled(settings)
? getResearchGuidanceForSurface("triage")
: "",
].filter((section) => section.trim()).join("\n\n"),
);
const triageContributions = this.options.pluginRunner
?.getPromptContributionsForSurface("triage")
?? [];
if (triageContributions.length > 0) {
planLog.log(`${task.id}: applied ${triageContributions.length} plugin prompt contributions for triage surface`);
}
const triagePluginContributions = buildPluginPromptSection(
"triage",
this.options.pluginRunner,
);
const triageSystemPromptFinal = triagePluginContributions
? `${triageSystemPrompt}\n\n${triagePluginContributions}`
: triageSystemPrompt;
pluginContributions: triagePluginContributions,
});
const triageSystemPromptFinal = collapsePromptLayers(triageLayers);
// Build skill selection context (assigned agent skills take precedence over role fallback)
const skillContext = await buildSessionSkillContext({
@@ -1054,6 +1053,7 @@ export class TriageProcessor {
pluginRunner: this.options.pluginRunner,
cwd: this.rootDir,
systemPrompt: triageSystemPromptFinal,
systemPromptLayers: triageLayers,
tools: "coding",
customTools,
onText: agentLogger.onText,
@@ -1294,6 +1294,7 @@ export class TriageProcessor {
pluginRunner: this.options.pluginRunner,
cwd: this.rootDir,
systemPrompt: triageSystemPromptFinal,
systemPromptLayers: triageLayers,
tools: "coding",
customTools,
onText: agentLogger.onText,