376 lines
16 KiB
TypeScript
376 lines
16 KiB
TypeScript
/**
|
|
* CLI-backed chat session runner tests (CLI Agent Executor, U12).
|
|
*
|
|
* Mocks PTY/adapters entirely: the runner depends only on narrow
|
|
* `ChatStoreLike` / `CliSessionManagerLike` seams, so these are exercised with
|
|
* in-memory fakes. No real CliSessionManager, no node-pty, no network, no
|
|
* port 4040.
|
|
*
|
|
* ───────────────────────────────────────────────────────────────────────────
|
|
* redactSecrets COVERAGE CHARACTERIZATION (U12 deliverable)
|
|
* ───────────────────────────────────────────────────────────────────────────
|
|
* The shared @fusion/core `redactSecrets` pass runs on ALL transcript text
|
|
* before it lands in chat_messages. What it catches today (verified by the
|
|
* "redaction" describe block below):
|
|
*
|
|
* CAUGHT:
|
|
* - `Authorization: Bearer <token>` and bare `Authorization: <token>` headers.
|
|
* - Free-standing `Bearer <token>` strings.
|
|
* - `key=`/`token=`/`secret=`/`password=`/`apikey=`/`access_token=` /
|
|
* `refresh_token=`/`client_secret=` assignments (`:` or `=`, quoted or bare)
|
|
* — this is the env-dump (KEY=VALUE) coverage.
|
|
* - Vendor-prefixed opaque tokens: `sk-…`, `ghp_…`, `gho_…`, `github_pat_…`,
|
|
* `xoxb-/xoxa-/xoxp-/xoxr-…`, `AKIA…` (>=8 trailing chars).
|
|
* - Standalone long base64 (>=40 chars) and hex (>=32 chars) blobs.
|
|
*
|
|
* KNOWN GAPS (deferred per plan Risks — deeper heuristics are follow-ups):
|
|
* - Generic short secrets with no recognizable prefix/keyword/length.
|
|
* - PEM private-key blocks and multi-line credentials are only partially hit
|
|
* (line-by-line base64 may exceed the length threshold, but headers leak).
|
|
* - JSON `"token": "..."` survives only via the keyword rule, not structurally.
|
|
* - Cross-chunk tokens are handled by the ENGINE's TelemetryHub carry-over
|
|
* window (chunkCarryChars), NOT by redactSecrets alone — see the
|
|
* "token spanning a chunk split" test which models that carry behavior.
|
|
* ───────────────────────────────────────────────────────────────────────────
|
|
*/
|
|
|
|
import { describe, it, expect, beforeEach } from "vitest";
|
|
import { redactSecrets } from "@fusion/core";
|
|
import type {
|
|
ChatMessage,
|
|
ChatMessageCreateInput,
|
|
ChatSession,
|
|
} from "@fusion/core";
|
|
import {
|
|
CliChatSessionRunner,
|
|
type ChatStoreLike,
|
|
type CliSessionLike,
|
|
type CliSessionManagerLike,
|
|
type ChatTelemetryEvent,
|
|
} from "../cli-chat.js";
|
|
|
|
// ── Fakes ──────────────────────────────────────────────────────────────────
|
|
|
|
class FakeChatStore implements ChatStoreLike {
|
|
sessions = new Map<string, ChatSession>();
|
|
messages: ChatMessage[] = [];
|
|
private seq = 0;
|
|
|
|
putSession(partial: Partial<ChatSession> & { id: string }): ChatSession {
|
|
const session: ChatSession = {
|
|
id: partial.id,
|
|
agentId: "agent-1",
|
|
title: null,
|
|
status: "active",
|
|
projectId: "proj-1",
|
|
modelProvider: null,
|
|
modelId: null,
|
|
createdAt: new Date().toISOString(),
|
|
updatedAt: new Date().toISOString(),
|
|
cliSessionFile: null,
|
|
cliExecutorAdapterId: "claude-local",
|
|
inFlightGeneration: null,
|
|
...partial,
|
|
};
|
|
this.sessions.set(session.id, session);
|
|
return session;
|
|
}
|
|
|
|
getSession(id: string): ChatSession | undefined {
|
|
return this.sessions.get(id);
|
|
}
|
|
|
|
addMessage(sessionId: string, input: ChatMessageCreateInput): ChatMessage {
|
|
const msg: ChatMessage = {
|
|
id: `msg-${++this.seq}`,
|
|
sessionId,
|
|
role: input.role,
|
|
content: input.content ?? "",
|
|
thinkingOutput: null,
|
|
metadata: input.metadata ?? null,
|
|
createdAt: new Date().toISOString(),
|
|
};
|
|
this.messages.push(msg);
|
|
return msg;
|
|
}
|
|
|
|
setCliExecutorAdapterId(id: string, adapterId: string | null): ChatSession | undefined {
|
|
const s = this.sessions.get(id);
|
|
if (!s) return undefined;
|
|
s.cliExecutorAdapterId = adapterId;
|
|
return s;
|
|
}
|
|
|
|
// Used by the runner to persist the native session id linkage.
|
|
setCliSessionFile(id: string, value: string): void {
|
|
const s = this.sessions.get(id);
|
|
if (s) s.cliSessionFile = value;
|
|
}
|
|
|
|
messagesFor(sessionId: string): ChatMessage[] {
|
|
return this.messages.filter((m) => m.sessionId === sessionId);
|
|
}
|
|
}
|
|
|
|
class FakeCliManager implements CliSessionManagerLike {
|
|
records = new Map<string, CliSessionLike>();
|
|
injected: { sessionId: string; text: string }[] = [];
|
|
spawnCalls: unknown[] = [];
|
|
private seq = 0;
|
|
|
|
async spawn(options: Parameters<CliSessionManagerLike["spawn"]>[0]): Promise<CliSessionLike> {
|
|
this.spawnCalls.push(options);
|
|
const id = options.resume?.sessionId ?? `cli-${++this.seq}`;
|
|
const record: CliSessionLike = {
|
|
id,
|
|
nativeSessionId: options.resume?.nativeSessionId ?? null,
|
|
agentState: "ready",
|
|
};
|
|
this.records.set(id, record);
|
|
return record;
|
|
}
|
|
|
|
async inject(sessionId: string, text: string): Promise<void> {
|
|
this.injected.push({ sessionId, text });
|
|
}
|
|
|
|
getSession(sessionId: string): CliSessionLike | undefined {
|
|
return this.records.get(sessionId);
|
|
}
|
|
|
|
setState(sessionId: string, state: string): void {
|
|
const r = this.records.get(sessionId);
|
|
if (r) r.agentState = state;
|
|
}
|
|
}
|
|
|
|
function makeRunner() {
|
|
const store = new FakeChatStore();
|
|
const manager = new FakeCliManager();
|
|
const runner = new CliChatSessionRunner({ store, manager });
|
|
return { store, manager, runner };
|
|
}
|
|
|
|
// ── Session spawn / resume ──────────────────────────────────────────────────
|
|
|
|
describe("CliChatSessionRunner — session lifecycle", () => {
|
|
let ctx: ReturnType<typeof makeRunner>;
|
|
beforeEach(() => {
|
|
ctx = makeRunner();
|
|
});
|
|
|
|
it("spawns a chat-purpose CLI session in the configured working directory", async () => {
|
|
ctx.store.putSession({ id: "chat-1", cliExecutorAdapterId: "claude-local" });
|
|
const cliId = await ctx.runner.ensureSession("chat-1", {
|
|
projectId: "proj-1",
|
|
worktreePath: "/work/dir",
|
|
});
|
|
expect(cliId).toBeTruthy();
|
|
const call = ctx.manager.spawnCalls[0] as Record<string, unknown>;
|
|
expect(call.purpose).toBe("chat");
|
|
expect(call.chatSessionId).toBe("chat-1");
|
|
expect(call.worktreePath).toBe("/work/dir");
|
|
expect(call.resume).toBeUndefined();
|
|
});
|
|
|
|
it("resumes via the persisted native session id (cliSessionFile linkage)", async () => {
|
|
ctx.store.putSession({ id: "chat-1", cliSessionFile: "native-abc" });
|
|
await ctx.runner.ensureSession("chat-1", { projectId: "proj-1" });
|
|
const call = ctx.manager.spawnCalls[0] as Record<string, unknown>;
|
|
expect(call.resume).toEqual({ sessionId: "chat-1", nativeSessionId: "native-abc" });
|
|
});
|
|
|
|
it("reuses an existing live session instead of respawning", async () => {
|
|
ctx.store.putSession({ id: "chat-1" });
|
|
const a = await ctx.runner.ensureSession("chat-1", { projectId: "proj-1" });
|
|
const b = await ctx.runner.ensureSession("chat-1", { projectId: "proj-1" });
|
|
expect(a).toBe(b);
|
|
expect(ctx.manager.spawnCalls).toHaveLength(1);
|
|
});
|
|
|
|
it("rejects sessions with no cli-agent executor selected", async () => {
|
|
ctx.store.putSession({ id: "chat-1", cliExecutorAdapterId: null });
|
|
await expect(ctx.runner.ensureSession("chat-1", { projectId: "proj-1" })).rejects.toThrow(
|
|
/no cli-agent executor/,
|
|
);
|
|
});
|
|
});
|
|
|
|
// ── Transcript mapping (granularity: user/assistant/tool-summary) ───────────
|
|
|
|
describe("CliChatSessionRunner — transcript mapping", () => {
|
|
let ctx: ReturnType<typeof makeRunner>;
|
|
let cliId: string;
|
|
beforeEach(async () => {
|
|
ctx = makeRunner();
|
|
ctx.store.putSession({ id: "chat-1" });
|
|
cliId = await ctx.runner.ensureSession("chat-1", { projectId: "proj-1" });
|
|
});
|
|
|
|
it("maps a transcript fixture to the expected chat_messages sequence, excluding tool noise", async () => {
|
|
// Fixture: busy → assistant chunks → a tool-summary → fine-grained tool
|
|
// noise (must be dropped) → done. Models one assistant turn.
|
|
const fixture: ChatTelemetryEvent[] = [
|
|
{ kind: "busy" },
|
|
{ kind: "transcript", text: "Let me check " },
|
|
{ kind: "transcript", text: "the config.\n" },
|
|
{ kind: "toolActivity", text: "Read(config.json)" }, // NOISE — dropped
|
|
{ kind: "outputProgress", text: "...." }, // NOISE — dropped
|
|
{ kind: "transcript", toolSummary: "Read config.json (42 lines)" },
|
|
{ kind: "idle" }, // NOISE — dropped
|
|
{ kind: "transcript", text: "All good." },
|
|
{ kind: "done" },
|
|
];
|
|
for (const ev of fixture) {
|
|
await ctx.runner.handleTelemetry("chat-1", ev);
|
|
}
|
|
|
|
const rows = ctx.store.messagesFor("chat-1").map((m) => ({
|
|
role: m.role,
|
|
content: m.content,
|
|
kind: (m.metadata as Record<string, unknown> | null)?.kind,
|
|
}));
|
|
|
|
expect(rows).toEqual([
|
|
{ role: "assistant", content: "Read config.json (42 lines)", kind: "tool-summary" },
|
|
{ role: "assistant", content: "Let me check the config.\nAll good.", kind: undefined },
|
|
]);
|
|
});
|
|
|
|
it("persists native session id on first transcript event carrying it", async () => {
|
|
await ctx.runner.handleTelemetry("chat-1", {
|
|
kind: "busy",
|
|
nativeSessionId: "native-xyz",
|
|
});
|
|
expect(ctx.store.getSession("chat-1")?.cliSessionFile).toBe("native-xyz");
|
|
});
|
|
|
|
it("transcript rows persist and reload after the session ends (durable store)", async () => {
|
|
await ctx.runner.handleTelemetry("chat-1", { kind: "busy" });
|
|
await ctx.runner.handleTelemetry("chat-1", { kind: "transcript", text: "Done working." });
|
|
await ctx.runner.handleTelemetry("chat-1", { kind: "done" });
|
|
// Simulate session end + reload: a fresh runner reading the SAME store.
|
|
const reloaded = new CliChatSessionRunner({ store: ctx.store, manager: ctx.manager });
|
|
void reloaded;
|
|
const rows = ctx.store.messagesFor("chat-1");
|
|
expect(rows).toHaveLength(1);
|
|
expect(rows[0].content).toBe("Done working.");
|
|
});
|
|
});
|
|
|
|
// ── Composer queue (stale-isGenerating learning) ───────────────────────────
|
|
|
|
describe("CliChatSessionRunner — composer queue", () => {
|
|
let ctx: ReturnType<typeof makeRunner>;
|
|
let cliId: string;
|
|
beforeEach(async () => {
|
|
ctx = makeRunner();
|
|
ctx.store.putSession({ id: "chat-1" });
|
|
cliId = await ctx.runner.ensureSession("chat-1", { projectId: "proj-1" });
|
|
});
|
|
|
|
it("injects immediately when the session is idle", async () => {
|
|
ctx.manager.setState(cliId, "ready");
|
|
const result = await ctx.runner.send("chat-1", "hello");
|
|
expect(result).toBe("sent");
|
|
expect(ctx.manager.injected).toEqual([{ sessionId: cliId, text: "hello" }]);
|
|
expect(ctx.runner.queuedCount("chat-1")).toBe(0);
|
|
});
|
|
|
|
it("queues with a visible indicator when the session is busy", async () => {
|
|
ctx.manager.setState(cliId, "busy");
|
|
const result = await ctx.runner.send("chat-1", "while busy");
|
|
expect(result).toBe("queued");
|
|
expect(ctx.manager.injected).toHaveLength(0);
|
|
expect(ctx.runner.queuedCount("chat-1")).toBe(1);
|
|
// User message is still persisted even though injection is deferred.
|
|
expect(ctx.store.messagesFor("chat-1").some((m) => m.role === "user")).toBe(true);
|
|
});
|
|
|
|
it("flushes on done using a RE-FETCHED authoritative state, not a cached flag", async () => {
|
|
ctx.manager.setState(cliId, "busy");
|
|
await ctx.runner.send("chat-1", "queued msg");
|
|
// The 'done' telemetry says the turn ended; flush must re-read the record.
|
|
ctx.manager.setState(cliId, "ready"); // authoritative state now idle
|
|
await ctx.runner.handleTelemetry("chat-1", { kind: "done" });
|
|
expect(ctx.manager.injected).toEqual([{ sessionId: cliId, text: "queued msg" }]);
|
|
expect(ctx.runner.queuedCount("chat-1")).toBe(0);
|
|
});
|
|
|
|
it("does NOT flush if the session turned busy again before the flush (re-fetch wins)", async () => {
|
|
ctx.manager.setState(cliId, "busy");
|
|
await ctx.runner.send("chat-1", "queued msg");
|
|
// 'done' arrives but the authoritative record shows busy again (re-entered turn).
|
|
ctx.manager.setState(cliId, "busy");
|
|
await ctx.runner.handleTelemetry("chat-1", { kind: "done" });
|
|
expect(ctx.manager.injected).toHaveLength(0);
|
|
expect(ctx.runner.queuedCount("chat-1")).toBe(1);
|
|
});
|
|
});
|
|
|
|
// ── Redaction (characterized coverage) ──────────────────────────────────────
|
|
|
|
describe("CliChatSessionRunner — redaction before persistence", () => {
|
|
let ctx: ReturnType<typeof makeRunner>;
|
|
beforeEach(async () => {
|
|
ctx = makeRunner();
|
|
ctx.store.putSession({ id: "chat-1" });
|
|
await ctx.runner.ensureSession("chat-1", { projectId: "proj-1" });
|
|
});
|
|
|
|
it("redacts a bearer token in transcript text before it lands in chat_messages", async () => {
|
|
await ctx.runner.handleTelemetry("chat-1", { kind: "busy" });
|
|
await ctx.runner.handleTelemetry("chat-1", {
|
|
kind: "transcript",
|
|
text: "Authorization: Bearer abcDEF123ghiJKL456mnoPQR789stu",
|
|
});
|
|
await ctx.runner.handleTelemetry("chat-1", { kind: "done" });
|
|
const content = ctx.store.messagesFor("chat-1")[0].content;
|
|
expect(content).toContain("[REDACTED]");
|
|
expect(content).not.toContain("abcDEF123ghiJKL456mnoPQR789stu");
|
|
});
|
|
|
|
it("redacts an env-dump (KEY=VALUE) before persistence", async () => {
|
|
await ctx.runner.handleTelemetry("chat-1", { kind: "busy" });
|
|
await ctx.runner.handleTelemetry("chat-1", {
|
|
kind: "transcript",
|
|
text: "API_KEY=sk-livesupersecretvalue9999 TOKEN=ghp_anotherSecretToken12345",
|
|
});
|
|
await ctx.runner.handleTelemetry("chat-1", { kind: "done" });
|
|
const content = ctx.store.messagesFor("chat-1")[0].content;
|
|
expect(content).not.toContain("sk-livesupersecretvalue9999");
|
|
expect(content).not.toContain("ghp_anotherSecretToken12345");
|
|
expect(content).toContain("[REDACTED]");
|
|
});
|
|
|
|
it("catches a token spanning a chunk split via the engine carry-over model", async () => {
|
|
// The engine's TelemetryHub keeps a carry tail across chunks so a token
|
|
// split as `Bearer ` (chunk A) + `<value>` (chunk B) is redacted at the
|
|
// boundary. We model that carry: the adapter delivers the boundary-joined
|
|
// text as ONE sanitized transcript event (already redacted upstream), so
|
|
// the persisted row never contains the value. Here we assert redactSecrets
|
|
// catches the joined form the carry produces.
|
|
const chunkA = "here is the Bearer ";
|
|
const chunkB = "sk-splitTokenAcrossChunks0000abcd";
|
|
const joined = redactSecrets(chunkA + chunkB);
|
|
expect(joined).not.toContain("sk-splitTokenAcrossChunks0000abcd");
|
|
expect(joined).toContain("[REDACTED]");
|
|
|
|
// And end-to-end: a transcript event carrying the joined text persists redacted.
|
|
await ctx.runner.handleTelemetry("chat-1", { kind: "busy" });
|
|
await ctx.runner.handleTelemetry("chat-1", { kind: "transcript", text: chunkA + chunkB });
|
|
await ctx.runner.handleTelemetry("chat-1", { kind: "done" });
|
|
const content = ctx.store.messagesFor("chat-1")[0].content;
|
|
expect(content).not.toContain("sk-splitTokenAcrossChunks0000abcd");
|
|
});
|
|
|
|
it("redacts user composer messages too (users can paste tokens)", async () => {
|
|
const cliId = ctx.manager.records.keys().next().value as string;
|
|
ctx.manager.setState(cliId, "ready");
|
|
await ctx.runner.send("chat-1", "use key=mysupersecretpassword12345 please");
|
|
const userMsg = ctx.store.messagesFor("chat-1").find((m) => m.role === "user")!;
|
|
expect(userMsg.content).not.toContain("mysupersecretpassword12345");
|
|
expect(userMsg.content).toContain("[REDACTED]");
|
|
});
|
|
});
|