root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

FN-9030: resolve computer state at project root

Browse Source 
Resolve computer-use state once per invocation at the containing Fusion project root.

- Add a state-root resolver shared by computer command adapters and snapshot storage.
- Preserve capture and replay snapshots across working directories while isolating projects.
- Cover root resolution and cross-directory state behavior, and document the storage contract.
- Add a patch changeset for the CLI fix.

Files changed:
 .../fn-9030-computer-use-project-local-state.md    |  7 ++
 docs/computer-use.md                               |  2 +-
 .../commands/__tests__/computer-commands.test.ts   | 76 ++++++++++++++++++++++
 .../__tests__/computer-snapshot-index.test.ts      | 14 ++++
 .../commands/__tests__/computer-state-root.test.ts | 49 ++++++++++++++
 packages/cli/src/commands/computer.ts              |  9 ++-
 packages/cli/src/commands/computer/state-root.ts   | 18 +++++
 7 files changed, 171 insertions(+), 4 deletions(-)

Fusion-Task-Id: FN-9030

Fusion-Task-Lineage: a0cb48ee-fd7f-4966-9930-cde273027eca

Co-authored-by: Fusion (runfusion.ai) <noreply@runfusion.ai>
This commit is contained in:
gsxdsm
2026-08-13 16:08:52 -07:00
parent cb4ab3b6da
commit d3e51f566e
7 changed files with 171 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
.changeset/fn-9030-computer-use-project-local-state.md Normal file
View File

@@ -0,0 +1,7 @@
---
"@runfusion/fusion": patch
---
summary: Keep computer-use snapshots available across project directories.
category: fix
dev: Resolve computer-use state once per invocation through resolveComputerStateRoot.

2
docs/computer-use.md
View File

@@ -130,7 +130,7 @@ Screenshots are always paths, never base64, `data:` URLs, or byte arrays. `--no-
## Durable snapshots and safe element replay
Snapshots are stored per project at `.fusion/computer-use/snapshots/<snapshotId>.json`; the app's latest pointer is `.fusion/computer-use/latest/<targetKeySlug>.json`. They are never shared across project roots. Capture atomically persists the record and pointer before returning `snapshotId`.
Snapshots are stored under the resolved Fusion project root, not the invoking working directory: `<projectRoot>/.fusion/computer-use/snapshots/<snapshotId>.json`; the app's latest pointer is `<projectRoot>/.fusion/computer-use/latest/<targetKeySlug>.json`. Each `fn computer` invocation resolves that root once by walking upward from its starting directory, and falls back to that resolved directory when no project root is found. Snapshots, pointers, and screenshots use the same root, so they are never shared across projects. Capture atomically persists the record and pointer before returning `snapshotId`.
A resolved app has app-scoped `targetKey` (`bundle:<bundleId>`, or `pid:<pid>`) and window-scoped `windowKey` (`<targetKey>#<windowId>`). There is one latest pointer per app, not per window. An action with no `--snapshot-id` uses that latest snapshot. Action window flags are optional assertions: a supplied selector that differs from the recorded window produces `SNAPSHOT_STALE` / `window-mismatch`.

76
packages/cli/src/commands/__tests__/computer-commands.test.ts
View File

@@ -1,5 +1,10 @@
import { access, mkdir, mkdtemp, readdir, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { describe, expect, it, vi } from "vitest";
import { runComputer } from "../computer.js";
import { ComputerSnapshotStore } from "../computer/snapshot-store.js";
import { ComputerUseError } from "../computer/contract.js";
import type { ComputerAdapter } from "../computer/adapter.js";
const app = { bundleId: "com.example.App", name: "App", pid: 1 };
@@ -14,6 +19,77 @@ const adapter: ComputerAdapter = {
};
describe("computer commands", () => {
it("emits one JSON envelope and persists a snapshot", async () => { const output: string[] = []; const root = await import("node:fs/promises").then((fs) => fs.mkdtemp("/tmp/fusion-computer-")); try { expect(await runComputer(["get-app-state", "--app", "App", "--no-screenshot", "--json"], { adapter, projectRoot: root, stdout: (x) => output.push(x) })).toBe(0); const envelope = JSON.parse(output[0]); expect(envelope).toMatchObject({ schemaVersion: 1, ok: true, command: "computer.get-app-state" }); expect(envelope.result.snapshot.snapshotId).toMatch(/^cs_/); } finally { await (await import("node:fs/promises")).rm(root, { recursive: true, force: true }); } });
it("keeps snapshots project-local when capture and replay use different directories", async () => {
const root = await mkdtemp(join(tmpdir(), "fusion-computer-project-"));
const nested = join(root, "pkg");
const captureOutput: string[] = [];
const actionOutput: string[] = [];
const clock = { now: () => new Date("2026-08-13T00:00:00.000Z") };
const localAdapter: ComputerAdapter = { ...adapter, captureState: async () => {
const state = await adapter.captureState({ kind: "name", value: "App", raw: "App" }, { screenshot: false });
state.snapshot.capturedAt = clock.now().toISOString();
return state;
} };
try {
await mkdir(nested);
await mkdir(join(root, ".fusion"));
await writeFile(join(root, ".fusion", "project.json"), JSON.stringify({ id: "proj_0123456789abcdef", createdAt: "2026-08-13T00:00:00.000Z" }));
expect(await runComputer(["get-app-state", "--app", "App", "--no-screenshot", "--json"], { adapter: localAdapter, projectRoot: nested, clock, stdout: (text) => captureOutput.push(text) })).toBe(0);
const snapshotId = JSON.parse(captureOutput[0]!).result.snapshot.snapshotId;
expect(await runComputer(["click", "--app", "App", "--element-index", "7", "--json"], { adapter: localAdapter, projectRoot: root, clock, stdout: (text) => actionOutput.push(text) })).toBe(0);
expect(JSON.parse(actionOutput[0]!)).toMatchObject({ ok: true, command: "computer.click", result: { snapshotId } });
expect(await readdir(join(root, ".fusion", "computer-use", "snapshots"))).toContain(`${snapshotId}.json`);
expect(await readdir(join(root, ".fusion", "computer-use", "latest"))).not.toHaveLength(0);
await expect(access(join(nested, ".fusion", "computer-use"))).rejects.toThrow();
} finally { await rm(root, { recursive: true, force: true }); }
});
it("fences stale IDs and replays latest click and set-value targets before actions", async () => {
const root = await mkdtemp(join(tmpdir(), "fusion-computer-fence-"));
const clock = { now: () => new Date("2026-08-13T00:00:00.000Z") };
const store = new ComputerSnapshotStore({ projectRoot: root, now: clock.now });
const snapshotElement = (index: number) => ({ index, role: "AXButton", title: "Go", value: null, label: null, enabled: true, focused: false, bounds: null, actions: [], locator: { kind: "ax-path" as const, path: "button[0]", role: "AXButton", subrole: null, identifier: null, title: "Go" } });
const first = await store.persist({ app, window: { windowId: "w", windowIndex: 0, title: "w", bounds: null, minimized: false }, elementCount: 8, elements: [snapshotElement(7)] });
const latest = await store.persist({ app, window: { windowId: "w", windowIndex: 0, title: "w", bounds: null, minimized: false }, elementCount: 8, elements: [snapshotElement(7)] });
const calls: string[] = [];
const click = vi.fn(async (input: Parameters<ComputerAdapter["click"]>[0]) => { calls.push("click"); return { action: "click" as const, app: input.app, snapshotId: input.snapshotId, elementIndex: input.element.element.index, fromElementIndex: null, toElementIndex: null, performed: true }; });
const setValue = vi.fn(async (input: Parameters<ComputerAdapter["set-value"]>[0]) => { calls.push("set-value"); return { action: "set-value" as const, app: input.app, snapshotId: input.snapshotId, elementIndex: input.element.element.index, fromElementIndex: null, toElementIndex: null, performed: true }; });
const replayAdapter: ComputerAdapter = { ...adapter,
resolveWindow: async () => { calls.push("window"); return { window: { windowId: "w", windowIndex: 0, title: "w", bounds: { x: 1, y: 2, width: 3, height: 4 }, minimized: false }, handle: "live-window" }; },
resolveLocator: async (_window, locator) => { calls.push("locator"); return { element: { ...snapshotElement(7), bounds: { x: 10, y: 20, width: 30, height: 40 }, locator }, handle: "live-element" }; },
click, "set-value": setValue,
};
const staleOutput: string[] = [];
try {
expect(await runComputer(["click", "--app", "App", "--element-index", "7", "--snapshot-id", first.snapshotId, "--json"], { adapter: replayAdapter, store, projectRoot: root, stdout: (text) => staleOutput.push(text) })).toBe(1);
expect(JSON.parse(staleOutput[0]!)).toMatchObject({ error: { code: "SNAPSHOT_STALE", details: { reason: "superseded" } } });
expect(click).not.toHaveBeenCalled();
expect(await runComputer(["set-value", "--app", "App", "--element-index", "7", "--snapshot-id", first.snapshotId, "--value", "safe", "--json"], { adapter: replayAdapter, store, projectRoot: root, stdout: () => undefined })).toBe(1);
expect(setValue).not.toHaveBeenCalled();
expect(await runComputer(["click", "--app", "App", "--element-index", "7", "--snapshot-id", latest.snapshotId, "--json"], { adapter: replayAdapter, store, projectRoot: root, stdout: () => undefined })).toBe(0);
expect(calls).toEqual(["window", "locator", "click"]);
expect(click).toHaveBeenLastCalledWith(expect.objectContaining({ snapshotId: latest.snapshotId, element: expect.objectContaining({ element: expect.objectContaining({ index: 7, locator: snapshotElement(7).locator, bounds: { x: 10, y: 20, width: 30, height: 40 } }) }) }));
calls.length = 0;
expect(await runComputer(["set-value", "--app", "App", "--element-index", "7", "--value", "safe", "--json"], { adapter: replayAdapter, store, projectRoot: root, stdout: () => undefined })).toBe(0);
expect(calls).toEqual(["window", "locator", "set-value"]);
} finally { await rm(root, { recursive: true, force: true }); }
});
it("maps vanished windows and locator replay failures to safe snapshot errors", async () => {
const root = await mkdtemp(join(tmpdir(), "fusion-computer-replay-failure-"));
const store = new ComputerSnapshotStore({ projectRoot: root, now: () => new Date("2026-08-13T00:00:00.000Z") });
await store.persist({ app, window: { windowId: "w", windowIndex: 0, title: "w", bounds: null, minimized: false }, elementCount: 8, elements: [{ index: 7, role: "AXButton", title: "Go", value: null, label: null, enabled: true, focused: false, bounds: null, actions: [], locator: { kind: "ax-path", path: "button[0]", role: "AXButton", subrole: null, identifier: null, title: "Go" } }] });
const output: string[] = [];
try {
const missingWindow: ComputerAdapter = { ...adapter, resolveWindow: async () => { throw new ComputerUseError("WINDOW_NOT_FOUND", "gone"); } };
expect(await runComputer(["click", "--app", "App", "--element-index", "7", "--json"], { adapter: missingWindow, store, projectRoot: root, stdout: (text) => output.push(text) })).toBe(1);
expect(JSON.parse(output.pop()!)).toMatchObject({ error: { code: "SNAPSHOT_STALE", details: { reason: "window-gone" } } });
const missingLocator: ComputerAdapter = { ...adapter, resolveLocator: async () => { throw new ComputerUseError("ELEMENT_UNRESOLVABLE", "gone", "Re-run fn computer get-app-state."); } };
expect(await runComputer(["click", "--app", "App", "--element-index", "7", "--json"], { adapter: missingLocator, store, projectRoot: root, stdout: (text) => output.push(text) })).toBe(1);
expect(JSON.parse(output.pop()!)).toMatchObject({ error: { code: "ELEMENT_UNRESOLVABLE", details: { elementIndex: 7 } } });
} finally { await rm(root, { recursive: true, force: true }); }
});
it("uses the snapshot replay path for element-scoped typing", async () => {
const root = await import("node:fs/promises").then((fs) => fs.mkdtemp("/tmp/fusion-computer-"));
let resolved = 0;

14
packages/cli/src/commands/__tests__/computer-snapshot-index.test.ts
View File

@@ -44,6 +44,20 @@ describe("ComputerSnapshotStore", () => {
expect(JSON.parse(await readFile(join(root, ".fusion/computer-use/snapshots", `${record.snapshotId}.json`), "utf8")).elements["7"].index).toBe(7);
});
it("retains app, window, and sparse locator identity across fresh store instances", async () => {
const { store, root } = await fixture();
const sparse = element(7);
const record = await store.persist({ app, window, elementCount: 9, elements: [sparse] });
const freshStore = new ComputerSnapshotStore({ projectRoot: root, now: () => new Date(record.capturedAt) });
const restored = await freshStore.resolve({ app, snapshotId: record.snapshotId });
expect(restored).toEqual(record);
expect(restored.elements["7"]?.locator).toEqual({ kind: "ax-path", path: "AXWindow[0]/AXButton[7]", role: "AXButton", subrole: null, identifier: null, title: "button-7" });
expect(restored.app).toEqual({ bundleId: "com.example.Editor", name: "Editor", pid: 41 });
expect(restored.window).toMatchObject({ windowId: "w-1", windowIndex: 1 });
expect(() => freshStore.getElement(restored, 8)).toThrow(expect.objectContaining({ code: "ELEMENT_INDEX_NOT_FOUND" }));
});
it("uses the required C9 freshness order and remediation", async () => {
const { store, setNow } = await fixture();
await expect(store.resolve({ app })).rejects.toMatchObject({ code: "SNAPSHOT_REQUIRED" });

49
packages/cli/src/commands/__tests__/computer-state-root.test.ts Normal file
View File

@@ -0,0 +1,49 @@
import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { isAbsolute, join, relative as relativePath, resolve } from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import { resolveComputerStateRoot } from "../computer/state-root.js";
const roots: string[] = [];
afterEach(async () => { await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); });
async function fixture(): Promise<string> {
const root = await mkdtemp(join(tmpdir(), "computer-state-root-"));
roots.push(root);
return root;
}
async function markProject(root: string): Promise<void> {
await mkdir(join(root, ".fusion"), { recursive: true });
await writeFile(join(root, ".fusion", "project.json"), JSON.stringify({ id: "proj_0123456789abcdef", createdAt: "2026-08-13T00:00:00.000Z" }));
}
describe("resolveComputerStateRoot", () => {
it("uses a project marker at the start directory", async () => {
const root = await fixture();
await markProject(root);
expect(resolveComputerStateRoot(root)).toBe(root);
});
it("walks to a project marker above a nested directory", async () => {
const root = await fixture();
const nested = join(root, "packages", "cli");
await markProject(root);
await mkdir(nested, { recursive: true });
expect(resolveComputerStateRoot(nested)).toBe(root);
});
it("falls back to the resolved start path when no marker exists", async () => {
const root = await fixture();
const nested = join(root, "unmarked");
await mkdir(nested);
expect(resolveComputerStateRoot(nested)).toBe(nested);
});
it("normalizes a relative start path to an absolute path", async () => {
const root = await fixture();
const relative = relativePath(process.cwd(), root);
expect(isAbsolute(relative)).toBe(false);
expect(resolveComputerStateRoot(relative)).toBe(resolve(relative));
});
});

9
packages/cli/src/commands/computer.ts
View File

@@ -3,6 +3,7 @@ import { MacosComputerAdapter } from "./computer/adapter-macos.js";
import { resolveComputerAdapter, type ComputerClock } from "./computer/adapter-registry.js";
import type { ComputerAdapter, ResolvedComputerElement, ResolvedComputerWindow } from "./computer/adapter.js";
import { createComputerSnapshotStore, type ComputerSnapshotStore } from "./computer/snapshot-store.js";
import { resolveComputerStateRoot } from "./computer/state-root.js";
import { COMPUTER_COMMAND_SURFACE, COMPUTER_SUBCOMMANDS, ComputerUseError, failureEnvelope, isValidSnapshotId, parseAppTarget, successEnvelope, validateResult, type AppRef, type CommandName, type ComputerSubcommand } from "./computer/contract.js";
export interface ComputerCommandOptions { platform?: string; projectRoot?: string; adapter?: ComputerAdapter; store?: ComputerSnapshotStore; clock?: ComputerClock; stdout?: (text: string) => void; stderr?: (text: string) => void; stdin?: () => Promise<string>; }
@@ -12,8 +13,8 @@ const emit = (value: unknown, json: boolean, options: ComputerCommandOptions): v
const fail = (name: CommandName, error: unknown, json: boolean, options: ComputerCommandOptions): number => { const e = error instanceof ComputerUseError ? error : new ComputerUseError("INTERNAL", "Computer command failed unexpectedly."); const envelope = failureEnvelope(name, e); if (json) emit(envelope, true, options); else (options.stderr ?? console.error)(`error: ${e.code}: ${e.message}${e.remediation ? `\n${e.remediation}` : ""}`); return 1; };
function value(args: string[], flag: string): string | undefined { const i = args.indexOf(flag); return i >= 0 && args[i + 1] && !args[i + 1].startsWith("--") ? args[i + 1] : undefined; }
function number(args: string[], flag: string): number | undefined { const raw = value(args, flag); if (raw === undefined) return undefined; const parsed = Number(raw); if (!Number.isInteger(parsed) || parsed < 0) throw new ComputerUseError("INVALID_ARGUMENTS", `Invalid ${flag}.`); return parsed; }
function adapterFor(options: ComputerCommandOptions): ComputerAdapter { if (options.adapter) return options.adapter; const root = options.projectRoot ?? process.cwd(); return resolveComputerAdapter({ platform: options.platform, projectRoot: root, clock: options.clock, macosAdapterFactory: ({ seam, clock, projectRoot }) => new MacosComputerAdapter({ seam, clock, projectRoot }) }); }
function storeFor(options: ComputerCommandOptions): ComputerSnapshotStore { return options.store ?? createComputerSnapshotStore({ projectRoot: options.projectRoot, now: options.clock ? () => options.clock!.now() : undefined }); }
function adapterFor(options: ComputerCommandOptions): ComputerAdapter { if (options.adapter) return options.adapter; return resolveComputerAdapter({ platform: options.platform, projectRoot: options.projectRoot!, clock: options.clock, macosAdapterFactory: ({ seam, clock, projectRoot }) => new MacosComputerAdapter({ seam, clock, projectRoot }) }); }
function storeFor(options: ComputerCommandOptions): ComputerSnapshotStore { return options.store ?? createComputerSnapshotStore({ projectRoot: options.projectRoot!, now: options.clock ? () => options.clock!.now() : undefined }); }
async function appFor(adapter: ComputerAdapter, raw: string): Promise<AppRef> { const target = parseAppTarget(raw); const apps = (await adapter.listApps()).apps; const matches = target.kind === "pid" ? apps.filter((x) => String(x.pid) === target.value) : (() => { const byBundle = apps.filter((x) => x.bundleId === target.value); return byBundle.length ? byBundle : apps.filter((x) => x.name === target.value); })(); if (!matches.length) throw new ComputerUseError("APP_NOT_FOUND", `No running app matches ${raw}.`); if (matches.length > 1) throw new ComputerUseError("AMBIGUOUS_APP", `More than one app matches ${raw}.`, "Use a bundle id or pid target.", { candidateCount: matches.length }); return matches[0]; }
async function requireElements(args: string[], indexes: readonly number[], adapter: ComputerAdapter, store: ComputerSnapshotStore, app: AppRef): Promise<{ record: Awaited<ReturnType<ComputerSnapshotStore["resolve"]>>; window: ResolvedComputerWindow; elements: ResolvedComputerElement[] }> {
if (!indexes.length) throw new ComputerUseError("INVALID_ARGUMENTS", "At least one --element-index is required.");
@@ -174,7 +175,9 @@ export async function runComputer(args: string[], options: ComputerCommandOption
try {
// C10 stage 2 is deliberately complete and precedes adapter construction, filesystem, and OS discovery.
validateFlags(name, handlerArgs);
const payload = await COMPUTER_HANDLERS[name](handlerArgs, options);
const projectRoot = resolveComputerStateRoot(options.projectRoot ?? process.cwd());
const resolvedOptions = { ...options, projectRoot };
const payload = await COMPUTER_HANDLERS[name](handlerArgs, resolvedOptions);
if (!validateResult(name, payload, handlerArgs.includes("--no-screenshot"))) throw new ComputerUseError("INTERNAL", "Computer command produced an invalid contract result.");
emit(successEnvelope(command(name), payload), json, options);
return 0;

18
packages/cli/src/commands/computer/state-root.ts Normal file
View File

@@ -0,0 +1,18 @@
import { resolve } from "node:path";
import { findKbDir } from "../../project-resolver.js";
/**
* FNXC:ComputerUse 2026-08-13-22:34:
* Computer-use state is project-local rather than cwd-local because capture and
* replay run in separate processes from different directories in one project.
* This function is called at exactly one runComputer boundary so snapshots,
* latest pointers, and screenshots always use the same state root.
*/
export function resolveComputerStateRoot(startPath: string): string {
const resolvedStartPath = resolve(startPath);
try {
return findKbDir(resolvedStartPath) ?? resolvedStartPath;
} catch {
return resolvedStartPath;
}
}