fusion/packages/cli/src/__tests__/skill-sync.test.ts

import { describe, it, expect } from "vitest";
import { readFileSync, existsSync, readdirSync } from "node:fs";
import { resolve, dirname, relative, join } from "node:path";
import { fileURLToPath } from "node:url";
import { spawnSync } from "node:child_process";

const __dirname = dirname(fileURLToPath(import.meta.url));
const cliRoot = resolve(__dirname, "../..");
const repoRoot = resolve(cliRoot, "../..");
const skillDir = resolve(cliRoot, "skill/fusion");
const extensionPath = resolve(cliRoot, "src/extension.ts");

const EXT_TOOLS_BEGIN =
  "<!-- BEGIN: extension-tools (auto-generated by scripts/sync-fusion-skill-tools.mjs — do not edit by hand) -->";
const EXT_TOOLS_END = "<!-- END: extension-tools -->";
const CAP_TABLE_BEGIN =
  "<!-- BEGIN: fusion-capabilities-tool-table (auto-generated by scripts/sync-fusion-skill-tools.mjs — do not edit by hand) -->";
const CAP_TABLE_END = "<!-- END: fusion-capabilities-tool-table -->";

const engineToolSourceFiles = [
  resolve(repoRoot, "packages/engine/src/agent-tools.ts"),
  resolve(repoRoot, "packages/engine/src/triage.ts"),
  resolve(repoRoot, "packages/engine/src/executor.ts"),
  resolve(repoRoot, "packages/engine/src/merger.ts"),
  resolve(repoRoot, "packages/engine/src/agent-heartbeat.ts"),
];

function findMatchingBrace(source: string, openIndex: number): number {
  let depth = 0;
  let inSingle = false;
  let inDouble = false;
  let inTemplate = false;
  let inLineComment = false;
  let inBlockComment = false;
  let escaped = false;

  for (let i = openIndex; i < source.length; i++) {
    const char = source[i];
    const next = source[i + 1];

    if (inLineComment) {
      if (char === "\n") inLineComment = false;
      continue;
    }
    if (inBlockComment) {
      if (char === "*" && next === "/") {
        inBlockComment = false;
        i++;
      }
      continue;
    }

    if (inSingle || inDouble || inTemplate) {
      if (escaped) {
        escaped = false;
        continue;
      }
      if (char === "\\") {
        escaped = true;
        continue;
      }
      if (inSingle && char === "'") inSingle = false;
      else if (inDouble && char === '"') inDouble = false;
      else if (inTemplate && char === "`") inTemplate = false;
      continue;
    }

    if (char === "/" && next === "/") {
      inLineComment = true;
      i++;
      continue;
    }
    if (char === "/" && next === "*") {
      inBlockComment = true;
      i++;
      continue;
    }

    if (char === "'") {
      inSingle = true;
      continue;
    }
    if (char === '"') {
      inDouble = true;
      continue;
    }
    if (char === "`") {
      inTemplate = true;
      continue;
    }

    if (char === "{") depth++;
    if (char === "}") {
      depth--;
      if (depth === 0) return i;
    }
  }

  throw new Error("Unbalanced braces");
}

function splitTopLevelProperties(objectBody: string): string[] {
  const props: string[] = [];
  let start = 0;
  let depthParen = 0;
  let depthBrace = 0;
  let depthBracket = 0;
  let inSingle = false;
  let inDouble = false;
  let inTemplate = false;
  let inLineComment = false;
  let inBlockComment = false;
  let escaped = false;

  for (let i = 0; i < objectBody.length; i++) {
    const ch = objectBody[i];
    const next = objectBody[i + 1];

    if (inLineComment) {
      if (ch === "\n") inLineComment = false;
      continue;
    }
    if (inBlockComment) {
      if (ch === "*" && next === "/") {
        inBlockComment = false;
        i++;
      }
      continue;
    }

    if (inSingle || inDouble || inTemplate) {
      if (escaped) {
        escaped = false;
        continue;
      }
      if (ch === "\\") {
        escaped = true;
        continue;
      }
      if (inSingle && ch === "'") inSingle = false;
      else if (inDouble && ch === '"') inDouble = false;
      else if (inTemplate && ch === "`") inTemplate = false;
      continue;
    }

    if (ch === "/" && next === "/") {
      inLineComment = true;
      i++;
      continue;
    }
    if (ch === "/" && next === "*") {
      inBlockComment = true;
      i++;
      continue;
    }

    if (ch === "'") {
      inSingle = true;
      continue;
    }
    if (ch === '"') {
      inDouble = true;
      continue;
    }
    if (ch === "`") {
      inTemplate = true;
      continue;
    }

    if (ch === "(") depthParen++;
    else if (ch === ")") depthParen--;
    else if (ch === "{") depthBrace++;
    else if (ch === "}") depthBrace--;
    else if (ch === "[") depthBracket++;
    else if (ch === "]") depthBracket--;

    if (ch === "," && depthParen === 0 && depthBrace === 0 && depthBracket === 0) {
      const prop = objectBody.slice(start, i).trim();
      if (prop) props.push(prop);
      start = i + 1;
    }
  }

  const tail = objectBody.slice(start).trim();
  if (tail) props.push(tail);
  return props;
}

function getRegisterToolBlocks(): Array<{ name: string; block: string }> {
  const src = readFileSync(extensionPath, "utf-8");
  const blocks: Array<{ name: string; block: string }> = [];
  const token = "pi.registerTool(";
  let from = 0;

  while (true) {
    const start = src.indexOf(token, from);
    if (start === -1) break;
    const braceStart = src.indexOf("{", start);
    const braceEnd = findMatchingBrace(src, braceStart);
    const block = src.slice(braceStart, braceEnd + 1);
    const nameMatch = block.match(/name:\s*"(fn_[a-z_]+)"/);
    if (nameMatch) {
      blocks.push({ name: nameMatch[1], block });
    }
    from = braceEnd + 1;
  }

  return blocks;
}

/**
 * Extract all tool names registered via pi.registerTool({ name: "..." })
 * from the extension source code.
 */
function getExtensionToolNames(): string[] {
  return getRegisterToolBlocks()
    .map((entry) => entry.name)
    .sort();
}

function getExtensionToolParamNames(): Map<string, string[]> {
  const result = new Map<string, string[]>();
  for (const { name, block } of getRegisterToolBlocks()) {
    const paramsStart = block.indexOf("parameters:");
    if (paramsStart === -1) {
      result.set(name, []);
      continue;
    }
    const objectStart = block.indexOf("Type.Object(", paramsStart);
    if (objectStart === -1) {
      result.set(name, []);
      continue;
    }
    const braceStart = block.indexOf("{", objectStart);
    const braceEnd = findMatchingBrace(block, braceStart);
    const body = block.slice(braceStart + 1, braceEnd);

    const params = splitTopLevelProperties(body)
      .map((prop) => prop.match(/^([A-Za-z_][A-Za-z0-9_]*)\s*:/)?.[1])
      .filter((value): value is string => Boolean(value))
      .sort();

    result.set(name, params);
  }
  return result;
}

/**
 * Extract tool names documented in extension-tools.md (### fn_* headings).
 */
function getDocumentedToolNames(): string[] {
  const doc = readFileSync(
    resolve(skillDir, "references/extension-tools.md"),
    "utf-8",
  );
  const matches = [...doc.matchAll(/^### (fn_[a-z_]+)/gm)];
  return matches.map((m) => m[1]).sort();
}

function getDocumentedToolParamNames(): Map<string, string[]> {
  const doc = readFileSync(resolve(skillDir, "references/extension-tools.md"), "utf-8");
  const map = new Map<string, string[]>();

  const toolRegex = /^### (fn_[a-z_]+)\n([\s\S]*?)(?=^### fn_|^## [A-Za-z]|^<!-- END: extension-tools -->)/gm;
  for (const match of doc.matchAll(toolRegex)) {
    const [, toolName, section] = match;
    const params = [...section.matchAll(/\| `([A-Za-z_][A-Za-z0-9_]*)` \|/g)]
      .map((m) => m[1])
      .sort();
    map.set(toolName, params);
  }

  return map;
}

/**
 * Extract tool names listed in SKILL.md under the tool categories.
 */
function getSkillMdToolNames(): string[] {
  const doc = readFileSync(resolve(skillDir, "SKILL.md"), "utf-8");
  const matches = [...doc.matchAll(/`(fn_[a-z_]+)`/g)];
  return [...new Set(matches.map((m) => m[1]))].sort();
}

/**
 * Extract tool names from the capabilities catalog table.
 */
function getCapabilitiesToolNames(): string[] {
  const doc = readFileSync(
    resolve(skillDir, "references/fusion-capabilities.md"),
    "utf-8",
  );
  const matches = [...doc.matchAll(/\| `(fn_[a-z_]+)` \|/g)];
  return matches.map((m) => m[1]).sort();
}

function getEngineSessionToolNames(): string[] {
  const names = new Set<string>();
  for (const path of engineToolSourceFiles) {
    const src = readFileSync(path, "utf-8");
    for (const match of src.matchAll(/name:\s*"(fn_[a-z_]+)"/g)) {
      names.add(match[1]);
    }
  }
  return [...names].sort();
}

function getDocumentedEngineToolNames(): string[] {
  const doc = readFileSync(resolve(skillDir, "references/engine-tools.md"), "utf-8");
  return [...new Set([...doc.matchAll(/`(fn_[a-z_]+)`/g)].map((m) => m[1]))].sort();
}

function collectMarkdownFiles(dir: string): string[] {
  const files: string[] = [];
  for (const entry of readdirSync(dir, { withFileTypes: true })) {
    const fullPath = join(dir, entry.name);
    if (entry.isDirectory()) {
      files.push(...collectMarkdownFiles(fullPath));
      continue;
    }
    if (entry.isFile() && entry.name.endsWith(".md")) {
      files.push(fullPath);
    }
  }
  return files.sort();
}

function escapeRegex(value: string): string {
  return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}

describe("Skill-Extension Sync", () => {
  it("skill directory structure exists", () => {
    expect(existsSync(resolve(skillDir, "SKILL.md"))).toBe(true);
    expect(existsSync(resolve(skillDir, "references"))).toBe(true);
    expect(existsSync(resolve(skillDir, "workflows"))).toBe(true);
    expect(
      existsSync(resolve(skillDir, "references/extension-tools.md")),
    ).toBe(true);
  });

  it("SKILL.md has valid frontmatter with name and description", () => {
    const content = readFileSync(resolve(skillDir, "SKILL.md"), "utf-8");
    expect(content).toMatch(/^---\n/);
    expect(content).toMatch(/\nname:\s*fusion\n/);
    expect(content).toMatch(/\ndescription:\s*.+\n/);
  });

  it("extension-tools.md documents exactly the same tools as extension.ts", () => {
    const extensionTools = getExtensionToolNames();
    const documentedTools = getDocumentedToolNames();

    const missingFromDocs = extensionTools.filter(
      (t) => !documentedTools.includes(t),
    );
    const extraInDocs = documentedTools.filter(
      (t) => !extensionTools.includes(t),
    );

    expect(missingFromDocs).toEqual([]);
    expect(extraInDocs).toEqual([]);
    expect(documentedTools).toEqual(extensionTools);
  });

  it("extension-tools.md tool parameter tables match extension.ts registrations", () => {
    const extensionTools = getExtensionToolParamNames();
    const documentedTools = getDocumentedToolParamNames();

    for (const [toolName, extensionParams] of extensionTools.entries()) {
      expect(documentedTools.has(toolName), `missing documented section for ${toolName}`).toBe(true);
      const documentedParams = documentedTools.get(toolName) ?? [];
      expect(documentedParams).toEqual(extensionParams);
    }
  });

  it("SKILL.md tool listing includes all registered tools", () => {
    const extensionTools = getExtensionToolNames();
    const skillTools = getSkillMdToolNames();

    const missingFromSkill = extensionTools.filter(
      (t) => !skillTools.includes(t),
    );
    expect(missingFromSkill).toEqual([]);
  });

  it("fusion-capabilities.md tool table includes all registered tools", () => {
    const extensionTools = getExtensionToolNames();
    const capTools = getCapabilitiesToolNames();

    const missingFromCaps = extensionTools.filter(
      (t) => !capTools.includes(t),
    );
    expect(missingFromCaps).toEqual([]);
    expect(capTools).toEqual(extensionTools);
  });

  it("fusion-capabilities.md tool table is auto-generated with markers", () => {
    const doc = readFileSync(resolve(skillDir, "references/fusion-capabilities.md"), "utf-8");
    expect(doc).toContain(CAP_TABLE_BEGIN);
    expect(doc).toContain(CAP_TABLE_END);
  });

  it("extension-tools.md has auto-generated markers", () => {
    const doc = readFileSync(resolve(skillDir, "references/extension-tools.md"), "utf-8");
    expect(doc).toContain(EXT_TOOLS_BEGIN);
    expect(doc).toContain(EXT_TOOLS_END);
  });

  it("engine-tools.md documents all engine session-scoped tools", () => {
    const engineTools = getEngineSessionToolNames();
    const documented = getDocumentedEngineToolNames();
    const missing = engineTools.filter((name) => !documented.includes(name));
    expect(missing).toEqual([]);
  });

  it("covers the full Fusion skill markdown surface", () => {
    const markdownFiles = collectMarkdownFiles(skillDir).map((filePath) =>
      relative(skillDir, filePath),
    );

    expect(markdownFiles).toEqual([
      "SKILL.md",
      "references/best-practices.md",
      "references/cli-commands.md",
      "references/engine-tools.md",
      "references/extension-tools.md",
      "references/fusion-capabilities.md",
      "references/skill-patterns.md",
      "references/task-structure.md",
      "workflows/dashboard-cli.md",
      "workflows/specifications.md",
      "workflows/task-lifecycle.md",
      "workflows/task-management.md",
    ]);
  });

  it("enforces fn_* naming across extension + all skill markdown for public tools", () => {
    const extensionTools = getExtensionToolNames();
    const publicSuffixes = extensionTools.map((toolName) =>
      toolName.replace(/^fn_/, ""),
    );

    const allowedUnprefixedInternalTools = new Set([
      "task_create",
      "task_update",
      "task_log",
      "task_done",
      "review_step",
      "spawn_agent",
    ]);

    const forbiddenSuffixes = publicSuffixes.filter(
      (suffix) => !allowedUnprefixedInternalTools.has(suffix),
    );

    const filesToScan = [extensionPath, ...collectMarkdownFiles(skillDir)];
    const violations: string[] = [];

    for (const filePath of filesToScan) {
      const content = readFileSync(filePath, "utf-8");
      const relativePath = relative(cliRoot, filePath);

      for (const suffix of forbiddenSuffixes) {
        const regex = new RegExp(`(?<!fn_)\\b${escapeRegex(suffix)}\\b`, "g");
        if (regex.test(content)) {
          violations.push(`${relativePath}: ${suffix}`);
        }
      }
    }

    expect(violations).toEqual([]);
  });

  it("/fn command is documented in the skill", () => {
    const skillMd = readFileSync(resolve(skillDir, "SKILL.md"), "utf-8");
    expect(skillMd).toContain("/fn");

    const dashboardCli = readFileSync(
      resolve(skillDir, "workflows/dashboard-cli.md"),
      "utf-8",
    );
    expect(dashboardCli).toContain("/fn");
  });

  it("SKILL.md and reference generated blocks match sync script output (no drift)", () => {
    const script = resolve(repoRoot, "scripts/sync-fusion-skill-tools.mjs");
    const result = spawnSync("node", [script, "--check"], {
      encoding: "utf-8",
    });
    if (result.status !== 0) {
      throw new Error(
        `sync-fusion-skill-tools --check failed:\n${result.stderr || result.stdout}`,
      );
    }
  });

  it("package.json includes skills in pi config and files array", () => {
    const pkg = JSON.parse(
      readFileSync(resolve(cliRoot, "package.json"), "utf-8"),
    );
    expect(pkg.pi.skills).toContain("./skill");
    expect(pkg.files).toContain("skill/**");
  });
});