FN-6350: accept labeled external integration evidence

Relax the spec-review evidence detector so complete labeled Markdown evidence passes validation. - Scan dedicated External Integration Evidence sections alongside existing prompt sections. - Recognize labeled repo, docs, release/download, CLI name, and checksum evidence with flexible Markdown formatting. - Add regression coverage for FN-6349-style evidence blocks and triage reviewer handoff. Files changed: ...alidation-external-integration-evidence.test.ts | 51 +++++++++++++++++++ ...triage-review-spec-external-integration.test.ts | 40 ++++++++++++++- .../external-integration-evidence.ts | 59 +++++++++++++++++++--- 3 files changed, 143 insertions(+), 7 deletions(-) Fusion-Task-Id: FN-6350 Fusion-Task-Lineage: 7d0d9a0a-1f51-41db-9434-d15496293c2c
2026-06-13 04:54:59 -07:00
parent 30e747bebc
commit 7eafa91dd7
3 changed files with 143 additions and 7 deletions
--- a/packages/engine/src/tests/spec-validation-external-integration-evidence.test.ts
+++ b/packages/engine/src/tests/spec-validation-external-integration-evidence.test.ts
@@ -1,6 +1,22 @@
 import { describe, expect, it } from "vitest";
 import { detectExternalIntegrationEvidenceGaps } from "../spec-validation/external-integration-evidence.js";

+const fn6349EvidenceBlock = `## Mission
+Validate released third-party external integration.
+
+## External Integration Evidence
+This task installs and runs the released third-party-distributed Fusion CLI (\`@runfusion/fusion\`) from the public npm registry. Provenance (verified via \`npm view @runfusion/fusion\` on 2026-06-13):
+
+- Canonical upstream repo URL: https://github.com/Runfusion/Fusion
+- Docs / homepage URL: https://github.com/Runfusion/Fusion#readme (npm package page: https://www.npmjs.com/package/@runfusion/fusion); in-repo author guide \`docs/plugins/external-authoring.md\`
+- Release / download URL: https://registry.npmjs.org/@runfusion/fusion/-/fusion-0.41.0.tgz
+- Binary / CLI name: \`fn\` (provided by the published \`@runfusion/fusion\` package; also invokable via \`npx @runfusion/fusion@latest\`)
+- Checksum (dist.integrity for 0.41.0): \`sha512-y8BSeK3XUgcE7ceTrz6F/zWQidaiADVgHSHHWKRzwjyR40xeUc8i5ZSolGd1zL/K9AxrBSkRErimkW1xqb/EBw==\` (marker: \`upstream-pending-verification\` if a newer release ships before validation)
+
+## Steps
+- Install and run the released third-party external integration.
+`;
+
 describe("detectExternalIntegrationEvidenceGaps", () => {
  it("returns empty findings when prompt has no external integration signals", () => {
    const prompt = `# Task\n## Mission\nRefactor retry budget counters in scheduler.\n## Steps\n- Update store logic.`;
@@ -24,6 +40,41 @@ describe("detectExternalIntegrationEvidenceGaps", () => {
    expect(detectExternalIntegrationEvidenceGaps({ promptContent: prompt })).toEqual([]);
  });

+  it("accepts FN-6349 labeled evidence in a dedicated external integration evidence section", () => {
+    expect(detectExternalIntegrationEvidenceGaps({ promptContent: fn6349EvidenceBlock })).toEqual([]);
+  });
+
+  it("accepts concrete labeled markdown evidence with backtick-wrapped URLs and sha256 digest", () => {
+    const prompt = `## Mission\nInstall third-party external CLI from an upstream release.\n\n## External-Integration Evidence\n- Canonical upstream repo: \`https://github.com/acme/tooling\`\n- Docs/homepage: \`https://docs.acme.test/tooling\`\n- Release/download: \`https://downloads.acme.test/tooling/tooling-1.2.3.tar.gz\`\n- Binary/CLI name: \`ac\`\n- Checksum: sha256-deadbeef\n\n## Steps\n- Download, probe, and run the external binary.`;
+
+    expect(detectExternalIntegrationEvidenceGaps({ promptContent: prompt })).toEqual([]);
+  });
+
+  it("accepts inline labeled evidence in pre-existing scanned sections", () => {
+    const prompt = `## Mission\nAdd third-party external tool install flow.\n\n## Context to Read First\n- Canonical upstream repo URL: https://github.com/acme/tooling\n- Docs URL: https://docs.acme.test/tooling\n- Release URL: https://github.com/acme/tooling/releases/download/v1.0.0/tooling.tgz\n- CLI name: \`ac\`\n- Checksum: upstream-pending-verification\n\n## Steps\n- Install, probe, and run the external binary.`;
+
+    expect(detectExternalIntegrationEvidenceGaps({ promptContent: prompt })).toEqual([]);
+  });
+
+  it("still requires checksum evidence when the FN-6349 block omits checksum and source markers", () => {
+    const prompt = fn6349EvidenceBlock.replace(
+      /- Checksum \(dist\.integrity for 0\.41\.0\):.*\n/,
+      "- Checksum (dist.integrity for 0.41.0):\n",
+    );
+
+    const findings = detectExternalIntegrationEvidenceGaps({ promptContent: prompt });
+    expect(findings.length).toBeGreaterThan(0);
+    expect(findings[0]?.missing).toContain("checksum-or-source-of-truth-evidence");
+  });
+
+  it("still requires an artifact URL and a backticked CLI name", () => {
+    const prompt = `## Mission\nAdd third-party external CLI install flow.\n\n## External Integration Evidence\n- Canonical upstream repo URL: https://github.com/acme/tooling\n- Docs / homepage URL: https://docs.acme.test/tooling\n- Release / download URL:\n- Binary / CLI name: ac\n- Checksum: sha512-deadbeef\n\n## Steps\n- Download and probe the external binary.`;
+
+    const findings = detectExternalIntegrationEvidenceGaps({ promptContent: prompt });
+    expect(findings.length).toBeGreaterThan(0);
+    expect(findings[0]?.missing).toEqual(expect.arrayContaining(["release-or-download-url", "binary-or-cli-name"]));
+  });
+
  it("treats duplicate-segment github URLs as missing canonical evidence", () => {
    const duplicateRepo = ["foo", "foo"].join("/");
    const prompt = `## Mission\nExternal tool install.\n## Steps\n- download release from https://github.com/${duplicateRepo}/releases/latest/download/foo.tgz\n- run and probe \`foo\``;
--- a/packages/engine/src/tests/triage-review-spec-external-integration.test.ts
+++ b/packages/engine/src/tests/triage-review-spec-external-integration.test.ts
@@ -1,4 +1,4 @@
-import { describe, it, expect, vi } from "vitest";
+import { beforeEach, describe, it, expect, vi } from "vitest";
 import { mkdtemp, mkdir, writeFile, rm } from "node:fs/promises";
 import { join } from "node:path";
 import { tmpdir } from "node:os";
@@ -62,6 +62,9 @@ const mockTaskDetail: TaskDetail = {
 };

 describe("triage fn_review_spec external integration evidence", () => {
+  beforeEach(() => {
+    mockReviewStep.mockReset();
+  });
  it("short-circuits to REVISE when evidence is incomplete", async () => {
    const rootDir = await mkdtemp(join(tmpdir(), "fusion-triage-ext-evidence-"));
    try {
@@ -98,6 +101,41 @@ describe("triage fn_review_spec external integration evidence", () => {
    }
  });

+  it("calls reviewer when dedicated labeled evidence section is complete", async () => {
+    const rootDir = await mkdtemp(join(tmpdir(), "fusion-triage-ext-evidence-labeled-ok-"));
+    try {
+      const taskId = "FN-5321";
+      const promptPath = `.fusion/tasks/${taskId}/PROMPT.md`;
+      await mkdir(join(rootDir, ".fusion", "tasks", taskId), { recursive: true });
+      await writeFile(
+        join(rootDir, promptPath),
+        "## Mission\nValidate released third-party external integration.\n\n## External Integration Evidence\n- Canonical upstream repo URL: https://github.com/Runfusion/Fusion\n- Docs / homepage URL: https://github.com/Runfusion/Fusion#readme (npm package page: https://www.npmjs.com/package/@runfusion/fusion)\n- Release / download URL: https://registry.npmjs.org/@runfusion/fusion/-/fusion-0.41.0.tgz\n- Binary / CLI name: `fn`\n- Checksum (dist.integrity for 0.41.0): `sha512-y8BSeK3XUgcE7ceTrz6F/zWQidaiADVgHSHHWKRzwjyR40xeUc8i5ZSolGd1zL/K9AxrBSkRErimkW1xqb/EBw==` (marker: `upstream-pending-verification`)\n\n## Steps\n- Install, download, probe, and run the released external binary.\n",
+      );
+
+      mockReviewStep.mockResolvedValueOnce({ verdict: "APPROVE", summary: "ok", review: "" });
+      const store = createMockStore({ getTask: vi.fn().mockResolvedValue({ ...mockTaskDetail, id: taskId }) });
+      const processor = new TriageProcessor(store, rootDir);
+      const verdictRef = { current: null as any };
+      const tool = (processor as any).createReviewSpecTool(
+        taskId,
+        promptPath,
+        { current: null },
+        { current: null },
+        verdictRef,
+        { current: "" },
+        {},
+        false,
+      );
+
+      const result = await tool.execute({});
+      expect(result.content[0]?.text).toBe("APPROVE");
+      expect(verdictRef.current).toBe("APPROVE");
+      expect(mockReviewStep).toHaveBeenCalledTimes(1);
+    } finally {
+      await rm(rootDir, { recursive: true, force: true });
+    }
+  });
+
  it("calls reviewer when evidence is complete", async () => {
    const rootDir = await mkdtemp(join(tmpdir(), "fusion-triage-ext-evidence-ok-"));
    try {
--- a/packages/engine/src/spec-validation/external-integration-evidence.ts
+++ b/packages/engine/src/spec-validation/external-integration-evidence.ts
@@ -22,7 +22,14 @@ export interface DetectExternalIntegrationEvidenceOptions {
  detectorOverrides?: ExternalIntegrationDetectorOverrides;
 }

-const SECTION_NAMES = ["Mission", "Steps", "File Scope", "Context to Read First"];
+const SECTION_NAMES = [
+  "Mission",
+  "Steps",
+  "File Scope",
+  "Context to Read First",
+  "External Integration Evidence",
+  "External-Integration Evidence",
+];
 const DEFAULT_TRIGGER_TOKENS = [
  "third-party",
  "third party",
@@ -57,12 +64,42 @@ function hasLikelyCliName(text: string): boolean {
  for (const match of codeMatches) {
    const idx = match.index ?? -1;
    if (idx < 0) continue;
-    const window = text.slice(Math.max(0, idx - 80), Math.min(text.length, idx + (match[0]?.length ?? 0) + 80));
+    const window = text.slice(
+      Math.max(0, idx - 80),
+      Math.min(text.length, idx + (match[0]?.length ?? 0) + 80),
+    );
    if (/\b(?:probe|invoke|run|spawn|which|where)\b/i.test(window)) return true;
+    const leadingText = text.slice(Math.max(0, idx - 80), idx);
+    if (/\b(?:(?:binary|cli)(?:\s*\/\s*|\s+or\s+)?(?:cli\s+)?name|cli\s+name)\s*:?\s*$/i.test(leadingText)) return true;
  }
  return false;
 }

+function collectHttpUrls(text: string): string[] {
+  return Array.from(text.matchAll(/https:\/\/[^\s)\]`"']+/gi)).map((m) =>
+    m[0].replace(/[),.;:!?]+$/, ""),
+  );
+}
+
+function hasLabeledUrl(text: string, labelPattern: RegExp, urlPattern: RegExp = /https:\/\//i): boolean {
+  return text
+    .split(/\r?\n/)
+    .some((line) => labelPattern.test(line) && collectHttpUrls(line).some((url) => urlPattern.test(url)));
+}
+
+function isReleaseOrDownloadUrl(url: string): boolean {
+  return (
+    /https:\/\/github\.com\/[^\s)\]`"']*releases\/[^\s)\]`"']+/i.test(url) ||
+    /https:\/\/[^\s)\]`"']*download[^\s)\]`"']*/i.test(url) ||
+    /https:\/\/registry\.npmjs\.org\/[^\s)\]`"']+\/-\/[^\s)\]`"']+\.tgz(?:$|[?#])/i.test(url) ||
+    /\.(?:tgz|tar\.gz)(?:$|[?#])/i.test(url)
+  );
+}
+
+function isLikelyDocsUrl(url: string): boolean {
+  return !/^https:\/\/github\.com\//i.test(url) && !isReleaseOrDownloadUrl(url);
+}
+
 function hasCanonicalGithubRepoUrl(text: string): boolean {
  const urls = Array.from(text.matchAll(/https:\/\/github\.com\/[^\s)\]`"']+/gi)).map((m) => m[0]);
  for (const url of urls) {
@@ -93,9 +130,17 @@ export function detectExternalIntegrationEvidenceGaps(
  const hints = collectHints(text, integrationPattern);
  const findingHints = hints.length > 0 ? hints : ["external-integration"];

-  const hasDocsUrl = /https:\/\/(?!github\.com\/)[^\s)\]`"']+/i.test(text);
-  const hasReleaseUrl = /https:\/\/github\.com\/[^\s)\]`"']*releases\/[^\s)\]`"']+/i.test(text) || /https:\/\/[^\s)\]`"']*download[^\s)\]`"']*/i.test(text);
-  const hasChecksumMarker = /\bsha256\b|pinned manifest|validateExternalIntegrationManifest|WORKTRUNK_PINNED_RELEASE|upstream-pending-verification/i.test(text);
+  const urls = collectHttpUrls(text);
+  const hasDocsUrl =
+    hasLabeledUrl(text, /\b(?:docs?|homepage)\b(?:\s*(?:\/|or)\s*\b(?:docs?|homepage)\b)?(?:\s+url)?\s*:/i) ||
+    urls.some(isLikelyDocsUrl);
+  const hasReleaseUrl =
+    hasLabeledUrl(text, /\b(?:release|download)\b(?:\s*(?:\/|or)\s*\b(?:release|download)\b)?(?:\s+url)?\s*:/i) ||
+    urls.some(isReleaseOrDownloadUrl);
+  const hasChecksumMarker =
+    /\bsha\d+\b|pinned manifest|validateExternalIntegrationManifest|WORKTRUNK_PINNED_RELEASE|upstream-pending-verification/i.test(
+      text,
+    );
  const hasCliName = hasLikelyCliName(text);
  const hasCanonicalRepo = hasCanonicalGithubRepoUrl(text);

@@ -119,7 +164,9 @@ export function formatExternalIntegrationEvidenceDiagnostic(
  const lines = ["REVISE — External-integration evidence gaps in PROMPT.md:"];
  for (const finding of findings) {
    lines.push(`  - ${finding.integrationHint}: missing ${finding.missing.join(", ")}`);
-    lines.push("    Fix: add canonical upstream repo/docs/release URL evidence, CLI name in backticks, and checksum or explicit upstream-pending-verification marker.");
+    lines.push(
+      "    Fix: add canonical upstream repo/docs/release URL evidence, CLI name in backticks, and checksum or explicit upstream-pending-verification marker.",
+    );
  }
  return lines.join("\n");
 }