From 7eafa91dd78d95967c910a1cb76134dc2a82e7f0 Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Sat, 13 Jun 2026 04:54:59 -0700 Subject: [PATCH] FN-6350: accept labeled external integration evidence Relax the spec-review evidence detector so complete labeled Markdown evidence passes validation. - Scan dedicated External Integration Evidence sections alongside existing prompt sections. - Recognize labeled repo, docs, release/download, CLI name, and checksum evidence with flexible Markdown formatting. - Add regression coverage for FN-6349-style evidence blocks and triage reviewer handoff. Files changed: ...alidation-external-integration-evidence.test.ts | 51 +++++++++++++++++++ ...triage-review-spec-external-integration.test.ts | 40 ++++++++++++++- .../external-integration-evidence.ts | 59 +++++++++++++++++++--- 3 files changed, 143 insertions(+), 7 deletions(-) Fusion-Task-Id: FN-6350 Fusion-Task-Lineage: 7d0d9a0a-1f51-41db-9434-d15496293c2c --- ...tion-external-integration-evidence.test.ts | 51 ++++++++++++++++ ...e-review-spec-external-integration.test.ts | 40 ++++++++++++- .../external-integration-evidence.ts | 59 +++++++++++++++++-- 3 files changed, 143 insertions(+), 7 deletions(-) diff --git a/packages/engine/src/__tests__/spec-validation-external-integration-evidence.test.ts b/packages/engine/src/__tests__/spec-validation-external-integration-evidence.test.ts index 46fcc4211e..5bd5ad25c4 100644 --- a/packages/engine/src/__tests__/spec-validation-external-integration-evidence.test.ts +++ b/packages/engine/src/__tests__/spec-validation-external-integration-evidence.test.ts @@ -1,6 +1,22 @@ import { describe, expect, it } from "vitest"; import { detectExternalIntegrationEvidenceGaps } from "../spec-validation/external-integration-evidence.js"; +const fn6349EvidenceBlock = `## Mission +Validate released third-party external integration. + +## External Integration Evidence +This task installs and runs the released third-party-distributed Fusion CLI (\`@runfusion/fusion\`) from the public npm registry. Provenance (verified via \`npm view @runfusion/fusion\` on 2026-06-13): + +- Canonical upstream repo URL: https://github.com/Runfusion/Fusion +- Docs / homepage URL: https://github.com/Runfusion/Fusion#readme (npm package page: https://www.npmjs.com/package/@runfusion/fusion); in-repo author guide \`docs/plugins/external-authoring.md\` +- Release / download URL: https://registry.npmjs.org/@runfusion/fusion/-/fusion-0.41.0.tgz +- Binary / CLI name: \`fn\` (provided by the published \`@runfusion/fusion\` package; also invokable via \`npx @runfusion/fusion@latest\`) +- Checksum (dist.integrity for 0.41.0): \`sha512-y8BSeK3XUgcE7ceTrz6F/zWQidaiADVgHSHHWKRzwjyR40xeUc8i5ZSolGd1zL/K9AxrBSkRErimkW1xqb/EBw==\` (marker: \`upstream-pending-verification\` if a newer release ships before validation) + +## Steps +- Install and run the released third-party external integration. +`; + describe("detectExternalIntegrationEvidenceGaps", () => { it("returns empty findings when prompt has no external integration signals", () => { const prompt = `# Task\n## Mission\nRefactor retry budget counters in scheduler.\n## Steps\n- Update store logic.`; @@ -24,6 +40,41 @@ describe("detectExternalIntegrationEvidenceGaps", () => { expect(detectExternalIntegrationEvidenceGaps({ promptContent: prompt })).toEqual([]); }); + it("accepts FN-6349 labeled evidence in a dedicated external integration evidence section", () => { + expect(detectExternalIntegrationEvidenceGaps({ promptContent: fn6349EvidenceBlock })).toEqual([]); + }); + + it("accepts concrete labeled markdown evidence with backtick-wrapped URLs and sha256 digest", () => { + const prompt = `## Mission\nInstall third-party external CLI from an upstream release.\n\n## External-Integration Evidence\n- Canonical upstream repo: \`https://github.com/acme/tooling\`\n- Docs/homepage: \`https://docs.acme.test/tooling\`\n- Release/download: \`https://downloads.acme.test/tooling/tooling-1.2.3.tar.gz\`\n- Binary/CLI name: \`ac\`\n- Checksum: sha256-deadbeef\n\n## Steps\n- Download, probe, and run the external binary.`; + + expect(detectExternalIntegrationEvidenceGaps({ promptContent: prompt })).toEqual([]); + }); + + it("accepts inline labeled evidence in pre-existing scanned sections", () => { + const prompt = `## Mission\nAdd third-party external tool install flow.\n\n## Context to Read First\n- Canonical upstream repo URL: https://github.com/acme/tooling\n- Docs URL: https://docs.acme.test/tooling\n- Release URL: https://github.com/acme/tooling/releases/download/v1.0.0/tooling.tgz\n- CLI name: \`ac\`\n- Checksum: upstream-pending-verification\n\n## Steps\n- Install, probe, and run the external binary.`; + + expect(detectExternalIntegrationEvidenceGaps({ promptContent: prompt })).toEqual([]); + }); + + it("still requires checksum evidence when the FN-6349 block omits checksum and source markers", () => { + const prompt = fn6349EvidenceBlock.replace( + /- Checksum \(dist\.integrity for 0\.41\.0\):.*\n/, + "- Checksum (dist.integrity for 0.41.0):\n", + ); + + const findings = detectExternalIntegrationEvidenceGaps({ promptContent: prompt }); + expect(findings.length).toBeGreaterThan(0); + expect(findings[0]?.missing).toContain("checksum-or-source-of-truth-evidence"); + }); + + it("still requires an artifact URL and a backticked CLI name", () => { + const prompt = `## Mission\nAdd third-party external CLI install flow.\n\n## External Integration Evidence\n- Canonical upstream repo URL: https://github.com/acme/tooling\n- Docs / homepage URL: https://docs.acme.test/tooling\n- Release / download URL:\n- Binary / CLI name: ac\n- Checksum: sha512-deadbeef\n\n## Steps\n- Download and probe the external binary.`; + + const findings = detectExternalIntegrationEvidenceGaps({ promptContent: prompt }); + expect(findings.length).toBeGreaterThan(0); + expect(findings[0]?.missing).toEqual(expect.arrayContaining(["release-or-download-url", "binary-or-cli-name"])); + }); + it("treats duplicate-segment github URLs as missing canonical evidence", () => { const duplicateRepo = ["foo", "foo"].join("/"); const prompt = `## Mission\nExternal tool install.\n## Steps\n- download release from https://github.com/${duplicateRepo}/releases/latest/download/foo.tgz\n- run and probe \`foo\``; diff --git a/packages/engine/src/__tests__/triage-review-spec-external-integration.test.ts b/packages/engine/src/__tests__/triage-review-spec-external-integration.test.ts index a96c59197a..a513c4f198 100644 --- a/packages/engine/src/__tests__/triage-review-spec-external-integration.test.ts +++ b/packages/engine/src/__tests__/triage-review-spec-external-integration.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, vi } from "vitest"; +import { beforeEach, describe, it, expect, vi } from "vitest"; import { mkdtemp, mkdir, writeFile, rm } from "node:fs/promises"; import { join } from "node:path"; import { tmpdir } from "node:os"; @@ -62,6 +62,9 @@ const mockTaskDetail: TaskDetail = { }; describe("triage fn_review_spec external integration evidence", () => { + beforeEach(() => { + mockReviewStep.mockReset(); + }); it("short-circuits to REVISE when evidence is incomplete", async () => { const rootDir = await mkdtemp(join(tmpdir(), "fusion-triage-ext-evidence-")); try { @@ -98,6 +101,41 @@ describe("triage fn_review_spec external integration evidence", () => { } }); + it("calls reviewer when dedicated labeled evidence section is complete", async () => { + const rootDir = await mkdtemp(join(tmpdir(), "fusion-triage-ext-evidence-labeled-ok-")); + try { + const taskId = "FN-5321"; + const promptPath = `.fusion/tasks/${taskId}/PROMPT.md`; + await mkdir(join(rootDir, ".fusion", "tasks", taskId), { recursive: true }); + await writeFile( + join(rootDir, promptPath), + "## Mission\nValidate released third-party external integration.\n\n## External Integration Evidence\n- Canonical upstream repo URL: https://github.com/Runfusion/Fusion\n- Docs / homepage URL: https://github.com/Runfusion/Fusion#readme (npm package page: https://www.npmjs.com/package/@runfusion/fusion)\n- Release / download URL: https://registry.npmjs.org/@runfusion/fusion/-/fusion-0.41.0.tgz\n- Binary / CLI name: `fn`\n- Checksum (dist.integrity for 0.41.0): `sha512-y8BSeK3XUgcE7ceTrz6F/zWQidaiADVgHSHHWKRzwjyR40xeUc8i5ZSolGd1zL/K9AxrBSkRErimkW1xqb/EBw==` (marker: `upstream-pending-verification`)\n\n## Steps\n- Install, download, probe, and run the released external binary.\n", + ); + + mockReviewStep.mockResolvedValueOnce({ verdict: "APPROVE", summary: "ok", review: "" }); + const store = createMockStore({ getTask: vi.fn().mockResolvedValue({ ...mockTaskDetail, id: taskId }) }); + const processor = new TriageProcessor(store, rootDir); + const verdictRef = { current: null as any }; + const tool = (processor as any).createReviewSpecTool( + taskId, + promptPath, + { current: null }, + { current: null }, + verdictRef, + { current: "" }, + {}, + false, + ); + + const result = await tool.execute({}); + expect(result.content[0]?.text).toBe("APPROVE"); + expect(verdictRef.current).toBe("APPROVE"); + expect(mockReviewStep).toHaveBeenCalledTimes(1); + } finally { + await rm(rootDir, { recursive: true, force: true }); + } + }); + it("calls reviewer when evidence is complete", async () => { const rootDir = await mkdtemp(join(tmpdir(), "fusion-triage-ext-evidence-ok-")); try { diff --git a/packages/engine/src/spec-validation/external-integration-evidence.ts b/packages/engine/src/spec-validation/external-integration-evidence.ts index 1b6e3d9184..31f6dd4e96 100644 --- a/packages/engine/src/spec-validation/external-integration-evidence.ts +++ b/packages/engine/src/spec-validation/external-integration-evidence.ts @@ -22,7 +22,14 @@ export interface DetectExternalIntegrationEvidenceOptions { detectorOverrides?: ExternalIntegrationDetectorOverrides; } -const SECTION_NAMES = ["Mission", "Steps", "File Scope", "Context to Read First"]; +const SECTION_NAMES = [ + "Mission", + "Steps", + "File Scope", + "Context to Read First", + "External Integration Evidence", + "External-Integration Evidence", +]; const DEFAULT_TRIGGER_TOKENS = [ "third-party", "third party", @@ -57,12 +64,42 @@ function hasLikelyCliName(text: string): boolean { for (const match of codeMatches) { const idx = match.index ?? -1; if (idx < 0) continue; - const window = text.slice(Math.max(0, idx - 80), Math.min(text.length, idx + (match[0]?.length ?? 0) + 80)); + const window = text.slice( + Math.max(0, idx - 80), + Math.min(text.length, idx + (match[0]?.length ?? 0) + 80), + ); if (/\b(?:probe|invoke|run|spawn|which|where)\b/i.test(window)) return true; + const leadingText = text.slice(Math.max(0, idx - 80), idx); + if (/\b(?:(?:binary|cli)(?:\s*\/\s*|\s+or\s+)?(?:cli\s+)?name|cli\s+name)\s*:?\s*$/i.test(leadingText)) return true; } return false; } +function collectHttpUrls(text: string): string[] { + return Array.from(text.matchAll(/https:\/\/[^\s)\]`"']+/gi)).map((m) => + m[0].replace(/[),.;:!?]+$/, ""), + ); +} + +function hasLabeledUrl(text: string, labelPattern: RegExp, urlPattern: RegExp = /https:\/\//i): boolean { + return text + .split(/\r?\n/) + .some((line) => labelPattern.test(line) && collectHttpUrls(line).some((url) => urlPattern.test(url))); +} + +function isReleaseOrDownloadUrl(url: string): boolean { + return ( + /https:\/\/github\.com\/[^\s)\]`"']*releases\/[^\s)\]`"']+/i.test(url) || + /https:\/\/[^\s)\]`"']*download[^\s)\]`"']*/i.test(url) || + /https:\/\/registry\.npmjs\.org\/[^\s)\]`"']+\/-\/[^\s)\]`"']+\.tgz(?:$|[?#])/i.test(url) || + /\.(?:tgz|tar\.gz)(?:$|[?#])/i.test(url) + ); +} + +function isLikelyDocsUrl(url: string): boolean { + return !/^https:\/\/github\.com\//i.test(url) && !isReleaseOrDownloadUrl(url); +} + function hasCanonicalGithubRepoUrl(text: string): boolean { const urls = Array.from(text.matchAll(/https:\/\/github\.com\/[^\s)\]`"']+/gi)).map((m) => m[0]); for (const url of urls) { @@ -93,9 +130,17 @@ export function detectExternalIntegrationEvidenceGaps( const hints = collectHints(text, integrationPattern); const findingHints = hints.length > 0 ? hints : ["external-integration"]; - const hasDocsUrl = /https:\/\/(?!github\.com\/)[^\s)\]`"']+/i.test(text); - const hasReleaseUrl = /https:\/\/github\.com\/[^\s)\]`"']*releases\/[^\s)\]`"']+/i.test(text) || /https:\/\/[^\s)\]`"']*download[^\s)\]`"']*/i.test(text); - const hasChecksumMarker = /\bsha256\b|pinned manifest|validateExternalIntegrationManifest|WORKTRUNK_PINNED_RELEASE|upstream-pending-verification/i.test(text); + const urls = collectHttpUrls(text); + const hasDocsUrl = + hasLabeledUrl(text, /\b(?:docs?|homepage)\b(?:\s*(?:\/|or)\s*\b(?:docs?|homepage)\b)?(?:\s+url)?\s*:/i) || + urls.some(isLikelyDocsUrl); + const hasReleaseUrl = + hasLabeledUrl(text, /\b(?:release|download)\b(?:\s*(?:\/|or)\s*\b(?:release|download)\b)?(?:\s+url)?\s*:/i) || + urls.some(isReleaseOrDownloadUrl); + const hasChecksumMarker = + /\bsha\d+\b|pinned manifest|validateExternalIntegrationManifest|WORKTRUNK_PINNED_RELEASE|upstream-pending-verification/i.test( + text, + ); const hasCliName = hasLikelyCliName(text); const hasCanonicalRepo = hasCanonicalGithubRepoUrl(text); @@ -119,7 +164,9 @@ export function formatExternalIntegrationEvidenceDiagnostic( const lines = ["REVISE — External-integration evidence gaps in PROMPT.md:"]; for (const finding of findings) { lines.push(` - ${finding.integrationHint}: missing ${finding.missing.join(", ")}`); - lines.push(" Fix: add canonical upstream repo/docs/release URL evidence, CLI name in backticks, and checksum or explicit upstream-pending-verification marker."); + lines.push( + " Fix: add canonical upstream repo/docs/release URL evidence, CLI name in backticks, and checksum or explicit upstream-pending-verification marker.", + ); } return lines.join("\n"); }