feat(FN-5152): complete Step 1 — add near-duplicate core helper

Fusion-Task-Id: FN-5152
Fusion-Task-Lineage: 954570d8-82e1-4519-a739-c52ee790d2db

packages/core/src/__tests__/near-duplicate.test.ts

@@ -0,0 +1,168 @@
+import { describe, expect, it } from "vitest";
+
+import { extractIntentSignature, findNearDuplicates } from "../near-duplicate.js";
+
+const fn5144Title = "Create PR dialog missing /pr/options /pr/preflight /pr/generate-metadata routes";
+const fn5144Description =
+  "The Create PR dialog calls POST /api/tasks/:id/pr/generate-metadata, GET /api/tasks/:id/pr/preflight, and GET /api/tasks/:id/pr/options but handlers are missing in packages/dashboard/src/routes/register-git-github.ts.";
+
+const fn5149Title = "Create PR modal routes: add /pr/options, /pr/preflight, /pr/generate-metadata endpoints";
+const fn5149Description =
+  "PrCreateModal currently 404s for GET /api/tasks/:id/pr/options, GET /api/tasks/:id/pr/preflight, and POST /api/tasks/:id/pr/generate-metadata. Wire routes in `register-git-github.ts` and reuse `PrPreflightResponse` contracts.";
+
+const fn5145Title = "Review tab Create PR button no-op when PrCreateModal mount is gated";
+const fn5145Description =
+  "TaskReviewTab create button toggles state but PrCreateModal mount is hidden by tab switch. Move `PrCreateModal` in `TaskDetailModal.tsx` and keep `task-review-create-pr` wiring.";
+
+const fn5150Title = "TaskReviewTab Create PR button no-op; lift PrCreateModal mount";
+const fn5150Description =
+  "Lift `PrCreateModal` to tab-agnostic mount in `TaskDetailModal.tsx`; `task-review-create-pr` in TaskReviewTab currently no-op.";
+
+describe("extractIntentSignature", () => {
+  it("extracts PR route paths from FN-5144 content", () => {
+    const sig = extractIntentSignature({
+      title: fn5144Title,
+      description: fn5144Description,
+    });
+
+    expect(sig.routePaths).toEqual(
+      expect.arrayContaining([
+        "/pr/options",
+        "/pr/preflight",
+        "/pr/generate-metadata",
+      ]),
+    );
+  });
+
+  it("extracts identifier and file tokens from FN-5149 content", () => {
+    const sig = extractIntentSignature({
+      title: fn5149Title,
+      description: `${fn5149Description} \`PrCreateModal\` \`PrPreflightResponse\` packages/dashboard/src/routes/register-git-github.ts`,
+    });
+
+    expect(sig.identifiers).toEqual(
+      expect.arrayContaining([
+        "prcreatemodal",
+        "prpreflightresponse",
+        "register-git-github.ts",
+      ]),
+    );
+  });
+});
+
+describe("findNearDuplicates", () => {
+  it("flags FN-5144 and FN-5149 pair via shared PR route tokens", () => {
+    const matches = findNearDuplicates(
+      { title: fn5144Title, description: fn5144Description },
+      [
+        {
+          id: "FN-5149",
+          title: fn5149Title,
+          description: fn5149Description,
+          column: "todo",
+          createdAt: Date.now(),
+        },
+      ],
+      { nowMs: Date.now() },
+    );
+
+    expect(matches[0]?.id).toBe("FN-5149");
+    expect(matches[0]?.sharedTokens).toEqual(
+      expect.arrayContaining(["/pr/options", "/pr/preflight", "/pr/generate-metadata"]),
+    );
+  });
+
+  it("flags FN-5145 and FN-5150 pair", () => {
+    const matches = findNearDuplicates(
+      { title: fn5145Title, description: fn5145Description },
+      [
+        {
+          id: "FN-5150",
+          title: fn5150Title,
+          description: fn5150Description,
+          column: "todo",
+          createdAt: Date.now(),
+        },
+      ],
+    );
+    expect(matches).toHaveLength(1);
+    expect(matches[0]?.id).toBe("FN-5150");
+  });
+
+  it("does not match on generic-large-file overlap only", () => {
+    const matches = findNearDuplicates(
+      {
+        title: "Add PR merge auto-rebase",
+        description: "Touch packages/dashboard/src/routes/register-git-github.ts only",
+      },
+      [
+        {
+          id: "FN-X",
+          title: "Fix PR comments pagination",
+          description: "Also touches packages/dashboard/src/routes/register-git-github.ts",
+          column: "todo",
+          createdAt: Date.now(),
+        },
+      ],
+    );
+    expect(matches).toEqual([]);
+  });
+
+  it("does not match different domains", () => {
+    const matches = findNearDuplicates(
+      { title: "Fix dashboard board scrolling", description: "Board snap bug" },
+      [
+        {
+          id: "FN-Y",
+          title: "Fix PR create dialog",
+          description: "pr modal bug",
+          column: "todo",
+          createdAt: Date.now(),
+        },
+      ],
+    );
+    expect(matches).toEqual([]);
+  });
+
+  it("respects 7-day window", () => {
+    const now = Date.now();
+    const matches = findNearDuplicates(
+      { title: fn5144Title, description: fn5144Description },
+      [
+        {
+          id: "FN-OLD",
+          title: fn5149Title,
+          description: fn5149Description,
+          column: "todo",
+          createdAt: now - 8 * 24 * 60 * 60 * 1000,
+        },
+      ],
+      { nowMs: now },
+    );
+    expect(matches).toEqual([]);
+  });
+
+  it("does not match title-only overlap with no shared high-signal tokens", () => {
+    const matches = findNearDuplicates(
+      { title: "Fix review task create bug", description: "plain words only" },
+      [
+        {
+          id: "FN-Z",
+          title: "Fix review task create issue",
+          description: "still plain words no routes or files",
+          column: "todo",
+          createdAt: Date.now(),
+        },
+      ],
+    );
+    expect(matches).toEqual([]);
+  });
+
+  it("returns [] for empty signal input", () => {
+    const matches = findNearDuplicates(
+      { title: "", description: "just english sentence no path tokens" },
+      [],
+    );
+    expect(matches).toEqual([]);
+  });
+});

packages/core/src/duplicate-detection.ts

@@ -30,7 +30,7 @@ export interface ContentFingerprintInput {
 const DEFAULT_THRESHOLD = 0.45;
 const DEFAULT_LIMIT = 5;
 const DEFAULT_EXCLUDE_COLUMNS: Column[] = ["done", "archived"];
-const STOPWORDS = new Set([
+export const STOPWORDS = new Set([
   "a",
   "an",
   "the",
@@ -74,7 +74,7 @@ export function computeContentFingerprint(
     .digest("hex");
 }
 
-function tokenize(value: string): string[] {
+export function tokenize(value: string): string[] {
   return value
     .toLowerCase()
     .split(/\W+/)

packages/core/src/index.ts

@@ -131,6 +131,8 @@ export {
   SelfDefeatingDependencyError,
 } from "./store.js";
 export {
+  STOPWORDS,
+  tokenize,
   computeContentFingerprint,
   findDuplicateMatches,
   type ContentFingerprintInput,
@@ -138,6 +140,14 @@ export {
   type DuplicateMatch,
   type DuplicateMatchInput,
 } from "./duplicate-detection.js";
+export {
+  extractIntentSignature,
+  findNearDuplicates,
+  type IntentSignature,
+  type NearDuplicateInput,
+  type NearDuplicateCandidate,
+  type NearDuplicateMatch,
+} from "./near-duplicate.js";
 export { getTaskDuplicateLineage } from "./duplicate-lineage.js";
 export {
   __getDeterministicGuardMutexSize,

packages/core/src/near-duplicate.ts

@@ -0,0 +1,205 @@
+import { STOPWORDS, tokenize } from "./duplicate-detection.js";
+import type { Column } from "./types.js";
+
+const DEFAULT_WINDOW_MS = 7 * 24 * 60 * 60 * 1000;
+const DEFAULT_LIMIT = 5;
+const DEFAULT_MIN_SHARED_TOKENS = 2;
+const DEFAULT_TITLE_THRESHOLD = 0.3;
+const GENERIC_LARGE_FILE_TITLE_THRESHOLD = 0.5;
+const MAX_TOKENS_PER_BUCKET = 32;
+
+// FN-5152: frequently touched broad files should not trigger near-duplicate matches by themselves.
+const GENERIC_LARGE_FILES = new Set([
+  "register-git-github.ts",
+  "register-task-workflow-routes.ts",
+  "store.ts",
+  "types.ts",
+  "styles.css",
+]);
+
+export interface IntentSignature {
+  routePaths: string[];
+  filePaths: string[];
+  identifiers: string[];
+  titleTokens: string[];
+}
+
+export interface NearDuplicateInput {
+  title?: string | null;
+  description: string;
+  fileScope?: string[];
+}
+
+export interface NearDuplicateCandidate {
+  id: string;
+  title: string;
+  description: string;
+  column: Column;
+  fileScope?: string[];
+  createdAt?: number;
+}
+
+export interface NearDuplicateMatch {
+  id: string;
+  score: number;
+  sharedTokens: string[];
+  titleScore: number;
+  reason: "near-duplicate-intent";
+}
+
+interface SignalToken {
+  token: string;
+  kind: "route" | "file" | "identifier";
+}
+
+function toUnique(values: string[], limit = MAX_TOKENS_PER_BUCKET): string[] {
+  return Array.from(new Set(values)).slice(0, limit);
+}
+
+function jaccard(left: string[], right: string[]): number {
+  const leftSet = new Set(left);
+  const rightSet = new Set(right);
+  if (leftSet.size === 0 || rightSet.size === 0) {
+    return 0;
+  }
+  let intersection = 0;
+  for (const token of leftSet) {
+    if (rightSet.has(token)) {
+      intersection += 1;
+    }
+  }
+  const union = leftSet.size + rightSet.size - intersection;
+  return union === 0 ? 0 : intersection / union;
+}
+
+function extractRoutePaths(text: string): string[] {
+  const matches = text.match(/\/[a-z][\w/:.-]+(?:\/[\w:.-]+)+/gi) ?? [];
+  const expanded: string[] = [];
+  for (const match of matches) {
+    const lower = match.toLowerCase();
+    expanded.push(lower);
+    const prTail = lower.match(/\/pr\/[\w:.-]+(?:\/[\w:.-]+)*/g);
+    if (prTail) {
+      expanded.push(...prTail);
+    }
+  }
+  const filtered = expanded
+    .filter((entry) => /\/[\w:.-]+\/[\w:.-]+/.test(entry))
+    .filter((entry) => /\b(pr|api|tasks|users|repos|reviews|merge|settings|workflow)\b/.test(entry));
+  return toUnique(filtered);
+}
+
+function extractFilePaths(text: string): string[] {
+  const matches = text.match(/(?:packages|app|scripts|docs|tests?)\/[\w./-]+\.(?:ts|tsx|js|mjs|cjs|css|md|json)/g) ?? [];
+  return toUnique(matches.map((entry) => entry.toLowerCase()));
+}
+
+function extractIdentifiers(text: string): string[] {
+  const backticked = Array.from(text.matchAll(/`([^`]+)`/g)).map((match) => match[1]?.trim() ?? "");
+  const codeShaped = text.match(/\b(?:[A-Z][A-Za-z0-9]+[A-Z][A-Za-z0-9]*|[a-z0-9]+_[a-z0-9_]+|[a-z0-9]+(?:-[a-z0-9]+)+(?:\.[a-z0-9]+)?|[a-z]+[A-Z][A-Za-z0-9]*)\b/g) ?? [];
+  const raw = [...backticked, ...codeShaped];
+  const filtered = raw
+    .map((value) => value.trim())
+    .filter((value) => value.length >= 4)
+    .filter((value) => !STOPWORDS.has(value.toLowerCase()))
+    .map((value) => value.toLowerCase());
+  return toUnique(filtered);
+}
+
+export function extractIntentSignature(input: NearDuplicateInput): IntentSignature {
+  const title = input.title ?? "";
+  const fileScope = input.fileScope ?? [];
+  const text = `${title}\n${input.description}\n${fileScope.join("\n")}`;
+  return {
+    routePaths: extractRoutePaths(text),
+    filePaths: extractFilePaths(text),
+    identifiers: extractIdentifiers(text),
+    titleTokens: toUnique(tokenize(title).filter((token) => token.length >= 3)),
+  };
+}
+
+function getSignalTokens(signature: IntentSignature): SignalToken[] {
+  return [
+    ...signature.routePaths.map((token) => ({ token, kind: "route" as const })),
+    ...signature.filePaths.map((token) => ({ token, kind: "file" as const })),
+    ...signature.identifiers.map((token) => ({ token, kind: "identifier" as const })),
+  ];
+}
+
+export function findNearDuplicates(
+  input: NearDuplicateInput,
+  candidates: NearDuplicateCandidate[],
+  opts?: {
+    minSharedTokens?: number;
+    titleThreshold?: number;
+    windowMs?: number;
+    nowMs?: number;
+    limit?: number;
+  },
+): NearDuplicateMatch[] {
+  const source = extractIntentSignature(input);
+  const sourceSignals = getSignalTokens(source);
+  if (sourceSignals.length === 0) {
+    return [];
+  }
+
+  const minSharedTokens = opts?.minSharedTokens ?? DEFAULT_MIN_SHARED_TOKENS;
+  const titleThreshold = opts?.titleThreshold ?? DEFAULT_TITLE_THRESHOLD;
+  const windowMs = opts?.windowMs ?? DEFAULT_WINDOW_MS;
+  const nowMs = opts?.nowMs ?? Date.now();
+  const cutoff = nowMs - windowMs;
+  const limit = opts?.limit ?? DEFAULT_LIMIT;
+
+  const matches: NearDuplicateMatch[] = [];
+
+  for (const candidate of candidates) {
+    if (candidate.createdAt != null && candidate.createdAt < cutoff) {
+      continue;
+    }
+
+    const candidateSignature = extractIntentSignature({
+      title: candidate.title,
+      description: candidate.description,
+      fileScope: candidate.fileScope,
+    });
+    const candidateSignals = getSignalTokens(candidateSignature);
+    if (candidateSignals.length === 0) {
+      continue;
+    }
+
+    const candidateTokenKinds = new Map(candidateSignals.map((entry) => [entry.token, entry.kind]));
+    const sharedTokens = toUnique(sourceSignals
+      .filter((entry) => candidateTokenKinds.has(entry.token))
+      .map((entry) => entry.token), 256);
+
+    const titleScore = jaccard(source.titleTokens, candidateSignature.titleTokens);
+
+    const sharedFileTokens = sharedTokens.filter((token) => candidateTokenKinds.get(token) === "file");
+    const allSharedAreFiles = sharedFileTokens.length === sharedTokens.length;
+    const allSharedAreGenericLargeFiles =
+      sharedFileTokens.length > 0 &&
+      sharedFileTokens.every((token) => GENERIC_LARGE_FILES.has(token.split("/").pop() ?? token));
+    const effectiveTitleThreshold =
+      allSharedAreFiles && allSharedAreGenericLargeFiles
+        ? Math.max(titleThreshold, GENERIC_LARGE_FILE_TITLE_THRESHOLD)
+        : titleThreshold;
+
+    if (sharedTokens.length < minSharedTokens || titleScore < effectiveTitleThreshold) {
+      continue;
+    }
+
+    const signalDenominator = Math.max(sourceSignals.length, candidateSignals.length, 1);
+    const score =
+      0.5 * (sharedTokens.length / signalDenominator) + 0.5 * titleScore;
+
+    matches.push({
+      id: candidate.id,
+      score,
+      sharedTokens,
+      titleScore,
+      reason: "near-duplicate-intent",
+    });
+  }
+
+  return matches.sort((a, b) => b.score - a.score).slice(0, limit);
+}