fusion/packages/core/src/duplicate-detection.ts

import { createHash } from "node:crypto";

import type { ColumnId } from "./types.js";

export interface DuplicateMatch {
  id: string;
  title: string;
  description: string;
  column: ColumnId;
  score: number;
}

export interface DuplicateMatchInput {
  title?: string;
  description: string;
}

export interface DuplicateCandidate {
  id: string;
  title: string;
  description: string;
  column: ColumnId;
}

export interface ContentFingerprintInput {
  title?: string | null;
  description: string;
}

const DEFAULT_THRESHOLD = 0.45;
const DEFAULT_LIMIT = 5;
/* DELIBERATE-LITERAL — the default for a caller that does not supply resolved terminal columns,
   reviewed 2026-07-31-04:20. Callers that CAN resolve should pass `excludeColumns`; this pair is the
   degraded mode, not the rule. */
const DEFAULT_EXCLUDE_COLUMNS: readonly string[] = ["done", "archived"];
export const STOPWORDS = new Set([
  "a",
  "an",
  "the",
  "and",
  "or",
  "of",
  "to",
  "for",
  "in",
  "is",
  "on",
  "with",
  "fn",
]);

function normalizeFingerprintPart(value: string): string {
  return value
    .toLowerCase()
    .replace(/[,.;:!?"'`(){}]+/g, "")
    .replaceAll("[", "")
    .replaceAll("]", "")
    .replace(/…$/u, "")
    .replace(/\s+/g, " ")
    .trim();
}

/**
 * Deterministic dedup fingerprint for task content.
 */
export function computeContentFingerprint(
  input: ContentFingerprintInput,
): string | null {
  const normalizedDescription = normalizeFingerprintPart(input.description);
  if (normalizedDescription.length === 0) {
    return null;
  }

  const normalizedTitle = normalizeFingerprintPart(input.title ?? "");
  return createHash("sha256")
    .update(`${normalizedTitle}\n${normalizedDescription}`)
    .digest("hex");
}

export function tokenize(value: string): string[] {
  return value
    .toLowerCase()
    .split(/\W+/)
    .map((token) => token.trim())
    .filter((token) => token.length > 0 && !STOPWORDS.has(token));
}

function toTrigrams(value: string): Set<string> {
  const normalized = value.toLowerCase().replace(/\s+/g, " ").trim();
  if (normalized.length < 3) {
    return new Set();
  }
  const trigrams = new Set<string>();
  for (let index = 0; index <= normalized.length - 3; index += 1) {
    trigrams.add(normalized.slice(index, index + 3));
  }
  return trigrams;
}

function jaccardScore(left: Set<string>, right: Set<string>): number {
  if (left.size === 0 || right.size === 0) {
    return 0;
  }
  let intersectionCount = 0;
  for (const token of left) {
    if (right.has(token)) {
      intersectionCount += 1;
    }
  }
  const unionCount = left.size + right.size - intersectionCount;
  return unionCount > 0 ? intersectionCount / unionCount : 0;
}

function hasTitleTrigramOverlap(left: string, right: string): boolean {
  const leftTrigrams = toTrigrams(left);
  const rightTrigrams = toTrigrams(right);
  if (leftTrigrams.size === 0 || rightTrigrams.size === 0) {
    return false;
  }
  for (const trigram of leftTrigrams) {
    if (rightTrigrams.has(trigram)) {
      return true;
    }
  }
  return false;
}

export function findDuplicateMatches(
  input: DuplicateMatchInput,
  candidates: DuplicateCandidate[],
  opts?: {
    threshold?: number;
    limit?: number;
    /*
    FNXC:WorkflowLifecycleColumns 2026-07-31-04:20 (batch-core feed):
    `readonly string[]`, not the legacy `Column[]` enum. A renamed board's terminal column is a
    perfectly valid id that the enum cannot express, so the enum made this parameter unusable by
    exactly the callers that need it — the type was the guard's real limit, not the logic.

    Widening only; every existing caller passes legacy ids, which remain assignable.
    */
    excludeColumns?: readonly string[];
  },
): DuplicateMatch[] {
  const description = input.description.trim();
  if (description.length === 0) {
    return [];
  }

  const threshold = opts?.threshold ?? DEFAULT_THRESHOLD;
  const limit = opts?.limit ?? DEFAULT_LIMIT;
  const excludedColumns = new Set<string>(opts?.excludeColumns ?? DEFAULT_EXCLUDE_COLUMNS);
  const sourceText = `${input.title ?? ""} ${description}`.trim();
  const sourceTokens = new Set(tokenize(sourceText));
  const sourceTitle = input.title ?? "";
  const sourceTitleTokens = new Set(tokenize(sourceTitle));

  const matches: DuplicateMatch[] = [];
  for (const candidate of candidates) {
    if (excludedColumns.has(candidate.column)) {
      continue;
    }

    const candidateText = `${candidate.title} ${candidate.description}`.trim();
    const candidateTokens = new Set(tokenize(candidateText));
    const candidateTitleTokens = new Set(tokenize(candidate.title));
    const contentScore = jaccardScore(sourceTokens, candidateTokens);
    const titleScore = jaccardScore(sourceTitleTokens, candidateTitleTokens);
    let score = Math.max(contentScore, contentScore * 0.7 + titleScore * 0.3);

    if (hasTitleTrigramOverlap(sourceTitle, candidate.title)) {
      score += 0.05;
    }

    if (score >= threshold) {
      matches.push({
        id: candidate.id,
        title: candidate.title,
        description: candidate.description,
        column: candidate.column,
        score,
      });
    }
  }

  return matches.sort((a, b) => b.score - a.score).slice(0, limit);
}