root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(FN-5152): complete Step 1 — add near-duplicate core helper

Browse Source 
Fusion-Task-Id: FN-5152
Fusion-Task-Lineage: 954570d8-82e1-4519-a739-c52ee790d2db
This commit is contained in:
Fusion (runfusion.ai)
2026-05-19 08:27:20 -07:00
committed by gsxdsm
parent 71a2ea1954
commit 7d01a666a9
4 changed files with 385 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
packages/core/src/duplicate-detection.ts
View File

@@ -30,7 +30,7 @@ export interface ContentFingerprintInput {
const DEFAULT_THRESHOLD = 0.45;
const DEFAULT_LIMIT = 5;
const DEFAULT_EXCLUDE_COLUMNS: Column[] = ["done", "archived"];
const STOPWORDS = new Set([
export const STOPWORDS = new Set([
"a",
"an",
"the",
@@ -74,7 +74,7 @@ export function computeContentFingerprint(
.digest("hex");
}
function tokenize(value: string): string[] {
export function tokenize(value: string): string[] {
return value
.toLowerCase()
.split(/\W+/)