root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(FN-5405): remove broad-scope triage heuristics and UI advisory chips/b

Browse Source 
Removes the broad-scope detection feature end-to-end: the TaskCard chip and TaskDetailModal advisory banner are gone from the dashboard, the triage heuristic that flagged tasks as broad-scope has been deleted from the engine along with its associated run-audit events, and documentation references ha

Fusion-Task-Id: FN-5405
This commit is contained in:
Fusion (runfusion.ai)
2026-05-21 15:06:17 -07:00
committed by gsxdsm
parent 5a76a89071
commit b8147dd3b1
15 changed files with 51 additions and 935 deletions
Download Patch File Download Diff File Expand all files Collapse all files

207
packages/engine/src/__tests__/reliability-interactions/broad-scope-triage-flag.test.ts
View File

@@ -1,207 +0,0 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
import { execSync } from "node:child_process";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { DEFAULT_SETTINGS, TaskStore } from "@fusion/core";
import * as broadScopeHeuristics from "../../triage-broad-scope-heuristics.js";
import { TriageProcessor } from "../../triage.js";
function git(cwd: string, command: string): string {
return execSync(command, { cwd, encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"] }).trim();
}
async function createFixture() {
const rootDir = await mkdtemp(join(tmpdir(), "fusion-broad-scope-triage-"));
git(rootDir, "git init -b main");
git(rootDir, 'git config user.email "test@example.com"');
git(rootDir, 'git config user.name "Test User"');
git(rootDir, "git commit --allow-empty -m init");
const store = new TaskStore(rootDir, undefined, { inMemoryDb: true });
await store.init();
await store.updateSettings({ ...DEFAULT_SETTINGS, requirePlanApproval: false });
const triage = new TriageProcessor(store, rootDir);
return {
rootDir,
store,
triage,
persistPrompt: async (taskId: string, prompt: string) => {
await mkdir(join(rootDir, ".fusion", "tasks", taskId), { recursive: true });
await writeFile(join(rootDir, ".fusion", "tasks", taskId, "PROMPT.md"), prompt, "utf-8");
},
cleanup: async () => {
store.close();
await rm(rootDir, { recursive: true, force: true });
},
};
}
function buildPrompt({ size, stepCount, fileScopeCount }: { size: "S" | "M" | "L"; stepCount: number; fileScopeCount: number }): string {
const steps = Array.from({ length: stepCount }, (_, index) => `### Step ${index + 1}: Step ${index + 1}\n- [ ] do work ${index + 1}`)
.join("\n\n");
const fileScope = Array.from({ length: fileScopeCount }, (_, index) => `- ` + "`" + `packages/engine/src/generated/file-${index + 1}.ts` + "`")
.join("\n");
return `# Task: FN-1 - test\n\n**Size:** ${size}\n\n## Review Level: 1\n\n## File Scope\n${fileScope}\n\n## Steps\n\n${steps}\n`;
}
describe("reliability interactions: broad-scope triage flag", () => {
const fixtures: Array<Awaited<ReturnType<typeof createFixture>>> = [];
afterEach(async () => {
vi.useRealTimers();
vi.restoreAllMocks();
while (fixtures.length) await fixtures.pop()!.cleanup();
});
it("adds broadScopeFlag metadata and preserves intentSignature/fileScope composition", async () => {
const fx = await createFixture();
fixtures.push(fx);
const task = await fx.store.createTask({
title: "Repair engine regression across generated files",
description: "Touches /api/tasks/:id/pr/options, auth.ts, and 30 failing files across the triage pipeline.",
});
const prompt = buildPrompt({ size: "L", stepCount: 12, fileScopeCount: 25 });
await fx.persistPrompt(task.id, prompt);
await (fx.triage as any).finalizeApprovedTask(task, prompt, await fx.store.getSettings(), {});
const updated = await fx.store.getTask(task.id);
expect(updated.sourceMetadata?.broadScopeFlag).toMatchObject({
score: 9,
reasons: expect.arrayContaining(["size-l", "steps-high", "file-scope-high", "failing-file-mentions-high", "size-l-with-many-steps"]),
signals: expect.objectContaining({
size: "L",
stepCount: 12,
fileScopeCount: 25,
failingFileMentions: 30,
}),
thresholds: expect.objectContaining({
stepsHigh: 12,
fileScopeHigh: 20,
failingFileMentionsHigh: 30,
sizeLStepsThreshold: 9,
}),
version: 1,
flaggedAt: expect.any(String),
});
expect(updated.sourceMetadata?.intentSignature).toBeTruthy();
expect(updated.sourceMetadata?.fileScope).toHaveLength(25);
});
it("keeps flagged tasks in todo because the flag is advisory only", async () => {
const fx = await createFixture();
fixtures.push(fx);
const task = await fx.store.createTask({
title: "Repair engine regression across generated files",
description: "Touches /api/tasks/:id/pr/options, auth.ts, and 30 failing files across the triage pipeline.",
});
const prompt = buildPrompt({ size: "L", stepCount: 12, fileScopeCount: 25 });
await fx.persistPrompt(task.id, prompt);
await (fx.triage as any).finalizeApprovedTask(task, prompt, await fx.store.getSettings(), {});
const updated = await fx.store.getTask(task.id);
expect(updated.column).toBe("todo");
});
it("emits a run-audit event with the broad-scope metadata payload", async () => {
const fx = await createFixture();
fixtures.push(fx);
const task = await fx.store.createTask({
title: "Repair engine regression across generated files",
description: "Touches /api/tasks/:id/pr/options, auth.ts, and 30 failing files across the triage pipeline.",
});
const prompt = buildPrompt({ size: "L", stepCount: 12, fileScopeCount: 25 });
await fx.persistPrompt(task.id, prompt);
await (fx.triage as any).finalizeApprovedTask(task, prompt, await fx.store.getSettings(), {});
const audit = fx.store.getRunAuditEvents({ taskId: task.id, limit: 20 });
expect(audit).toEqual(expect.arrayContaining([
expect.objectContaining({
mutationType: "task:broad-scope-flagged-at-triage",
metadata: expect.objectContaining({
score: 9,
reasons: expect.arrayContaining(["size-l", "steps-high", "file-scope-high", "failing-file-mentions-high", "size-l-with-many-steps"]),
signals: expect.objectContaining({ size: "L", stepCount: 12, fileScopeCount: 25, failingFileMentions: 30 }),
thresholds: expect.objectContaining({
stepsHigh: 12,
fileScopeHigh: 20,
failingFileMentionsHigh: 30,
sizeLStepsThreshold: 9,
}),
version: 1,
}),
}),
]));
});
it("appends an operator log entry when the flag fires", async () => {
const fx = await createFixture();
fixtures.push(fx);
const task = await fx.store.createTask({
title: "Repair engine regression across generated files",
description: "Touches /api/tasks/:id/pr/options, auth.ts, and 30 failing files across the triage pipeline.",
});
const prompt = buildPrompt({ size: "L", stepCount: 12, fileScopeCount: 25 });
await fx.persistPrompt(task.id, prompt);
await (fx.triage as any).finalizeApprovedTask(task, prompt, await fx.store.getSettings(), {});
const updated = await fx.store.getTask(task.id);
expect(updated.log.some((entry) => entry.action === "Broad-scope triage flag")).toBe(true);
});
it("does not add flag metadata, audit, or log entry for small narrow tasks", async () => {
const fx = await createFixture();
fixtures.push(fx);
const task = await fx.store.createTask({
title: "Fix one narrow regression",
description: "Touches auth.ts only.",
});
const prompt = buildPrompt({ size: "S", stepCount: 4, fileScopeCount: 3 });
await fx.persistPrompt(task.id, prompt);
await (fx.triage as any).finalizeApprovedTask(task, prompt, await fx.store.getSettings(), {});
const updated = await fx.store.getTask(task.id);
expect(updated.column).toBe("todo");
expect(updated.sourceMetadata?.broadScopeFlag).toBeUndefined();
expect(updated.log.some((entry) => entry.action === "Broad-scope triage flag")).toBe(false);
const audit = fx.store.getRunAuditEvents({ taskId: task.id, limit: 20 });
expect(audit.some((entry) => entry.mutationType === "task:broad-scope-flagged-at-triage")).toBe(false);
});
it("fails open when signal extraction throws", async () => {
const fx = await createFixture();
fixtures.push(fx);
const task = await fx.store.createTask({
title: "Repair engine regression across generated files",
description: "Touches /api/tasks/:id/pr/options, auth.ts, and 30 failing files across the triage pipeline.",
});
const prompt = buildPrompt({ size: "L", stepCount: 12, fileScopeCount: 25 });
await fx.persistPrompt(task.id, prompt);
vi.spyOn(broadScopeHeuristics, "extractBroadScopeSignals").mockImplementation(() => {
throw new Error("boom");
});
await (fx.triage as any).finalizeApprovedTask(task, prompt, await fx.store.getSettings(), {});
const updated = await fx.store.getTask(task.id);
expect(updated.column).toBe("todo");
expect(updated.sourceMetadata?.broadScopeFlag).toBeUndefined();
expect(updated.log.some((entry) => entry.action === "Broad-scope triage flag")).toBe(false);
const audit = fx.store.getRunAuditEvents({ taskId: task.id, limit: 20 });
expect(audit.some((entry) => entry.mutationType === "task:broad-scope-flagged-at-triage")).toBe(false);
});
});

120
packages/engine/src/__tests__/triage-broad-scope-heuristics.test.ts
View File

@@ -1,120 +0,0 @@
import { describe, expect, it } from "vitest";
import {
BROAD_SCOPE_FLAG_VERSION,
decideBroadScopeFlag,
extractBroadScopeSignals,
} from "../triage-broad-scope-heuristics.js";
describe("triage broad-scope heuristics", () => {
describe("extractBroadScopeSignals", () => {
it("uses the largest matching multi-digit failing-file mention", () => {
const signals = extractBroadScopeSignals({
size: "M",
stepCount: 5,
fileScopeCount: 4,
descriptionText: "Touches 12 failing files, 30 broken tests, and 21 files overall. Ignore 7 failing files.",
});
expect(signals).toMatchObject({
size: "M",
stepCount: 5,
fileScopeCount: 4,
failingFileMentions: 30,
});
});
it("caps pathological counts at 9999", () => {
const signals = extractBroadScopeSignals({
size: "L",
stepCount: 14,
fileScopeCount: 24,
descriptionText: "Spec mentions 12345 failing files across 200 broken tests.",
});
expect(signals.failingFileMentions).toBe(9999);
});
it("returns zero when there are no qualifying mentions", () => {
const signals = extractBroadScopeSignals({
size: "S",
stepCount: 2,
fileScopeCount: 1,
descriptionText: "Only 9 failing files are listed, plus one broken test.",
});
expect(signals.failingFileMentions).toBe(0);
});
});
describe("decideBroadScopeFlag", () => {
it("does not flag small low-scope tasks", () => {
const decision = decideBroadScopeFlag({
size: "S",
stepCount: 4,
fileScopeCount: 3,
failingFileMentions: 0,
});
expect(decision.flagged).toBe(false);
expect(decision.score).toBe(0);
expect(decision.reasons).toEqual([]);
});
it("does not flag size L alone", () => {
const decision = decideBroadScopeFlag({
size: "L",
stepCount: 4,
fileScopeCount: 3,
failingFileMentions: 0,
});
expect(decision.flagged).toBe(false);
expect(decision.score).toBe(2);
expect(decision.reasons).toEqual(["size-l"]);
});
it("flags size L tasks with many steps", () => {
const decision = decideBroadScopeFlag({
size: "L",
stepCount: 12,
fileScopeCount: 3,
failingFileMentions: 0,
});
expect(decision.flagged).toBe(true);
expect(decision.score).toBe(5);
expect(decision.reasons).toEqual(["size-l", "steps-high", "size-l-with-many-steps"]);
});
it("does not flag high file scope alone", () => {
const decision = decideBroadScopeFlag({
size: "M",
stepCount: 5,
fileScopeCount: 21,
failingFileMentions: 0,
});
expect(decision.flagged).toBe(false);
expect(decision.score).toBe(2);
expect(decision.reasons).toEqual(["file-scope-high"]);
});
it("flags when multiple strong signals combine", () => {
const decision = decideBroadScopeFlag({
size: "M",
stepCount: 5,
fileScopeCount: 21,
failingFileMentions: 30,
});
expect(decision.flagged).toBe(true);
expect(decision.score).toBe(4);
expect(decision.reasons).toEqual(["file-scope-high", "failing-file-mentions-high"]);
});
});
it("exports the initial heuristic version", () => {
expect(BROAD_SCOPE_FLAG_VERSION).toBe(1);
});
});

2
packages/engine/src/run-audit.ts
View File

@@ -268,10 +268,8 @@ export type DatabaseMutationType =
| "task:auto-recover-worktree-metadata-skipped-active"
// task:auto-archived-ghost-bug metadata: { findings: Array<{ construct: { kind: string; raw: string; filePath?: string; line?: number }; matched: boolean; probeError?: string; output?: string }>; reason: string }
// task:auto-archived-duplicate metadata: { siblingTaskIds: string[]; scores: Record<string, number> }
// task:broad-scope-flagged-at-triage metadata: { score: number; reasons: string[]; signals: { size: "S"|"M"|"L"|null; stepCount: number; fileScopeCount: number; failingFileMentions: number }; thresholds: { stepsHigh: number; fileScopeHigh: number; failingFileMentionsHigh: number; sizeLStepsThreshold: number }; version: number }
| "task:auto-archived-ghost-bug"
| "task:auto-archived-duplicate"
| "task:broad-scope-flagged-at-triage"
| "task:auto-reconciled-self-defeating-dep"
| "task:dependency-cycle-rejected"
| "task:dependency-cycle-detected"

90
packages/engine/src/triage-broad-scope-heuristics.ts
View File

@@ -1,90 +0,0 @@
export const BROAD_SCOPE_FLAG_VERSION = 1;
export const DEFAULT_BROAD_SCOPE_THRESHOLDS = {
stepsHigh: 12,
fileScopeHigh: 20,
failingFileMentionsHigh: 30,
sizeLStepsThreshold: 9,
} as const;
export interface BroadScopeSignals {
size: "S" | "M" | "L" | null;
stepCount: number;
fileScopeCount: number;
failingFileMentions: number;
}
export interface BroadScopeFlagDecision {
flagged: boolean;
score: number;
reasons: string[];
signals: BroadScopeSignals;
thresholds: typeof DEFAULT_BROAD_SCOPE_THRESHOLDS;
version: number;
}
export function extractBroadScopeSignals(input: {
size: "S" | "M" | "L" | null;
stepCount: number;
fileScopeCount: number;
descriptionText: string;
}): BroadScopeSignals {
const matches = input.descriptionText.matchAll(/\b(\d{2,})\s+(failing|broken|test|file)s?\b/gi);
let failingFileMentions = 0;
for (const match of matches) {
const value = Number.parseInt(match[1] ?? "0", 10);
if (Number.isFinite(value)) {
failingFileMentions = Math.max(failingFileMentions, Math.min(value, 9999));
}
}
return {
size: input.size,
stepCount: input.stepCount,
fileScopeCount: input.fileScopeCount,
failingFileMentions,
};
}
export function decideBroadScopeFlag(
signals: BroadScopeSignals,
thresholds: Partial<typeof DEFAULT_BROAD_SCOPE_THRESHOLDS> = {},
): BroadScopeFlagDecision {
const resolvedThresholds = {
...DEFAULT_BROAD_SCOPE_THRESHOLDS,
...thresholds,
};
const reasons: string[] = [];
let score = 0;
if (signals.size === "L") {
score += 2;
reasons.push("size-l");
}
if (signals.stepCount >= resolvedThresholds.stepsHigh) {
score += 2;
reasons.push("steps-high");
}
if (signals.fileScopeCount >= resolvedThresholds.fileScopeHigh) {
score += 2;
reasons.push("file-scope-high");
}
if (signals.failingFileMentions >= resolvedThresholds.failingFileMentionsHigh) {
score += 2;
reasons.push("failing-file-mentions-high");
}
if (signals.size === "L" && signals.stepCount >= resolvedThresholds.sizeLStepsThreshold) {
score += 1;
reasons.push("size-l-with-many-steps");
}
return {
flagged: score >= 3,
score,
reasons,
signals,
thresholds: resolvedThresholds,
version: BROAD_SCOPE_FLAG_VERSION,
};
}

91
packages/engine/src/triage.ts
View File

@@ -79,11 +79,6 @@ import {
isResearchToolSurfaceEnabled,
} from "./tool-availability.js";
import { runGhostBugPreflight } from "./triage-preflight.js";
import {
BROAD_SCOPE_FLAG_VERSION,
decideBroadScopeFlag,
extractBroadScopeSignals,
} from "./triage-broad-scope-heuristics.js";
import { archiveAsGhostBug } from "./self-healing.js";
import { createRunAuditor, generateSyntheticRunId } from "./run-audit.js";
@@ -266,6 +261,13 @@ For tasks you assess as Size M or L, consider whether splitting into 2-5 child t
- Coordination overhead (worktrees, dependency wiring, merge sequencing) is real — only split when the parallelism or scope-clarity benefit clearly outweighs it
- If you decide not to split an M/L task, proceed with a normal PROMPT.md specification
**Broad-scope decomposition signals:**
- Size L tasks, especially when the planned step count would reach 9 or more.
- Plans whose implementation-step count would reach 12 or more (additive signal — counts even when the surrounding "more than 7/10 steps" threshold above has not yet fired).
- Tasks whose declared \`## File Scope\` would list 20 or more entries.
- Descriptions that quantify large remediation batches (for example "47 failing tests", "30+ broken files") at or above 30 items — treat as a strong signal that the work should be partitioned by subsystem or file group before specifying.
- When two or more of the signals above fire together, default to splitting via \`fn_task_create\`. If you still choose to keep the task as a single unit, justify the decision explicitly in the PROMPT.md \`## Mission\` paragraph.
## Triage tools
You have these extra tools during triage:
- \`fn_task_list\` — list existing active tasks
@@ -2416,54 +2418,6 @@ export class TriageProcessor {
} catch {
// Fail open on persisted PROMPT.md parsing and keep using the in-memory parse.
}
type BroadScopeFlagRecord = {
score: number;
reasons: string[];
signals: {
size: "S" | "M" | "L" | null;
stepCount: number;
fileScopeCount: number;
failingFileMentions: number;
};
thresholds: {
stepsHigh: number;
fileScopeHigh: number;
failingFileMentionsHigh: number;
sizeLStepsThreshold: number;
};
version: number;
flaggedAt: string;
};
let broadScopeFlagRecord: BroadScopeFlagRecord | null = null;
try {
const broadScopeSignals = extractBroadScopeSignals({
size: taskUpdates.size ?? task.size ?? null,
stepCount: parsedSteps.length,
fileScopeCount: parsedFileScope.length,
descriptionText: task.description ?? "",
});
const broadScopeDecision = decideBroadScopeFlag(broadScopeSignals);
if (broadScopeDecision.flagged) {
broadScopeFlagRecord = {
score: broadScopeDecision.score,
reasons: broadScopeDecision.reasons,
signals: broadScopeDecision.signals,
thresholds: broadScopeDecision.thresholds,
version: BROAD_SCOPE_FLAG_VERSION,
flaggedAt: new Date().toISOString(),
};
taskUpdates.sourceMetadataPatch = {
...(taskUpdates.sourceMetadataPatch ?? {}),
broadScopeFlag: broadScopeFlagRecord,
};
planLog.warn(
`${task.id}: broad-scope flag at triage — score=${broadScopeDecision.score}, reasons=${broadScopeDecision.reasons.join(",")}`,
);
}
} catch (error: unknown) {
const message = error instanceof Error ? error.message : String(error);
planLog.warn(`${task.id}: broad-scope heuristic failed open: ${message}`);
}
let taskIntentSignature: ReturnType<typeof extractIntentSignature> = {
routePaths: [],
filePaths: [],
@@ -2500,37 +2454,6 @@ export class TriageProcessor {
await this.store.updateTask(task.id, taskUpdates);
if (broadScopeFlagRecord) {
try {
await this.store.logEntry(
task.id,
"Broad-scope triage flag",
`Heuristics suggest this task may benefit from decomposition (score=${broadScopeFlagRecord.score}; signals: ${broadScopeFlagRecord.reasons.join(", ")}). Consider creating child tasks via fn_task_create or marking breakIntoSubtasks=true before execution.`,
);
const auditor = createRunAuditor(this.store, {
taskId: task.id,
agentId: task.assignedAgentId ?? "triage",
runId: generateSyntheticRunId("triage", task.id),
phase: "triage",
source: "triage",
});
await auditor.database({
type: "task:broad-scope-flagged-at-triage",
target: task.id,
metadata: {
score: broadScopeFlagRecord.score,
reasons: broadScopeFlagRecord.reasons,
signals: broadScopeFlagRecord.signals,
thresholds: broadScopeFlagRecord.thresholds,
version: broadScopeFlagRecord.version,
},
});
} catch (error: unknown) {
const message = error instanceof Error ? error.message : String(error);
planLog.warn(`${task.id}: broad-scope heuristic failed open: ${message}`);
}
}
try {
const preflightDecision = await Promise.race([
runGhostBugPreflight(