feat(prefetch/faz6): anti-poison — generic-model skip + per-araç kategori tavanı
Some checks failed
QA Gate (P0/P1) / Test affected app (pull_request) Has been cancelled

5 Opel aracı (model="Opel" decode çözememiş) generic ROOT kataloğa düşüp
419k/102k/21k kategori + 0 PARÇA üretti → 97k kuyruğu zehirledi. İki guard:
- isGenericModel (model==brand veya boş) → processInit skip + markPoison
- CATEGORY_CAP (3000, env PREFETCH_CATEGORY_CAP): processChildren progress.total
  bu tavanı aşınca drill'i durdurur + markPoison
markPoison → prefetch:poison:<id> (30g TTL) + scheduled/progress temizler;
processInit & tryPick poison'ı atlar. +3 spec (generic skip / cap / scan skip).
Full api suite 321 pass. Çöp kategori temizliği ayrıca yapılacak.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-06-23 06:33:04 +03:00
parent d119b1eb2c
commit 95061dfa53
2 changed files with 127 additions and 4 deletions

View File

@@ -28,11 +28,13 @@ function makeDeps(opts: { waiting: number; limitResults: unknown[][] }) {
// the production nest build, which compiles spec files.
exists: vi.fn(async (..._args: unknown[]) => false), // no cooldown / guard / complete marker
get: vi.fn(async () => null), // no no-result residue
set: vi.fn(async () => undefined),
set: vi.fn(async (..._args: unknown[]) => undefined),
del: vi.fn(async (..._args: unknown[]) => undefined),
incr: vi.fn(async (..._args: unknown[]) => 1), // per-source rate window counter
expire: vi.fn(async () => undefined),
ttl: vi.fn(async () => -2), // checkCooldown: no cooldown key
setNx: vi.fn(async () => true), // scan lock acquired by default
getJson: vi.fn(async () => null), // Phase-2 cursor empty
getJson: vi.fn(async (..._args: unknown[]): Promise<unknown> => null), // cursor / progress empty
setJson: vi.fn(async () => undefined),
};
const posthog = { payload: vi.fn(async () => ({})) }; // compiled-in defaults
@@ -222,6 +224,58 @@ describe("PrefetchWorkerService — fast lane (lifo) + backlog gating", () => {
});
});
describe("poison guards (Faz 6: generic-model + category cap)", () => {
type PI = { processInit: (j: unknown) => Promise<void> };
type PC = { processChildren: (j: unknown) => Promise<void> };
it("processInit skips + marks poison for a generic-model vehicle (model == brand)", async () => {
const { service, queue, redis } = makeDeps({
waiting: 0,
limitResults: [[{ id: "op1", brandName: "Opel", model: "Opel" }]], // vehicle lookup
});
await (service as never as PI).processInit({
data: { vehicleId: "op1", source: "pl24" },
} as never);
const poisonSet = redis.set.mock.calls.some((c) =>
String(c[0]).includes("prefetch:poison:op1"),
);
expect(poisonSet).toBe(true);
expect(queue.add).not.toHaveBeenCalled(); // tree never drilled
});
it("processChildren marks poison once progress.total exceeds CATEGORY_CAP", async () => {
const { service, queue, redis } = makeDeps({ waiting: 0, limitResults: [] });
redis.getJson.mockImplementation(async (...a: unknown[]) =>
String(a[0]).includes("progress") ? { total: 5000 } : null,
);
await (service as never as PC).processChildren({
data: { vehicleId: "op1", categoryId: "c1", source: "pl24", depth: 1 },
} as never);
const poisonSet = redis.set.mock.calls.some((c) =>
String(c[0]).includes("prefetch:poison:op1"),
);
expect(poisonSet).toBe(true);
expect(queue.add).not.toHaveBeenCalled(); // stopped drilling
});
it("scan skips a poison-marked vehicle", async () => {
const { service, queue, redis } = makeDeps({
waiting: 10,
limitResults: [
[],
[{ id: "op1", source: "pl24", createdAt: new Date("2026-01-01T00:00:00Z") }],
],
});
redis.exists.mockImplementation(async (...a: unknown[]) =>
String(a[0]).includes("prefetch:poison:"),
);
await (
service as never as { processBackfillScan: () => Promise<void> }
).processBackfillScan();
expect(queue.add).not.toHaveBeenCalled();
});
});
describe("queueCategoryJob — depth cap (no-op guard)", () => {
// pl24 linkPath with no leaf marker (/bom/, /partinfo/ …) → treated as a
// non-leaf folder that would normally queue a prefetch-children job.

View File

@@ -113,6 +113,18 @@ const NORESULT_TTL_S = (Number(process.env.PREFETCH_NORESULT_TTL_DAYS) || 7) * 8
* forever). The TTL re-validates periodically so a later upstream catalog change
* is eventually picked up (default 21 days). Env-tunable. */
const COMPLETE_TTL_S = (Number(process.env.PREFETCH_COMPLETE_TTL_DAYS) || 21) * 86_400;
/**
* Per-vehicle category ceiling — anti-poison guard. A correctly-decoded PL24
* vehicle has ~100-800 categories (p95 744). A vehicle whose tree blows past
* this is almost always a decode that failed to resolve the model and landed on
* the generic ROOT catalog (the whole brand universe — e.g. 5 Opels with
* model="Opel" produced 420k/102k/21k categories and ZERO parts). Once a
* vehicle's discovered tree exceeds this, stop drilling it and mark it poison so
* it never gets re-picked. Env-tunable. */
const CATEGORY_CAP = Number(process.env.PREFETCH_CATEGORY_CAP) || 3000;
/** TTL for the poison marker (generic-model / over-cap vehicles). Long, because
* the underlying cause is a decode bug — re-validate monthly. */
const POISON_TTL_S = (Number(process.env.PREFETCH_POISON_TTL_DAYS) || 30) * 86_400;
@Injectable()
export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy {
@@ -263,9 +275,15 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy {
await checkCooldown(this.redis, source);
checkTimeWindow(source);
// Already flagged as poison (over-cap / generic model on a prior run) — skip.
if (await this.redis.exists(this.poisonKey(vehicleId))) {
this.logger.debug(`[prefetch] Skip poison vehicle=${vehicleId}`);
return;
}
// Verify vehicle still exists
const [vehicle] = await this.db
.select({ id: vehicles.id })
.select({ id: vehicles.id, brandName: vehicles.brandName, model: vehicles.model })
.from(vehicles)
.where(eq(vehicles.id, vehicleId))
.limit(1);
@@ -275,7 +293,20 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy {
return;
}
await initProgress(this.redis, vehicleId);
// Generic-model guard: when VIN decode fails to resolve the model it stays
// equal to the brand (e.g. model "Opel" for brand "Opel") and the catalog
// fetch lands on the brand's ROOT tree — the whole model universe, hundreds
// of thousands of part-less categories. Don't drill it; mark poison until the
// decode is fixed. A real model is always more specific than the brand.
if (this.isGenericModel(vehicle.brandName, vehicle.model)) {
this.logger.warn(
`[prefetch] Skip generic-model vehicle=${vehicleId} (brand=${vehicle.brandName}, model=${vehicle.model}) — marking poison`,
);
await this.markPoison(vehicleId);
return;
}
await initProgress(this.redis, vehicleId); // resets progress.total — the cap's tree-size proxy
// Get all top-level categories for this vehicle
let topCategories = await this.db
@@ -389,6 +420,19 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy {
return;
}
// Anti-poison cap: progress.total tracks every sub-job queued for this vehicle
// (≈ its discovered tree size). Once it blows past the ceiling the vehicle is a
// generic-catalog explosion — stop drilling and mark it poison so the rest of
// its (part-less) tree is never fetched and it's never re-picked.
const prog = await this.redis.getJson<{ total: number }>(`prefetch:progress:${vehicleId}`);
if ((prog?.total ?? 0) >= CATEGORY_CAP) {
this.logger.warn(
`[prefetch] Category cap ${CATEGORY_CAP} hit for vehicle=${vehicleId} (tree≈${prog?.total}) — marking poison, stop drilling`,
);
await this.markPoison(vehicleId);
return;
}
try {
const children = await this.categoriesService.getChildren(categoryId);
@@ -514,6 +558,8 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy {
// every wave; the marker's TTL re-validates them periodically. Phase-1
// (zero-parts) vehicles never carry this marker, so this is a no-op there.
if (await this.redis.exists(this.completeKey(v.id))) return;
// Skip poison vehicles (generic-model / over-cap catalog explosions).
if (await this.redis.exists(this.poisonKey(v.id))) return;
seen.add(v.id);
picked.push({ id: v.id, source: v.source, fast });
};
@@ -773,6 +819,29 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy {
return `prefetch:complete:${vehicleId}`;
}
private poisonKey(vehicleId: string): string {
return `prefetch:poison:${vehicleId}`;
}
/**
* True when VIN decode failed to resolve a model: it stays equal to the brand
* (case-insensitive) or is empty. Such a vehicle can't be catalog-scoped and
* fetches the brand's whole root tree — see CATEGORY_CAP.
*/
private isGenericModel(brandName: string | null, model: string | null): boolean {
const m = (model ?? "").trim();
if (!m) return true;
return m.toLowerCase() === (brandName ?? "").trim().toLowerCase();
}
/** Flag a vehicle as poison (generic-model / over-cap) so init/scan skip it,
* and clear its in-flight + progress state. */
private async markPoison(vehicleId: string): Promise<void> {
await this.redis.set(this.poisonKey(vehicleId), "1", POISON_TTL_S);
await this.redis.del(`prefetch:scheduled:${vehicleId}`);
await this.redis.del(`prefetch:progress:${vehicleId}`);
}
/**
* Per-source fixed-window (60s) rate limit. Increments the source's window
* counter; once it exceeds the source's ceiling, throws RateLimitError