feat(pl24): Faz 3 — backfill anahtarı + Mitsubishi parça-listesi düzeltmesi
Some checks failed
QA Gate (P0/P1) / Test affected app (pull_request) Has been cancelled

Faz 3 backfill'i açmadan önce hacmi ölçtüm ve raporun işaret ettiğinden
çok daha büyük bir israf çıktı.

**Mitsubishi'nin parça listesi grup sanılıyordu.**
`/p5mitsubishi/extern/details/vinDetails` yanıtı `partno`/`qty` taşıyan
bir PARÇA listesi (canlı doğrulama: 16 kayıt), ama her kaydın kendi
linki `partInfoTable` ve ne wid ne yol sınıflandırıcıda karşılık
buluyordu. Sonuç (prod ölçümü): 2.193 parça listesi grup düğümüne
döndü, içlerindeki 19.576 tekil parça ("SCREW,LOCK CYLINDER",
"BOLT,STEERING COLUMN WASHER") kategori olarak kaydedildi. Bu 19.576
sahte düğümün TOPLAM 2 tanesinde parça var ve hepsi her prefetch
turunda yeniden çekiliyor. İkisi de %100 Mitsubishi.

- `detailsTable` artık yaprak, `isPl24PartDetailNode()` ile
  `partInfoTable` hiç kuyruklanmıyor.
- `isLeafLinkPath` artık `linkWid`'i de geçiriyor. Okuma yolu bu
  güvenilir sinyali hep kullanıyordu ama kuyruklama yolu düşürüyordu —
  sınıflandırıcı `detailsTable`'ı öğrense bile burada yine grup
  sayılacaktı.
- Migration 0036: 19.576 sahte kategori siliniyor. Okuma yolu bir
  düğümün ÖNCE çocuklarına baktığı için bu silme düzeltmenin parçası,
  ayrı temizlik değil. Kuru çalıştırma: 19.576 kategori, 9 araç, 2
  parça, Mitsubishi dışı 0.

**Backfill anahtarı.** `PL24_BACKFILL_ENABLED` eklendi, varsayılan
KAPALI. Eski `PL24_TR_DISABLED` adı "tr hesabı öldü" diyordu ama işi
"toplu yükü tek sağ kalan hesaptan uzak tut"tu. Eski değişken hâlâ
kapatabiliyor — yarım deploy musluğu sessizce açamasın. Kullanıcı
tetikli fast-lane bu anahtardan etkilenmiyor.

9 yeni test. api 647 test geçiyor.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-09-20 10:51:42 +03:00
parent e8ffa6a5db
commit 142ec3a150
7 changed files with 230 additions and 18 deletions

View File

@@ -0,0 +1,20 @@
-- Mitsubishi'nin parça listesi grup sanılıyordu.
--
-- `/p5mitsubishi/extern/details/vinDetails` yanıtı `partno`/`qty` taşıyan bir
-- PARÇA listesi, ama her kaydın kendi linki `partInfoTable` (parça-detay) ve ne
-- wid ne de yol sınıflandırıcıda karşılık buluyordu. Sonuç: 2.193 parça listesi
-- grup düğümüne dönüştü ve içlerindeki 19.576 tekil parça ("SCREW,LOCK CYLINDER",
-- "BOLT,STEERING COLUMN WASHER") kategori olarak kaydedildi. Ölçüm (prod,
-- 2026-09-20): bu 19.576 sahte düğümün TOPLAM 2 tanesinde parça var, ve her
-- prefetch turunda yeniden çekiliyorlar.
--
-- Sınıflandırıcı düzeltildi (detailsTable artık yaprak, partInfoTable hiç
-- kuyruklanmıyor), ama okuma yolu bir düğümün ÖNCE çocuklarına bakıyor: sahte
-- çocuklar dururken parça listesi asla çekilmez. Bu yüzden satırların silinmesi
-- düzeltmenin parçası, ayrı bir temizlik değil.
--
-- Güvenli: yalnız pl24 kaynaklı ve yalnız bu iki imzayı taşıyan satırlar; ikisi
-- de prod'da %100 Mitsubishi. Silinen ~2 parça satırı üst listeden yeniden gelir.
DELETE FROM "categories"
WHERE "source" = 'pl24'
AND ("link_wid" = 'partInfoTable' OR "link_path" ILIKE '%/details/vinpartinfo%');

View File

@@ -253,6 +253,13 @@
"when": 1786435493000,
"tag": "0035_subscription_dunning",
"breakpoints": true
},
{
"idx": 36,
"version": "7",
"when": 1786521893000,
"tag": "0036_pl24_mitsubishi_partinfo_cleanup",
"breakpoints": true
}
]
}

View File

@@ -1,5 +1,10 @@
import { describe, expect, it } from "vitest";
import { isPl24GroupNode, isPl24LeafNode, isStalePl24Architecture } from "./pl24-tree";
import {
isPl24GroupNode,
isPl24LeafNode,
isPl24PartDetailNode,
isStalePl24Architecture,
} from "./pl24-tree";
// Canlı P5 yanıtlarından (2026-09-16 keşfi, plv2-artefakt/) alınan gerçek
// wid + path çiftleri. Bu dosya "0 parça" sınıfı hatanın regresyon kilidi.
@@ -137,3 +142,45 @@ describe("isStalePl24Architecture — P4→P5 göç tespiti", () => {
expect(isStalePl24Architecture({ catalogPath: "/psa/x", currentApiPath: null })).toBe(false);
});
});
/**
* Mitsubishi: parça listesi grup sanılıyordu (plv2.md, bulgu consumers-13).
*
* `/p5mitsubishi/extern/details/vinDetails` yanıtı `partno`/`qty` taşıyan bir
* PARÇA listesi (canlı doğrulama 2026-09-20: 16 kayıt), ama her kaydın linki
* `partInfoTable`. Ne wid ne yol sınıflandırıcıda karşılık bulmuyordu → 2.193
* parça listesi gruba, içlerindeki 19.576 tekil parça kategoriye dönüşmüştü;
* hepsinde toplam 2 parça vardı ve her prefetch turunda yeniden çekiliyorlardı.
*/
describe("Mitsubishi detailsTable / partInfoTable", () => {
const detailsPath =
"/p5mitsubishi/extern/details/vinDetails?bomDetails=133_110D00125Y&mainGroup=33";
const partInfoPath = "/p5mitsubishi/extern/details/vinpartinfo?bomDetails=142_7103K22Y5T";
it("detailsTable bir parça listesi — yaprak", () => {
expect(isPl24LeafNode({ linkWid: "detailsTable", linkPath: detailsPath })).toBe(true);
expect(isPl24GroupNode({ linkWid: "detailsTable", linkPath: detailsPath })).toBe(false);
});
it("partInfoTable tekil parça detayı — ne grup ne liste", () => {
expect(isPl24PartDetailNode({ linkWid: "partInfoTable", linkPath: partInfoPath })).toBe(true);
expect(isPl24GroupNode({ linkWid: "partInfoTable", linkPath: partInfoPath })).toBe(false);
});
it("wid yoksa yol kalıbı parça-detayını yine yakalar", () => {
expect(isPl24PartDetailNode({ linkPath: partInfoPath })).toBe(true);
});
it("gerçek parça uçlarını parça-detayı sanmaz", () => {
// Volvo'nun partinfo yaprağı ve VW'nin bom listesi etkilenmemeli.
expect(
isPl24PartDetailNode({
linkWid: "partinfo",
linkPath: "/p5volvo/extern/partinfo/vin?partno=1",
}),
).toBe(false);
expect(
isPl24PartDetailNode({ linkWid: "bomlist", linkPath: "/p5vwag/extern/bom/vin?x=1" }),
).toBe(false);
});
});

View File

@@ -22,8 +22,44 @@
* drill. Path matching stays as a fallback for stored rows without a wid.
*/
/** `link.wid` values that identify a parts (BOM) node across every P5 backend. */
const LEAF_WIDS = new Set(["bomlist", "bomoverviewlist", "servicepartsitemstable", "partinfo"]);
/**
* `link.wid` values that identify a parts (BOM) node across every P5 backend.
*
* `detailstable` is Mitsubishi's: `/p5mitsubishi/extern/details/vinDetails`
* answers with 16-ish records carrying `partno`/`qty`, i.e. it IS the parts
* list — but each record's own link is a `partInfoTable` per-part detail, and
* neither the wid nor the path matched anything here, so the whole list was
* drilled as a group. Prod on 2026-09-20: 2,193 Mitsubishi parts lists turned
* into group nodes and their 19,576 individual parts ("SCREW,LOCK CYLINDER",
* "BOLT,STEERING COLUMN WASHER") became categories — 19,576 fake tree nodes
* with 2 parts between them, each re-fetched on every prefetch pass.
*/
const LEAF_WIDS = new Set([
"bomlist",
"bomoverviewlist",
"servicepartsitemstable",
"partinfo",
"detailstable",
]);
/**
* Nodes that describe ONE part rather than a list of them. They are neither a
* group to drill nor a list to fetch: the parent's own response already carried
* the part. Queueing them buys nothing and costs one upstream request each —
* 19,576 of them on prod before this was recognised.
*/
const PART_DETAIL_WIDS = new Set(["partinfotable"]);
const PART_DETAIL_PATH = /\/details\/vinpartinfo\b/i;
/** True when this node is a single part's detail view, not a listing. */
export function isPl24PartDetailNode(opts: {
linkPath?: string | null;
linkWid?: string | null;
}): boolean {
const wid = opts.linkWid?.toLowerCase().trim();
if (wid && PART_DETAIL_WIDS.has(wid)) return true;
return PART_DETAIL_PATH.test(opts.linkPath ?? "");
}
/** `link.wid` values that identify a drillable group node. */
const GROUP_WID_PATTERN =
@@ -61,6 +97,8 @@ export function isPl24GroupNode(opts: {
hasSubgroups?: boolean | null;
}): boolean {
if (!opts.linkPath && !opts.linkWid) return false;
// A per-part detail node is not a group; drilling it returns nothing.
if (isPl24PartDetailNode(opts)) return false;
return !isPl24LeafNode(opts);
}

View File

@@ -225,7 +225,18 @@ describe("PrefetchWorkerService — fast lane (lifo) + backlog gating", () => {
});
describe("checkSourceRate — per-source rate limit", () => {
type CSR = { checkSourceRate: (s: string) => Promise<void> };
type CSR = { checkSourceRate: (s: string, lane?: "main" | "fast") => Promise<void> };
// pl24's MAIN lane is parked whenever background backfill is off, which is
// the default — so these ceiling tests turn it on explicitly to exercise the
// rate limiter itself rather than the backfill gate (covered separately).
beforeEach(() => {
process.env.PL24_BACKFILL_ENABLED = "true";
process.env.PL24_TR_DISABLED = undefined;
});
afterEach(() => {
process.env.PL24_BACKFILL_ENABLED = undefined;
});
it("passes when under the source ceiling", async () => {
const { service, redis } = makeDeps({ waiting: 0, limitResults: [] });
@@ -241,6 +252,24 @@ describe("PrefetchWorkerService — fast lane (lifo) + backlog gating", () => {
});
});
it("pl24 MAIN lane parkta iken hiç sayaç harcamaz (varsayılan)", async () => {
process.env.PL24_BACKFILL_ENABLED = undefined;
const { service, redis } = makeDeps({ waiting: 0, limitResults: [] });
await expect((service as never as CSR).checkSourceRate("pl24", "main")).rejects.toMatchObject(
{ cause: "source-rate" },
);
expect(redis.incr).not.toHaveBeenCalled();
});
it("kullanıcı (fast) şeridi backfill anahtarından etkilenmez", async () => {
process.env.PL24_BACKFILL_ENABLED = undefined;
const { service, redis } = makeDeps({ waiting: 0, limitResults: [] });
redis.incr.mockResolvedValueOnce(1);
await expect(
(service as never as CSR).checkSourceRate("pl24", "fast"),
).resolves.toBeUndefined();
});
it("is unlimited (no counter) for a source without a configured ceiling", async () => {
const { service, redis } = makeDeps({ waiting: 0, limitResults: [] });
await (service as never as CSR).checkSourceRate("unknown-source");
@@ -339,3 +368,36 @@ describe("PrefetchWorkerService — PL24 derinlik tavanı", () => {
expect(fn("parts-catalogs", true)).toBeGreaterThan(1);
});
});
/**
* PL24 arka plan backfill anahtarı (Faz 3) + Mitsubishi parça-detay kırpması.
*/
describe("PL24 backfill anahtarı", () => {
const ENV = { ...process.env };
afterEach(() => {
process.env = { ...ENV };
});
it("varsayılan KAPALI — değişken hiç yoksa arka plan akmaz", async () => {
const { __testables } = await import("./prefetch-worker.service");
process.env.PL24_BACKFILL_ENABLED = undefined;
process.env.PL24_TR_DISABLED = undefined;
expect(__testables.isPl24BackfillEnabled()).toBe(false);
});
it("yalnız açık 'true' ile açılır", async () => {
const { __testables } = await import("./prefetch-worker.service");
process.env.PL24_TR_DISABLED = undefined;
process.env.PL24_BACKFILL_ENABLED = "true";
expect(__testables.isPl24BackfillEnabled()).toBe(true);
process.env.PL24_BACKFILL_ENABLED = "1";
expect(__testables.isPl24BackfillEnabled()).toBe(false);
});
it("eski PL24_TR_DISABLED hâlâ kapatabilir (yarım deploy musluğu açamaz)", async () => {
const { __testables } = await import("./prefetch-worker.service");
process.env.PL24_BACKFILL_ENABLED = "true";
process.env.PL24_TR_DISABLED = "true";
expect(__testables.isPl24BackfillEnabled()).toBe(false);
});
});

View File

@@ -10,7 +10,7 @@ import { and, asc, eq, gt, inArray, isNull, notExists, sql } from "drizzle-orm";
import { CategoriesService } from "../categories/categories.service";
import { DATABASE, type Database } from "../database/database.provider";
import { categories, parts, vehicles } from "../database/schema/core";
import { isPl24LeafNode } from "../integrations/pl24/pl24-tree";
import { isPl24LeafNode, isPl24PartDetailNode } from "../integrations/pl24/pl24-tree";
import { PostHogService } from "../posthog/posthog.service";
import { RedisService } from "../redis/redis.service";
import { QUEUE_NAMES, getBullConnection } from "./bull.config";
@@ -101,6 +101,25 @@ const EST_JOBS_PER_VEHICLE = Number(process.env.PREFETCH_EST_JOBS_PER_VEHICLE) |
*/
const DAILY_FAST_RESERVE = 0.2;
/** Only these decode sources have catalogs worth prefetching. */
/**
* Whether the PL24 *background* backfill lane may run. The user-triggered fast
* lane is never gated by this.
*
* Default is OFF. Two PL24 accounts were banned while bulk background load ran
* against them, so the background lane has to be switched on deliberately and
* watched, never inherited from an unset variable.
*
* `PL24_BACKFILL_ENABLED` replaces the old `PL24_TR_DISABLED`, whose name said
* "the tr account is dead" while its actual job was "keep bulk load off the one
* surviving account". The old variable is still honoured so a half-applied
* deploy cannot silently open the tap: it can only keep the lane closed.
*/
function isPl24BackfillEnabled(): boolean {
if (process.env.PL24_BACKFILL_ENABLED !== "true") return false;
// Legacy kill switch still wins while it is explicitly set.
return process.env.PL24_TR_DISABLED !== "true";
}
const BACKFILL_SOURCES = ["pl24", "emex", "parts-catalogs"];
/** Redis key holding the rolling rescan cursor (last createdAt seen). */
const BACKFILL_CURSOR_KEY = "prefetch:backfill:cursor";
@@ -204,7 +223,7 @@ function jitter(ms: number): number {
}
/** Test-only surface for the pure helpers above. */
export const __testables = { maxDepthFor, jitter };
export const __testables = { maxDepthFor, jitter, isPl24BackfillEnabled };
// ── Phase-1 residue exclusion ──
/**
@@ -524,7 +543,7 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy {
// inflated progress.total makes the chain never reach "finished".
queued += await this.queueCategoryJob(child, vehicleId, source, 1, fast);
}
} else if (this.isLeafLinkPath(cat.linkPath, cat.source, cat.hasSubgroups)) {
} else if (this.isLeafLinkPath(cat.linkPath, cat.source, cat.hasSubgroups, cat.linkWid)) {
// Leaf — check if parts already fetched
const [partCheck] = await this.db
.select({ id: parts.id })
@@ -740,10 +759,9 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy {
// runaway was built. This also preserves the fast-lane reserve for real users.
const eligible: string[] = [];
for (const s of BACKFILL_SOURCES) {
// PL24_TR_DISABLED: tr is dead on PL24's side and ALL pl24 traffic maps to
// the sole surviving de account — keep background backfill off it entirely
// (fast/user lane still flows) so bulk load can't burn the last account.
if (s === "pl24" && process.env.PL24_TR_DISABLED === "true") continue;
// Background backfill is opt-in per source; pl24 defaults to OFF so bulk
// load can never burn the one surviving account by accident.
if (s === "pl24" && !isPl24BackfillEnabled()) continue;
if (await this.redis.exists(`prefetch:activity:${s}`)) continue;
if (cfg.businessHoursOnly !== false && !isWithinTimeWindow(s)) continue;
const mainLimit = this.dailyMainLimit(s);
@@ -894,6 +912,7 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy {
cat: {
id: string;
linkPath: string | null;
linkWid?: string | null;
source: string;
unavailable: boolean;
hasSubgroups?: boolean | null;
@@ -905,7 +924,18 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy {
): Promise<number> {
if (cat.unavailable) return 0;
if (this.isLeafLinkPath(cat.linkPath, cat.source, cat.hasSubgroups)) {
// A per-part detail node (Mitsubishi `partInfoTable` /details/vinpartinfo) is
// neither a group nor a listing: its parent's response already carried the
// part. Queueing it costs one upstream request and returns nothing. Prod had
// 19,576 of these, with 2 parts between them.
if (
cat.source === "pl24" &&
isPl24PartDetailNode({ linkPath: cat.linkPath, linkWid: cat.linkWid })
) {
return 0;
}
if (this.isLeafLinkPath(cat.linkPath, cat.source, cat.hasSubgroups, cat.linkWid)) {
// Leaf — check if already has parts
const [partCheck] = await this.db
.select({ id: parts.id })
@@ -970,6 +1000,7 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy {
linkPath: string | null,
source: string,
hasSubgroups?: boolean | null,
linkWid?: string | null,
): boolean {
if (!linkPath) return false;
// EMEX: Vehicle.aspx group nodes are parents to drill; Unit.aspx (hierarchical
@@ -988,8 +1019,13 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy {
// PL24: one shared classifier (integrations/pl24/pl24-tree). The old inline
// list was case-sensitive, so p5psa/p5volvo's camelCase `/details/vin/
// bomDetails` was never recognised as a leaf and its parts were never
// prefetched (still true today for Mitsubishi/Fiat/Renault).
return isPl24LeafNode({ linkPath, hasSubgroups });
// prefetched.
// `linkWid` is passed through on purpose: it is the reliable cross-brand
// marker and the read path has always used it, but this queueing path used
// to drop it and fall back to path matching alone. That is why Mitsubishi's
// `detailsTable` parts list was queued as a group here even after the shared
// classifier learned about it.
return isPl24LeafNode({ linkPath, hasSubgroups, linkWid });
}
/**
@@ -1186,10 +1222,10 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy {
// (observed: fresh BMW init couldn't get a single pcat slot). Totals per
// source stay the same as before, so upstream load is unchanged.
const total = SOURCE_RATE_MAX[source] ?? 0;
// PL24_TR_DISABLED: park already-queued pl24 MAIN-lane jobs (long defer, no
// attempt consumed) — the eligibility scan stops producing new ones, this
// stops the existing backlog from draining through the de account.
if (source === "pl24" && lane === "main" && process.env.PL24_TR_DISABLED === "true") {
// Park already-queued pl24 MAIN-lane jobs while background backfill is off
// (long defer, no attempt consumed) — the eligibility scan stops producing
// new ones, this stops an existing backlog from draining through the account.
if (source === "pl24" && lane === "main" && !isPl24BackfillEnabled()) {
throw new RateLimitError(15 * 60_000, "source-rate");
}
if (total <= 0) return;