Merge pull request 'feat(pl24): Faz 3 — backfill anahtarı + Mitsubishi parça-listesi düzeltmesi' (#270) from dev into main

apps/api/drizzle/0036_pl24_mitsubishi_partinfo_cleanup.sql

@@ -0,0 +1,20 @@
+-- Mitsubishi'nin parça listesi grup sanılıyordu.
+--
+-- `/p5mitsubishi/extern/details/vinDetails` yanıtı `partno`/`qty` taşıyan bir
+-- PARÇA listesi, ama her kaydın kendi linki `partInfoTable` (parça-detay) ve ne
+-- wid ne de yol sınıflandırıcıda karşılık buluyordu. Sonuç: 2.193 parça listesi
+-- grup düğümüne dönüştü ve içlerindeki 19.576 tekil parça ("SCREW,LOCK CYLINDER",
+-- "BOLT,STEERING COLUMN WASHER") kategori olarak kaydedildi. Ölçüm (prod,
+-- 2026-09-20): bu 19.576 sahte düğümün TOPLAM 2 tanesinde parça var, ve her
+-- prefetch turunda yeniden çekiliyorlar.
+--
+-- Sınıflandırıcı düzeltildi (detailsTable artık yaprak, partInfoTable hiç
+-- kuyruklanmıyor), ama okuma yolu bir düğümün ÖNCE çocuklarına bakıyor: sahte
+-- çocuklar dururken parça listesi asla çekilmez. Bu yüzden satırların silinmesi
+-- düzeltmenin parçası, ayrı bir temizlik değil.
+--
+-- Güvenli: yalnız pl24 kaynaklı ve yalnız bu iki imzayı taşıyan satırlar; ikisi
+-- de prod'da %100 Mitsubishi. Silinen ~2 parça satırı üst listeden yeniden gelir.
+DELETE FROM "categories"
+WHERE "source" = 'pl24'
+  AND ("link_wid" = 'partInfoTable' OR "link_path" ILIKE '%/details/vinpartinfo%');

apps/api/drizzle/meta/_journal.json

@@ -253,6 +253,13 @@
       "when": 1786435493000,
       "tag": "0035_subscription_dunning",
       "breakpoints": true
+    },
+    {
+      "idx": 36,
+      "version": "7",
+      "when": 1786521893000,
+      "tag": "0036_pl24_mitsubishi_partinfo_cleanup",
+      "breakpoints": true
     }
   ]
 }

apps/api/src/integrations/pl24/pl24-tree.spec.ts

@@ -1,5 +1,10 @@
 import { describe, expect, it } from "vitest";
-import { isPl24GroupNode, isPl24LeafNode, isStalePl24Architecture } from "./pl24-tree";
+import {
+  isPl24GroupNode,
+  isPl24LeafNode,
+  isPl24PartDetailNode,
+  isStalePl24Architecture,
+} from "./pl24-tree";
 
 // Canlı P5 yanıtlarından (2026-09-16 keşfi, plv2-artefakt/) alınan gerçek
 // wid + path çiftleri. Bu dosya "0 parça" sınıfı hatanın regresyon kilidi.
@@ -137,3 +142,45 @@ describe("isStalePl24Architecture — P4→P5 göç tespiti", () => {
     expect(isStalePl24Architecture({ catalogPath: "/psa/x", currentApiPath: null })).toBe(false);
   });
 });
+
+/**
+ * Mitsubishi: parça listesi grup sanılıyordu (plv2.md, bulgu consumers-13).
+ *
+ * `/p5mitsubishi/extern/details/vinDetails` yanıtı `partno`/`qty` taşıyan bir
+ * PARÇA listesi (canlı doğrulama 2026-09-20: 16 kayıt), ama her kaydın linki
+ * `partInfoTable`. Ne wid ne yol sınıflandırıcıda karşılık bulmuyordu → 2.193
+ * parça listesi gruba, içlerindeki 19.576 tekil parça kategoriye dönüşmüştü;
+ * hepsinde toplam 2 parça vardı ve her prefetch turunda yeniden çekiliyorlardı.
+ */
+describe("Mitsubishi detailsTable / partInfoTable", () => {
+  const detailsPath =
+    "/p5mitsubishi/extern/details/vinDetails?bomDetails=133_110D00125Y&mainGroup=33";
+  const partInfoPath = "/p5mitsubishi/extern/details/vinpartinfo?bomDetails=142_7103K22Y5T";
+
+  it("detailsTable bir parça listesi — yaprak", () => {
+    expect(isPl24LeafNode({ linkWid: "detailsTable", linkPath: detailsPath })).toBe(true);
+    expect(isPl24GroupNode({ linkWid: "detailsTable", linkPath: detailsPath })).toBe(false);
+  });
+
+  it("partInfoTable tekil parça detayı — ne grup ne liste", () => {
+    expect(isPl24PartDetailNode({ linkWid: "partInfoTable", linkPath: partInfoPath })).toBe(true);
+    expect(isPl24GroupNode({ linkWid: "partInfoTable", linkPath: partInfoPath })).toBe(false);
+  });
+
+  it("wid yoksa yol kalıbı parça-detayını yine yakalar", () => {
+    expect(isPl24PartDetailNode({ linkPath: partInfoPath })).toBe(true);
+  });
+
+  it("gerçek parça uçlarını parça-detayı sanmaz", () => {
+    // Volvo'nun partinfo yaprağı ve VW'nin bom listesi etkilenmemeli.
+    expect(
+      isPl24PartDetailNode({
+        linkWid: "partinfo",
+        linkPath: "/p5volvo/extern/partinfo/vin?partno=1",
+      }),
+    ).toBe(false);
+    expect(
+      isPl24PartDetailNode({ linkWid: "bomlist", linkPath: "/p5vwag/extern/bom/vin?x=1" }),
+    ).toBe(false);
+  });
+});

apps/api/src/integrations/pl24/pl24-tree.ts

@@ -22,8 +22,44 @@
  * drill. Path matching stays as a fallback for stored rows without a wid.
  */
 
-/** `link.wid` values that identify a parts (BOM) node across every P5 backend. */
-const LEAF_WIDS = new Set(["bomlist", "bomoverviewlist", "servicepartsitemstable", "partinfo"]);
+/**
+ * `link.wid` values that identify a parts (BOM) node across every P5 backend.
+ *
+ * `detailstable` is Mitsubishi's: `/p5mitsubishi/extern/details/vinDetails`
+ * answers with 16-ish records carrying `partno`/`qty`, i.e. it IS the parts
+ * list — but each record's own link is a `partInfoTable` per-part detail, and
+ * neither the wid nor the path matched anything here, so the whole list was
+ * drilled as a group. Prod on 2026-09-20: 2,193 Mitsubishi parts lists turned
+ * into group nodes and their 19,576 individual parts ("SCREW,LOCK CYLINDER",
+ * "BOLT,STEERING COLUMN WASHER") became categories — 19,576 fake tree nodes
+ * with 2 parts between them, each re-fetched on every prefetch pass.
+ */
+const LEAF_WIDS = new Set([
+  "bomlist",
+  "bomoverviewlist",
+  "servicepartsitemstable",
+  "partinfo",
+  "detailstable",
+]);
+
+/**
+ * Nodes that describe ONE part rather than a list of them. They are neither a
+ * group to drill nor a list to fetch: the parent's own response already carried
+ * the part. Queueing them buys nothing and costs one upstream request each —
+ * 19,576 of them on prod before this was recognised.
+ */
+const PART_DETAIL_WIDS = new Set(["partinfotable"]);
+const PART_DETAIL_PATH = /\/details\/vinpartinfo\b/i;
+
+/** True when this node is a single part's detail view, not a listing. */
+export function isPl24PartDetailNode(opts: {
+  linkPath?: string | null;
+  linkWid?: string | null;
+}): boolean {
+  const wid = opts.linkWid?.toLowerCase().trim();
+  if (wid && PART_DETAIL_WIDS.has(wid)) return true;
+  return PART_DETAIL_PATH.test(opts.linkPath ?? "");
+}
 
 /** `link.wid` values that identify a drillable group node. */
 const GROUP_WID_PATTERN =
@@ -61,6 +97,8 @@ export function isPl24GroupNode(opts: {
   hasSubgroups?: boolean | null;
 }): boolean {
   if (!opts.linkPath && !opts.linkWid) return false;
+  // A per-part detail node is not a group; drilling it returns nothing.
+  if (isPl24PartDetailNode(opts)) return false;
   return !isPl24LeafNode(opts);
 }
 

apps/api/src/jobs/prefetch-worker.service.spec.ts

@@ -225,7 +225,18 @@ describe("PrefetchWorkerService — fast lane (lifo) + backlog gating", () => {
   });
 
   describe("checkSourceRate — per-source rate limit", () => {
-    type CSR = { checkSourceRate: (s: string) => Promise<void> };
+    type CSR = { checkSourceRate: (s: string, lane?: "main" | "fast") => Promise<void> };
+
+    // pl24's MAIN lane is parked whenever background backfill is off, which is
+    // the default — so these ceiling tests turn it on explicitly to exercise the
+    // rate limiter itself rather than the backfill gate (covered separately).
+    beforeEach(() => {
+      process.env.PL24_BACKFILL_ENABLED = "true";
+      process.env.PL24_TR_DISABLED = undefined;
+    });
+    afterEach(() => {
+      process.env.PL24_BACKFILL_ENABLED = undefined;
+    });
 
     it("passes when under the source ceiling", async () => {
       const { service, redis } = makeDeps({ waiting: 0, limitResults: [] });
@@ -241,6 +252,24 @@ describe("PrefetchWorkerService — fast lane (lifo) + backlog gating", () => {
       });
     });
 
+    it("pl24 MAIN lane parkta iken hiç sayaç harcamaz (varsayılan)", async () => {
+      process.env.PL24_BACKFILL_ENABLED = undefined;
+      const { service, redis } = makeDeps({ waiting: 0, limitResults: [] });
+      await expect((service as never as CSR).checkSourceRate("pl24", "main")).rejects.toMatchObject(
+        { cause: "source-rate" },
+      );
+      expect(redis.incr).not.toHaveBeenCalled();
+    });
+
+    it("kullanıcı (fast) şeridi backfill anahtarından etkilenmez", async () => {
+      process.env.PL24_BACKFILL_ENABLED = undefined;
+      const { service, redis } = makeDeps({ waiting: 0, limitResults: [] });
+      redis.incr.mockResolvedValueOnce(1);
+      await expect(
+        (service as never as CSR).checkSourceRate("pl24", "fast"),
+      ).resolves.toBeUndefined();
+    });
+
     it("is unlimited (no counter) for a source without a configured ceiling", async () => {
       const { service, redis } = makeDeps({ waiting: 0, limitResults: [] });
       await (service as never as CSR).checkSourceRate("unknown-source");
@@ -339,3 +368,36 @@ describe("PrefetchWorkerService — PL24 derinlik tavanı", () => {
     expect(fn("parts-catalogs", true)).toBeGreaterThan(1);
   });
 });
+
+/**
+ * PL24 arka plan backfill anahtarı (Faz 3) + Mitsubishi parça-detay kırpması.
+ */
+describe("PL24 backfill anahtarı", () => {
+  const ENV = { ...process.env };
+  afterEach(() => {
+    process.env = { ...ENV };
+  });
+
+  it("varsayılan KAPALI — değişken hiç yoksa arka plan akmaz", async () => {
+    const { __testables } = await import("./prefetch-worker.service");
+    process.env.PL24_BACKFILL_ENABLED = undefined;
+    process.env.PL24_TR_DISABLED = undefined;
+    expect(__testables.isPl24BackfillEnabled()).toBe(false);
+  });
+
+  it("yalnız açık 'true' ile açılır", async () => {
+    const { __testables } = await import("./prefetch-worker.service");
+    process.env.PL24_TR_DISABLED = undefined;
+    process.env.PL24_BACKFILL_ENABLED = "true";
+    expect(__testables.isPl24BackfillEnabled()).toBe(true);
+    process.env.PL24_BACKFILL_ENABLED = "1";
+    expect(__testables.isPl24BackfillEnabled()).toBe(false);
+  });
+
+  it("eski PL24_TR_DISABLED hâlâ kapatabilir (yarım deploy musluğu açamaz)", async () => {
+    const { __testables } = await import("./prefetch-worker.service");
+    process.env.PL24_BACKFILL_ENABLED = "true";
+    process.env.PL24_TR_DISABLED = "true";
+    expect(__testables.isPl24BackfillEnabled()).toBe(false);
+  });
+});

apps/api/src/jobs/prefetch-worker.service.ts

@@ -10,7 +10,7 @@ import { and, asc, eq, gt, inArray, isNull, notExists, sql } from "drizzle-orm";
 import { CategoriesService } from "../categories/categories.service";
 import { DATABASE, type Database } from "../database/database.provider";
 import { categories, parts, vehicles } from "../database/schema/core";
-import { isPl24LeafNode } from "../integrations/pl24/pl24-tree";
+import { isPl24LeafNode, isPl24PartDetailNode } from "../integrations/pl24/pl24-tree";
 import { PostHogService } from "../posthog/posthog.service";
 import { RedisService } from "../redis/redis.service";
 import { QUEUE_NAMES, getBullConnection } from "./bull.config";
@@ -101,6 +101,25 @@ const EST_JOBS_PER_VEHICLE = Number(process.env.PREFETCH_EST_JOBS_PER_VEHICLE) |
  */
 const DAILY_FAST_RESERVE = 0.2;
 /** Only these decode sources have catalogs worth prefetching. */
+/**
+ * Whether the PL24 *background* backfill lane may run. The user-triggered fast
+ * lane is never gated by this.
+ *
+ * Default is OFF. Two PL24 accounts were banned while bulk background load ran
+ * against them, so the background lane has to be switched on deliberately and
+ * watched, never inherited from an unset variable.
+ *
+ * `PL24_BACKFILL_ENABLED` replaces the old `PL24_TR_DISABLED`, whose name said
+ * "the tr account is dead" while its actual job was "keep bulk load off the one
+ * surviving account". The old variable is still honoured so a half-applied
+ * deploy cannot silently open the tap: it can only keep the lane closed.
+ */
+function isPl24BackfillEnabled(): boolean {
+  if (process.env.PL24_BACKFILL_ENABLED !== "true") return false;
+  // Legacy kill switch still wins while it is explicitly set.
+  return process.env.PL24_TR_DISABLED !== "true";
+}
+
 const BACKFILL_SOURCES = ["pl24", "emex", "parts-catalogs"];
 /** Redis key holding the rolling rescan cursor (last createdAt seen). */
 const BACKFILL_CURSOR_KEY = "prefetch:backfill:cursor";
@@ -204,7 +223,7 @@ function jitter(ms: number): number {
 }
 
 /** Test-only surface for the pure helpers above. */
-export const __testables = { maxDepthFor, jitter };
+export const __testables = { maxDepthFor, jitter, isPl24BackfillEnabled };
 
 // ── Phase-1 residue exclusion ──
 /**
@@ -524,7 +543,7 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy {
           // inflated progress.total makes the chain never reach "finished".
           queued += await this.queueCategoryJob(child, vehicleId, source, 1, fast);
         }
-      } else if (this.isLeafLinkPath(cat.linkPath, cat.source, cat.hasSubgroups)) {
+      } else if (this.isLeafLinkPath(cat.linkPath, cat.source, cat.hasSubgroups, cat.linkWid)) {
         // Leaf — check if parts already fetched
         const [partCheck] = await this.db
           .select({ id: parts.id })
@@ -740,10 +759,9 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy {
     // runaway was built. This also preserves the fast-lane reserve for real users.
     const eligible: string[] = [];
     for (const s of BACKFILL_SOURCES) {
-      // PL24_TR_DISABLED: tr is dead on PL24's side and ALL pl24 traffic maps to
-      // the sole surviving de account — keep background backfill off it entirely
-      // (fast/user lane still flows) so bulk load can't burn the last account.
-      if (s === "pl24" && process.env.PL24_TR_DISABLED === "true") continue;
+      // Background backfill is opt-in per source; pl24 defaults to OFF so bulk
+      // load can never burn the one surviving account by accident.
+      if (s === "pl24" && !isPl24BackfillEnabled()) continue;
       if (await this.redis.exists(`prefetch:activity:${s}`)) continue;
       if (cfg.businessHoursOnly !== false && !isWithinTimeWindow(s)) continue;
       const mainLimit = this.dailyMainLimit(s);
@@ -894,6 +912,7 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy {
     cat: {
       id: string;
       linkPath: string | null;
+      linkWid?: string | null;
       source: string;
       unavailable: boolean;
       hasSubgroups?: boolean | null;
@@ -905,7 +924,18 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy {
   ): Promise<number> {
     if (cat.unavailable) return 0;
 
-    if (this.isLeafLinkPath(cat.linkPath, cat.source, cat.hasSubgroups)) {
+    // A per-part detail node (Mitsubishi `partInfoTable` /details/vinpartinfo) is
+    // neither a group nor a listing: its parent's response already carried the
+    // part. Queueing it costs one upstream request and returns nothing. Prod had
+    // 19,576 of these, with 2 parts between them.
+    if (
+      cat.source === "pl24" &&
+      isPl24PartDetailNode({ linkPath: cat.linkPath, linkWid: cat.linkWid })
+    ) {
+      return 0;
+    }
+
+    if (this.isLeafLinkPath(cat.linkPath, cat.source, cat.hasSubgroups, cat.linkWid)) {
       // Leaf — check if already has parts
       const [partCheck] = await this.db
         .select({ id: parts.id })
@@ -970,6 +1000,7 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy {
     linkPath: string | null,
     source: string,
     hasSubgroups?: boolean | null,
+    linkWid?: string | null,
   ): boolean {
     if (!linkPath) return false;
     // EMEX: Vehicle.aspx group nodes are parents to drill; Unit.aspx (hierarchical
@@ -988,8 +1019,13 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy {
     // PL24: one shared classifier (integrations/pl24/pl24-tree). The old inline
     // list was case-sensitive, so p5psa/p5volvo's camelCase `/details/vin/
     // bomDetails` was never recognised as a leaf and its parts were never
-    // prefetched (still true today for Mitsubishi/Fiat/Renault).
-    return isPl24LeafNode({ linkPath, hasSubgroups });
+    // prefetched.
+    // `linkWid` is passed through on purpose: it is the reliable cross-brand
+    // marker and the read path has always used it, but this queueing path used
+    // to drop it and fall back to path matching alone. That is why Mitsubishi's
+    // `detailsTable` parts list was queued as a group here even after the shared
+    // classifier learned about it.
+    return isPl24LeafNode({ linkPath, hasSubgroups, linkWid });
   }
 
   /**
@@ -1186,10 +1222,10 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy {
     // (observed: fresh BMW init couldn't get a single pcat slot). Totals per
     // source stay the same as before, so upstream load is unchanged.
     const total = SOURCE_RATE_MAX[source] ?? 0;
-    // PL24_TR_DISABLED: park already-queued pl24 MAIN-lane jobs (long defer, no
-    // attempt consumed) — the eligibility scan stops producing new ones, this
-    // stops the existing backlog from draining through the de account.
-    if (source === "pl24" && lane === "main" && process.env.PL24_TR_DISABLED === "true") {
+    // Park already-queued pl24 MAIN-lane jobs while background backfill is off
+    // (long defer, no attempt consumed) — the eligibility scan stops producing
+    // new ones, this stops an existing backlog from draining through the account.
+    if (source === "pl24" && lane === "main" && !isPl24BackfillEnabled()) {
       throw new RateLimitError(15 * 60_000, "source-rate");
     }
     if (total <= 0) return;

docker-compose.coolify.yml

@@ -49,6 +49,7 @@ services:
       - PL24_PASSWORD_2=${PL24_PASSWORD_2:-}
       - PL24_PROXY_DE=${PL24_PROXY_DE:-}
       - PL24_TR_DISABLED=${PL24_TR_DISABLED:-}
+      - PL24_BACKFILL_ENABLED=${PL24_BACKFILL_ENABLED:-}
       - EMEX_USERNAME=${EMEX_USERNAME:-}
       - EMEX_PASSWORD=${EMEX_PASSWORD:-}
       - EMEX_USE_PROXY=${EMEX_USE_PROXY:-false}
@@ -207,6 +208,7 @@ services:
       - PL24_PASSWORD_2=${PL24_PASSWORD_2:-}
       - PL24_PROXY_DE=${PL24_PROXY_DE:-}
       - PL24_TR_DISABLED=${PL24_TR_DISABLED:-}
+      - PL24_BACKFILL_ENABLED=${PL24_BACKFILL_ENABLED:-}
       - EMEX_USERNAME=${EMEX_USERNAME:-}
       - EMEX_PASSWORD=${EMEX_PASSWORD:-}
       - EMEX_USE_PROXY=${EMEX_USE_PROXY:-false}