diff --git a/apps/api/src/catalog/catalog-browse-heal.spec.ts b/apps/api/src/catalog/catalog-browse-heal.spec.ts new file mode 100644 index 0000000..57f1be2 --- /dev/null +++ b/apps/api/src/catalog/catalog-browse-heal.spec.ts @@ -0,0 +1,192 @@ +import { describe, expect, it, vi } from "vitest"; +import { isStaleBrowseArchitecture } from "../integrations/pl24/pl24-tree"; +import { CatalogService } from "./catalog.service"; + +/** + * Regression lock for the pinned-browse-rows bug (plv2.md, finding consumers-09). + * + * `catalog_vehicles` is a permanent cache: `getModels` returns early as soon as + * a brand has any row, so `fetchVehicleList` never runs again for that brand. + * The 188 rows listed while PSA and Volvo were still P4 (127 LEGACY_PSA + 61 + * LEGACY_VOLVO on prod, 2026-09-20) were therefore stuck serving the frozen + * 2024-02-13 PSA snapshot and a Volvo endpoint that answers HTTP 503. + */ + +type Row = { + id: string; + serviceName: string; + architecture: string | null; + brandId: string | null; +}; + +const psaRow = (id: string): Row => ({ + id, + serviceName: "peugeot_parts", + architecture: "LEGACY_PSA", + brandId: "b1", +}); +const freshRow = (id: string): Row => ({ + id, + serviceName: "peugeot_parts", + architecture: "P5_MODERN", + brandId: "b1", +}); + +function makeService(opts: { + lockFree?: boolean; + fetched?: Array<{ model: string; vehicleId: string }>; + fetchThrows?: boolean; + refetched?: Row[]; +}) { + const deleted: string[][] = []; + const inserted: unknown[][] = []; + + const tx = { + insert: () => ({ + values: (v: unknown[]) => { + inserted.push(v); + return { onConflictDoNothing: async () => undefined }; + }, + }), + delete: () => ({ + where: async (cond: unknown) => { + // drizzle's inArray() puts its bound values in a `queryChunks` entry + // that is itself an array of Param objects; pull the plain ids back out + // so a test can assert WHICH rows were removed, not just how many. + const chunks = (cond as { queryChunks?: unknown[] })?.queryChunks ?? []; + const params = chunks.find((c): c is unknown[] => Array.isArray(c)) ?? []; + const ids = params + .map((param) => (param as { value?: unknown })?.value) + .filter((v): v is string => typeof v === "string"); + deleted.push(ids); + return undefined; + }, + }), + }; + + const db = { + select: () => ({ from: () => ({ where: async () => opts.refetched ?? [] }) }), + transaction: async (cb: (t: typeof tx) => Promise) => cb(tx), + }; + const redis = { setNx: vi.fn(async () => opts.lockFree !== false) }; + const pl24Service = { + fetchVehicleList: vi.fn(async () => { + if (opts.fetchThrows) throw new Error("upstream 503"); + return opts.fetched ?? []; + }), + }; + + const svc = new CatalogService( + db as never, + redis as never, + pl24Service as never, + {} as never, + ) as never as { + healStaleBrowseRows: (brand: string, rows: Row[], where: unknown[]) => Promise; + }; + return { svc, redis, pl24Service, deleted, inserted }; +} + +describe("isStaleBrowseArchitecture", () => { + it("flags a row whose stored architecture is not the current one", () => { + expect( + isStaleBrowseArchitecture({ + storedArchitecture: "LEGACY_PSA", + currentArchitecture: "P5_MODERN", + }), + ).toBe(true); + expect( + isStaleBrowseArchitecture({ + storedArchitecture: "LEGACY_VOLVO", + currentArchitecture: "P5_MODERN", + }), + ).toBe(true); + }); + + it("leaves a matching row alone", () => { + expect( + isStaleBrowseArchitecture({ + storedArchitecture: "P5_MODERN", + currentArchitecture: "P5_MODERN", + }), + ).toBe(false); + // Brands that are still P4 upstream must not be touched. + expect( + isStaleBrowseArchitecture({ + storedArchitecture: "LEGACY_FORD", + currentArchitecture: "LEGACY_FORD", + }), + ).toBe(false); + }); + + it("does nothing without both sides (unknown service / unlabelled row)", () => { + expect( + isStaleBrowseArchitecture({ storedArchitecture: null, currentArchitecture: "P5_MODERN" }), + ).toBe(false); + expect( + isStaleBrowseArchitecture({ storedArchitecture: "LEGACY_PSA", currentArchitecture: null }), + ).toBe(false); + }); +}); + +describe("healStaleBrowseRows", () => { + it("does not touch PL24 when every row is current", async () => { + const { svc, pl24Service, redis } = makeService({}); + const out = await svc.healStaleBrowseRows("Peugeot", [freshRow("a"), freshRow("b")], []); + expect(out).toBeNull(); + expect(pl24Service.fetchVehicleList).not.toHaveBeenCalled(); + expect(redis.setNx).not.toHaveBeenCalled(); + }); + + it("re-lists a stale brand and replaces its rows", async () => { + const refetched = [freshRow("new1"), freshRow("new2")]; + const { svc, pl24Service, deleted, inserted } = makeService({ + fetched: [ + { model: "208", vehicleId: "p5-208" }, + { model: "308", vehicleId: "p5-308" }, + ], + refetched, + }); + const out = await svc.healStaleBrowseRows("Peugeot", [psaRow("old1"), psaRow("old2")], []); + expect(pl24Service.fetchVehicleList).toHaveBeenCalledWith("peugeot_parts"); + expect(inserted).toHaveLength(1); + expect(inserted[0]).toHaveLength(2); + expect(deleted).toHaveLength(1); + expect(out).toEqual(refetched); + }); + + it("deletes only the stale rows when a service holds a mix", async () => { + // A row already re-listed under the new architecture conflicts on insert and + // is skipped, so deleting it would drop it for good. + const { svc, deleted } = makeService({ + fetched: [{ model: "208", vehicleId: "p5-208" }], + refetched: [freshRow("keep")], + }); + await svc.healStaleBrowseRows("Peugeot", [psaRow("old1"), freshRow("keep")], []); + expect(deleted).toHaveLength(1); + expect(deleted).toEqual([["old1"]]); + }); + + it("keeps the stale rows when the re-list throws", async () => { + const { svc, deleted, inserted } = makeService({ fetchThrows: true }); + const out = await svc.healStaleBrowseRows("Volvo", [psaRow("old1")], []); + expect(out).toBeNull(); + expect(inserted).toHaveLength(0); + expect(deleted).toHaveLength(0); + }); + + it("keeps the stale rows when upstream returns an empty model list", async () => { + const { svc, deleted, inserted } = makeService({ fetched: [] }); + const out = await svc.healStaleBrowseRows("Volvo", [psaRow("old1")], []); + expect(out).toBeNull(); + expect(inserted).toHaveLength(0); + expect(deleted).toHaveLength(0); + }); + + it("attempts at most once a day per brand", async () => { + const { svc, pl24Service } = makeService({ lockFree: false }); + const out = await svc.healStaleBrowseRows("Peugeot", [psaRow("old1")], []); + expect(out).toBeNull(); + expect(pl24Service.fetchVehicleList).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/api/src/catalog/catalog.service.ts b/apps/api/src/catalog/catalog.service.ts index c567c36..35e3ebb 100644 --- a/apps/api/src/catalog/catalog.service.ts +++ b/apps/api/src/catalog/catalog.service.ts @@ -19,7 +19,7 @@ import { userBrands, userSubscriptions, } from "../database/schema/core"; -import { isPl24LeafNode } from "../integrations/pl24/pl24-tree"; +import { isPl24LeafNode, isStaleBrowseArchitecture } from "../integrations/pl24/pl24-tree"; import { PL24Service } from "../integrations/pl24/pl24.service"; import { type PL24DecodedCategory, @@ -170,7 +170,8 @@ export class CatalogService { .where(and(...whereConditions)); if (dbVehicles.length > 0) { - return dbVehicles; + const healed = await this.healStaleBrowseRows(brandName, dbVehicles, whereConditions); + return healed ?? dbVehicles; } // Fetch from PL24 for each service @@ -225,6 +226,118 @@ export class CatalogService { return allVehicles; } + /** + * Re-list a brand whose stored browse rows were created under a retired PL24 + * architecture, and replace them with the current ones. + * + * WHY (plv2.md, finding consumers-09): `getModels` treats `catalog_vehicles` + * as a permanent cache — one row for the brand and `fetchVehicleList` is never + * called again. The rows listed while PSA and Volvo were still P4 are pinned + * forever, so Peugeot/Citroën browse keeps serving the frozen 2024-02-13 + * snapshot and Volvo/Polestar browse keeps calling an endpoint that answers + * HTTP 503. A code fix alone never reaches these rows. + * + * Deliberately conservative, mirroring `healStalePl24Architecture` on the + * VIN side: + * - only rows whose stored architecture disagrees with the service table; + * - one attempt per brand per day (Redis lock) so a permanently failing + * re-list cannot hammer the one surviving account on every page view; + * - a failed or empty re-list leaves the old rows in place — stale data + * beats an empty catalog; + * - the stale rows are deleted only once the replacements are committed, + * and per service, so one broken sub-catalog cannot wipe a working one. + * + * Deleting a browse row cascades to its categories and parts. That is + * intended: those rows describe the retired tree and would otherwise survive + * as unreachable orphans under the new listing. + * + * Returns the refreshed rows, or null when nothing was migrated (caller keeps + * what it already had). + */ + private async healStaleBrowseRows( + brandName: string, + dbVehicles: (typeof catalogVehicles.$inferSelect)[], + whereConditions: ReturnType[], + ): Promise<(typeof catalogVehicles.$inferSelect)[] | null> { + const isStale = (v: typeof catalogVehicles.$inferSelect) => + isStaleBrowseArchitecture({ + storedArchitecture: v.architecture, + currentArchitecture: PL24_SERVICE_CATALOGS[v.serviceName]?.architecture, + }); + const staleServices = [...new Set(dbVehicles.filter(isStale).map((v) => v.serviceName))]; + if (staleServices.length === 0) return null; + + if (!(await this.redis.setNx(`pl24:browse-heal:${brandName}`, "1", 86_400))) return null; + + this.logger.log( + `[pl24-browse-heal] ${brandName}: ${staleServices.join(", ")} listed under a retired architecture — re-listing`, + ); + + let migrated = 0; + for (const svc of staleServices) { + // ONLY the stale rows. A service can hold a mix — some rows already + // re-listed under the new architecture — and those must survive: the + // insert below skips them on conflict, so deleting them here would drop + // them for good. + const staleIds = dbVehicles + .filter((v) => v.serviceName === svc && isStale(v)) + .map((v) => v.id); + try { + const fetched = await this.pl24Service.fetchVehicleList(svc); + if (fetched.length === 0) { + this.logger.warn( + `[pl24-browse-heal] ${svc}: upstream returned no models — keeping ${staleIds.length} stale row(s)`, + ); + continue; + } + const config = PL24_SERVICE_CATALOGS[svc]; + const brandId = dbVehicles.find((v) => v.serviceName === svc)?.brandId ?? null; + await this.db.transaction(async (tx) => { + await tx + .insert(catalogVehicles) + .values( + fetched.map((v) => ({ + source: "pl24" as const, + serviceName: svc, + brandName, + brandId, + model: v.model, + year: v.year || null, + engine: v.engine || null, + bodyType: v.bodyType || null, + transmission: v.transmission || null, + market: v.market || null, + serviceVehicleId: v.vehicleId, + catalogPath: v.catalogPath || null, + architecture: config?.architecture || "P5_MODERN", + metadata: v.metadata || null, + categoriesFetched: false, + updatedAt: new Date(), + })), + ) + .onConflictDoNothing(); + if (staleIds.length > 0) { + await tx.delete(catalogVehicles).where(inArray(catalogVehicles.id, staleIds)); + } + }); + migrated += staleIds.length; + this.logger.log( + `[pl24-browse-heal] ${svc}: ${staleIds.length} stale row(s) → ${fetched.length} fresh model(s)`, + ); + } catch (err) { + this.logger.warn( + `[pl24-browse-heal] ${svc} re-list failed, keeping stale rows: ${(err as Error).message}`, + ); + } + } + + if (migrated === 0) return null; + return await this.db + .select() + .from(catalogVehicles) + .where(and(...whereConditions)); + } + /** * Get a single catalog vehicle by ID. */ diff --git a/apps/api/src/integrations/pl24/__fixtures__/p5_volvo_en.json b/apps/api/src/integrations/pl24/__fixtures__/p5_volvo_en.json new file mode 100644 index 0000000..08538d7 --- /dev/null +++ b/apps/api/src/integrations/pl24/__fixtures__/p5_volvo_en.json @@ -0,0 +1,198 @@ +{ + "link": { + "wid": "mainGroupTable", + "path": "/p5volvo/extern/groups/vin/mainGroup?lang=en&model=308&modelYear=1620&partnerGroup=46&serviceName=volvo_parts&upds=2026-08-24--11-02&vin=YV1AS84ABD1168166" + }, + "segments": { + "vinfoBasic": { + "records": [ + { + "values": { + "description": "Vehicle Identification No.", + "value": "YV1AS84ABD1168166" + } + }, + { + "values": { + "description": "Year", + "value": "2013" + } + }, + { + "values": { + "description": "Model", + "value": "S80 (07\\-)" + } + }, + { + "values": { + "description": "Km/h or mph", + "value": "K" + } + }, + { + "values": { + "description": "Factory code", + "value": "21" + } + }, + { + "values": { + "description": "Steering gear prod no", + "value": "31360538" + } + }, + { + "values": { + "description": "Structure week", + "value": "201236" + } + }, + { + "values": { + "description": "Type", + "value": "S80" + } + }, + { + "values": { + "description": "Chassis", + "value": "168166" + } + }, + { + "values": { + "description": "Partner group", + "value": "Europe" + } + }, + { + "values": { + "description": "Upholstery/interior code", + "value": "210100" + } + }, + { + "values": { + "description": "Upholstery/interior", + "value": "LEATHER/ANTHR/QRTZCEIL/NV" + } + }, + { + "values": { + "description": "Exterior color", + "value": "61400" + } + }, + { + "values": { + "description": "Exterior color", + "value": "WHITE SOLID ICE WHITE" + } + }, + { + "values": { + "description": "Body style code", + "value": "0" + } + }, + { + "values": { + "description": "Body style", + "value": "Sedan" + } + }, + { + "values": { + "description": "Special vehicle code", + "value": " " + } + }, + { + "values": { + "description": "Special vehicles", + "value": " " + } + }, + { + "values": { + "description": "Sales type", + "value": "42" + } + }, + { + "values": { + "description": "Sales type", + "value": "SALES VERSION 42" + } + }, + { + "values": { + "description": "Market code", + "value": "49" + } + }, + { + "values": { + "description": "Market", + "value": "TR" + } + }, + { + "values": { + "description": "Engine Code", + "value": "84" + } + }, + { + "values": { + "description": "Engine", + "value": "D4162T" + } + }, + { + "values": { + "description": "Engine part no", + "value": "6906309" + } + }, + { + "values": { + "description": "Engine serial no", + "value": "00000000000004138845 / 0ELD61 2208122233587" + } + }, + { + "values": { + "description": "Transmission Code", + "value": "B" + } + }, + { + "values": { + "description": "Transmission", + "value": "6\\-PSHIFT 2WD / MPS6" + } + }, + { + "values": { + "description": "Transmission part no", + "value": "1285041" + } + }, + { + "values": { + "description": "Transmission serial no", + "value": "00AWBB1 170812170654" + } + }, + { + "values": { + "description": "Chassis code", + "value": "35659B7A6276" + } + } + ] + } + } +} diff --git a/apps/api/src/integrations/pl24/__fixtures__/p5_volvo_tr.json b/apps/api/src/integrations/pl24/__fixtures__/p5_volvo_tr.json new file mode 100644 index 0000000..5221916 --- /dev/null +++ b/apps/api/src/integrations/pl24/__fixtures__/p5_volvo_tr.json @@ -0,0 +1,198 @@ +{ + "link": { + "wid": "mainGroupTable", + "path": "/p5volvo/extern/groups/vin/mainGroup?lang=tr&model=308&modelYear=1620&partnerGroup=46&serviceName=volvo_parts&upds=2026-08-24--11-02&vin=YV1AS84ABD1168166" + }, + "segments": { + "vinfoBasic": { + "records": [ + { + "values": { + "description": "Sasi numarasi", + "value": "YV1AS84ABD1168166" + } + }, + { + "values": { + "description": "Model yili", + "value": "2013" + } + }, + { + "values": { + "description": "Model", + "value": "S80 (07\\-)" + } + }, + { + "values": { + "description": "Hız Birimi", + "value": "K" + } + }, + { + "values": { + "description": "Fabrika Kodu", + "value": "21" + } + }, + { + "values": { + "description": "Direksiyon Kutusu Üretim No", + "value": "31360538" + } + }, + { + "values": { + "description": "Üretim Haftası", + "value": "201236" + } + }, + { + "values": { + "description": "Türü", + "value": "S80" + } + }, + { + "values": { + "description": "Şasi", + "value": "168166" + } + }, + { + "values": { + "description": "Ortak grubu", + "value": "Europe" + } + }, + { + "values": { + "description": "Döşeme/iç mekan kodu", + "value": "210100" + } + }, + { + "values": { + "description": "Döşeme", + "value": "LEATHER/ANTHR/QRTZCEIL/NV" + } + }, + { + "values": { + "description": "Dis rengi", + "value": "61400" + } + }, + { + "values": { + "description": "Dis rengi", + "value": "WHITE SOLID ICE WHITE" + } + }, + { + "values": { + "description": "Karoseri Tipi Kodu", + "value": "0" + } + }, + { + "values": { + "description": "Kaporta Stili", + "value": "Sedan" + } + }, + { + "values": { + "description": "Özel Araç Kodu", + "value": " " + } + }, + { + "values": { + "description": "Özel araçlar", + "value": " " + } + }, + { + "values": { + "description": "Satis tipi", + "value": "42" + } + }, + { + "values": { + "description": "Satis tipi", + "value": "SALES VERSION 42" + } + }, + { + "values": { + "description": "Piyasa Kodu", + "value": "49" + } + }, + { + "values": { + "description": "Market", + "value": "TR" + } + }, + { + "values": { + "description": "Motor kodu", + "value": "84" + } + }, + { + "values": { + "description": "Motor", + "value": "D4162T" + } + }, + { + "values": { + "description": "Motor Parça No", + "value": "6906309" + } + }, + { + "values": { + "description": "Motor Seri Numarası", + "value": "00000000000004138845 / 0ELD61 2208122233587" + } + }, + { + "values": { + "description": "Şanzıman kodu", + "value": "B" + } + }, + { + "values": { + "description": "Şanzıman", + "value": "6\\-PSHIFT 2WD / MPS6" + } + }, + { + "values": { + "description": "Şanzıman Parça No", + "value": "1285041" + } + }, + { + "values": { + "description": "Şanzıman Seri No", + "value": "00AWBB1 170812170654" + } + }, + { + "values": { + "description": "Şasi Kodu", + "value": "35659B7A6276" + } + } + ] + } + } +} diff --git a/apps/api/src/integrations/pl24/pl24-tree.ts b/apps/api/src/integrations/pl24/pl24-tree.ts index 7a5e9ac..7b35b91 100644 --- a/apps/api/src/integrations/pl24/pl24-tree.ts +++ b/apps/api/src/integrations/pl24/pl24-tree.ts @@ -86,3 +86,28 @@ export function isStalePl24Architecture(opts: { if (!storedIsLegacyPsaOrVolvo) return false; return current.startsWith("/p5"); } + +/** + * True when a stored `catalog_vehicles` browse row was listed under an + * architecture the service table no longer uses. + * + * Browse rows are a PERMANENT cache: `CatalogService.getModels` returns early + * whenever the brand already has rows, so `fetchVehicleList` is never called + * again for that brand. The 188 rows listed while PSA and Volvo were still P4 + * (127 LEGACY_PSA + 61 LEGACY_VOLVO on prod, 2026-09-20) are therefore pinned + * forever: Peugeot/Citroën browse serves the frozen 2024-02-13 snapshot and + * Volvo/Polestar browse serves an endpoint that answers HTTP 503. Detecting the + * mismatch at list time lets the brand re-list itself once, the same way + * `isStalePl24Architecture` heals a VIN-decoded vehicle. + */ +export function isStaleBrowseArchitecture(opts: { + storedArchitecture?: string | null; + currentArchitecture?: string | null; +}): boolean { + const stored = opts.storedArchitecture?.trim(); + const current = opts.currentArchitecture?.trim(); + // An unknown service (no config) or an unlabelled row is left alone: without a + // current architecture to compare against there is nothing to migrate TO. + if (!stored || !current) return false; + return stored !== current; +} diff --git a/apps/api/src/integrations/pl24/pl24-volvo-vinfo.spec.ts b/apps/api/src/integrations/pl24/pl24-volvo-vinfo.spec.ts new file mode 100644 index 0000000..9f577df --- /dev/null +++ b/apps/api/src/integrations/pl24/pl24-volvo-vinfo.spec.ts @@ -0,0 +1,68 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { PL24Service } from "./pl24.service"; + +/** + * Volvo P5 vinfoBasic regression lock (plv2.md, finding p5core-09). + * + * Volvo emits BOTH a bare internal code and a readable description for the same + * attribute — "Şanzıman kodu" = "B" next to "Şanzıman" = "6-PSHIFT 2WD / MPS6", + * and "Satis tipi" = "42" next to "Türü" = "S80". The key order the parser needs + * for VAG (whose "Şanzıman kodu" IS the useful value, e.g. "MQ200") therefore + * rendered a single letter as the gearbox and a bare number as the series. + * + * Fixtures are the real 2026-09-16 discovery captures for VIN + * YV1AS84ABD1168166 (S80), trimmed to the segments the parser reads, in both + * the Turkish and the English label locale. + */ + +const fixture = (name: string) => + JSON.parse(readFileSync(join(__dirname, "__fixtures__", `${name}.json`), "utf-8")); + +// parseVehicleResponse only touches the response and the service name. +const parse = (data: unknown, serviceName = "volvo_parts") => { + const svc = Object.create(PL24Service.prototype) as unknown as { + parseVehicleResponse: (v: string, d: unknown, s: string) => Record; + isDaimlerService: (s: string) => boolean; + }; + svc.isDaimlerService = () => false; + return svc.parseVehicleResponse("YV1AS84ABD1168166", data, serviceName); +}; + +describe("Volvo P5 vinfoBasic — kod yerine açıklama", () => { + it("Türkçe etiketlerde şanzımanı okunur değerinden alır", () => { + const out = parse(fixture("p5_volvo_tr")); + expect(out.transmission).toBe("6-PSHIFT 2WD / MPS6"); + // Tek harflik "Şanzıman kodu" artık kazanmıyor. + expect(out.transmission).not.toBe("B"); + }); + + it("İngilizce etiketlerde de aynı sonucu verir", () => { + const out = parse(fixture("p5_volvo_en")); + expect(out.transmission).toBe("6-PSHIFT 2WD / MPS6"); + }); + + it("seri, çıplak satış kodu yerine gerçek tipi döner", () => { + expect(parse(fixture("p5_volvo_tr")).series).toBe("S80"); + expect(parse(fixture("p5_volvo_en")).series).toBe("S80"); + expect(parse(fixture("p5_volvo_tr")).series).not.toBe("42"); + }); + + it("kaporta stili iki dilde de çözülür ve '0' kodu sızmaz", () => { + expect(parse(fixture("p5_volvo_tr")).bodyType).toBe("Sedan"); + expect(parse(fixture("p5_volvo_en")).bodyType).toBe("Sedan"); + }); + + it("motor alanları bozulmadan kalır", () => { + const out = parse(fixture("p5_volvo_tr")); + expect(out.engineCode).toBe("84"); + expect(out.engineType).toBe("D4162T"); + }); + + it("model ve yıl korunur", () => { + const out = parse(fixture("p5_volvo_tr")); + expect(out.model).toBe("S80 (07-)"); + expect(out.year).toBe(2013); + }); +}); diff --git a/apps/api/src/integrations/pl24/pl24.service.ts b/apps/api/src/integrations/pl24/pl24.service.ts index 24e4f06..3bf511d 100644 --- a/apps/api/src/integrations/pl24/pl24.service.ts +++ b/apps/api/src/integrations/pl24/pl24.service.ts @@ -1184,6 +1184,31 @@ export class PL24Service { return null; }; + /** + * Like `lookup`, but prefers a human-readable value over a bare internal + * code when the record carries both. + * + * Volvo's vinfoBasic has BOTH "Şanzıman kodu" ("B") and "Şanzıman" + * ("6-PSHIFT 2WD / MPS6"), and BOTH "Satis tipi" ("42") and a second + * "Satis tipi" ("SALES VERSION 42"). The code-first key order that VAG needs + * (its "Şanzıman kodu" IS the useful value, e.g. "MQ200") therefore rendered + * a single letter as the Volvo gearbox and a bare number as its series. + * + * Falls back to the first present value, so a backend that only ever emits + * codes behaves exactly as before. + */ + const lookupDescriptive = (...keys: string[]): string | null => { + let firstPresent: string | null = null; + for (const k of keys) { + const v = vehicleData[k]?.trim(); + if (!v) continue; + if (firstPresent === null) firstPresent = v; + // Two characters or fewer, or digits only → an index, not a description. + if (v.length > 2 && !/^\d+$/.test(v)) return v; + } + return firstPresent; + }; + // Extract prNr records for richer vehicle attributes const prNrRecords = segments.prNr?.records || []; const prNrByCode: Record = {}; @@ -1226,13 +1251,17 @@ export class PL24Service { // normalizeLabel folds ı→i and strips diacritics, so "Şanzıman kodu" and // "ŞANZIMAN KODU" both arrive as "sanziman_kodu". PSA uses "AKTARMA // SİSTEMLERİ" ("5 MEKANİK VİTES KUTUSU"), Subaru "Mission". - const transmissionCode = lookup( + const transmissionCode = lookupDescriptive( "sanziman_kodu", "transmission_code", "vites_kutusu", "atm,mtm", "aktarma_sistemleri", "sanziman", + // Volvo in the English locale: "Transmission code" ("B") plus the readable + // "Transmission" ("6-PSHIFT 2WD / MPS6"). Without the plain key the + // English response falls back to the single-letter code. + "transmission", "sanziman_numarasi", "mission", ); @@ -1242,7 +1271,10 @@ export class PL24Service { const bodyType = Object.entries(prNrByCode).find(([code]) => code.startsWith("K8"))?.[1] || // PSA "GÖVDE TİPİ" ("4 KAPILI SEDAN"), Volvo "Kaporta Stili" ("Sedan"). - lookup("karoseri", "body", "body_type", "govde_tipi", "kaporta_stili") || + // Volvo: TR "Kaporta Stili" / EN "Body style" ("Sedan"). Its + // "Karoseri Tipi Kodu" / "Body style code" is a bare "0" — never matched + // here because the lookup is exact-key, and that is deliberate. + lookup("karoseri", "body", "body_type", "govde_tipi", "kaporta_stili", "body_style") || null; // Engine description from prNr D3* (Motor nitelikleri) @@ -1301,7 +1333,10 @@ export class PL24Service { damToModelYear(lookup("dam")) || extractModelYear(vin) || 0, - series: lookup("seri", "satis_tipi", "sales_type", "turu"), + // Volvo's "Satis tipi" is the bare sales code ("42") with the readable + // form ("SALES VERSION 42") in a duplicate row, and its "Türü" is the real + // series ("S80") — so prefer whichever of these is actually descriptive. + series: lookupDescriptive("seri", "satis_tipi", "sales_type", "turu", "type"), bodyType, engineCode: engineCode || diff --git a/apps/api/src/integrations/pl24/pl24.types.spec.ts b/apps/api/src/integrations/pl24/pl24.types.spec.ts index 260ad11..36bb3a8 100644 --- a/apps/api/src/integrations/pl24/pl24.types.spec.ts +++ b/apps/api/src/integrations/pl24/pl24.types.spec.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { PL24_WMI_SERVICE_MAP, SERVICE_TO_BRAND } from "./pl24.types"; +import { PL24_SERVICE_CATALOGS, PL24_WMI_SERVICE_MAP, SERVICE_TO_BRAND } from "./pl24.types"; // Q6 routing additions (undecoded-vin-rca.md). Both target services are already // live in prod (nissan_parts: JN1 success; mercedesvans_parts: WDF decodes today), @@ -45,3 +45,53 @@ describe("PL24_WMI_SERVICE_MAP — kapsam dışı WMI'ler", () => { expect(PL24_WMI_SERVICE_MAP.VXK).toBe("psa_opel_parts"); }); }); + +/** + * 2026-09-20 canlı WMI servisi taraması (plv2.md, bulgu types_wmi-06/09). + * Prod'daki 1.406 haritasız araçtan hangilerinin PL24'te gerçekten karşılığı + * olduğu `/pl24-wmi/ext/api/2.0/decode` ile tek tek soruldu; bu test o cevapları + * kilitliyor — özellikle "yok" cevaplarını, çünkü onları haritaya eklemek + * boşuna istek üretir. + */ +describe("WMI haritası — canlı WMI servisi taraması (2026-09-20)", () => { + it("Renault yeniden açıldı (VIN tanımlama askısı kalktı)", () => { + expect(PL24_WMI_SERVICE_MAP.VF1).toBe("renault_parts"); + expect(PL24_WMI_SERVICE_MAP.VF6).toBe("renault_parts"); + expect(PL24_WMI_SERVICE_MAP.VNE).toBe("renault_parts"); + }); + + it("canlı servisin çözdüğü yeni WMI'ler haritada", () => { + expect(PL24_WMI_SERVICE_MAP.NLH).toBe("hyundai_parts"); + expect(PL24_WMI_SERVICE_MAP.TMA).toBe("hyundai_parts"); + expect(PL24_WMI_SERVICE_MAP.NLJ).toBe("hyundai_parts"); + expect(PL24_WMI_SERVICE_MAP.KMF).toBe("hyundai_parts"); + expect(PL24_WMI_SERVICE_MAP.KNE).toBe("kia_parts"); + expect(PL24_WMI_SERVICE_MAP.KNC).toBe("kia_parts"); + expect(PL24_WMI_SERVICE_MAP.MMC).toBe("mmc_parts"); + expect(PL24_WMI_SERVICE_MAP.XMC).toBe("mmc_parts"); + expect(PL24_WMI_SERVICE_MAP.JSA).toBe("suzuki_parts"); + }); + + it("NMB binek Mercedes değil, kamyon kataloğuna gider", () => { + expect(PL24_WMI_SERVICE_MAP.NMB).toBe("mercedestrucks_parts"); + // Binek WMI'leri bozulmadı. + expect(PL24_WMI_SERVICE_MAP.WDD).toBe("mercedes_parts"); + }); + + it("PL24'te olmayan WMI'ler haritaya EKLENMEDİ (HTTP 410)", () => { + // Honda ve Chevrolet'nin PL24'te kataloğu yok; eklemek boşuna istek olurdu. + for (const wmi of ["JHM", "SHH", "SHS", "MAK", "NLA", "KL1", "NM4", "VR7"]) { + expect(PL24_WMI_SERVICE_MAP[wmi]).toBeUndefined(); + } + }); + + it("JMZ (Mazda) eklenmedi — servis Ford kataloğu öneriyor, marka tutarsız", () => { + expect(PL24_WMI_SERVICE_MAP.JMZ).toBeUndefined(); + }); + + it("eşlenen her servisin katalog tanımı var", () => { + for (const [wmi, svc] of Object.entries(PL24_WMI_SERVICE_MAP)) { + expect(PL24_SERVICE_CATALOGS[svc], `${wmi} → ${svc}`).toBeDefined(); + } + }); +}); diff --git a/apps/api/src/integrations/pl24/pl24.types.ts b/apps/api/src/integrations/pl24/pl24.types.ts index eebea4a..46d1f3b 100644 --- a/apps/api/src/integrations/pl24/pl24.types.ts +++ b/apps/api/src/integrations/pl24/pl24.types.ts @@ -540,6 +540,10 @@ export const PL24_WMI_SERVICE_MAP: Record = { // Mercedes-Benz WDB: "mercedes_parts", + // NMB: canlı WMI servisi bunu mercedes_parts'a DEĞİL mercedestrucks_parts'a + // çözüyor (error:false) — 30 prod aracı "Mercedes-Benz" etiketliydi ama binek + // kataloğunda yok. + NMB: "mercedestrucks_parts", WDD: "mercedes_parts", WDC: "mercedes_parts", W1K: "mercedes_parts", @@ -578,15 +582,21 @@ export const PL24_WMI_SERVICE_MAP: Record = { JTJ: "lexus_parts", "2T2": "lexus_parts", - // Renault — DISABLED: PL24 has suspended Renault VIN identification ("Bu marka - // için şasi numarası tanımlamasının belirsiz bir süre için mevcut olmayacağını - // üzülerek bildiririz."). renault_parts authorizes fine but every decode throws - // that message → wasted ~1s call AND it trips the PL24 circuit breaker, which - // then skips PL24 for ALL brands. PCAT + EMEX cover Renault. Re-enable when PL24 - // restores Renault VIN decode. - // VF1: "renault_parts", - // VF6: "renault_parts", - // VNE: "renault_parts", + // Renault — RE-ENABLED 2026-09-20. It was disabled while PL24 had suspended + // Renault VIN identification ("Bu marka için şasi numarası tanımlamasının + // belirsiz bir süre için mevcut olmayacağını üzülerek bildiririz."), which both + // wasted a call per decode and, back then, tripped the global PL24 breaker. + // BOTH reasons are gone, each verified against prod rather than assumed: + // 1. The suspension is over — live directAccess on /p5renault for a real + // customer VIN (VF14SRCL458170337) returns resultStatus + // VEHICLE_IDENTIFIED, "SYMBOL II/LOGAN II", and the WMI service answers + // {service: renault_parts, valid: true, error: false}. + // 2. The breaker no longer counts definitive upstream negatives, only + // transient transport faults (see vehicles.service `isTransient`). + // 450 prod vehicles carry VF1 and had no PL24 catalog at all. + VF1: "renault_parts", + VF6: "renault_parts", + VNE: "renault_parts", // Dacia UU1: "dacia_parts", @@ -606,6 +616,10 @@ export const PL24_WMI_SERVICE_MAP: Record = { WMH: "man_parts", // Mitsubishi + // XMC / MMC: canlı WMI servisi ikisini de mmc_parts'a çözüyor ve P5 doğruluyor + // (error:false) — 26 prod aracı haritasızdı. + MMC: "mmc_parts", // Mitsubishi Japan + XMC: "mmc_parts", // Mitsubishi (diğer pazarlar) JMB: "mmc_parts", JMY: "mmc_parts", MMB: "mmc_parts", @@ -615,6 +629,7 @@ export const PL24_WMI_SERVICE_MAP: Record = { JS2: "suzuki_parts", JS3: "suzuki_parts", TSM: "suzuki_parts", + JSA: "suzuki_parts", // Suzuki (canlı WMI: suzuki_parts, error:false) MA3: "suzuki_parts", MBH: "suzuki_parts", @@ -638,10 +653,19 @@ export const PL24_WMI_SERVICE_MAP: Record = { // Hyundai KMH: "hyundai_parts", // Hyundai Korea Motor House + // Aşağıdakiler 2026-09-20'de canlı WMI servisinden alındı; hepsi hyundai_parts. + // TMA iki marka döndürüyor (hyundai_parts + kia_parts) — Hyundai Assan (Türkiye) + // üretimi olduğu için hyundai seçildi. + NLH: "hyundai_parts", // Hyundai Assan / Türkiye (106 prod aracı) + TMA: "hyundai_parts", // Hyundai Türkiye (49) + NLJ: "hyundai_parts", // Hyundai (8) + KMF: "hyundai_parts", // Hyundai (7) TMK: "hyundai_parts", // Hyundai (Turkey/other markets) // Kia KNA: "kia_parts", // Kia (worldwide production) + KNE: "kia_parts", // Kia (canlı WMI, 31 prod aracı) + KNC: "kia_parts", // Kia (canlı WMI, 8) U5Y: "kia_parts", // Kia Slovakia // Nissan @@ -682,6 +706,13 @@ export const PL24_WMI_SERVICE_MAP: Record = { // kaydedilmişti. Haritada tutmak yalnız boşuna PL24 isteği üretir ve // pcat/emex/vinpin fallback'ini geciktirir. + // PL24'TE HİÇ OLMAYANLAR — canlı WMI servisi HTTP 410 "no brands found for WMI" + // dedi (2026-09-20), tıpkı NM4 ve VR7 gibi. Haritaya eklenmemeleri kasıtlı: + // eklemek yalnız boşuna istek üretir ve pcat/emex/vinpin fallback'ini geciktirir. + // JHM, SHH, SHS, MAK, NLA (Honda) · KL1 (Chevrolet) + // JMZ (Mazda) 410 DEĞİL ama fordp/fordt döndürüyor (Ford-Mazda platform ortaklığı + // dönemi); marka tutarsız olduğu için eklenmedi — bir Mazda 3 Ford kataloğunda yok. + // Volvo YV1: "volvo_parts", // Volvo Cars (Sweden) YV4: "volvo_parts", // Volvo Cars (specific models) diff --git a/apps/api/src/jobs/prefetch-utils.ts b/apps/api/src/jobs/prefetch-utils.ts index 45bfbcb..9332035 100644 --- a/apps/api/src/jobs/prefetch-utils.ts +++ b/apps/api/src/jobs/prefetch-utils.ts @@ -74,14 +74,21 @@ export async function checkCooldown(redis: RedisService, source: string): Promis } } -/** Current hour (0–23) in Europe/Istanbul. */ -function currentIstanbulHour(): number { +/** Europe/Istanbul hour (0–23) at an arbitrary instant. */ +function istanbulHourAt(ms: number): number { const hourStr = new Intl.DateTimeFormat("en-US", { timeZone: "Europe/Istanbul", hour: "numeric", hour12: false, - }).format(new Date()); - return Number.parseInt(hourStr, 10); + }).format(new Date(ms)); + // `% 24` because the h24 hour cycle renders midnight as "24", which would put + // the hour outside every window and silently park the source forever. + return Number.parseInt(hourStr, 10) % 24; +} + +/** Current hour (0–23) in Europe/Istanbul. */ +function currentIstanbulHour(): number { + return istanbulHourAt(Date.now()); } /** @@ -115,6 +122,35 @@ export function checkTimeWindow(source: string): void { } } +/** + * The first instant at or after `fromMs` that falls inside `source`'s scrape + * window. Returns `fromMs` unchanged when the window is disabled (the default + * 0–24) or when `fromMs` is already inside it. + * + * WHY (plv2.md, finding consumers_jobs-03 — the budget/window deadlock): + * `checkSourceDailyBudget` defers a spent source to the next UTC midnight. With + * PREFETCH_PL24_START=9 that midnight lands at 03:00 Europe/Istanbul — six hours + * BEFORE the window opens. The woken job therefore did no work, threw + * `time-window`, and was deferred again to 09:00 — by which point the fresh + * daily budget had already been spent by the same stampede of no-op wake-ups. + * Measured on prod 2026-09-20: 600/600 pl24 budget consumed, 0 catalog requests + * and 0 new categories for the whole day. Landing the deferral inside the window + * breaks the cycle. + */ +export function alignToWindow(source: string, fromMs: number): number { + if (source !== "pl24") return fromMs; + if (PL24_WINDOW_START <= 0 && PL24_WINDOW_END >= 24) return fromMs; + let t = fromMs; + // Step by the hour rather than constructing a local-midnight date: DST-safe and + // free of month/year rollover edge cases. 48 steps covers any window shape. + for (let i = 0; i < 48; i++) { + const h = istanbulHourAt(t); + if (h >= PL24_WINDOW_START && h < PL24_WINDOW_END) return t; + t += 3_600_000; + } + return t; +} + /** * Milliseconds until the next 09:00 Europe/Istanbul. */ diff --git a/apps/api/src/jobs/prefetch-window-budget.spec.ts b/apps/api/src/jobs/prefetch-window-budget.spec.ts new file mode 100644 index 0000000..a9702e0 --- /dev/null +++ b/apps/api/src/jobs/prefetch-window-budget.spec.ts @@ -0,0 +1,286 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +/** + * Regression lock for the PL24 budget/window DEADLOCK (plv2.md, consumers_jobs-03). + * + * Two bugs combined to kill PL24 prefetch outright on prod: + * 1. `checkSourceDailyBudget` was debited in `process()` while the + * business-hours gate still sat at the top of each handler — so a job that + * woke outside the window paid a budget unit to do nothing. + * 2. A spent source was deferred to the next UTC midnight, which is 03:00 + * Europe/Istanbul — six hours BEFORE a 09:00 window opens. Every deferred + * job therefore woke early, burned a unit of the fresh daily budget, hit + * the window gate and was deferred again. + * + * Measured on prod 2026-09-20 before the fix: `prefetch:daily:pl24` at 600/600 + * with ZERO catalog requests and ZERO new pl24 categories for the whole day. + * + * The window constants are read at module load, so every case imports the + * modules fresh with the env already set. + */ + +const REAL_ENV = { ...process.env }; + +/** 2026-09-20 00:00 UTC = 03:00 Europe/Istanbul — outside a 09:00–18:00 window. */ +const OUTSIDE_MS = Date.UTC(2026, 8, 20, 0, 0, 0); +/** 2026-09-20 09:00 UTC = 12:00 Europe/Istanbul — inside it. */ +const INSIDE_MS = Date.UTC(2026, 8, 20, 9, 0, 0); + +function istanbulHour(ms: number): number { + return ( + Number.parseInt( + new Intl.DateTimeFormat("en-US", { + timeZone: "Europe/Istanbul", + hour: "numeric", + hour12: false, + }).format(new Date(ms)), + 10, + ) % 24 + ); +} + +async function loadModules(env: Record) { + vi.resetModules(); + for (const [k, v] of Object.entries(env)) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + const utils = await import("./prefetch-utils"); + const worker = await import("./prefetch-worker.service"); + return { utils, worker }; +} + +/** Minimal deps: only what `process()` touches before dispatching a job. */ +function makeDeps(redisOverrides: Record = {}) { + const incrCalls: string[] = []; + const redis = { + exists: vi.fn(async (..._a: unknown[]) => false), + get: vi.fn(async (..._a: unknown[]): Promise => null), + set: vi.fn(async (..._a: unknown[]) => undefined), + del: vi.fn(async (..._a: unknown[]) => undefined), + incr: vi.fn(async (k: string) => { + incrCalls.push(k); + return 1; + }), + expire: vi.fn(async (..._a: unknown[]) => undefined), + ttl: vi.fn(async (..._a: unknown[]) => -2), // no cooldown key + setNx: vi.fn(async (..._a: unknown[]) => true), + getJson: vi.fn(async (..._a: unknown[]): Promise => null), + setJson: vi.fn(async (..._a: unknown[]) => undefined), + ...redisOverrides, + }; + const queue = { + name: "catalog-prefetch", + add: vi.fn(async (..._a: unknown[]) => undefined), + getJob: vi.fn(async (..._a: unknown[]): Promise => null), + }; + const categoriesService = { + getCategoryWithParts: vi.fn(async (..._a: unknown[]) => ({ parts: [] })), + getChildren: vi.fn(async (..._a: unknown[]) => []), + }; + return { redis, queue, categoriesService, incrCalls }; +} + +function makeJob(name: string, data: Record) { + return { + name, + data, + moveToDelayed: vi.fn(async (..._a: unknown[]) => undefined), + attemptsMade: 0, + }; +} + +const dailyIncrs = (keys: string[]) => keys.filter((k) => k.startsWith("prefetch:daily:pl24")); + +describe("alignToWindow", () => { + afterEach(() => { + process.env = { ...REAL_ENV }; + }); + + it("pushes a pre-window instant into the configured window", async () => { + const { utils } = await loadModules({ PREFETCH_PL24_START: "9", PREFETCH_PL24_END: "18" }); + const aligned = utils.alignToWindow("pl24", OUTSIDE_MS); + expect(aligned).toBeGreaterThan(OUTSIDE_MS); + const h = istanbulHour(aligned); + expect(h).toBeGreaterThanOrEqual(9); + expect(h).toBeLessThan(18); + }); + + it("leaves an in-window instant untouched", async () => { + const { utils } = await loadModules({ PREFETCH_PL24_START: "9", PREFETCH_PL24_END: "18" }); + expect(utils.alignToWindow("pl24", INSIDE_MS)).toBe(INSIDE_MS); + }); + + it("is a no-op for sources that have no window", async () => { + const { utils } = await loadModules({ PREFETCH_PL24_START: "9", PREFETCH_PL24_END: "18" }); + expect(utils.alignToWindow("emex", OUTSIDE_MS)).toBe(OUTSIDE_MS); + expect(utils.alignToWindow("parts-catalogs", OUTSIDE_MS)).toBe(OUTSIDE_MS); + }); + + it("is a no-op when the window is disabled (the default 0–24)", async () => { + const { utils } = await loadModules({ + PREFETCH_PL24_START: undefined, + PREFETCH_PL24_END: undefined, + }); + expect(utils.alignToWindow("pl24", OUTSIDE_MS)).toBe(OUTSIDE_MS); + }); +}); + +describe("process() gate order — window before daily budget", () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + afterEach(() => { + vi.useRealTimers(); + process.env = { ...REAL_ENV }; + }); + + it("does NOT debit the daily budget for a job deferred by the window", async () => { + const { worker } = await loadModules({ + PREFETCH_PL24_START: "9", + PREFETCH_PL24_END: "18", + PL24_TR_DISABLED: undefined, + }); + vi.setSystemTime(OUTSIDE_MS); + const { redis, queue, categoriesService, incrCalls } = makeDeps(); + const svc = new worker.PrefetchWorkerService( + queue as never, + queue as never, + categoriesService as never, + redis as never, + { payload: vi.fn(async () => ({})) } as never, + {} as never, + ); + const job = makeJob("prefetch-parts", { + vehicleId: "v1", + categoryId: "c1", + source: "pl24", + fast: true, + }); + + await expect( + (svc as never as { process: (j: unknown, t?: string) => Promise }).process(job, "tok"), + ).rejects.toThrow(); + + expect(job.moveToDelayed).toHaveBeenCalled(); + expect(dailyIncrs(incrCalls)).toHaveLength(0); + // The per-minute counter is not charged either: the window gate is pure + // clock arithmetic and runs before any Redis write. + expect(incrCalls).toHaveLength(0); + // …and the handler never ran, so nothing was fetched upstream either. + expect(categoriesService.getCategoryWithParts).not.toHaveBeenCalled(); + }); + + it("defers the window-blocked job to an instant inside the window", async () => { + const { worker } = await loadModules({ + PREFETCH_PL24_START: "9", + PREFETCH_PL24_END: "18", + PL24_TR_DISABLED: undefined, + }); + vi.setSystemTime(OUTSIDE_MS); + const { redis, queue, categoriesService } = makeDeps(); + const svc = new worker.PrefetchWorkerService( + queue as never, + queue as never, + categoriesService as never, + redis as never, + { payload: vi.fn(async () => ({})) } as never, + {} as never, + ); + const job = makeJob("prefetch-parts", { + vehicleId: "v1", + categoryId: "c1", + source: "pl24", + fast: true, + }); + await expect( + (svc as never as { process: (j: unknown, t?: string) => Promise }).process(job, "tok"), + ).rejects.toThrow(); + + const [when] = job.moveToDelayed.mock.calls[0] as [number]; + const h = istanbulHour(when); + expect(h).toBeGreaterThanOrEqual(9); + expect(h).toBeLessThan(18); + }); + + it("debits the daily budget once the window is open", async () => { + const { worker } = await loadModules({ + PREFETCH_PL24_START: "9", + PREFETCH_PL24_END: "18", + PL24_TR_DISABLED: undefined, + }); + vi.setSystemTime(INSIDE_MS); + const { redis, queue, categoriesService, incrCalls } = makeDeps(); + const svc = new worker.PrefetchWorkerService( + queue as never, + queue as never, + categoriesService as never, + redis as never, + { payload: vi.fn(async () => ({})) } as never, + {} as never, + ); + const job = makeJob("prefetch-parts", { + vehicleId: "v1", + categoryId: "c1", + source: "pl24", + fast: true, + }); + + await (svc as never as { process: (j: unknown, t?: string) => Promise }).process( + job, + "tok", + ); + + expect(dailyIncrs(incrCalls)).toHaveLength(1); + expect(categoriesService.getCategoryWithParts).toHaveBeenCalledWith("c1"); + }); +}); + +describe("checkSourceDailyBudget — spent source retries inside the window", () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + afterEach(() => { + vi.useRealTimers(); + process.env = { ...REAL_ENV }; + }); + + it("never parks a spent source at 03:00 Istanbul again", async () => { + const { worker } = await loadModules({ + PREFETCH_PL24_START: "9", + PREFETCH_PL24_END: "18", + PREFETCH_DAILY_PL24: "600", + }); + vi.setSystemTime(INSIDE_MS); + // Counter already at the fast-lane ceiling → the next job must be deferred. + const { redis, queue, categoriesService } = makeDeps({ + get: vi.fn(async (k: string) => (String(k).startsWith("prefetch:daily:pl24") ? "600" : null)), + }); + const svc = new worker.PrefetchWorkerService( + queue as never, + queue as never, + categoriesService as never, + redis as never, + { payload: vi.fn(async () => ({})) } as never, + {} as never, + ); + const job = makeJob("prefetch-parts", { + vehicleId: "v1", + categoryId: "c1", + source: "pl24", + fast: true, + }); + + await expect( + (svc as never as { process: (j: unknown, t?: string) => Promise }).process(job, "tok"), + ).rejects.toThrow(); + + const [when] = job.moveToDelayed.mock.calls[0] as [number]; + // Past the UTC rollover… + expect(when).toBeGreaterThan(Date.UTC(2026, 8, 21, 0, 0, 0)); + // …and inside the window, not at 03:00 Istanbul like the old deferral. + const h = istanbulHour(when); + expect(h).toBeGreaterThanOrEqual(9); + expect(h).toBeLessThan(18); + }); +}); diff --git a/apps/api/src/jobs/prefetch-worker.service.ts b/apps/api/src/jobs/prefetch-worker.service.ts index 4e61ca3..fcde616 100644 --- a/apps/api/src/jobs/prefetch-worker.service.ts +++ b/apps/api/src/jobs/prefetch-worker.service.ts @@ -17,6 +17,7 @@ import { QUEUE_NAMES, getBullConnection } from "./bull.config"; import { backfillContext } from "./prefetch-context"; import { RateLimitError, + alignToWindow, checkCooldown, checkTimeWindow, initProgress, @@ -354,12 +355,30 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy { job.name === "prefetch-parts") ) { const lane = (job.data as { fast?: boolean }).fast ? "fast" : "main"; + // GATE ORDER IS LOAD-BEARING — cheapest first, and every gate that can + // reject the job must run BEFORE any counter is debited. + // + // The business-hours window and the cooldown used to sit at the top of + // each handler, i.e. AFTER the per-minute counter and the daily budget + // had already been charged. So a job that woke outside the window paid a + // budget unit to do nothing. Combined with a deferral target of "next UTC + // midnight" (= 03:00 Europe/Istanbul, six hours before a 09:00 window + // opens) that closed a loop: the whole daily allowance was burned by + // no-op wake-ups before the window ever opened, so the source never ran + // again. Measured on prod 2026-09-20 — pl24 at 600/600 with 0 catalog + // requests and 0 new categories for the day. + // + // 1. Window: pure clock arithmetic, no I/O, and an out-of-window job can + // never do useful work — so nothing else is worth spending on it. + checkTimeWindow(data.source); + // 2. Cooldown: one Redis TTL read. Pauses the whole worker (see catch). + await checkCooldown(this.redis, data.source); + // 3. Per-minute ceiling. await this.checkSourceRate(data.source, lane); - // Daily budget AFTER the per-minute gate: a job deferred on the minute - // ceiling above never reaches here, so rate-limited retries don't inflate - // the daily counter — only jobs about to do real work are counted. The - // lane decides which threshold applies (backfill stops at the main limit, - // the user's fast lane may use the full budget). + // 4. Daily budget last: a job deferred on any gate above never reaches + // here, so only jobs about to do real work are counted. The lane + // decides which threshold applies (backfill stops at the main limit, + // the user's fast lane may use the full budget). await this.checkSourceDailyBudget(data.source, lane); } if (data.source === "parts-catalogs" && PCAT_PACE_MS > 0) { @@ -421,8 +440,8 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy { const { vehicleId, source, fast = false } = job.data; this.logger.log(`[prefetch] Init for vehicle=${vehicleId}, source=${source}`); - await checkCooldown(this.redis, source); - checkTimeWindow(source); + // Cooldown + time-window are enforced in process() before the daily budget + // is debited — see the comment there; re-checking here would be a no-op. // Already flagged as poison (tree exceeded CATEGORY_CAP on a prior run) — skip. if (await this.redis.exists(this.poisonKey(vehicleId))) { @@ -563,8 +582,8 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy { const { vehicleId, categoryId, source, depth, fast = false } = job.data; this.logger.log(`[prefetch] Children for category=${categoryId}, depth=${depth}`); - await checkCooldown(this.redis, source); - checkTimeWindow(source); + // Cooldown + time-window are enforced in process() before the daily budget + // is debited — see the comment there; re-checking here would be a no-op. const depthCeiling = maxDepthFor(source, fast); if (depth >= depthCeiling) { @@ -631,8 +650,8 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy { const { vehicleId, categoryId, source } = job.data; this.logger.log(`[prefetch] Parts for category=${categoryId}`); - await checkCooldown(this.redis, source); - checkTimeWindow(source); + // Cooldown + time-window are enforced in process() before the daily budget + // is debited — see the comment there; re-checking here would be a no-op. try { await this.categoriesService.getCategoryWithParts(categoryId); @@ -1214,11 +1233,17 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy { // deferred job wakes in the SAME millisecond (observed: 11495 jobs all at // 00:00:01 UTC) — the promotion lands as one burst and the pressure signal // flaps. Other sources keep flowing (per-job defer, not a worker pause). - const msLeft = dayMs - (now % dayMs) + 1000 + Math.floor(Math.random() * 45 * 60_000); + const rollover = now + dayMs - (now % dayMs) + 1000 + Math.floor(Math.random() * 45 * 60_000); + // Land the retry INSIDE the source's scrape window. The UTC rollover alone + // is 03:00 Europe/Istanbul, so with a 09:00 window every deferred job woke + // six hours early, failed the window check and was deferred again — the + // other half of the deadlock fixed in process(). alignToWindow is a no-op + // when no window is configured (the default). + const msLeft = Math.max(1000, alignToWindow(source, rollover) - now); if (n === limit) { this.logger.warn( `[prefetch] ${source} daily budget hit (lane=${lane}, ${n}/${limit} of ${max}) — ` + - `deferring ~${Math.round(msLeft / 3_600_000)}h until the window rolls`, + `deferring ~${Math.round(msLeft / 3_600_000)}h to the next in-window slot`, ); } throw new RateLimitError(msLeft, "source-rate");