Merge pull request 'fix(pl24): bütçe kilitlenmesi, sabitlenmiş browse satırları, Volvo alanları + WMI kapsamı' (#268) from dev into main
This commit was merged in pull request #268.
This commit is contained in:
192
apps/api/src/catalog/catalog-browse-heal.spec.ts
Normal file
192
apps/api/src/catalog/catalog-browse-heal.spec.ts
Normal file
@@ -0,0 +1,192 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { isStaleBrowseArchitecture } from "../integrations/pl24/pl24-tree";
|
||||
import { CatalogService } from "./catalog.service";
|
||||
|
||||
/**
|
||||
* Regression lock for the pinned-browse-rows bug (plv2.md, finding consumers-09).
|
||||
*
|
||||
* `catalog_vehicles` is a permanent cache: `getModels` returns early as soon as
|
||||
* a brand has any row, so `fetchVehicleList` never runs again for that brand.
|
||||
* The 188 rows listed while PSA and Volvo were still P4 (127 LEGACY_PSA + 61
|
||||
* LEGACY_VOLVO on prod, 2026-09-20) were therefore stuck serving the frozen
|
||||
* 2024-02-13 PSA snapshot and a Volvo endpoint that answers HTTP 503.
|
||||
*/
|
||||
|
||||
type Row = {
|
||||
id: string;
|
||||
serviceName: string;
|
||||
architecture: string | null;
|
||||
brandId: string | null;
|
||||
};
|
||||
|
||||
const psaRow = (id: string): Row => ({
|
||||
id,
|
||||
serviceName: "peugeot_parts",
|
||||
architecture: "LEGACY_PSA",
|
||||
brandId: "b1",
|
||||
});
|
||||
const freshRow = (id: string): Row => ({
|
||||
id,
|
||||
serviceName: "peugeot_parts",
|
||||
architecture: "P5_MODERN",
|
||||
brandId: "b1",
|
||||
});
|
||||
|
||||
function makeService(opts: {
|
||||
lockFree?: boolean;
|
||||
fetched?: Array<{ model: string; vehicleId: string }>;
|
||||
fetchThrows?: boolean;
|
||||
refetched?: Row[];
|
||||
}) {
|
||||
const deleted: string[][] = [];
|
||||
const inserted: unknown[][] = [];
|
||||
|
||||
const tx = {
|
||||
insert: () => ({
|
||||
values: (v: unknown[]) => {
|
||||
inserted.push(v);
|
||||
return { onConflictDoNothing: async () => undefined };
|
||||
},
|
||||
}),
|
||||
delete: () => ({
|
||||
where: async (cond: unknown) => {
|
||||
// drizzle's inArray() puts its bound values in a `queryChunks` entry
|
||||
// that is itself an array of Param objects; pull the plain ids back out
|
||||
// so a test can assert WHICH rows were removed, not just how many.
|
||||
const chunks = (cond as { queryChunks?: unknown[] })?.queryChunks ?? [];
|
||||
const params = chunks.find((c): c is unknown[] => Array.isArray(c)) ?? [];
|
||||
const ids = params
|
||||
.map((param) => (param as { value?: unknown })?.value)
|
||||
.filter((v): v is string => typeof v === "string");
|
||||
deleted.push(ids);
|
||||
return undefined;
|
||||
},
|
||||
}),
|
||||
};
|
||||
|
||||
const db = {
|
||||
select: () => ({ from: () => ({ where: async () => opts.refetched ?? [] }) }),
|
||||
transaction: async (cb: (t: typeof tx) => Promise<void>) => cb(tx),
|
||||
};
|
||||
const redis = { setNx: vi.fn(async () => opts.lockFree !== false) };
|
||||
const pl24Service = {
|
||||
fetchVehicleList: vi.fn(async () => {
|
||||
if (opts.fetchThrows) throw new Error("upstream 503");
|
||||
return opts.fetched ?? [];
|
||||
}),
|
||||
};
|
||||
|
||||
const svc = new CatalogService(
|
||||
db as never,
|
||||
redis as never,
|
||||
pl24Service as never,
|
||||
{} as never,
|
||||
) as never as {
|
||||
healStaleBrowseRows: (brand: string, rows: Row[], where: unknown[]) => Promise<Row[] | null>;
|
||||
};
|
||||
return { svc, redis, pl24Service, deleted, inserted };
|
||||
}
|
||||
|
||||
describe("isStaleBrowseArchitecture", () => {
|
||||
it("flags a row whose stored architecture is not the current one", () => {
|
||||
expect(
|
||||
isStaleBrowseArchitecture({
|
||||
storedArchitecture: "LEGACY_PSA",
|
||||
currentArchitecture: "P5_MODERN",
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(
|
||||
isStaleBrowseArchitecture({
|
||||
storedArchitecture: "LEGACY_VOLVO",
|
||||
currentArchitecture: "P5_MODERN",
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("leaves a matching row alone", () => {
|
||||
expect(
|
||||
isStaleBrowseArchitecture({
|
||||
storedArchitecture: "P5_MODERN",
|
||||
currentArchitecture: "P5_MODERN",
|
||||
}),
|
||||
).toBe(false);
|
||||
// Brands that are still P4 upstream must not be touched.
|
||||
expect(
|
||||
isStaleBrowseArchitecture({
|
||||
storedArchitecture: "LEGACY_FORD",
|
||||
currentArchitecture: "LEGACY_FORD",
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("does nothing without both sides (unknown service / unlabelled row)", () => {
|
||||
expect(
|
||||
isStaleBrowseArchitecture({ storedArchitecture: null, currentArchitecture: "P5_MODERN" }),
|
||||
).toBe(false);
|
||||
expect(
|
||||
isStaleBrowseArchitecture({ storedArchitecture: "LEGACY_PSA", currentArchitecture: null }),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("healStaleBrowseRows", () => {
|
||||
it("does not touch PL24 when every row is current", async () => {
|
||||
const { svc, pl24Service, redis } = makeService({});
|
||||
const out = await svc.healStaleBrowseRows("Peugeot", [freshRow("a"), freshRow("b")], []);
|
||||
expect(out).toBeNull();
|
||||
expect(pl24Service.fetchVehicleList).not.toHaveBeenCalled();
|
||||
expect(redis.setNx).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("re-lists a stale brand and replaces its rows", async () => {
|
||||
const refetched = [freshRow("new1"), freshRow("new2")];
|
||||
const { svc, pl24Service, deleted, inserted } = makeService({
|
||||
fetched: [
|
||||
{ model: "208", vehicleId: "p5-208" },
|
||||
{ model: "308", vehicleId: "p5-308" },
|
||||
],
|
||||
refetched,
|
||||
});
|
||||
const out = await svc.healStaleBrowseRows("Peugeot", [psaRow("old1"), psaRow("old2")], []);
|
||||
expect(pl24Service.fetchVehicleList).toHaveBeenCalledWith("peugeot_parts");
|
||||
expect(inserted).toHaveLength(1);
|
||||
expect(inserted[0]).toHaveLength(2);
|
||||
expect(deleted).toHaveLength(1);
|
||||
expect(out).toEqual(refetched);
|
||||
});
|
||||
|
||||
it("deletes only the stale rows when a service holds a mix", async () => {
|
||||
// A row already re-listed under the new architecture conflicts on insert and
|
||||
// is skipped, so deleting it would drop it for good.
|
||||
const { svc, deleted } = makeService({
|
||||
fetched: [{ model: "208", vehicleId: "p5-208" }],
|
||||
refetched: [freshRow("keep")],
|
||||
});
|
||||
await svc.healStaleBrowseRows("Peugeot", [psaRow("old1"), freshRow("keep")], []);
|
||||
expect(deleted).toHaveLength(1);
|
||||
expect(deleted).toEqual([["old1"]]);
|
||||
});
|
||||
|
||||
it("keeps the stale rows when the re-list throws", async () => {
|
||||
const { svc, deleted, inserted } = makeService({ fetchThrows: true });
|
||||
const out = await svc.healStaleBrowseRows("Volvo", [psaRow("old1")], []);
|
||||
expect(out).toBeNull();
|
||||
expect(inserted).toHaveLength(0);
|
||||
expect(deleted).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("keeps the stale rows when upstream returns an empty model list", async () => {
|
||||
const { svc, deleted, inserted } = makeService({ fetched: [] });
|
||||
const out = await svc.healStaleBrowseRows("Volvo", [psaRow("old1")], []);
|
||||
expect(out).toBeNull();
|
||||
expect(inserted).toHaveLength(0);
|
||||
expect(deleted).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("attempts at most once a day per brand", async () => {
|
||||
const { svc, pl24Service } = makeService({ lockFree: false });
|
||||
const out = await svc.healStaleBrowseRows("Peugeot", [psaRow("old1")], []);
|
||||
expect(out).toBeNull();
|
||||
expect(pl24Service.fetchVehicleList).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -19,7 +19,7 @@ import {
|
||||
userBrands,
|
||||
userSubscriptions,
|
||||
} from "../database/schema/core";
|
||||
import { isPl24LeafNode } from "../integrations/pl24/pl24-tree";
|
||||
import { isPl24LeafNode, isStaleBrowseArchitecture } from "../integrations/pl24/pl24-tree";
|
||||
import { PL24Service } from "../integrations/pl24/pl24.service";
|
||||
import {
|
||||
type PL24DecodedCategory,
|
||||
@@ -170,7 +170,8 @@ export class CatalogService {
|
||||
.where(and(...whereConditions));
|
||||
|
||||
if (dbVehicles.length > 0) {
|
||||
return dbVehicles;
|
||||
const healed = await this.healStaleBrowseRows(brandName, dbVehicles, whereConditions);
|
||||
return healed ?? dbVehicles;
|
||||
}
|
||||
|
||||
// Fetch from PL24 for each service
|
||||
@@ -225,6 +226,118 @@ export class CatalogService {
|
||||
return allVehicles;
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-list a brand whose stored browse rows were created under a retired PL24
|
||||
* architecture, and replace them with the current ones.
|
||||
*
|
||||
* WHY (plv2.md, finding consumers-09): `getModels` treats `catalog_vehicles`
|
||||
* as a permanent cache — one row for the brand and `fetchVehicleList` is never
|
||||
* called again. The rows listed while PSA and Volvo were still P4 are pinned
|
||||
* forever, so Peugeot/Citroën browse keeps serving the frozen 2024-02-13
|
||||
* snapshot and Volvo/Polestar browse keeps calling an endpoint that answers
|
||||
* HTTP 503. A code fix alone never reaches these rows.
|
||||
*
|
||||
* Deliberately conservative, mirroring `healStalePl24Architecture` on the
|
||||
* VIN side:
|
||||
* - only rows whose stored architecture disagrees with the service table;
|
||||
* - one attempt per brand per day (Redis lock) so a permanently failing
|
||||
* re-list cannot hammer the one surviving account on every page view;
|
||||
* - a failed or empty re-list leaves the old rows in place — stale data
|
||||
* beats an empty catalog;
|
||||
* - the stale rows are deleted only once the replacements are committed,
|
||||
* and per service, so one broken sub-catalog cannot wipe a working one.
|
||||
*
|
||||
* Deleting a browse row cascades to its categories and parts. That is
|
||||
* intended: those rows describe the retired tree and would otherwise survive
|
||||
* as unreachable orphans under the new listing.
|
||||
*
|
||||
* Returns the refreshed rows, or null when nothing was migrated (caller keeps
|
||||
* what it already had).
|
||||
*/
|
||||
private async healStaleBrowseRows(
|
||||
brandName: string,
|
||||
dbVehicles: (typeof catalogVehicles.$inferSelect)[],
|
||||
whereConditions: ReturnType<typeof eq>[],
|
||||
): Promise<(typeof catalogVehicles.$inferSelect)[] | null> {
|
||||
const isStale = (v: typeof catalogVehicles.$inferSelect) =>
|
||||
isStaleBrowseArchitecture({
|
||||
storedArchitecture: v.architecture,
|
||||
currentArchitecture: PL24_SERVICE_CATALOGS[v.serviceName]?.architecture,
|
||||
});
|
||||
const staleServices = [...new Set(dbVehicles.filter(isStale).map((v) => v.serviceName))];
|
||||
if (staleServices.length === 0) return null;
|
||||
|
||||
if (!(await this.redis.setNx(`pl24:browse-heal:${brandName}`, "1", 86_400))) return null;
|
||||
|
||||
this.logger.log(
|
||||
`[pl24-browse-heal] ${brandName}: ${staleServices.join(", ")} listed under a retired architecture — re-listing`,
|
||||
);
|
||||
|
||||
let migrated = 0;
|
||||
for (const svc of staleServices) {
|
||||
// ONLY the stale rows. A service can hold a mix — some rows already
|
||||
// re-listed under the new architecture — and those must survive: the
|
||||
// insert below skips them on conflict, so deleting them here would drop
|
||||
// them for good.
|
||||
const staleIds = dbVehicles
|
||||
.filter((v) => v.serviceName === svc && isStale(v))
|
||||
.map((v) => v.id);
|
||||
try {
|
||||
const fetched = await this.pl24Service.fetchVehicleList(svc);
|
||||
if (fetched.length === 0) {
|
||||
this.logger.warn(
|
||||
`[pl24-browse-heal] ${svc}: upstream returned no models — keeping ${staleIds.length} stale row(s)`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
const config = PL24_SERVICE_CATALOGS[svc];
|
||||
const brandId = dbVehicles.find((v) => v.serviceName === svc)?.brandId ?? null;
|
||||
await this.db.transaction(async (tx) => {
|
||||
await tx
|
||||
.insert(catalogVehicles)
|
||||
.values(
|
||||
fetched.map((v) => ({
|
||||
source: "pl24" as const,
|
||||
serviceName: svc,
|
||||
brandName,
|
||||
brandId,
|
||||
model: v.model,
|
||||
year: v.year || null,
|
||||
engine: v.engine || null,
|
||||
bodyType: v.bodyType || null,
|
||||
transmission: v.transmission || null,
|
||||
market: v.market || null,
|
||||
serviceVehicleId: v.vehicleId,
|
||||
catalogPath: v.catalogPath || null,
|
||||
architecture: config?.architecture || "P5_MODERN",
|
||||
metadata: v.metadata || null,
|
||||
categoriesFetched: false,
|
||||
updatedAt: new Date(),
|
||||
})),
|
||||
)
|
||||
.onConflictDoNothing();
|
||||
if (staleIds.length > 0) {
|
||||
await tx.delete(catalogVehicles).where(inArray(catalogVehicles.id, staleIds));
|
||||
}
|
||||
});
|
||||
migrated += staleIds.length;
|
||||
this.logger.log(
|
||||
`[pl24-browse-heal] ${svc}: ${staleIds.length} stale row(s) → ${fetched.length} fresh model(s)`,
|
||||
);
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
`[pl24-browse-heal] ${svc} re-list failed, keeping stale rows: ${(err as Error).message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (migrated === 0) return null;
|
||||
return await this.db
|
||||
.select()
|
||||
.from(catalogVehicles)
|
||||
.where(and(...whereConditions));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a single catalog vehicle by ID.
|
||||
*/
|
||||
|
||||
198
apps/api/src/integrations/pl24/__fixtures__/p5_volvo_en.json
Normal file
198
apps/api/src/integrations/pl24/__fixtures__/p5_volvo_en.json
Normal file
@@ -0,0 +1,198 @@
|
||||
{
|
||||
"link": {
|
||||
"wid": "mainGroupTable",
|
||||
"path": "/p5volvo/extern/groups/vin/mainGroup?lang=en&model=308&modelYear=1620&partnerGroup=46&serviceName=volvo_parts&upds=2026-08-24--11-02&vin=YV1AS84ABD1168166"
|
||||
},
|
||||
"segments": {
|
||||
"vinfoBasic": {
|
||||
"records": [
|
||||
{
|
||||
"values": {
|
||||
"description": "Vehicle Identification No.",
|
||||
"value": "YV1AS84ABD1168166"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Year",
|
||||
"value": "2013"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Model",
|
||||
"value": "S80 (07\\-)"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Km/h or mph",
|
||||
"value": "K"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Factory code",
|
||||
"value": "21"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Steering gear prod no",
|
||||
"value": "31360538"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Structure week",
|
||||
"value": "201236"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Type",
|
||||
"value": "S80"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Chassis",
|
||||
"value": "168166"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Partner group",
|
||||
"value": "Europe"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Upholstery/interior code",
|
||||
"value": "210100"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Upholstery/interior",
|
||||
"value": "LEATHER/ANTHR/QRTZCEIL/NV"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Exterior color",
|
||||
"value": "61400"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Exterior color",
|
||||
"value": "WHITE SOLID ICE WHITE"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Body style code",
|
||||
"value": "0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Body style",
|
||||
"value": "Sedan"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Special vehicle code",
|
||||
"value": " "
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Special vehicles",
|
||||
"value": " "
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Sales type",
|
||||
"value": "42"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Sales type",
|
||||
"value": "SALES VERSION 42"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Market code",
|
||||
"value": "49"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Market",
|
||||
"value": "TR"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Engine Code",
|
||||
"value": "84"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Engine",
|
||||
"value": "D4162T"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Engine part no",
|
||||
"value": "6906309"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Engine serial no",
|
||||
"value": "00000000000004138845 / 0ELD61 2208122233587"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Transmission Code",
|
||||
"value": "B"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Transmission",
|
||||
"value": "6\\-PSHIFT 2WD / MPS6"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Transmission part no",
|
||||
"value": "1285041"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Transmission serial no",
|
||||
"value": "00AWBB1 170812170654"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Chassis code",
|
||||
"value": "35659B7A6276"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
198
apps/api/src/integrations/pl24/__fixtures__/p5_volvo_tr.json
Normal file
198
apps/api/src/integrations/pl24/__fixtures__/p5_volvo_tr.json
Normal file
@@ -0,0 +1,198 @@
|
||||
{
|
||||
"link": {
|
||||
"wid": "mainGroupTable",
|
||||
"path": "/p5volvo/extern/groups/vin/mainGroup?lang=tr&model=308&modelYear=1620&partnerGroup=46&serviceName=volvo_parts&upds=2026-08-24--11-02&vin=YV1AS84ABD1168166"
|
||||
},
|
||||
"segments": {
|
||||
"vinfoBasic": {
|
||||
"records": [
|
||||
{
|
||||
"values": {
|
||||
"description": "Sasi numarasi",
|
||||
"value": "YV1AS84ABD1168166"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Model yili",
|
||||
"value": "2013"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Model",
|
||||
"value": "S80 (07\\-)"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Hız Birimi",
|
||||
"value": "K"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Fabrika Kodu",
|
||||
"value": "21"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Direksiyon Kutusu Üretim No",
|
||||
"value": "31360538"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Üretim Haftası",
|
||||
"value": "201236"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Türü",
|
||||
"value": "S80"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Şasi",
|
||||
"value": "168166"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Ortak grubu",
|
||||
"value": "Europe"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Döşeme/iç mekan kodu",
|
||||
"value": "210100"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Döşeme",
|
||||
"value": "LEATHER/ANTHR/QRTZCEIL/NV"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Dis rengi",
|
||||
"value": "61400"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Dis rengi",
|
||||
"value": "WHITE SOLID ICE WHITE"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Karoseri Tipi Kodu",
|
||||
"value": "0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Kaporta Stili",
|
||||
"value": "Sedan"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Özel Araç Kodu",
|
||||
"value": " "
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Özel araçlar",
|
||||
"value": " "
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Satis tipi",
|
||||
"value": "42"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Satis tipi",
|
||||
"value": "SALES VERSION 42"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Piyasa Kodu",
|
||||
"value": "49"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Market",
|
||||
"value": "TR"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Motor kodu",
|
||||
"value": "84"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Motor",
|
||||
"value": "D4162T"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Motor Parça No",
|
||||
"value": "6906309"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Motor Seri Numarası",
|
||||
"value": "00000000000004138845 / 0ELD61 2208122233587"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Şanzıman kodu",
|
||||
"value": "B"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Şanzıman",
|
||||
"value": "6\\-PSHIFT 2WD / MPS6"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Şanzıman Parça No",
|
||||
"value": "1285041"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Şanzıman Seri No",
|
||||
"value": "00AWBB1 170812170654"
|
||||
}
|
||||
},
|
||||
{
|
||||
"values": {
|
||||
"description": "Şasi Kodu",
|
||||
"value": "35659B7A6276"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -86,3 +86,28 @@ export function isStalePl24Architecture(opts: {
|
||||
if (!storedIsLegacyPsaOrVolvo) return false;
|
||||
return current.startsWith("/p5");
|
||||
}
|
||||
|
||||
/**
|
||||
* True when a stored `catalog_vehicles` browse row was listed under an
|
||||
* architecture the service table no longer uses.
|
||||
*
|
||||
* Browse rows are a PERMANENT cache: `CatalogService.getModels` returns early
|
||||
* whenever the brand already has rows, so `fetchVehicleList` is never called
|
||||
* again for that brand. The 188 rows listed while PSA and Volvo were still P4
|
||||
* (127 LEGACY_PSA + 61 LEGACY_VOLVO on prod, 2026-09-20) are therefore pinned
|
||||
* forever: Peugeot/Citroën browse serves the frozen 2024-02-13 snapshot and
|
||||
* Volvo/Polestar browse serves an endpoint that answers HTTP 503. Detecting the
|
||||
* mismatch at list time lets the brand re-list itself once, the same way
|
||||
* `isStalePl24Architecture` heals a VIN-decoded vehicle.
|
||||
*/
|
||||
export function isStaleBrowseArchitecture(opts: {
|
||||
storedArchitecture?: string | null;
|
||||
currentArchitecture?: string | null;
|
||||
}): boolean {
|
||||
const stored = opts.storedArchitecture?.trim();
|
||||
const current = opts.currentArchitecture?.trim();
|
||||
// An unknown service (no config) or an unlabelled row is left alone: without a
|
||||
// current architecture to compare against there is nothing to migrate TO.
|
||||
if (!stored || !current) return false;
|
||||
return stored !== current;
|
||||
}
|
||||
|
||||
68
apps/api/src/integrations/pl24/pl24-volvo-vinfo.spec.ts
Normal file
68
apps/api/src/integrations/pl24/pl24-volvo-vinfo.spec.ts
Normal file
@@ -0,0 +1,68 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { PL24Service } from "./pl24.service";
|
||||
|
||||
/**
|
||||
* Volvo P5 vinfoBasic regression lock (plv2.md, finding p5core-09).
|
||||
*
|
||||
* Volvo emits BOTH a bare internal code and a readable description for the same
|
||||
* attribute — "Şanzıman kodu" = "B" next to "Şanzıman" = "6-PSHIFT 2WD / MPS6",
|
||||
* and "Satis tipi" = "42" next to "Türü" = "S80". The key order the parser needs
|
||||
* for VAG (whose "Şanzıman kodu" IS the useful value, e.g. "MQ200") therefore
|
||||
* rendered a single letter as the gearbox and a bare number as the series.
|
||||
*
|
||||
* Fixtures are the real 2026-09-16 discovery captures for VIN
|
||||
* YV1AS84ABD1168166 (S80), trimmed to the segments the parser reads, in both
|
||||
* the Turkish and the English label locale.
|
||||
*/
|
||||
|
||||
const fixture = (name: string) =>
|
||||
JSON.parse(readFileSync(join(__dirname, "__fixtures__", `${name}.json`), "utf-8"));
|
||||
|
||||
// parseVehicleResponse only touches the response and the service name.
|
||||
const parse = (data: unknown, serviceName = "volvo_parts") => {
|
||||
const svc = Object.create(PL24Service.prototype) as unknown as {
|
||||
parseVehicleResponse: (v: string, d: unknown, s: string) => Record<string, unknown>;
|
||||
isDaimlerService: (s: string) => boolean;
|
||||
};
|
||||
svc.isDaimlerService = () => false;
|
||||
return svc.parseVehicleResponse("YV1AS84ABD1168166", data, serviceName);
|
||||
};
|
||||
|
||||
describe("Volvo P5 vinfoBasic — kod yerine açıklama", () => {
|
||||
it("Türkçe etiketlerde şanzımanı okunur değerinden alır", () => {
|
||||
const out = parse(fixture("p5_volvo_tr"));
|
||||
expect(out.transmission).toBe("6-PSHIFT 2WD / MPS6");
|
||||
// Tek harflik "Şanzıman kodu" artık kazanmıyor.
|
||||
expect(out.transmission).not.toBe("B");
|
||||
});
|
||||
|
||||
it("İngilizce etiketlerde de aynı sonucu verir", () => {
|
||||
const out = parse(fixture("p5_volvo_en"));
|
||||
expect(out.transmission).toBe("6-PSHIFT 2WD / MPS6");
|
||||
});
|
||||
|
||||
it("seri, çıplak satış kodu yerine gerçek tipi döner", () => {
|
||||
expect(parse(fixture("p5_volvo_tr")).series).toBe("S80");
|
||||
expect(parse(fixture("p5_volvo_en")).series).toBe("S80");
|
||||
expect(parse(fixture("p5_volvo_tr")).series).not.toBe("42");
|
||||
});
|
||||
|
||||
it("kaporta stili iki dilde de çözülür ve '0' kodu sızmaz", () => {
|
||||
expect(parse(fixture("p5_volvo_tr")).bodyType).toBe("Sedan");
|
||||
expect(parse(fixture("p5_volvo_en")).bodyType).toBe("Sedan");
|
||||
});
|
||||
|
||||
it("motor alanları bozulmadan kalır", () => {
|
||||
const out = parse(fixture("p5_volvo_tr"));
|
||||
expect(out.engineCode).toBe("84");
|
||||
expect(out.engineType).toBe("D4162T");
|
||||
});
|
||||
|
||||
it("model ve yıl korunur", () => {
|
||||
const out = parse(fixture("p5_volvo_tr"));
|
||||
expect(out.model).toBe("S80 (07-)");
|
||||
expect(out.year).toBe(2013);
|
||||
});
|
||||
});
|
||||
@@ -1184,6 +1184,31 @@ export class PL24Service {
|
||||
return null;
|
||||
};
|
||||
|
||||
/**
|
||||
* Like `lookup`, but prefers a human-readable value over a bare internal
|
||||
* code when the record carries both.
|
||||
*
|
||||
* Volvo's vinfoBasic has BOTH "Şanzıman kodu" ("B") and "Şanzıman"
|
||||
* ("6-PSHIFT 2WD / MPS6"), and BOTH "Satis tipi" ("42") and a second
|
||||
* "Satis tipi" ("SALES VERSION 42"). The code-first key order that VAG needs
|
||||
* (its "Şanzıman kodu" IS the useful value, e.g. "MQ200") therefore rendered
|
||||
* a single letter as the Volvo gearbox and a bare number as its series.
|
||||
*
|
||||
* Falls back to the first present value, so a backend that only ever emits
|
||||
* codes behaves exactly as before.
|
||||
*/
|
||||
const lookupDescriptive = (...keys: string[]): string | null => {
|
||||
let firstPresent: string | null = null;
|
||||
for (const k of keys) {
|
||||
const v = vehicleData[k]?.trim();
|
||||
if (!v) continue;
|
||||
if (firstPresent === null) firstPresent = v;
|
||||
// Two characters or fewer, or digits only → an index, not a description.
|
||||
if (v.length > 2 && !/^\d+$/.test(v)) return v;
|
||||
}
|
||||
return firstPresent;
|
||||
};
|
||||
|
||||
// Extract prNr records for richer vehicle attributes
|
||||
const prNrRecords = segments.prNr?.records || [];
|
||||
const prNrByCode: Record<string, string> = {};
|
||||
@@ -1226,13 +1251,17 @@ export class PL24Service {
|
||||
// normalizeLabel folds ı→i and strips diacritics, so "Şanzıman kodu" and
|
||||
// "ŞANZIMAN KODU" both arrive as "sanziman_kodu". PSA uses "AKTARMA
|
||||
// SİSTEMLERİ" ("5 MEKANİK VİTES KUTUSU"), Subaru "Mission".
|
||||
const transmissionCode = lookup(
|
||||
const transmissionCode = lookupDescriptive(
|
||||
"sanziman_kodu",
|
||||
"transmission_code",
|
||||
"vites_kutusu",
|
||||
"atm,mtm",
|
||||
"aktarma_sistemleri",
|
||||
"sanziman",
|
||||
// Volvo in the English locale: "Transmission code" ("B") plus the readable
|
||||
// "Transmission" ("6-PSHIFT 2WD / MPS6"). Without the plain key the
|
||||
// English response falls back to the single-letter code.
|
||||
"transmission",
|
||||
"sanziman_numarasi",
|
||||
"mission",
|
||||
);
|
||||
@@ -1242,7 +1271,10 @@ export class PL24Service {
|
||||
const bodyType =
|
||||
Object.entries(prNrByCode).find(([code]) => code.startsWith("K8"))?.[1] ||
|
||||
// PSA "GÖVDE TİPİ" ("4 KAPILI SEDAN"), Volvo "Kaporta Stili" ("Sedan").
|
||||
lookup("karoseri", "body", "body_type", "govde_tipi", "kaporta_stili") ||
|
||||
// Volvo: TR "Kaporta Stili" / EN "Body style" ("Sedan"). Its
|
||||
// "Karoseri Tipi Kodu" / "Body style code" is a bare "0" — never matched
|
||||
// here because the lookup is exact-key, and that is deliberate.
|
||||
lookup("karoseri", "body", "body_type", "govde_tipi", "kaporta_stili", "body_style") ||
|
||||
null;
|
||||
|
||||
// Engine description from prNr D3* (Motor nitelikleri)
|
||||
@@ -1301,7 +1333,10 @@ export class PL24Service {
|
||||
damToModelYear(lookup("dam")) ||
|
||||
extractModelYear(vin) ||
|
||||
0,
|
||||
series: lookup("seri", "satis_tipi", "sales_type", "turu"),
|
||||
// Volvo's "Satis tipi" is the bare sales code ("42") with the readable
|
||||
// form ("SALES VERSION 42") in a duplicate row, and its "Türü" is the real
|
||||
// series ("S80") — so prefer whichever of these is actually descriptive.
|
||||
series: lookupDescriptive("seri", "satis_tipi", "sales_type", "turu", "type"),
|
||||
bodyType,
|
||||
engineCode:
|
||||
engineCode ||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { PL24_WMI_SERVICE_MAP, SERVICE_TO_BRAND } from "./pl24.types";
|
||||
import { PL24_SERVICE_CATALOGS, PL24_WMI_SERVICE_MAP, SERVICE_TO_BRAND } from "./pl24.types";
|
||||
|
||||
// Q6 routing additions (undecoded-vin-rca.md). Both target services are already
|
||||
// live in prod (nissan_parts: JN1 success; mercedesvans_parts: WDF decodes today),
|
||||
@@ -45,3 +45,53 @@ describe("PL24_WMI_SERVICE_MAP — kapsam dışı WMI'ler", () => {
|
||||
expect(PL24_WMI_SERVICE_MAP.VXK).toBe("psa_opel_parts");
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* 2026-09-20 canlı WMI servisi taraması (plv2.md, bulgu types_wmi-06/09).
|
||||
* Prod'daki 1.406 haritasız araçtan hangilerinin PL24'te gerçekten karşılığı
|
||||
* olduğu `/pl24-wmi/ext/api/2.0/decode` ile tek tek soruldu; bu test o cevapları
|
||||
* kilitliyor — özellikle "yok" cevaplarını, çünkü onları haritaya eklemek
|
||||
* boşuna istek üretir.
|
||||
*/
|
||||
describe("WMI haritası — canlı WMI servisi taraması (2026-09-20)", () => {
|
||||
it("Renault yeniden açıldı (VIN tanımlama askısı kalktı)", () => {
|
||||
expect(PL24_WMI_SERVICE_MAP.VF1).toBe("renault_parts");
|
||||
expect(PL24_WMI_SERVICE_MAP.VF6).toBe("renault_parts");
|
||||
expect(PL24_WMI_SERVICE_MAP.VNE).toBe("renault_parts");
|
||||
});
|
||||
|
||||
it("canlı servisin çözdüğü yeni WMI'ler haritada", () => {
|
||||
expect(PL24_WMI_SERVICE_MAP.NLH).toBe("hyundai_parts");
|
||||
expect(PL24_WMI_SERVICE_MAP.TMA).toBe("hyundai_parts");
|
||||
expect(PL24_WMI_SERVICE_MAP.NLJ).toBe("hyundai_parts");
|
||||
expect(PL24_WMI_SERVICE_MAP.KMF).toBe("hyundai_parts");
|
||||
expect(PL24_WMI_SERVICE_MAP.KNE).toBe("kia_parts");
|
||||
expect(PL24_WMI_SERVICE_MAP.KNC).toBe("kia_parts");
|
||||
expect(PL24_WMI_SERVICE_MAP.MMC).toBe("mmc_parts");
|
||||
expect(PL24_WMI_SERVICE_MAP.XMC).toBe("mmc_parts");
|
||||
expect(PL24_WMI_SERVICE_MAP.JSA).toBe("suzuki_parts");
|
||||
});
|
||||
|
||||
it("NMB binek Mercedes değil, kamyon kataloğuna gider", () => {
|
||||
expect(PL24_WMI_SERVICE_MAP.NMB).toBe("mercedestrucks_parts");
|
||||
// Binek WMI'leri bozulmadı.
|
||||
expect(PL24_WMI_SERVICE_MAP.WDD).toBe("mercedes_parts");
|
||||
});
|
||||
|
||||
it("PL24'te olmayan WMI'ler haritaya EKLENMEDİ (HTTP 410)", () => {
|
||||
// Honda ve Chevrolet'nin PL24'te kataloğu yok; eklemek boşuna istek olurdu.
|
||||
for (const wmi of ["JHM", "SHH", "SHS", "MAK", "NLA", "KL1", "NM4", "VR7"]) {
|
||||
expect(PL24_WMI_SERVICE_MAP[wmi]).toBeUndefined();
|
||||
}
|
||||
});
|
||||
|
||||
it("JMZ (Mazda) eklenmedi — servis Ford kataloğu öneriyor, marka tutarsız", () => {
|
||||
expect(PL24_WMI_SERVICE_MAP.JMZ).toBeUndefined();
|
||||
});
|
||||
|
||||
it("eşlenen her servisin katalog tanımı var", () => {
|
||||
for (const [wmi, svc] of Object.entries(PL24_WMI_SERVICE_MAP)) {
|
||||
expect(PL24_SERVICE_CATALOGS[svc], `${wmi} → ${svc}`).toBeDefined();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -540,6 +540,10 @@ export const PL24_WMI_SERVICE_MAP: Record<string, string> = {
|
||||
|
||||
// Mercedes-Benz
|
||||
WDB: "mercedes_parts",
|
||||
// NMB: canlı WMI servisi bunu mercedes_parts'a DEĞİL mercedestrucks_parts'a
|
||||
// çözüyor (error:false) — 30 prod aracı "Mercedes-Benz" etiketliydi ama binek
|
||||
// kataloğunda yok.
|
||||
NMB: "mercedestrucks_parts",
|
||||
WDD: "mercedes_parts",
|
||||
WDC: "mercedes_parts",
|
||||
W1K: "mercedes_parts",
|
||||
@@ -578,15 +582,21 @@ export const PL24_WMI_SERVICE_MAP: Record<string, string> = {
|
||||
JTJ: "lexus_parts",
|
||||
"2T2": "lexus_parts",
|
||||
|
||||
// Renault — DISABLED: PL24 has suspended Renault VIN identification ("Bu marka
|
||||
// için şasi numarası tanımlamasının belirsiz bir süre için mevcut olmayacağını
|
||||
// üzülerek bildiririz."). renault_parts authorizes fine but every decode throws
|
||||
// that message → wasted ~1s call AND it trips the PL24 circuit breaker, which
|
||||
// then skips PL24 for ALL brands. PCAT + EMEX cover Renault. Re-enable when PL24
|
||||
// restores Renault VIN decode.
|
||||
// VF1: "renault_parts",
|
||||
// VF6: "renault_parts",
|
||||
// VNE: "renault_parts",
|
||||
// Renault — RE-ENABLED 2026-09-20. It was disabled while PL24 had suspended
|
||||
// Renault VIN identification ("Bu marka için şasi numarası tanımlamasının
|
||||
// belirsiz bir süre için mevcut olmayacağını üzülerek bildiririz."), which both
|
||||
// wasted a call per decode and, back then, tripped the global PL24 breaker.
|
||||
// BOTH reasons are gone, each verified against prod rather than assumed:
|
||||
// 1. The suspension is over — live directAccess on /p5renault for a real
|
||||
// customer VIN (VF14SRCL458170337) returns resultStatus
|
||||
// VEHICLE_IDENTIFIED, "SYMBOL II/LOGAN II", and the WMI service answers
|
||||
// {service: renault_parts, valid: true, error: false}.
|
||||
// 2. The breaker no longer counts definitive upstream negatives, only
|
||||
// transient transport faults (see vehicles.service `isTransient`).
|
||||
// 450 prod vehicles carry VF1 and had no PL24 catalog at all.
|
||||
VF1: "renault_parts",
|
||||
VF6: "renault_parts",
|
||||
VNE: "renault_parts",
|
||||
|
||||
// Dacia
|
||||
UU1: "dacia_parts",
|
||||
@@ -606,6 +616,10 @@ export const PL24_WMI_SERVICE_MAP: Record<string, string> = {
|
||||
WMH: "man_parts",
|
||||
|
||||
// Mitsubishi
|
||||
// XMC / MMC: canlı WMI servisi ikisini de mmc_parts'a çözüyor ve P5 doğruluyor
|
||||
// (error:false) — 26 prod aracı haritasızdı.
|
||||
MMC: "mmc_parts", // Mitsubishi Japan
|
||||
XMC: "mmc_parts", // Mitsubishi (diğer pazarlar)
|
||||
JMB: "mmc_parts",
|
||||
JMY: "mmc_parts",
|
||||
MMB: "mmc_parts",
|
||||
@@ -615,6 +629,7 @@ export const PL24_WMI_SERVICE_MAP: Record<string, string> = {
|
||||
JS2: "suzuki_parts",
|
||||
JS3: "suzuki_parts",
|
||||
TSM: "suzuki_parts",
|
||||
JSA: "suzuki_parts", // Suzuki (canlı WMI: suzuki_parts, error:false)
|
||||
MA3: "suzuki_parts",
|
||||
MBH: "suzuki_parts",
|
||||
|
||||
@@ -638,10 +653,19 @@ export const PL24_WMI_SERVICE_MAP: Record<string, string> = {
|
||||
|
||||
// Hyundai
|
||||
KMH: "hyundai_parts", // Hyundai Korea Motor House
|
||||
// Aşağıdakiler 2026-09-20'de canlı WMI servisinden alındı; hepsi hyundai_parts.
|
||||
// TMA iki marka döndürüyor (hyundai_parts + kia_parts) — Hyundai Assan (Türkiye)
|
||||
// üretimi olduğu için hyundai seçildi.
|
||||
NLH: "hyundai_parts", // Hyundai Assan / Türkiye (106 prod aracı)
|
||||
TMA: "hyundai_parts", // Hyundai Türkiye (49)
|
||||
NLJ: "hyundai_parts", // Hyundai (8)
|
||||
KMF: "hyundai_parts", // Hyundai (7)
|
||||
TMK: "hyundai_parts", // Hyundai (Turkey/other markets)
|
||||
|
||||
// Kia
|
||||
KNA: "kia_parts", // Kia (worldwide production)
|
||||
KNE: "kia_parts", // Kia (canlı WMI, 31 prod aracı)
|
||||
KNC: "kia_parts", // Kia (canlı WMI, 8)
|
||||
U5Y: "kia_parts", // Kia Slovakia
|
||||
|
||||
// Nissan
|
||||
@@ -682,6 +706,13 @@ export const PL24_WMI_SERVICE_MAP: Record<string, string> = {
|
||||
// kaydedilmişti. Haritada tutmak yalnız boşuna PL24 isteği üretir ve
|
||||
// pcat/emex/vinpin fallback'ini geciktirir.
|
||||
|
||||
// PL24'TE HİÇ OLMAYANLAR — canlı WMI servisi HTTP 410 "no brands found for WMI"
|
||||
// dedi (2026-09-20), tıpkı NM4 ve VR7 gibi. Haritaya eklenmemeleri kasıtlı:
|
||||
// eklemek yalnız boşuna istek üretir ve pcat/emex/vinpin fallback'ini geciktirir.
|
||||
// JHM, SHH, SHS, MAK, NLA (Honda) · KL1 (Chevrolet)
|
||||
// JMZ (Mazda) 410 DEĞİL ama fordp/fordt döndürüyor (Ford-Mazda platform ortaklığı
|
||||
// dönemi); marka tutarsız olduğu için eklenmedi — bir Mazda 3 Ford kataloğunda yok.
|
||||
|
||||
// Volvo
|
||||
YV1: "volvo_parts", // Volvo Cars (Sweden)
|
||||
YV4: "volvo_parts", // Volvo Cars (specific models)
|
||||
|
||||
@@ -74,14 +74,21 @@ export async function checkCooldown(redis: RedisService, source: string): Promis
|
||||
}
|
||||
}
|
||||
|
||||
/** Current hour (0–23) in Europe/Istanbul. */
|
||||
function currentIstanbulHour(): number {
|
||||
/** Europe/Istanbul hour (0–23) at an arbitrary instant. */
|
||||
function istanbulHourAt(ms: number): number {
|
||||
const hourStr = new Intl.DateTimeFormat("en-US", {
|
||||
timeZone: "Europe/Istanbul",
|
||||
hour: "numeric",
|
||||
hour12: false,
|
||||
}).format(new Date());
|
||||
return Number.parseInt(hourStr, 10);
|
||||
}).format(new Date(ms));
|
||||
// `% 24` because the h24 hour cycle renders midnight as "24", which would put
|
||||
// the hour outside every window and silently park the source forever.
|
||||
return Number.parseInt(hourStr, 10) % 24;
|
||||
}
|
||||
|
||||
/** Current hour (0–23) in Europe/Istanbul. */
|
||||
function currentIstanbulHour(): number {
|
||||
return istanbulHourAt(Date.now());
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -115,6 +122,35 @@ export function checkTimeWindow(source: string): void {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The first instant at or after `fromMs` that falls inside `source`'s scrape
|
||||
* window. Returns `fromMs` unchanged when the window is disabled (the default
|
||||
* 0–24) or when `fromMs` is already inside it.
|
||||
*
|
||||
* WHY (plv2.md, finding consumers_jobs-03 — the budget/window deadlock):
|
||||
* `checkSourceDailyBudget` defers a spent source to the next UTC midnight. With
|
||||
* PREFETCH_PL24_START=9 that midnight lands at 03:00 Europe/Istanbul — six hours
|
||||
* BEFORE the window opens. The woken job therefore did no work, threw
|
||||
* `time-window`, and was deferred again to 09:00 — by which point the fresh
|
||||
* daily budget had already been spent by the same stampede of no-op wake-ups.
|
||||
* Measured on prod 2026-09-20: 600/600 pl24 budget consumed, 0 catalog requests
|
||||
* and 0 new categories for the whole day. Landing the deferral inside the window
|
||||
* breaks the cycle.
|
||||
*/
|
||||
export function alignToWindow(source: string, fromMs: number): number {
|
||||
if (source !== "pl24") return fromMs;
|
||||
if (PL24_WINDOW_START <= 0 && PL24_WINDOW_END >= 24) return fromMs;
|
||||
let t = fromMs;
|
||||
// Step by the hour rather than constructing a local-midnight date: DST-safe and
|
||||
// free of month/year rollover edge cases. 48 steps covers any window shape.
|
||||
for (let i = 0; i < 48; i++) {
|
||||
const h = istanbulHourAt(t);
|
||||
if (h >= PL24_WINDOW_START && h < PL24_WINDOW_END) return t;
|
||||
t += 3_600_000;
|
||||
}
|
||||
return t;
|
||||
}
|
||||
|
||||
/**
|
||||
* Milliseconds until the next 09:00 Europe/Istanbul.
|
||||
*/
|
||||
|
||||
286
apps/api/src/jobs/prefetch-window-budget.spec.ts
Normal file
286
apps/api/src/jobs/prefetch-window-budget.spec.ts
Normal file
@@ -0,0 +1,286 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
/**
|
||||
* Regression lock for the PL24 budget/window DEADLOCK (plv2.md, consumers_jobs-03).
|
||||
*
|
||||
* Two bugs combined to kill PL24 prefetch outright on prod:
|
||||
* 1. `checkSourceDailyBudget` was debited in `process()` while the
|
||||
* business-hours gate still sat at the top of each handler — so a job that
|
||||
* woke outside the window paid a budget unit to do nothing.
|
||||
* 2. A spent source was deferred to the next UTC midnight, which is 03:00
|
||||
* Europe/Istanbul — six hours BEFORE a 09:00 window opens. Every deferred
|
||||
* job therefore woke early, burned a unit of the fresh daily budget, hit
|
||||
* the window gate and was deferred again.
|
||||
*
|
||||
* Measured on prod 2026-09-20 before the fix: `prefetch:daily:pl24` at 600/600
|
||||
* with ZERO catalog requests and ZERO new pl24 categories for the whole day.
|
||||
*
|
||||
* The window constants are read at module load, so every case imports the
|
||||
* modules fresh with the env already set.
|
||||
*/
|
||||
|
||||
const REAL_ENV = { ...process.env };
|
||||
|
||||
/** 2026-09-20 00:00 UTC = 03:00 Europe/Istanbul — outside a 09:00–18:00 window. */
|
||||
const OUTSIDE_MS = Date.UTC(2026, 8, 20, 0, 0, 0);
|
||||
/** 2026-09-20 09:00 UTC = 12:00 Europe/Istanbul — inside it. */
|
||||
const INSIDE_MS = Date.UTC(2026, 8, 20, 9, 0, 0);
|
||||
|
||||
function istanbulHour(ms: number): number {
|
||||
return (
|
||||
Number.parseInt(
|
||||
new Intl.DateTimeFormat("en-US", {
|
||||
timeZone: "Europe/Istanbul",
|
||||
hour: "numeric",
|
||||
hour12: false,
|
||||
}).format(new Date(ms)),
|
||||
10,
|
||||
) % 24
|
||||
);
|
||||
}
|
||||
|
||||
async function loadModules(env: Record<string, string | undefined>) {
|
||||
vi.resetModules();
|
||||
for (const [k, v] of Object.entries(env)) {
|
||||
if (v === undefined) delete process.env[k];
|
||||
else process.env[k] = v;
|
||||
}
|
||||
const utils = await import("./prefetch-utils");
|
||||
const worker = await import("./prefetch-worker.service");
|
||||
return { utils, worker };
|
||||
}
|
||||
|
||||
/** Minimal deps: only what `process()` touches before dispatching a job. */
|
||||
function makeDeps(redisOverrides: Record<string, unknown> = {}) {
|
||||
const incrCalls: string[] = [];
|
||||
const redis = {
|
||||
exists: vi.fn(async (..._a: unknown[]) => false),
|
||||
get: vi.fn(async (..._a: unknown[]): Promise<string | null> => null),
|
||||
set: vi.fn(async (..._a: unknown[]) => undefined),
|
||||
del: vi.fn(async (..._a: unknown[]) => undefined),
|
||||
incr: vi.fn(async (k: string) => {
|
||||
incrCalls.push(k);
|
||||
return 1;
|
||||
}),
|
||||
expire: vi.fn(async (..._a: unknown[]) => undefined),
|
||||
ttl: vi.fn(async (..._a: unknown[]) => -2), // no cooldown key
|
||||
setNx: vi.fn(async (..._a: unknown[]) => true),
|
||||
getJson: vi.fn(async (..._a: unknown[]): Promise<unknown> => null),
|
||||
setJson: vi.fn(async (..._a: unknown[]) => undefined),
|
||||
...redisOverrides,
|
||||
};
|
||||
const queue = {
|
||||
name: "catalog-prefetch",
|
||||
add: vi.fn(async (..._a: unknown[]) => undefined),
|
||||
getJob: vi.fn(async (..._a: unknown[]): Promise<unknown> => null),
|
||||
};
|
||||
const categoriesService = {
|
||||
getCategoryWithParts: vi.fn(async (..._a: unknown[]) => ({ parts: [] })),
|
||||
getChildren: vi.fn(async (..._a: unknown[]) => []),
|
||||
};
|
||||
return { redis, queue, categoriesService, incrCalls };
|
||||
}
|
||||
|
||||
function makeJob(name: string, data: Record<string, unknown>) {
|
||||
return {
|
||||
name,
|
||||
data,
|
||||
moveToDelayed: vi.fn(async (..._a: unknown[]) => undefined),
|
||||
attemptsMade: 0,
|
||||
};
|
||||
}
|
||||
|
||||
const dailyIncrs = (keys: string[]) => keys.filter((k) => k.startsWith("prefetch:daily:pl24"));
|
||||
|
||||
describe("alignToWindow", () => {
|
||||
afterEach(() => {
|
||||
process.env = { ...REAL_ENV };
|
||||
});
|
||||
|
||||
it("pushes a pre-window instant into the configured window", async () => {
|
||||
const { utils } = await loadModules({ PREFETCH_PL24_START: "9", PREFETCH_PL24_END: "18" });
|
||||
const aligned = utils.alignToWindow("pl24", OUTSIDE_MS);
|
||||
expect(aligned).toBeGreaterThan(OUTSIDE_MS);
|
||||
const h = istanbulHour(aligned);
|
||||
expect(h).toBeGreaterThanOrEqual(9);
|
||||
expect(h).toBeLessThan(18);
|
||||
});
|
||||
|
||||
it("leaves an in-window instant untouched", async () => {
|
||||
const { utils } = await loadModules({ PREFETCH_PL24_START: "9", PREFETCH_PL24_END: "18" });
|
||||
expect(utils.alignToWindow("pl24", INSIDE_MS)).toBe(INSIDE_MS);
|
||||
});
|
||||
|
||||
it("is a no-op for sources that have no window", async () => {
|
||||
const { utils } = await loadModules({ PREFETCH_PL24_START: "9", PREFETCH_PL24_END: "18" });
|
||||
expect(utils.alignToWindow("emex", OUTSIDE_MS)).toBe(OUTSIDE_MS);
|
||||
expect(utils.alignToWindow("parts-catalogs", OUTSIDE_MS)).toBe(OUTSIDE_MS);
|
||||
});
|
||||
|
||||
it("is a no-op when the window is disabled (the default 0–24)", async () => {
|
||||
const { utils } = await loadModules({
|
||||
PREFETCH_PL24_START: undefined,
|
||||
PREFETCH_PL24_END: undefined,
|
||||
});
|
||||
expect(utils.alignToWindow("pl24", OUTSIDE_MS)).toBe(OUTSIDE_MS);
|
||||
});
|
||||
});
|
||||
|
||||
describe("process() gate order — window before daily budget", () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
});
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
process.env = { ...REAL_ENV };
|
||||
});
|
||||
|
||||
it("does NOT debit the daily budget for a job deferred by the window", async () => {
|
||||
const { worker } = await loadModules({
|
||||
PREFETCH_PL24_START: "9",
|
||||
PREFETCH_PL24_END: "18",
|
||||
PL24_TR_DISABLED: undefined,
|
||||
});
|
||||
vi.setSystemTime(OUTSIDE_MS);
|
||||
const { redis, queue, categoriesService, incrCalls } = makeDeps();
|
||||
const svc = new worker.PrefetchWorkerService(
|
||||
queue as never,
|
||||
queue as never,
|
||||
categoriesService as never,
|
||||
redis as never,
|
||||
{ payload: vi.fn(async () => ({})) } as never,
|
||||
{} as never,
|
||||
);
|
||||
const job = makeJob("prefetch-parts", {
|
||||
vehicleId: "v1",
|
||||
categoryId: "c1",
|
||||
source: "pl24",
|
||||
fast: true,
|
||||
});
|
||||
|
||||
await expect(
|
||||
(svc as never as { process: (j: unknown, t?: string) => Promise<void> }).process(job, "tok"),
|
||||
).rejects.toThrow();
|
||||
|
||||
expect(job.moveToDelayed).toHaveBeenCalled();
|
||||
expect(dailyIncrs(incrCalls)).toHaveLength(0);
|
||||
// The per-minute counter is not charged either: the window gate is pure
|
||||
// clock arithmetic and runs before any Redis write.
|
||||
expect(incrCalls).toHaveLength(0);
|
||||
// …and the handler never ran, so nothing was fetched upstream either.
|
||||
expect(categoriesService.getCategoryWithParts).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("defers the window-blocked job to an instant inside the window", async () => {
|
||||
const { worker } = await loadModules({
|
||||
PREFETCH_PL24_START: "9",
|
||||
PREFETCH_PL24_END: "18",
|
||||
PL24_TR_DISABLED: undefined,
|
||||
});
|
||||
vi.setSystemTime(OUTSIDE_MS);
|
||||
const { redis, queue, categoriesService } = makeDeps();
|
||||
const svc = new worker.PrefetchWorkerService(
|
||||
queue as never,
|
||||
queue as never,
|
||||
categoriesService as never,
|
||||
redis as never,
|
||||
{ payload: vi.fn(async () => ({})) } as never,
|
||||
{} as never,
|
||||
);
|
||||
const job = makeJob("prefetch-parts", {
|
||||
vehicleId: "v1",
|
||||
categoryId: "c1",
|
||||
source: "pl24",
|
||||
fast: true,
|
||||
});
|
||||
await expect(
|
||||
(svc as never as { process: (j: unknown, t?: string) => Promise<void> }).process(job, "tok"),
|
||||
).rejects.toThrow();
|
||||
|
||||
const [when] = job.moveToDelayed.mock.calls[0] as [number];
|
||||
const h = istanbulHour(when);
|
||||
expect(h).toBeGreaterThanOrEqual(9);
|
||||
expect(h).toBeLessThan(18);
|
||||
});
|
||||
|
||||
it("debits the daily budget once the window is open", async () => {
|
||||
const { worker } = await loadModules({
|
||||
PREFETCH_PL24_START: "9",
|
||||
PREFETCH_PL24_END: "18",
|
||||
PL24_TR_DISABLED: undefined,
|
||||
});
|
||||
vi.setSystemTime(INSIDE_MS);
|
||||
const { redis, queue, categoriesService, incrCalls } = makeDeps();
|
||||
const svc = new worker.PrefetchWorkerService(
|
||||
queue as never,
|
||||
queue as never,
|
||||
categoriesService as never,
|
||||
redis as never,
|
||||
{ payload: vi.fn(async () => ({})) } as never,
|
||||
{} as never,
|
||||
);
|
||||
const job = makeJob("prefetch-parts", {
|
||||
vehicleId: "v1",
|
||||
categoryId: "c1",
|
||||
source: "pl24",
|
||||
fast: true,
|
||||
});
|
||||
|
||||
await (svc as never as { process: (j: unknown, t?: string) => Promise<void> }).process(
|
||||
job,
|
||||
"tok",
|
||||
);
|
||||
|
||||
expect(dailyIncrs(incrCalls)).toHaveLength(1);
|
||||
expect(categoriesService.getCategoryWithParts).toHaveBeenCalledWith("c1");
|
||||
});
|
||||
});
|
||||
|
||||
describe("checkSourceDailyBudget — spent source retries inside the window", () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
});
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
process.env = { ...REAL_ENV };
|
||||
});
|
||||
|
||||
it("never parks a spent source at 03:00 Istanbul again", async () => {
|
||||
const { worker } = await loadModules({
|
||||
PREFETCH_PL24_START: "9",
|
||||
PREFETCH_PL24_END: "18",
|
||||
PREFETCH_DAILY_PL24: "600",
|
||||
});
|
||||
vi.setSystemTime(INSIDE_MS);
|
||||
// Counter already at the fast-lane ceiling → the next job must be deferred.
|
||||
const { redis, queue, categoriesService } = makeDeps({
|
||||
get: vi.fn(async (k: string) => (String(k).startsWith("prefetch:daily:pl24") ? "600" : null)),
|
||||
});
|
||||
const svc = new worker.PrefetchWorkerService(
|
||||
queue as never,
|
||||
queue as never,
|
||||
categoriesService as never,
|
||||
redis as never,
|
||||
{ payload: vi.fn(async () => ({})) } as never,
|
||||
{} as never,
|
||||
);
|
||||
const job = makeJob("prefetch-parts", {
|
||||
vehicleId: "v1",
|
||||
categoryId: "c1",
|
||||
source: "pl24",
|
||||
fast: true,
|
||||
});
|
||||
|
||||
await expect(
|
||||
(svc as never as { process: (j: unknown, t?: string) => Promise<void> }).process(job, "tok"),
|
||||
).rejects.toThrow();
|
||||
|
||||
const [when] = job.moveToDelayed.mock.calls[0] as [number];
|
||||
// Past the UTC rollover…
|
||||
expect(when).toBeGreaterThan(Date.UTC(2026, 8, 21, 0, 0, 0));
|
||||
// …and inside the window, not at 03:00 Istanbul like the old deferral.
|
||||
const h = istanbulHour(when);
|
||||
expect(h).toBeGreaterThanOrEqual(9);
|
||||
expect(h).toBeLessThan(18);
|
||||
});
|
||||
});
|
||||
@@ -17,6 +17,7 @@ import { QUEUE_NAMES, getBullConnection } from "./bull.config";
|
||||
import { backfillContext } from "./prefetch-context";
|
||||
import {
|
||||
RateLimitError,
|
||||
alignToWindow,
|
||||
checkCooldown,
|
||||
checkTimeWindow,
|
||||
initProgress,
|
||||
@@ -354,12 +355,30 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy {
|
||||
job.name === "prefetch-parts")
|
||||
) {
|
||||
const lane = (job.data as { fast?: boolean }).fast ? "fast" : "main";
|
||||
// GATE ORDER IS LOAD-BEARING — cheapest first, and every gate that can
|
||||
// reject the job must run BEFORE any counter is debited.
|
||||
//
|
||||
// The business-hours window and the cooldown used to sit at the top of
|
||||
// each handler, i.e. AFTER the per-minute counter and the daily budget
|
||||
// had already been charged. So a job that woke outside the window paid a
|
||||
// budget unit to do nothing. Combined with a deferral target of "next UTC
|
||||
// midnight" (= 03:00 Europe/Istanbul, six hours before a 09:00 window
|
||||
// opens) that closed a loop: the whole daily allowance was burned by
|
||||
// no-op wake-ups before the window ever opened, so the source never ran
|
||||
// again. Measured on prod 2026-09-20 — pl24 at 600/600 with 0 catalog
|
||||
// requests and 0 new categories for the day.
|
||||
//
|
||||
// 1. Window: pure clock arithmetic, no I/O, and an out-of-window job can
|
||||
// never do useful work — so nothing else is worth spending on it.
|
||||
checkTimeWindow(data.source);
|
||||
// 2. Cooldown: one Redis TTL read. Pauses the whole worker (see catch).
|
||||
await checkCooldown(this.redis, data.source);
|
||||
// 3. Per-minute ceiling.
|
||||
await this.checkSourceRate(data.source, lane);
|
||||
// Daily budget AFTER the per-minute gate: a job deferred on the minute
|
||||
// ceiling above never reaches here, so rate-limited retries don't inflate
|
||||
// the daily counter — only jobs about to do real work are counted. The
|
||||
// lane decides which threshold applies (backfill stops at the main limit,
|
||||
// the user's fast lane may use the full budget).
|
||||
// 4. Daily budget last: a job deferred on any gate above never reaches
|
||||
// here, so only jobs about to do real work are counted. The lane
|
||||
// decides which threshold applies (backfill stops at the main limit,
|
||||
// the user's fast lane may use the full budget).
|
||||
await this.checkSourceDailyBudget(data.source, lane);
|
||||
}
|
||||
if (data.source === "parts-catalogs" && PCAT_PACE_MS > 0) {
|
||||
@@ -421,8 +440,8 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy {
|
||||
const { vehicleId, source, fast = false } = job.data;
|
||||
this.logger.log(`[prefetch] Init for vehicle=${vehicleId}, source=${source}`);
|
||||
|
||||
await checkCooldown(this.redis, source);
|
||||
checkTimeWindow(source);
|
||||
// Cooldown + time-window are enforced in process() before the daily budget
|
||||
// is debited — see the comment there; re-checking here would be a no-op.
|
||||
|
||||
// Already flagged as poison (tree exceeded CATEGORY_CAP on a prior run) — skip.
|
||||
if (await this.redis.exists(this.poisonKey(vehicleId))) {
|
||||
@@ -563,8 +582,8 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy {
|
||||
const { vehicleId, categoryId, source, depth, fast = false } = job.data;
|
||||
this.logger.log(`[prefetch] Children for category=${categoryId}, depth=${depth}`);
|
||||
|
||||
await checkCooldown(this.redis, source);
|
||||
checkTimeWindow(source);
|
||||
// Cooldown + time-window are enforced in process() before the daily budget
|
||||
// is debited — see the comment there; re-checking here would be a no-op.
|
||||
|
||||
const depthCeiling = maxDepthFor(source, fast);
|
||||
if (depth >= depthCeiling) {
|
||||
@@ -631,8 +650,8 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy {
|
||||
const { vehicleId, categoryId, source } = job.data;
|
||||
this.logger.log(`[prefetch] Parts for category=${categoryId}`);
|
||||
|
||||
await checkCooldown(this.redis, source);
|
||||
checkTimeWindow(source);
|
||||
// Cooldown + time-window are enforced in process() before the daily budget
|
||||
// is debited — see the comment there; re-checking here would be a no-op.
|
||||
|
||||
try {
|
||||
await this.categoriesService.getCategoryWithParts(categoryId);
|
||||
@@ -1214,11 +1233,17 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy {
|
||||
// deferred job wakes in the SAME millisecond (observed: 11495 jobs all at
|
||||
// 00:00:01 UTC) — the promotion lands as one burst and the pressure signal
|
||||
// flaps. Other sources keep flowing (per-job defer, not a worker pause).
|
||||
const msLeft = dayMs - (now % dayMs) + 1000 + Math.floor(Math.random() * 45 * 60_000);
|
||||
const rollover = now + dayMs - (now % dayMs) + 1000 + Math.floor(Math.random() * 45 * 60_000);
|
||||
// Land the retry INSIDE the source's scrape window. The UTC rollover alone
|
||||
// is 03:00 Europe/Istanbul, so with a 09:00 window every deferred job woke
|
||||
// six hours early, failed the window check and was deferred again — the
|
||||
// other half of the deadlock fixed in process(). alignToWindow is a no-op
|
||||
// when no window is configured (the default).
|
||||
const msLeft = Math.max(1000, alignToWindow(source, rollover) - now);
|
||||
if (n === limit) {
|
||||
this.logger.warn(
|
||||
`[prefetch] ${source} daily budget hit (lane=${lane}, ${n}/${limit} of ${max}) — ` +
|
||||
`deferring ~${Math.round(msLeft / 3_600_000)}h until the window rolls`,
|
||||
`deferring ~${Math.round(msLeft / 3_600_000)}h to the next in-window slot`,
|
||||
);
|
||||
}
|
||||
throw new RateLimitError(msLeft, "source-rate");
|
||||
|
||||
Reference in New Issue
Block a user