From 02ec3856d52fed4da429b6703a443f14b61a1042 Mon Sep 17 00:00:00 2001 From: semih Date: Thu, 17 Sep 2026 13:44:59 +0300 Subject: [PATCH 1/3] =?UTF-8?q?perf(pl24):=20reaktif=20drill=20derinli?= =?UTF-8?q?=C4=9Fini=20s=C4=B1n=C4=B1rla,=20backfill'i=20jitter'l=C4=B1=20?= =?UTF-8?q?aral=C4=B1kla=20yava=C5=9Flat?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Faz 1 / adım 2b (analiz: /home/s/ss/plv2.md, bulgu consumers_jobs-03). SORUN: Ban'ı süren hacim backfill değil, her yeni decode'da çalışan fast-lane TAM AĞAÇ drill'iydi. 2026-08-18→09-04 arasında (PL24_TR_DISABLED main lane'i park etmişken) yeni kategorilerin %100'ü aynı gün decode edilen araçlardan geldi: günde 3-17 decode → 1.4k-6.8k kategori (bir Passat 1.251, bir L200 2.323). Araç başına drill derinliği sınırsızdı ve PL24 istekleri arasında hiç bekleme yoktu. DEĞİŞİKLİK: - `PREFETCH_PL24_FAST_DEPTH` (varsayılan 1): PL24 reaktif/fast lane yalnız üst gruplar + doğrudan çocuklarını gezer. Daha derini ya kullanıcı o düğümü açtığında (lazy) ya da bütçeli backfill lane'inde çekilir. Diğer kaynaklar ve PL24 main lane MAX_DEPTH kullanmaya devam eder. Fast lane'in derinlik tavanına ulaşması normal durum olduğu için log seviyesi warn değil log. - `PREFETCH_PL24_DELAY_MS` (varsayılan 8000) + ±%50 jitter: PL24 BACKFILL işlerine per-job bekleme. Kullanıcının fast lane'i asla yavaşlatılmaz. Metronom gibi düzenli akış otomasyon imzasıdır; jitter onu kırar. - Telemetri düzeltmesi: `proxy_logs`'a ham hesap etiketi ("tr") yerine gerçekte kullanılan hesap yazılıyor (PL24_TR_DISABLED altında "de"). Yeni `PL24AuthService.activeAccount()`. Test: `__testables` ile maxDepthFor/jitter dışa açıldı + derinlik tavanı testi. Etkilenen paketler: 198 test geçti. tsc + biome temiz. Co-Authored-By: Claude Opus 5 (1M context) --- .../integrations/pl24/pl24-auth.service.ts | 9 ++++ .../api/src/integrations/pl24/pl24.service.ts | 5 +- .../src/jobs/prefetch-worker.service.spec.ts | 25 +++++++++ apps/api/src/jobs/prefetch-worker.service.ts | 53 ++++++++++++++++++- 4 files changed, 88 insertions(+), 4 deletions(-) diff --git a/apps/api/src/integrations/pl24/pl24-auth.service.ts b/apps/api/src/integrations/pl24/pl24-auth.service.ts index 2a2c455..a22a0e5 100644 --- a/apps/api/src/integrations/pl24/pl24-auth.service.ts +++ b/apps/api/src/integrations/pl24/pl24-auth.service.ts @@ -659,6 +659,15 @@ export class PL24AuthService implements OnModuleInit { }; } + /** + * The account a request will really use (tr → de under PL24_TR_DISABLED). + * Telemetry must log this, not the raw argument, or `proxy_logs` claims the + * dead account served traffic. + */ + activeAccount(account: PL24Account): PL24Account { + return this.effectiveAccount(account); + } + /** Return ProxyAgent for account 'de', null for 'tr'. */ async getProxyAgent4Account(account: PL24Account): Promise { if (this.effectiveAccount(account) !== "de") return null; diff --git a/apps/api/src/integrations/pl24/pl24.service.ts b/apps/api/src/integrations/pl24/pl24.service.ts index cb79ead..8914af9 100644 --- a/apps/api/src/integrations/pl24/pl24.service.ts +++ b/apps/api/src/integrations/pl24/pl24.service.ts @@ -1003,13 +1003,14 @@ export class PL24Service { } await this.budget.consume("catalog"); const startedAt = Date.now(); + const activeAccount = this.authService.activeAccount(account); try { const response = await fetch(url, opts); this.budget.record({ kind: "catalog", url, proxied, - account, + account: activeAccount, statusCode: response.status, success: response.ok, startedAt, @@ -1020,7 +1021,7 @@ export class PL24Service { kind: "catalog", url, proxied, - account, + account: activeAccount, success: false, startedAt, error, diff --git a/apps/api/src/jobs/prefetch-worker.service.spec.ts b/apps/api/src/jobs/prefetch-worker.service.spec.ts index ddb333b..65c9843 100644 --- a/apps/api/src/jobs/prefetch-worker.service.spec.ts +++ b/apps/api/src/jobs/prefetch-worker.service.spec.ts @@ -314,3 +314,28 @@ describe("PrefetchWorkerService — fast lane (lifo) + backlog gating", () => { }); }); }); + +// ── PL24 reaktif drill derinliği + backfill pacing (plv2 Faz 1 / adım 2b) ── +// Ban'ı süren hacim, her yeni decode'da tüm ağacın gezilmesiydi (bir Passat = +// 1.251 kategori). Fast lane artık PL24'te 1. seviyede durur; derin drill ya +// kullanıcı tıklamasıyla ya da bütçeli backfill lane'inde olur. +describe("PrefetchWorkerService — PL24 derinlik tavanı", () => { + const load = async () => { + const mod = await import("./prefetch-worker.service"); + return mod as unknown as { + __testables?: { maxDepthFor(source: string, fast: boolean): number }; + }; + }; + + it("pl24 fast lane 1. seviyede durur, diğer kaynaklar tam derinlik kullanır", async () => { + // maxDepthFor modül-özel; davranışı dolaylı doğrula: env varsayılanları + process.env.PREFETCH_PL24_FAST_DEPTH = ""; + process.env.PREFETCH_MAX_DEPTH = ""; + const mod = await load(); + const fn = mod.__testables?.maxDepthFor; + if (!fn) return; // testable export yoksa atla (davranış e2e'de doğrulanır) + expect(fn("pl24", true)).toBe(1); + expect(fn("pl24", false)).toBeGreaterThan(1); + expect(fn("parts-catalogs", true)).toBeGreaterThan(1); + }); +}); diff --git a/apps/api/src/jobs/prefetch-worker.service.ts b/apps/api/src/jobs/prefetch-worker.service.ts index 0ead731..c93187c 100644 --- a/apps/api/src/jobs/prefetch-worker.service.ts +++ b/apps/api/src/jobs/prefetch-worker.service.ts @@ -170,6 +170,40 @@ const SOURCE_DAILY_MAX: Record = { */ const PCAT_PACE_MS = Number(process.env.PREFETCH_PCAT_DELAY_MS) || 1_500; +/** + * Per-job pacing for PL24 BACKFILL jobs (main lane only — a user waiting on a + * fresh decode must never be slowed down). Both account bans followed days of + * thousands of back-to-back PL24 calls; a paced, jittered stream looks nothing + * like that. Set 0 to disable. + */ +const PL24_PACE_MS = Number(process.env.PREFETCH_PL24_DELAY_MS) || 8_000; + +/** + * How deep the REACTIVE (fast-lane) drill may go for PL24. + * + * A freshly decoded vehicle used to be walked to the bottom immediately: one + * Passat produced 1,251 categories, one L200 2,323 — 1.4k-6.8k categories/day + * from 3-17 decodes, which is exactly the volume that preceded both bans + * (plv2.md §2.2). Depth 1 = top groups and their direct children; anything + * deeper is fetched lazily when the user actually opens that node, or by the + * budgeted backfill lane. Other sources keep MAX_DEPTH. + */ +const PL24_FAST_MAX_DEPTH = Number(process.env.PREFETCH_PL24_FAST_DEPTH) || 1; + +/** Depth ceiling for this source+lane. */ +function maxDepthFor(source: string, fast: boolean): number { + if (source === "pl24" && fast) return PL24_FAST_MAX_DEPTH; + return MAX_DEPTH; +} + +/** Jittered pace so our request stream is not a metronome. */ +function jitter(ms: number): number { + return Math.round(ms * (0.5 + Math.random())); +} + +/** Test-only surface for the pure helpers above. */ +export const __testables = { maxDepthFor, jitter }; + // ── Phase-1 residue exclusion ── /** * Skip a zero-parts vehicle once this many backfill attempts have completed with @@ -330,6 +364,15 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy { if (data.source === "parts-catalogs" && PCAT_PACE_MS > 0) { await new Promise((r) => setTimeout(r, PCAT_PACE_MS)); } + // PL24 backfill only: pace + jitter. The fast (user) lane is never delayed. + if ( + data.source === "pl24" && + PL24_PACE_MS > 0 && + !(job.data as { fast?: boolean }).fast && + (job.name === "prefetch-children" || job.name === "prefetch-parts") + ) { + await new Promise((r) => setTimeout(r, jitter(PL24_PACE_MS))); + } if (job.name === "backfill-scan") { return await this.processBackfillScan(); @@ -522,8 +565,14 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy { await checkCooldown(this.redis, source); checkTimeWindow(source); - if (depth >= MAX_DEPTH) { - this.logger.warn(`[prefetch] Max depth reached for category=${categoryId}`); + const depthCeiling = maxDepthFor(source, fast); + if (depth >= depthCeiling) { + // For the PL24 fast lane this is the normal stopping point, not a problem: + // deeper nodes are drilled lazily on user click or by the backfill lane. + const level = source === "pl24" && fast ? "log" : "warn"; + this.logger[level]( + `[prefetch] Depth ceiling ${depthCeiling} reached for category=${categoryId} (source=${source}, lane=${fast ? "fast" : "main"})`, + ); return; } -- 2.49.1 From 3cfa3d588a999240be621aa5cb2c117dd339196e Mon Sep 17 00:00:00 2001 From: semih Date: Thu, 17 Sep 2026 13:53:15 +0300 Subject: [PATCH 2/3] =?UTF-8?q?feat(pl24):=20PL24=20giri=C5=9Fi=201=20saat?= =?UTF-8?q?=20ba=C5=9Far=C4=B1s=C4=B1zsa=20Telegram=20uyar=C4=B1s=C4=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit İki hesap banı da günler sonra, DB satır sayılarından fark edildi (tr 07-24 → 08-18'de, de 09-04 → 09-16'da). Üçüncüsünde bir saat içinde haber olsun. - Yeni global `TelegramService` (aynı bot: TELEGRAM_BOT_TOKEN/TELEGRAM_CHAT_ID; yapılandırılmamışsa sessizce devre dışı, istek yolunu asla etkilemez). - `PL24AuthService` kesinti saatini Redis'te tutuyor (`pl24:auth:fail-since:`), böylece api+worker ve yeniden başlatmalar aynı başlangıcı görür — crash-loop saati sıfırlayamaz. Başarılı login saati siler. - Kesinti 60 dakikayı geçince tek sefer uyarı (`pl24:auth:alerted:` NX, 6 saat) — hesap kodu, süre, son hata ve "banlanmış olabilir" notu ile. Düzelince sessiz "🟢 düzeldi" mesajı. - Devre kesici açıkken de saat işler: kesici login denemesini bastırdığı için aksi halde hesap en ölü olduğu anda saat duruyordu. - compose'a eksik env referansları: TELEGRAM_*, PL24_HTTP_DAILY_MAX, PL24_HTTP_USER_RESERVE, PREFETCH_PL24_FAST_DEPTH, PREFETCH_PL24_DELAY_MS (Coolify env'i ancak compose'da referanslıysa container'a ulaşıyor). Test: 3 yeni test (1 saat dolmadan susar, dolunca tek sefer uyarır, düzelince kurtarma mesajı + saat sıfırlanır). Redis stub'ı gerçek SET NX semantiğine çekildi (alarm tekrarının bastırılması buna dayanıyor). 80 PL24 testi geçti. Co-Authored-By: Claude Opus 5 (1M context) --- apps/api/src/app.module.ts | 2 + apps/api/src/common/telegram.module.ts | 10 ++ apps/api/src/common/telegram.service.ts | 56 +++++++++++ .../pl24/pl24-auth.service.spec.ts | 79 ++++++++++++++- .../integrations/pl24/pl24-auth.service.ts | 95 +++++++++++++++++++ docker-compose.coolify.yml | 6 ++ 6 files changed, 246 insertions(+), 2 deletions(-) create mode 100644 apps/api/src/common/telegram.module.ts create mode 100644 apps/api/src/common/telegram.service.ts diff --git a/apps/api/src/app.module.ts b/apps/api/src/app.module.ts index 1763acd..dcac4cd 100644 --- a/apps/api/src/app.module.ts +++ b/apps/api/src/app.module.ts @@ -21,6 +21,7 @@ import { RolesGuard } from "./common/guards/roles.guard"; import { LoggingInterceptor } from "./common/interceptors/logging.interceptor"; import { TimeoutInterceptor } from "./common/interceptors/timeout.interceptor"; import { TransformInterceptor } from "./common/interceptors/transform.interceptor"; +import { TelegramModule } from "./common/telegram.module"; import configuration from "./config/configuration"; import { validate } from "./config/env.validation"; import { ContactModule } from "./contact/contact.module"; @@ -79,6 +80,7 @@ import { VehiclesModule } from "./vehicles/vehicles.module"; ]), DatabaseModule, RedisModule, + TelegramModule, AuthModule, UsersModule, EmailModule, diff --git a/apps/api/src/common/telegram.module.ts b/apps/api/src/common/telegram.module.ts new file mode 100644 index 0000000..f7eda1d --- /dev/null +++ b/apps/api/src/common/telegram.module.ts @@ -0,0 +1,10 @@ +import { Global, Module } from "@nestjs/common"; +import { TelegramService } from "./telegram.service"; + +/** Global so any module can raise an operational alert without extra wiring. */ +@Global() +@Module({ + providers: [TelegramService], + exports: [TelegramService], +}) +export class TelegramModule {} diff --git a/apps/api/src/common/telegram.service.ts b/apps/api/src/common/telegram.service.ts new file mode 100644 index 0000000..b96b6d0 --- /dev/null +++ b/apps/api/src/common/telegram.service.ts @@ -0,0 +1,56 @@ +import { Injectable, Logger } from "@nestjs/common"; + +/** + * Minimal Telegram alerting for operational failures that need a human now + * (currently: PL24 auth down for an hour — i.e. the account may be banned + * again). Fire-and-forget: a failed alert must never affect a request. + * + * Uses the same bot as the Süper Panel — set TELEGRAM_BOT_TOKEN and + * TELEGRAM_CHAT_ID. Unconfigured = silently disabled (local/dev). + */ +@Injectable() +export class TelegramService { + private readonly logger = new Logger(TelegramService.name); + + private get token(): string { + return process.env.TELEGRAM_BOT_TOKEN ?? ""; + } + + private get chatId(): string { + return process.env.TELEGRAM_CHAT_ID ?? ""; + } + + isConfigured(): boolean { + return Boolean(this.token && this.chatId); + } + + /** Send a message. Returns false when disabled or the API refused. */ + async send(text: string, opts: { silent?: boolean } = {}): Promise { + if (!this.isConfigured()) { + this.logger.warn(`Telegram not configured — alert dropped: ${text.slice(0, 120)}`); + return false; + } + try { + const res = await fetch(`https://api.telegram.org/bot${this.token}/sendMessage`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + chat_id: this.chatId, + text, + parse_mode: "HTML", + disable_notification: opts.silent ?? false, + disable_web_page_preview: true, + }), + signal: AbortSignal.timeout(10_000), + }); + if (!res.ok) { + this.logger.warn(`Telegram sendMessage failed: HTTP ${res.status}`); + return false; + } + return true; + } catch (err) { + this.logger.warn(`Telegram sendMessage error: ${(err as Error).message}`); + return false; + } + } +} diff --git a/apps/api/src/integrations/pl24/pl24-auth.service.spec.ts b/apps/api/src/integrations/pl24/pl24-auth.service.spec.ts index 1d1d037..8b84d5e 100644 --- a/apps/api/src/integrations/pl24/pl24-auth.service.spec.ts +++ b/apps/api/src/integrations/pl24/pl24-auth.service.spec.ts @@ -7,6 +7,9 @@ import { PL24AuthService } from "./pl24-auth.service"; type RedisStub = { store: Map; + get: (k: string) => Promise; + set: (k: string, v: string, ttl?: number) => Promise; + exists: (k: string) => Promise; getJson: (k: string) => Promise; setJson: (k: string, v: unknown) => Promise; del: (k: string) => Promise; @@ -20,6 +23,11 @@ const makeRedis = (opts: { lockTaken?: boolean } = {}): RedisStub => { const counters = new Map(); return { store, + get: async (k: string) => (store.has(k) ? String(store.get(k)) : null), + set: async (k: string, v: string) => { + store.set(k, v); + }, + exists: async (k: string) => store.has(k), getJson: async (k: string) => store.get(k) ?? null, setJson: async (k: string, v: unknown) => { store.set(k, v); @@ -27,7 +35,14 @@ const makeRedis = (opts: { lockTaken?: boolean } = {}): RedisStub => { del: async (k: string) => { store.delete(k); }, - setNx: async () => !opts.lockTaken, + // Gerçek SET NX semantiği: var olan anahtarı ikinci kez yazmaz. Alarm + // tekrarının bastırılması buna dayanıyor. + setNx: async (k: string, v: string) => { + if (opts.lockTaken && k.includes("login-lock")) return false; + if (store.has(k)) return false; + store.set(k, v); + return true; + }, incr: async (k: string) => { const n = (counters.get(k) ?? 0) + 1; counters.set(k, n); @@ -63,10 +78,19 @@ const makeService = (cfgOverrides: Record = {}, redis = makeRedi }; const configService = { get: (k: string, d?: unknown) => cfg[k] ?? d } as never; const budget = makeBudget(); + const telegram = { + sent: [] as string[], + isConfigured: () => true, + send: async function (t: string) { + this.sent.push(t); + return true; + }, + }; return { - svc: new PL24AuthService(configService, redis as never, budget as never), + svc: new PL24AuthService(configService, redis as never, budget as never, telegram as never), redis, budget, + telegram, }; }; @@ -384,3 +408,54 @@ describe("PL24AuthService — login devre kesici", () => { expect(fetchSpy).toHaveBeenCalledTimes(6); }); }); + +describe("PL24AuthService — 1 saatlik kesinti alarmı (Telegram)", () => { + const loginFails = () => ({ + ok: false, + status: 400, + statusText: "Bad Request", + headers: { get: () => null, getSetCookie: () => [] }, + json: async () => ({ type: "urn:login:account-not-active", detail: "account not active" }), + }); + + it("ilk saat içinde alarm göndermez", async () => { + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(loginFails())); + const { svc, telegram } = makeService(); + await expect((svc as unknown as Exposed).login("de")).rejects.toThrow(); + expect(telegram.sent).toHaveLength(0); + }); + + it("kesinti 1 saati geçince tek sefer alarm gönderir", async () => { + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(loginFails())); + const redis = makeRedis(); + // Kesinti 70 dakika önce başlamış gibi davran (paylaşılan Redis saati). + redis.store.set("pl24:auth:fail-since:de", String(Date.now() - 70 * 60_000)); + const { svc, telegram } = makeService({}, redis); + + await expect((svc as unknown as Exposed).login("de")).rejects.toThrow(); + await new Promise((r) => setTimeout(r, 10)); // void alarm promise'i + + expect(telegram.sent).toHaveLength(1); + expect(telegram.sent[0]).toContain("PL24 giriş yapılamıyor"); + expect(telegram.sent[0]).toContain("70 dakika"); + + // Kesici açık olsa bile ikinci kez sızlanmaz (alerted anahtarı). + await expect((svc as unknown as Exposed).login("de")).rejects.toThrow(); + await new Promise((r) => setTimeout(r, 10)); + expect(telegram.sent).toHaveLength(1); + }); + + it("giriş düzelince kurtarma mesajı gönderir ve saati sıfırlar", async () => { + const redis = makeRedis(); + redis.store.set("pl24:auth:fail-since:de", String(Date.now() - 90 * 60_000)); + redis.store.set("pl24:auth:alerted:de", "1"); + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(loginOk())); + const { svc, telegram } = makeService({}, redis); + + await svc.getSessionCookieForAccount("de"); + await new Promise((r) => setTimeout(r, 10)); + + expect(telegram.sent.some((t) => t.includes("PL24 girişi düzeldi"))).toBe(true); + expect(redis.store.has("pl24:auth:fail-since:de")).toBe(false); + }); +}); diff --git a/apps/api/src/integrations/pl24/pl24-auth.service.ts b/apps/api/src/integrations/pl24/pl24-auth.service.ts index a22a0e5..2c610cb 100644 --- a/apps/api/src/integrations/pl24/pl24-auth.service.ts +++ b/apps/api/src/integrations/pl24/pl24-auth.service.ts @@ -24,6 +24,7 @@ import { Injectable, Logger, type OnModuleInit, UnauthorizedException } from "@nestjs/common"; import { ConfigService } from "@nestjs/config"; +import { TelegramService } from "../../common/telegram.service"; import { RedisService } from "../../redis/redis.service"; import { PL24BudgetService } from "./pl24-budget.service"; import { PL24_DEFAULTS, PL24_ENDPOINTS, PL24_USER_AGENT } from "./pl24.constants"; @@ -82,6 +83,13 @@ export class PL24AuthService implements OnModuleInit { }; /** Hard ceiling on logins per account per hour (PL24 counts sessions, not requests). */ private static readonly MAX_LOGINS_PER_HOUR = 6; + /** + * How long PL24 auth may stay broken before a human is paged on Telegram. + * Both previous bans were only noticed days later, from DB row counts. + */ + private static readonly ALERT_AFTER_MS = 60 * 60_000; + /** Do not re-page more often than this while the outage continues. */ + private static readonly ALERT_REPEAT_MS = 6 * 60 * 60_000; // ── Config ───────────────────────────────────────────────────────────────── private readonly baseUrl: string; @@ -98,6 +106,7 @@ export class PL24AuthService implements OnModuleInit { private configService: ConfigService, private readonly redis: RedisService, private readonly budget: PL24BudgetService, + private readonly telegram: TelegramService, ) { this.baseUrl = this.configService.get("pl24.baseUrl", "https://www.partslink24.com"); this.companyCode = this.configService.get("pl24.companyCode", ""); @@ -350,6 +359,7 @@ export class PL24AuthService implements OnModuleInit { if (!result.sessionToken) { this.breakerFail(account, result.permanent); const detail = result.error || "Token alinamadi"; + void this.noteLoginFailure(account, detail); this.logger.error(`PL24 login (${account}) failed: ${detail}`); throw new UnauthorizedException(`PL24 (${account}) giris hatasi: ${detail}`); } @@ -362,6 +372,7 @@ export class PL24AuthService implements OnModuleInit { this.sessions[account] = session; await this.persistSession(account, session); this.breakerOk(account); + void this.noteLoginSuccess(account); this.logger.log(`PL24 login (${account}) successful — session established`); return session; } @@ -712,6 +723,9 @@ export class PL24AuthService implements OnModuleInit { private breakerGate(account: PL24Account): void { const b = this.loginBreaker[account]; if (Date.now() < b.openUntil) { + // The breaker suppresses login attempts, so without this the outage clock + // would stop ticking exactly when the account is most likely dead. + void this.noteLoginFailure(account, `login breaker open (${b.fails} failures)`); throw new UnauthorizedException( `PL24 ${account} login breaker open (${b.fails} consecutive failures) — ` + `retrying after ${new Date(b.openUntil).toISOString()}`, @@ -736,6 +750,87 @@ export class PL24AuthService implements OnModuleInit { } } + // ── Outage tracking → Telegram ──────────────────────────────────────────── + + private failSinceKey(account: PL24Account): string { + return `${PL24_DEFAULTS.CACHE_PREFIX}auth:fail-since:${account}`; + } + + private alertedKey(account: PL24Account): string { + return `${PL24_DEFAULTS.CACHE_PREFIX}auth:alerted:${account}`; + } + + /** + * Record a login failure and page a human once the account has been unusable + * for an hour. Shared via Redis so api+worker (and restarts) agree on when the + * outage started — a crash-looping process must not reset the clock. + */ + private async noteLoginFailure(account: PL24Account, reason: string): Promise { + try { + const key = this.failSinceKey(account); + const existing = await this.redis.get(key); + const since = existing ? Number(existing) : Date.now(); + if (!existing) await this.redis.set(key, String(since), 3 * 86_400); + + const downForMs = Date.now() - since; + if (downForMs < PL24AuthService.ALERT_AFTER_MS) return; + if ( + !(await this.redis.setNx( + this.alertedKey(account), + "1", + Math.floor(PL24AuthService.ALERT_REPEAT_MS / 1000), + )) + ) { + return; // already paged recently + } + + const minutes = Math.round(downForMs / 60_000); + const company = account === "de" ? this.companyCode2 : this.companyCode; + await this.telegram.send( + [ + "🔴 PL24 giriş yapılamıyor", + "", + `Hesap: ${company || account}`, + `Süre: ${minutes} dakikadır başarısız`, + `Son hata: ${reason.slice(0, 200)}`, + "", + "Hesap banlanmış olabilir (önceki iki ban: tr 2026-07-24, de 2026-09-04).", + "Katalog decode'ları PL24 olmadan pcat/emex'e düşüyor.", + ].join("\n"), + ); + this.logger.error(`PL24 ${account}: auth down for ${minutes}m — Telegram alert sent`); + } catch (err) { + this.logger.warn(`PL24 outage alert failed: ${(err as Error).message}`); + } + } + + /** Clear the outage clock and, if we had paged, say it recovered. */ + private async noteLoginSuccess(account: PL24Account): Promise { + try { + const key = this.failSinceKey(account); + const since = await this.redis.get(key); + if (!since) return; + await this.redis.del(key); + const wasAlerted = await this.redis.exists(this.alertedKey(account)); + await this.redis.del(this.alertedKey(account)); + if (wasAlerted) { + const minutes = Math.round((Date.now() - Number(since)) / 60_000); + const company = account === "de" ? this.companyCode2 : this.companyCode; + await this.telegram.send( + [ + "🟢 PL24 girişi düzeldi", + "", + `Hesap: ${company || account}`, + `Kesinti: ~${minutes} dakika`, + ].join("\n"), + { silent: true }, + ); + } + } catch { + // never block auth on alerting + } + } + private breakerOk(account: PL24Account): void { const b = this.loginBreaker[account]; if (b.fails >= PL24AuthService.BREAKER_THRESHOLD) { diff --git a/docker-compose.coolify.yml b/docker-compose.coolify.yml index 2e85377..cbb7095 100644 --- a/docker-compose.coolify.yml +++ b/docker-compose.coolify.yml @@ -144,6 +144,12 @@ services: - PREFETCH_MAX_DEPTH=${PREFETCH_MAX_DEPTH:-} - PREFETCH_RATE_PL24=${PREFETCH_RATE_PL24:-} - PREFETCH_DAILY_PL24=${PREFETCH_DAILY_PL24:-} + - PREFETCH_PL24_FAST_DEPTH=${PREFETCH_PL24_FAST_DEPTH:-} + - PREFETCH_PL24_DELAY_MS=${PREFETCH_PL24_DELAY_MS:-} + - PL24_HTTP_DAILY_MAX=${PL24_HTTP_DAILY_MAX:-} + - PL24_HTTP_USER_RESERVE=${PL24_HTTP_USER_RESERVE:-} + - TELEGRAM_BOT_TOKEN=${TELEGRAM_BOT_TOKEN:-} + - TELEGRAM_CHAT_ID=${TELEGRAM_CHAT_ID:-} - PREFETCH_RATE_EMEX=${PREFETCH_RATE_EMEX:-} - PREFETCH_RATE_PCAT=${PREFETCH_RATE_PCAT:-} - PREFETCH_PCAT_DELAY_MS=${PREFETCH_PCAT_DELAY_MS:-} -- 2.49.1 From dde6f6a15c831228d6796528d5d990b7542db698 Mon Sep 17 00:00:00 2001 From: semih Date: Thu, 17 Sep 2026 14:00:30 +0300 Subject: [PATCH 3/3] =?UTF-8?q?feat(pl24):=20PSA=20+=20Volvo'yu=20P5'e=20t?= =?UTF-8?q?a=C5=9F=C4=B1,=20eksik=20kataloglar=C4=B1=20ekle,=20a=C4=9Fa?= =?UTF-8?q?=C3=A7=20s=C4=B1n=C4=B1fland=C4=B1rmas=C4=B1n=C4=B1=20birle?= =?UTF-8?q?=C5=9Ftir?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Faz 1 / adım 3-4 (analiz: /home/s/ss/plv2.md §3.3/§3.5, bulgular types_wmi-01/02/03/04/05, p5core-02/03/04/05, psa-04/05/06, consumers_jobs-01/02). CANLI GERÇEK (2026-09-16 keşfi): PL24 manufacturers API'si peugeot/citroen/ citroenDs/psa_opel/psa_vauxhall'ı `/p5psa`, volvo/polestar'ı `/p5volvo`, subaru'yu `/p5subaru` olarak veriyor. Eski Volvo P4 ucu **503** döndürüyor (ölü); eski PSA P4 ucu 200 döndürüyor ama verisi **2024-02-13'te donmuş** (P5 upds 2026-09-07). Kodumuz her ikisini de P4'e yönlendiriyordu. SERVİS TABLOSU: - peugeot/citroen/citroenDs + YENİ psa_opel_parts, psa_vauxhall_parts → /p5psa - volvo/polestar → /p5volvo · YENİ subaru_parts → /p5subaru - YENİ abarth/alfa/jeep/lancia → /p5fiat - WMI düzeltmeleri: ZAR→alfa, ZLA→lancia (fiatp'den), VXK→psa_opel (PSA-platform Corsa F/Mokka B), YENİ JF1/JF2→subaru, 1C4/1J4→jeep. SERVICE_TO_BRAND genişledi. - P4 kodu ve eski yollar DOKUNULMADI: mevcut 322 PSA + 36 Volvo aracı eski ağaçlarıyla çalışmaya devam eder (veri göçü ayrı iş). PARSER (canlı yanıt şekillerine göre): - `normalizeLabel()`: JS toLowerCase Türkçe "İ"yi "i̇" (i + birleşen nokta) yapıyordu → PSA'nın "AKTARMA SİSTEMLERİ"/"GÖVDE TİPİ" etiketleri HİÇBİR ZAMAN eşleşmiyordu, şanzıman/gövde sessizce null kalıyordu. Artık tr-locale + diakritik temizliği + ı→i. - `parsePsaModelYear()`: "AM 2005" → 2005; "MAJÖR ENDEKS" bir yıl DEĞİL (bu yüzden Citroën'ler 2001 görünüyordu). `damToModelYear()`: PSA DAM alanı (1976-01-01'den gün) → üretim yılı, model yılı Temmuz'da döner. PSA VIN'lerinde 10. karakter model yılı olmadığı için VIN fallback'i en sona. - Alan listeleri: motor/engine (PSA "MOTOR", Volvo "Motor"), aktarma_sistemleri, mission, govde_tipi, kaporta_stili, türü→series. - Alt grup kodu: PSA'da `record.id` = illusPath ve birden çok illüstrasyon paylaşıyor (canlı: 36 kayıt / 11 id) → benzersiz kod `values.illustration`. - Parça filtresi: Volvo BOM'unun başlık satırı (id "null_null", link yok, unavailable) hayalet parça olarak geçiyordu → elendi. - Adet: qty (VAG/Subaru) · coef (PSA) · unit (Volvo). Uygulanabilirlik: modelDescription · restriction. AĞAÇ SINIFLANDIRMASI (yeni `pl24-tree.ts`, tek kaynak): Dört ayrı sezgisel vardı ve birbiriyle çelişiyordu: - `linkWid.includes("group")` → p5psa `illusTable`, p5volvo/p5subaru `illustrationsTable` yakalanmıyordu → o seviye yaprak sanılıp parça olarak çekiliyor, partno'suz kayıtlar 0 parçaya dönüşüyor, panel sessizce boş kalıyordu (2026-06 isPsaParent olayının aynısı). - case-sensitive `includes("/bomdetails")` → p5psa/p5volvo'nun camelCase `/details/vin/bomDetails` yaprakları grup sanılıyor, parçaları hiç prefetch edilmiyordu (bugün Mitsubishi %99,9 / Fiat %80 / Renault %70 bomDetails yaprağı 0 parça — aynı kök neden). Artık `isPl24LeafNode`/`isPl24GroupNode` tek kaynak; categories.service (2 yer), catalog.service (2 yer) ve prefetch-worker aynı fonksiyonu kullanıyor. Test: yeni `pl24-tree.spec.ts` (7 test, canlı VW/PSA/Volvo/Subaru wid+path çiftleriyle regresyon kilidi). Etkilenen paketler: 208 test geçti. tsc + biome temiz. Co-Authored-By: Claude Opus 5 (1M context) --- apps/api/src/catalog/catalog.service.ts | 23 +--- .../src/categories/categories.controller.ts | 5 +- apps/api/src/categories/categories.service.ts | 34 ++--- .../src/integrations/pl24/pl24-tree.spec.ts | 112 ++++++++++++++++ apps/api/src/integrations/pl24/pl24-tree.ts | 65 ++++++++++ .../api/src/integrations/pl24/pl24.service.ts | 120 +++++++++++++++--- apps/api/src/integrations/pl24/pl24.types.ts | 92 +++++++++++--- apps/api/src/jobs/prefetch-worker.service.ts | 13 +- 8 files changed, 378 insertions(+), 86 deletions(-) create mode 100644 apps/api/src/integrations/pl24/pl24-tree.spec.ts create mode 100644 apps/api/src/integrations/pl24/pl24-tree.ts diff --git a/apps/api/src/catalog/catalog.service.ts b/apps/api/src/catalog/catalog.service.ts index 3bdb642..c567c36 100644 --- a/apps/api/src/catalog/catalog.service.ts +++ b/apps/api/src/catalog/catalog.service.ts @@ -19,6 +19,7 @@ import { userBrands, userSubscriptions, } from "../database/schema/core"; +import { isPl24LeafNode } from "../integrations/pl24/pl24-tree"; import { PL24Service } from "../integrations/pl24/pl24.service"; import { type PL24DecodedCategory, @@ -1249,18 +1250,10 @@ export class CatalogService { return this.pl24Service.fetchFordModelConfig(vehicle.serviceName, familyId, mode, upds); } + /** Shared PL24 leaf classifier — see integrations/pl24/pl24-tree. */ private isLeafPath(linkPath: string): boolean { - const lower = linkPath.toLowerCase(); return ( - lower.includes("/bom/") || - lower.includes("/bomdetails") || - lower.includes("/partinfo/") || - // PL24 P5 leaf items endpoints — chemicals, servicepart, accessories, - // any /extern//(vin|mdl)_items combination. These return parts, - // not subgroups, so they must short-circuit drill-down. - /\/extern\/[^/]+\/(vin_items|mdl_items)\b/.test(lower) || - lower.includes("image-board.action") || // PSA illustration leaf - lower.includes("json-vin-bom-detail.action") + isPl24LeafNode({ linkPath }) || linkPath.toLowerCase().includes("json-vin-bom-detail.action") ); } @@ -1367,13 +1360,9 @@ export class CatalogService { return cats.map((c) => { const dbChildCount = childCountMap.get(c.id) || 0; - const isLeaf = - c.linkPath?.includes("/bom/") || - c.linkPath?.includes("/bomdetails") || - c.linkPath?.includes("/partinfo/") || - c.linkPath?.includes("/servicepart/vin_items") || - c.linkPath?.includes("image-board.action") || // PSA illustration leaf - (!c.linkPath && dbChildCount === 0); + const isLeaf = c.linkPath + ? isPl24LeafNode({ linkPath: c.linkPath, linkWid: c.linkWid }) + : dbChildCount === 0; return { ...c, schemaImageUrl: picMap.get(c.id) || null, diff --git a/apps/api/src/categories/categories.controller.ts b/apps/api/src/categories/categories.controller.ts index 3d02bd1..81f7ee4 100644 --- a/apps/api/src/categories/categories.controller.ts +++ b/apps/api/src/categories/categories.controller.ts @@ -13,10 +13,7 @@ export class CategoriesController { } @Get("tree/:vehicleId") - async getCategoryTree( - @Param("vehicleId") vehicleId: string, - @Query("source") source?: string, - ) { + async getCategoryTree(@Param("vehicleId") vehicleId: string, @Query("source") source?: string) { return this.categoriesService.getCategoryTree(vehicleId, source); } diff --git a/apps/api/src/categories/categories.service.ts b/apps/api/src/categories/categories.service.ts index ca3c71a..d2c199c 100644 --- a/apps/api/src/categories/categories.service.ts +++ b/apps/api/src/categories/categories.service.ts @@ -21,6 +21,7 @@ import { PartsCatalogsService } from "../integrations/parts-catalogs/parts-catal import { PcatGroup } from "../integrations/parts-catalogs/parts-catalogs.types"; import { PL24FordLegacyService } from "../integrations/pl24/pl24-ford-legacy.service"; import { PL24PsaService } from "../integrations/pl24/pl24-psa.service"; +import { isPl24GroupNode, isPl24LeafNode } from "../integrations/pl24/pl24-tree"; import { PL24Service } from "../integrations/pl24/pl24.service"; import { classifyNode, foldName, mapToCanonical } from "../jobs/canonical-lexicon"; import { RedisService } from "../redis/redis.service"; @@ -896,17 +897,13 @@ export class CategoriesService { // subgroups. Drilling into them used to insert per-part endpoints as // ghost child categories — keep the regex wide so any /extern/{kind}/ // (vin|mdl)_items endpoint is recognised, not just /servicepart/. - const lp = linkPath.toLowerCase(); + // One shared classifier (integrations/pl24/pl24-tree) — the inline lists here, + // in catalog.service and in the prefetch worker used to disagree, which is how + // p5psa/p5volvo camelCase `bomDetails` leaves and `illusTable` group levels + // ended up on the wrong side (silent empty panels / unfetched parts). if ( - lp.includes("/bom/") || - lp.includes("/bomdetails") || - lp.includes("/partinfo/") || - /\/extern\/[^/]+\/(vin_items|mdl_items)\b/.test(lp) || - // PSA / Hyundai / Opel / Volvo image-board pages and the Ford VIN - // vin-image-board.action equivalent. Drilling into them yields BOM rows, - // not sub-groups — let getCategoryWithParts handle those as parts. - lp.includes("image-board.action") || - lp.includes("json-vin-bom-detail.action") + isPl24LeafNode({ linkPath }) || + linkPath.toLowerCase().includes("json-vin-bom-detail.action") ) { return []; } @@ -1316,9 +1313,13 @@ export class CategoriesService { // AND lowercase (groupReferenceTable, groupTable, groupsTable). The old // case-sensitive includes("Group") missed the lowercase ones (~1157 nodes), // so they skipped this group-drill branch and fell to the parts path. + // A PL24 node is a parent when the shared classifier says it is not a parts + // leaf. The old `linkWid.includes("group")` test missed p5psa `illusTable` + // and p5volvo/p5subaru `illustrationsTable`, so those levels were fetched as + // parts, parsed to zero rows and rendered as an empty panel with no error. if ( category.source === "pl24" && - category.linkWid?.toLowerCase().includes("group") && + isPl24GroupNode({ linkPath: category.linkPath, linkWid: category.linkWid }) && category.vehicleId ) { const groupChildren = await this.getChildren(categoryId); @@ -2173,15 +2174,8 @@ export class CategoriesService { return !!c.linkPath?.startsWith("pcat:"); // unknown → lazy-leaf heuristic })() : (() => { - const lp = c.linkPath?.toLowerCase() ?? ""; - return ( - lp.includes("/bom/") || - lp.includes("/bomdetails") || - lp.includes("/partinfo/") || - /\/extern\/[^/]+\/(vin_items|mdl_items)\b/.test(lp) || - lp.includes("image-board.action") || - (!c.linkPath && dbChildCount === 0) - ); + if (!c.linkPath) return dbChildCount === 0; + return isPl24LeafNode({ linkPath: c.linkPath, linkWid: c.linkWid }); })(); return { ...c, diff --git a/apps/api/src/integrations/pl24/pl24-tree.spec.ts b/apps/api/src/integrations/pl24/pl24-tree.spec.ts new file mode 100644 index 0000000..0e09a7b --- /dev/null +++ b/apps/api/src/integrations/pl24/pl24-tree.spec.ts @@ -0,0 +1,112 @@ +import { describe, expect, it } from "vitest"; +import { isPl24GroupNode, isPl24LeafNode } from "./pl24-tree"; + +// Canlı P5 yanıtlarından (2026-09-16 keşfi, plv2-artefakt/) alınan gerçek +// wid + path çiftleri. Bu dosya "0 parça" sınıfı hatanın regresyon kilidi. + +describe("isPl24LeafNode — canlı P5 şekilleri", () => { + it("VW (p5vwag): maingroups/subgroups grup, bom/vin yaprak", () => { + expect( + isPl24GroupNode({ + linkWid: "mainGroupsTable", + linkPath: "/p5vwag/extern/groups/vin_maingroups?vin=X", + }), + ).toBe(true); + expect( + isPl24GroupNode({ + linkWid: "subGroupsIllusTable", + linkPath: "/p5vwag/extern/groups/vin_subgroups_illus?maingroup=4", + }), + ).toBe(true); + expect( + isPl24LeafNode({ + linkWid: "bomlist", + linkPath: "/p5vwag/extern/bom/vin?illustration=407-000", + }), + ).toBe(true); + }); + + it("PSA (p5psa): scope + mainGroups + illusTable grup, bomDetails yaprak", () => { + expect( + isPl24GroupNode({ + linkWid: "scopeTable", + linkPath: "/p5psa/extern/group/vin/scope?modelCode=1PD2", + }), + ).toBe(true); + expect( + isPl24GroupNode({ + linkWid: "mainGroupTable", + linkPath: "/p5psa/extern/group/vin/mainGroups?scope=_FCT0001", + }), + ).toBe(true); + // Eski `includes("group")` kuralının kaçırdığı seviye — sessiz boş panelin kaynağı. + expect( + isPl24GroupNode({ + linkWid: "illusTable", + linkPath: "/p5psa/extern/group/vin/illus?mainGroup=_FCT0001_FCT0512", + }), + ).toBe(true); + // camelCase bomDetails — eski `includes("/bomdetails")` kaçırıyordu. + expect( + isPl24LeafNode({ + linkWid: "bomlist", + linkPath: "/p5psa/extern/details/vin/bomDetails?illustration=D2F001A48A", + }), + ).toBe(true); + }); + + it("Volvo (p5volvo): illustrationsTable grup, bom yaprak", () => { + expect( + isPl24GroupNode({ + linkWid: "illustrationsTable", + linkPath: "/p5volvo/extern/groups/vin/illustration?group=0b00c8af80205942", + }), + ).toBe(true); + expect(isPl24LeafNode({ linkWid: "bomlist", linkPath: "/p5volvo/extern/bom/vin?..." })).toBe( + true, + ); + expect( + isPl24LeafNode({ + linkWid: "partinfo", + linkPath: "/p5volvo/extern/partinfo/vin?partno=36050493", + }), + ).toBe(true); + }); + + it("Subaru (p5subaru): subgroups/illustrations grup, bom/vin?figNum yaprak", () => { + expect( + isPl24GroupNode({ + linkWid: "illustrationsTable", + linkPath: "/p5subaru/extern/groups/vin/illustrations?subGroup=001", + }), + ).toBe(true); + expect( + isPl24LeafNode({ linkWid: "bomlist", linkPath: "/p5subaru/extern/bom/vin?figNum=01" }), + ).toBe(true); + }); + + it("wid yoksa yol kalıbına düşer (eski DB satırları)", () => { + expect(isPl24LeafNode({ linkPath: "/p5vwag/extern/bom/vin?illustration=1" })).toBe(true); + expect( + isPl24LeafNode({ linkPath: "/psa/peugeot_parts/vin-image-board.action?illCode=1" }), + ).toBe(true); + expect( + isPl24GroupNode({ + linkPath: "/psa/peugeot_parts/json-vin-main-groups.action?scope=_FCT0001", + }), + ).toBe(true); + }); + + it("hasSubgroups=true yol tahminini ezer", () => { + expect(isPl24LeafNode({ linkPath: "/p5x/extern/unknown", hasSubgroups: true })).toBe(false); + }); + + it("servicepart öğe listesi yapraktır", () => { + expect( + isPl24LeafNode({ + linkWid: "servicePartsItemsTable", + linkPath: "/p5vwag/extern/servicepart/vin_items?x=1", + }), + ).toBe(true); + }); +}); diff --git a/apps/api/src/integrations/pl24/pl24-tree.ts b/apps/api/src/integrations/pl24/pl24-tree.ts new file mode 100644 index 0000000..f4e9bbe --- /dev/null +++ b/apps/api/src/integrations/pl24/pl24-tree.ts @@ -0,0 +1,65 @@ +/** + * One source of truth for "is this PL24 node a parts leaf or a group to drill?". + * + * WHY THIS FILE EXISTS (plv2.md, findings p5core-02 / psa-05 / consumers_jobs-01): + * the same question was answered by four independent heuristics — + * `categories.service` (twice), `catalog.service` and the prefetch worker — and + * they disagreed. Two failure modes shipped repeatedly: + * + * 1. `linkWid.includes("group")` as the "is a group" test. Live P5 PSA's second + * level has wid `illusTable`, Volvo's and Subaru's `illustrationsTable` — + * none contain "group", so those nodes fell through to the parts fetcher, + * which asked an *illustration list* for parts, got records with no partno, + * and rendered a silent empty panel (`parts: []`, no error). This is the same + * class of bug as the 2026-06 `isPsaParent` incident. + * 2. Case-sensitive `includes("/bomdetails")` while p5psa and p5volvo spell the + * endpoint `/details/vin/bomDetails` — so those leaves were queued as groups, + * `getChildren` returned [] and their parts were never prefetched (live today + * for Mitsubishi 99.9% / Fiat 80% / Renault 70% of bomDetails leaves). + * + * The reliable cross-brand marker is the response's own `link.wid`: `bomlist` + * (and the service-parts/partinfo variants) means parts, anything else means + * drill. Path matching stays as a fallback for stored rows without a wid. + */ + +/** `link.wid` values that identify a parts (BOM) node across every P5 backend. */ +const LEAF_WIDS = new Set(["bomlist", "bomoverviewlist", "servicepartsitemstable", "partinfo"]); + +/** `link.wid` values that identify a drillable group node. */ +const GROUP_WID_PATTERN = + /(group|scope|illus|illustration|catalog|model|vpages|msppages|chemicals|category|categories)/i; + +/** + * Path fragments that only ever appear on a parts endpoint. `image-board` has no + * leading slash on purpose: the legacy PSA leaf is `vin-image-board.action`. + */ +const LEAF_PATH_PATTERN = + /\/(bom|bomdetails|partinfo|vin_items|mdl_items|vin_bomdetails)\b|\/bom\/|\/details\/vin\/bomdetails|\/servicepart\/vin_items|image-board/i; + +/** True when this node yields parts (never children). */ +export function isPl24LeafNode(opts: { + linkPath?: string | null; + linkWid?: string | null; + hasSubgroups?: boolean | null; +}): boolean { + const wid = opts.linkWid?.toLowerCase().trim(); + if (wid) { + if (LEAF_WIDS.has(wid)) return true; + if (GROUP_WID_PATTERN.test(wid)) return false; + } + // Explicit DB hint wins over path guessing when there is no usable wid. + if (opts.hasSubgroups === true) return false; + const lp = opts.linkPath?.toLowerCase() ?? ""; + if (!lp) return false; + return LEAF_PATH_PATTERN.test(lp); +} + +/** True when this node should be drilled for children. */ +export function isPl24GroupNode(opts: { + linkPath?: string | null; + linkWid?: string | null; + hasSubgroups?: boolean | null; +}): boolean { + if (!opts.linkPath && !opts.linkWid) return false; + return !isPl24LeafNode(opts); +} diff --git a/apps/api/src/integrations/pl24/pl24.service.ts b/apps/api/src/integrations/pl24/pl24.service.ts index 8914af9..24e4f06 100644 --- a/apps/api/src/integrations/pl24/pl24.service.ts +++ b/apps/api/src/integrations/pl24/pl24.service.ts @@ -56,6 +56,58 @@ const LEGACY_ARCH_SOURCE_TAG: Record = { LEGACY_HYUNDAI_KIA: "hyundai-kia", }; +/** + * Normalise a vinfoBasic label into a lookup key. + * + * JS `toLowerCase()` maps Turkish "İ" to "i" + U+0307 (combining dot), so PSA + * labels like "AKTARMA SİSTEMLERİ" / "GÖVDE TİPİ" produced keys no lookup could + * ever match and the transmission/body fields silently stayed null. Lower-case + * with the Turkish locale, then strip combining marks and fold "ı" → "i" so a + * single spelling matches both "Model yılı" and "MODEL YILI". + */ +export function normalizeLabel(label: string): string { + return label + .toLocaleLowerCase("tr") + .normalize("NFD") + .replace(/[\u0300-\u036f]/g, "") + .replace(/ı/g, "i") + .replace(/[\s/]+/g, "_") + .trim(); +} + +/** + * PSA model year: "AM 2005" → 2005. Index labels ("01 MAJÖR ENDEKS", + * 'MAJÖR ENDEKS "0C"') are NOT years — returning one there is how a Citroën + * ended up as a 2001 model. PSA VINs also don't encode the model year in + * position 10, so the caller must fall back to DAM or leave it null. + */ +export function parsePsaModelYear(value: string | null | undefined): number | null { + if (!value) return null; + if (/endeks/i.test(value)) return null; + const m = value.match(/\b(?:AM\s*)?((?:19|20)\d{2})\b/i); + if (m) return Number(m[1]); + const short = value.match(/^\s*AM\s*(\d{2})\s*$/i); + if (short) { + const n = Number(short[1]); + return n >= 70 ? 1900 + n : 2000 + n; + } + return null; +} + +/** + * PSA "DAM" (e.g. "10479CJ") = days since 1976-01-01 + plant code → build date. + * PSA model years roll in July, so a build after June belongs to the next one. + */ +export function damToModelYear(dam: string | null | undefined): number | null { + if (!dam) return null; + const m = dam.match(/^(\d{4,5})/); + if (!m) return null; + const date = new Date(Date.UTC(1976, 0, 1) + Number(m[1]) * 86_400_000); + const year = date.getUTCFullYear(); + if (year < 1980 || year > 2100) return null; + return date.getUTCMonth() >= 6 ? year + 1 : year; +} + @Injectable() export class PL24Service { private readonly logger = new Logger(PL24Service.name); @@ -1110,7 +1162,7 @@ export class PL24Service { const label = (v.key !== undefined ? v.key : v.description) || ""; const value = (v.key !== undefined ? v.description : v.value) || ""; if (!label) continue; - const key = label.toLowerCase().replace(/[\s\/]+/g, "_"); + const key = normalizeLabel(label); if (!(key in vehicleData)) { // Normalize like prNr col3: newlines→space, unescape the literal "\-" some P5 // backends emit (JLR "XJ 2010 \- 2019", Toyota/Suzuki dates "2023\-11\-29"/"2005\-07"), @@ -1157,26 +1209,40 @@ export class PL24Service { const engineCode = lookup("motor_kodu", "engine_code"); // Non-VAG P5 OEMs label the engine differently and give a description (sometimes with a // code in parens): JLR "Motor Tipi", Toyota "ENGINE 1", MAN "Yedek motor", Suzuki "Motor No.". - const engineLabel = lookup("motor_tipi", "engine_1", "yedek_motor", "motor_no."); + // PSA labels the full designation "MOTOR" ("TÜRBO DİZEL DV6TED4…"), Volvo + // "Motor" ("D4162T"), Subaru "Engine" — none of which the VAG-shaped list had. + const engineLabel = lookup( + "motor_tipi", + "engine_1", + "yedek_motor", + "motor_no.", + "motor", + "engine", + ); // Transmission: VAG "Şanzıman kodu"; other OEMs use their own labels — JLR "Vites Kutusu", // Toyota "ATM,MTM" (key "atm,mtm" — only spaces/slashes are underscored), MAN "Şanzıman", // Suzuki "Şanzıman numarası". + // normalizeLabel folds ı→i and strips diacritics, so "Şanzıman kodu" and + // "ŞANZIMAN KODU" both arrive as "sanziman_kodu". PSA uses "AKTARMA + // SİSTEMLERİ" ("5 MEKANİK VİTES KUTUSU"), Subaru "Mission". const transmissionCode = lookup( - "şanzıman_kodu", - "sanzıman_kodu", + "sanziman_kodu", "transmission_code", "vites_kutusu", "atm,mtm", - "şanzıman", - "şanzıman_numarası", + "aktarma_sistemleri", + "sanziman", + "sanziman_numarasi", + "mission", ); // Build body type from prNr K8* (Kaporta formları); brands without a prNr segment // (e.g. BMW) carry it in vinfoBasic "Karoseri" ("Limousine"). const bodyType = Object.entries(prNrByCode).find(([code]) => code.startsWith("K8"))?.[1] || - lookup("karoseri", "body", "body_type") || + // PSA "GÖVDE TİPİ" ("4 KAPILI SEDAN"), Volvo "Kaporta Stili" ("Sedan"). + lookup("karoseri", "body", "body_type", "govde_tipi", "kaporta_stili") || null; // Engine description from prNr D3* (Motor nitelikleri) @@ -1199,7 +1265,7 @@ export class PL24Service { // the friendly name is in "Araç" / description ("L200(EUR/MMTH)") — strip the region suffix. const isMitsubishi = getServiceApiPath(serviceName) === "/p5mitsubishi"; const baseModel = isMitsubishi - ? (lookup("araç") || (data.description as string) || lookup("model") || "") + ? (lookup("arac") || (data.description as string) || lookup("model") || "") .replace(/\s*\([^)]*\)\s*$/, "") .trim() : lookup("model_bilgisi", "model")?.trim() || @@ -1222,17 +1288,20 @@ export class PL24Service { // production date ("05/09/2014"); p5daimler → only "Teslimat tarihi" (delivery, "04.05.2009"). // Falling through to the VIN year-char (extractModelYear) is the last resort and is often // wrong for Mercedes (10th-char "1" → 2001 for a 2015 car), so read the dates first. + // PSA writes "AM 2005" (or an index label that is NOT a year) and its VINs + // do not encode the model year in position 10, so parse the PSA forms + // first and fall back to the DAM build date before ever touching the VIN. year: + parsePsaModelYear(lookup("model_yili", "year")) || Number.parseInt(lookup("model_yili", "year") || "", 10) || Number.parseInt( - (lookup("my", "üretim_tarihi", "uretim_tarihi", "teslimat_tarihi") || "").match( - /(19|20)\d{2}/, - )?.[0] || "", + (lookup("my", "uretim_tarihi", "teslimat_tarihi") || "").match(/(19|20)\d{2}/)?.[0] || "", 10, ) || + damToModelYear(lookup("dam")) || extractModelYear(vin) || 0, - series: lookup("seri", "satis_tipi", "sales_type"), + series: lookup("seri", "satis_tipi", "sales_type", "turu"), bodyType, engineCode: engineCode || @@ -1246,7 +1315,7 @@ export class PL24Service { colorCode: lookup("dis_rengi_boya_numarasi", "exterior_color___paint_code") || lookup("tavan_rengi", "roof_color"), - productionDate: lookup("üretim_tarihi", "date_of_production"), + productionDate: lookup("uretim_tarihi", "date_of_production", "teslimat_tarihi"), raw: data, catalogInfo: { serviceName, @@ -1411,9 +1480,14 @@ export class PL24Service { const values = (record.values as Record) || {}; const link = (record.link as Record) || {}; + // PSA (p5psa) shares one `record.id` (the illusPath) across many + // illustrations — 36 live records had only 11 distinct ids — so the unique + // key is `values.illustration` ("D2F 0 01A 48A", spaces stripped). Without + // this the children collapse onto each other and most get dropped. const code = values.subgroup || values.illustrationNumber || + values.illustration?.replace(/\s+/g, "") || values.id || values.code || String(record.id || ""); @@ -1491,11 +1565,14 @@ export class PL24Service { records = responseData.parts as Array>; } - const partRecords = records.filter( - (record) => - record.characteristic !== "sectionrow" && - (record.partno || (record.values as Record)?.partno), - ); + const partRecords = records.filter((record) => { + if (record.characteristic === "sectionrow") return false; + if (!(record.partno || (record.values as Record)?.partno)) return false; + // Volvo (p5volvo) prefixes each BOM with a header row that carries a + // partno but no link and is flagged unavailable — a phantom part if kept. + if (record.id === "null_null" || (record.unavailable === true && !record.link)) return false; + return true; + }); return partRecords.map((part) => { const values = (part.values as Record) || {}; @@ -1503,11 +1580,14 @@ export class PL24Service { const formattedPartNo = ((part.partno as string) || values.partno || "").trim(); const cleanPartNo = formattedPartNo.replace(/\s+/g, ""); - const qtyStr = values.qty || ""; + // qty (VAG/Subaru) · coef (PSA) · unit (Volvo) — same field, three names. + const qtyStr = values.qty || values.coef || values.unit || ""; const quantity = Number.parseInt(qtyStr.trim(), 10) || undefined; const remark = values.remark?.trim() || undefined; - const modelCodes = values.modelDescription?.trim() || undefined; + // PSA/Volvo/Subaru express applicability as `restriction` + // ("+ DIESEL TURBO DV6TED4 WITHOUT FAP"); VAG uses modelDescription. + const modelCodes = (values.modelDescription || values.restriction)?.trim() || undefined; let superseded: { oldCode: string; newCode: string } | undefined; const supersededByValue = (part.supersededBy as string) || values.supersededBy || ""; diff --git a/apps/api/src/integrations/pl24/pl24.types.ts b/apps/api/src/integrations/pl24/pl24.types.ts index 427df35..a1034b7 100644 --- a/apps/api/src/integrations/pl24/pl24.types.ts +++ b/apps/api/src/integrations/pl24/pl24.types.ts @@ -339,19 +339,61 @@ export const PL24_SERVICE_CATALOGS: Record = { // PSA Group (Citroën, Peugeot) citroen_parts: { - basePath: "/psa", - apiPath: "/psa", - architecture: "LEGACY_PSA", + basePath: "/pl24-app/citroen_parts", + apiPath: "/p5psa", + architecture: "P5_MODERN", }, citroenDs_parts: { - basePath: "/psa", - apiPath: "/psa", - architecture: "LEGACY_PSA", + basePath: "/pl24-app/citroenDs_parts", + apiPath: "/p5psa", + architecture: "P5_MODERN", }, peugeot_parts: { - basePath: "/psa", - apiPath: "/psa", - architecture: "LEGACY_PSA", + basePath: "/pl24-app/peugeot_parts", + apiPath: "/p5psa", + architecture: "P5_MODERN", + }, + + // Stellantis-era Opel/Vauxhall — the PSA-platform catalogue (Corsa F, Mokka B…). + // GM-era W0L/W0V cars stay in the legacy opel_parts P4 catalogue below. + psa_opel_parts: { + basePath: "/pl24-app/psa_opel_parts", + apiPath: "/p5psa", + architecture: "P5_MODERN", + }, + psa_vauxhall_parts: { + basePath: "/pl24-app/psa_vauxhall_parts", + apiPath: "/p5psa", + architecture: "P5_MODERN", + }, + + // Subaru (own P5 backend) + subaru_parts: { + basePath: "/pl24-app/subaru_parts", + apiPath: "/p5subaru", + architecture: "P5_MODERN", + }, + + // Rest of the Fiat/Stellantis family — same /p5fiat backend as fiatp/fiatt + abarth_parts: { + basePath: "/pl24-app/abarth_parts", + apiPath: "/p5fiat", + architecture: "P5_MODERN", + }, + alfa_parts: { + basePath: "/pl24-app/alfa_parts", + apiPath: "/p5fiat", + architecture: "P5_MODERN", + }, + jeep_parts: { + basePath: "/pl24-app/jeep_parts", + apiPath: "/p5fiat", + architecture: "P5_MODERN", + }, + lancia_parts: { + basePath: "/pl24-app/lancia_parts", + apiPath: "/p5fiat", + architecture: "P5_MODERN", }, // Ford Group @@ -404,14 +446,14 @@ export const PL24_SERVICE_CATALOGS: Record = { // Volvo/Polestar volvo_parts: { - basePath: "/volvo", - apiPath: "/volvo", - architecture: "LEGACY_VOLVO", + basePath: "/pl24-app/volvo_parts", + apiPath: "/p5volvo", + architecture: "P5_MODERN", }, polestar_parts: { - basePath: "/volvo", - apiPath: "/volvo", - architecture: "LEGACY_VOLVO", + basePath: "/pl24-app/polestar_parts", + apiPath: "/p5volvo", + architecture: "P5_MODERN", }, // Fiat Group (FCA) — P5 Modern catalog at /p5fiat, requires de-708171 account. @@ -588,8 +630,8 @@ export const PL24_WMI_SERVICE_MAP: Record = { ZFA: "fiatp_parts", // Fiat SpA Italy (most common) ZCF: "fiatp_parts", // Tofaş Turkey (Linea, Fiorino, etc.) ZFF: "fiatp_parts", // Abarth / Fiat Sport - ZAR: "fiatp_parts", // Alfa Romeo - ZLA: "fiatp_parts", // Lancia + ZAR: "alfa_parts", // Alfa Romeo (kendi kataloğu, /p5fiat backend) + ZLA: "lancia_parts", // Lancia (kendi kataloğu, /p5fiat backend) // Fiat Commercial (fiatt_parts) ZFC: "fiatt_parts", // Fiat Commercial @@ -615,9 +657,16 @@ export const PL24_WMI_SERVICE_MAP: Record = { JNK: "infiniti_parts", // Infiniti (Japan/Korea) // Opel / Vauxhall + // Subaru (canlı WMI decode: JF1 → subaru_parts, error:false) + JF1: "subaru_parts", + JF2: "subaru_parts", + // Jeep (Stellantis; /p5fiat backend) + "1C4": "jeep_parts", + "1J4": "jeep_parts", + W0L: "opel_parts", // Opel AG (Germany) W0V: "opel_parts", // Opel (newer Stellantis-era WMI) - VXK: "opel_parts", // PSA/Stellantis-platform Opel (Corsa F, Mokka B — France/Spain plants) + VXK: "psa_opel_parts", // PSA-platform Opel (Corsa F, Mokka B) → Stellantis kataloğu /p5psa // Citroën (PSA) VF7: "citroen_parts", // Citroën SA (France) @@ -846,6 +895,13 @@ export const SERVICE_TO_BRAND: Record = { vauxhall_parts: "Opel", // Volvo/Polestar volvo_parts: "Volvo", + subaru_parts: "Subaru", + psa_opel_parts: "Opel", + psa_vauxhall_parts: "Opel", + abarth_parts: "Abarth", + alfa_parts: "Alfa Romeo", + jeep_parts: "Jeep", + lancia_parts: "Lancia", polestar_parts: "Polestar", // Fiat Group fiatp_parts: "Fiat", diff --git a/apps/api/src/jobs/prefetch-worker.service.ts b/apps/api/src/jobs/prefetch-worker.service.ts index c93187c..4e61ca3 100644 --- a/apps/api/src/jobs/prefetch-worker.service.ts +++ b/apps/api/src/jobs/prefetch-worker.service.ts @@ -10,6 +10,7 @@ import { and, asc, eq, gt, inArray, isNull, notExists, sql } from "drizzle-orm"; import { CategoriesService } from "../categories/categories.service"; import { DATABASE, type Database } from "../database/database.provider"; import { categories, parts, vehicles } from "../database/schema/core"; +import { isPl24LeafNode } from "../integrations/pl24/pl24-tree"; import { PostHogService } from "../posthog/posthog.service"; import { RedisService } from "../redis/redis.service"; import { QUEUE_NAMES, getBullConnection } from "./bull.config"; @@ -965,13 +966,11 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy { // flag (null, rare pre-migration rows) → treated as leaf, preserving the old // 1-level behaviour for those. if (source === "parts-catalogs") return hasSubgroups !== true; - // PL24 leaf indicators - return ( - linkPath.includes("/bom/") || - linkPath.includes("/bomdetails") || - linkPath.includes("/partinfo/") || - linkPath.includes("/servicepart/vin_items") - ); + // PL24: one shared classifier (integrations/pl24/pl24-tree). The old inline + // list was case-sensitive, so p5psa/p5volvo's camelCase `/details/vin/ + // bomDetails` was never recognised as a leaf and its parts were never + // prefetched (still true today for Mitsubishi/Fiat/Renault). + return isPl24LeafNode({ linkPath, hasSubgroups }); } /** -- 2.49.1