Merge pull request 'dev' (#79) from dev into main

Reviewed-on: #79

apps/api/drizzle/0010_dedupe_parts.sql

@@ -0,0 +1,21 @@
+-- Phase 1: remove existing duplicate rows.
+-- "Duplicate" = same (vehicle_id, category_id, oem_code, name, position).
+-- Backfill + reactive prefetch can re-drill the same category multiple times;
+-- without a unique constraint that re-insert kept all rows, leaving 8–34%
+-- duplicate per source (~108k extra rows of ~588k). Keep the oldest row per
+-- group (preserves original created_at) and drop the rest.
+DELETE FROM "parts" WHERE id IN (
+  SELECT id FROM (
+    SELECT id, row_number() OVER (
+      PARTITION BY vehicle_id, category_id, oem_code, name, position
+      ORDER BY created_at ASC, id ASC
+    ) AS rn FROM "parts"
+  ) t WHERE rn > 1
+);
+--> statement-breakpoint
+
+-- Phase 2: prevent future dupes. NULLS NOT DISTINCT (Postgres 15+) so NULL
+-- position / NULL vehicle_id collapse like equal values rather than each
+-- counting as a separate "distinct" row.
+CREATE UNIQUE INDEX IF NOT EXISTS "parts_dedup_idx" ON "parts"
+  (vehicle_id, category_id, oem_code, name, position) NULLS NOT DISTINCT;

apps/api/drizzle/meta/_journal.json

@@ -71,6 +71,13 @@
       "when": 1779703007129,
       "tag": "0009_referral_rewards_rework",
       "breakpoints": true
+    },
+    {
+      "idx": 10,
+      "version": "7",
+      "when": 1780281600000,
+      "tag": "0010_dedupe_parts",
+      "breakpoints": true
     }
   ]
 }

apps/api/src/catalog/catalog.service.ts

@@ -958,7 +958,11 @@ export class CatalogService {
             source: "pl24" as const,
           }));
 
-          dbParts = await this.db.insert(parts).values(insertData).returning();
+          dbParts = await this.db
+            .insert(parts)
+            .values(insertData)
+            .onConflictDoNothing()
+            .returning();
         }
 
         if (needImage) {

apps/api/src/categories/categories.service.ts

@@ -155,10 +155,14 @@ export class CategoriesService {
             // Strip P4 header/breadcrumb nav crumbs (Portal, "Model seçimi",
             // current VIN) — these slipped past the older decoder filter and
             // sit in cached rawData; they are never real part categories.
+            // NOTE: vin-group.action?group1=… are NOT crumbs — they are Volvo's
+            // real top groups (VIN-indexed catalog). Only treat vin-group.action
+            // as a crumb when it lacks group1=, else every Volvo category is lost.
             const NAV_CRUMB_RE = /(portal|vehicle|vin-group|logout)\.action/i;
             const seenNames = new Set<string>();
             const uniqueCats = decodedCats.filter((c) => {
-              if (c.linkPath && NAV_CRUMB_RE.test(c.linkPath)) return false;
+              if (c.linkPath && NAV_CRUMB_RE.test(c.linkPath) && !c.linkPath.includes("group1="))
+                return false;
               const name = c.nameTr || c.nameEn;
               if (seenNames.has(name)) return false;
               seenNames.add(name);
@@ -891,6 +895,34 @@ export class CategoriesService {
         : { ...base, loadError: true };
     }
 
+    // Volvo VIN-indexed catalog: vin-group.action?group1=…[&group2=…] nodes are
+    // parents whose subgroups live in the same HTML (PL24's old JSON
+    // json-vin-main-group.action path now 404s). They carry no link_wid, so the
+    // group-node branch above misses them. Drill: if subgroups come back it's a
+    // parent; if none, fall through to the leaf parts path (the deepest group
+    // level carries an image-board + parts).
+    if (
+      category.source === "pl24" &&
+      category.linkPath?.includes("vin-group.action") &&
+      category.linkPath?.includes("group1=") &&
+      category.vehicleId
+    ) {
+      const volvoChildren = await this.getChildren(categoryId);
+      if (volvoChildren.length > 0) {
+        return {
+          id: category.id,
+          name: category.name,
+          description: category.nameOriginal || null,
+          parentId: category.parentId || null,
+          parts: [],
+          schemaPics: [],
+          hotspots: [],
+          children: volvoChildren,
+        };
+      }
+      // No subgroups → leaf; fall through to the parts path below.
+    }
+
     // Leaf category — get or fetch parts
     let loadError = false;
     let discoveredChildren: any[] = [];
@@ -992,7 +1024,11 @@ export class CategoriesService {
               }));
 
               if (allParts.length > 0) {
-                dbParts = await this.db.insert(parts).values(allParts).returning();
+                dbParts = await this.db
+                  .insert(parts)
+                  .values(allParts)
+                  .onConflictDoNothing()
+                  .returning();
                 this.logger.log(
                   `Stored ${dbParts.length} PartsCatalogs parts for category ${categoryId}`,
                 );
@@ -1142,7 +1178,11 @@ export class CategoriesService {
               };
             });
 
-            dbParts = await this.db.insert(parts).values(insertData).returning();
+            dbParts = await this.db
+              .insert(parts)
+              .values(insertData)
+              .onConflictDoNothing()
+              .returning();
             this.logger.log(`Stored ${dbParts.length} EMEX parts for category ${categoryId}`);
           }
 
@@ -1238,7 +1278,11 @@ export class CategoriesService {
                 source: "pl24" as const,
               }));
 
-              dbParts = await this.db.insert(parts).values(insertData).returning();
+              dbParts = await this.db
+                .insert(parts)
+                .values(insertData)
+                .onConflictDoNothing()
+                .returning();
             }
 
             // Store schema image if available

apps/api/src/database/schema/core.ts

@@ -434,6 +434,11 @@ export const parts = pgTable(
     index("parts_catalog_vehicle_id_idx").on(table.catalogVehicleId),
     index("parts_category_id_idx").on(table.categoryId),
     index("parts_oem_code_idx").on(table.oemCode),
+    // NOTE: composite UNIQUE INDEX `parts_dedup_idx` over
+    // (vehicle_id, category_id, oem_code, name, position) NULLS NOT DISTINCT
+    // is created by migration 0010_dedupe_parts. drizzle-orm 0.41 doesn't
+    // expose .nullsNotDistinct() on the index builder, so the constraint
+    // is owned by raw SQL — keep this comment in sync if you change it.
   ],
 );
 

apps/api/src/integrations/pl24/pl24-ford-legacy.service.ts

@@ -130,6 +130,59 @@ export class PL24FordLegacyService {
       const html = await this.fetchP4Page(linkPath, serviceName, false, account);
       if (!html) return [];
 
+      // Volvo (VIN-indexed legacy catalog): vin-group.action?group1=…[&group2=…]
+      // HTML pages list the next tree level as <a href> links — either a deeper
+      // vin-group.action?…&groupN=… (sub-group) or a vin-image-board.action?… leaf
+      // (the illustration that carries the BOM). PL24's old JSON
+      // json-vin-*-group.action endpoints now 404, so we scrape the HTML here.
+      // For sub-groups keep only links one group-level deeper than the current
+      // path; drop sibling/parent nav and the open-VIN-dialog crumb
+      // (openVinDialog=true — note the real links carry openVinDialog=false).
+      if (linkPath.includes("vin-group.action")) {
+        const cfg = getServiceConfig(serviceName);
+        const basePath = cfg ? `${cfg.basePath}/${serviceName}` : `/volvo/${serviceName}`;
+        const parentDepth = [...linkPath.matchAll(/\bgroup\d+=/g)].length;
+        const seenHref = new Set<string>();
+        const subs: PL24MainGroup[] = [];
+        const anchorRe =
+          /<a[^>]+href="((?:[^"]*vin-group\.action\?[^"]*group\d+=|[^"]*vin-image-board\.action\?)[^"]*)"[^>]*>([\s\S]*?)<\/a>/gi;
+        for (const m of html.matchAll(anchorRe)) {
+          const href = m[1].replace(/&amp;/g, "&");
+          if (href.includes("openVinDialog=true")) continue;
+          const isImageBoard = href.includes("vin-image-board.action");
+          if (!isImageBoard) {
+            const childDepth = [...href.matchAll(/\bgroup\d+=/g)].length;
+            if (childDepth <= parentDepth) continue; // sibling/parent group nav
+          }
+          if (seenHref.has(href)) continue;
+          seenHref.add(href);
+          const name = m[2]
+            .replace(/<[^>]+>/g, " ")
+            .replace(/&nbsp;/g, " ")
+            .replace(/&Ouml;/g, "Ö")
+            .replace(/&ouml;/g, "ö")
+            .replace(/&Auml;/g, "Ä")
+            .replace(/&auml;/g, "ä")
+            .replace(/&Uuml;/g, "Ü")
+            .replace(/&uuml;/g, "ü")
+            .replace(/&szlig;/g, "ß")
+            .replace(/&quot;/g, '"')
+            .replace(/&#39;/g, "'")
+            .replace(/&amp;/g, "&")
+            .replace(/&#(\d+);/g, (_, n) => String.fromCharCode(Number(n)))
+            .replace(/\s+/g, " ")
+            .trim()
+            .replace(/^\d+\s+/, "");
+          if (name.length < 2) continue;
+          const lp = href.startsWith("/") ? href : `${basePath}/${href}`;
+          subs.push({ id: href, code: href, name, linkPath: lp });
+        }
+        if (subs.length > 0) {
+          await this.redis.setJson(cacheKey, subs, 86400);
+        }
+        return subs;
+      }
+
       // Opel/Ford: json-main-group.action → { vCfgData: [...] } (Opel) OR { maingroups: [...] } (Ford)
       if (linkPath.includes("json-main-group.action")) {
         const basePath = linkPath.substring(0, linkPath.lastIndexOf("/") + 1);
@@ -313,11 +366,18 @@ export class PL24FordLegacyService {
       let parts: PL24Part[];
       const isVinImageBoard = linkPath.includes("vin-image-board.action");
       if (isVinImageBoard) {
-        // Ford/Volvo VIN flow: rows have pncHierCode + jsonUrl pointing to
+        // Ford VIN flow: rows have pncHierCode + jsonUrl pointing to
         // json-vin-bom-detail.action. Each detail JSON can return MULTIPLE
         // valid partno entries (e.g. left/right variant) — keep all of them.
         const pncRows = this.parseFordVinPncRows(html);
-        parts = await this.fetchFordVinBomParts(pncRows, serviceName);
+        if (pncRows.length > 0) {
+          parts = await this.fetchFordVinBomParts(pncRows, serviceName);
+        } else {
+          // Volvo vin-image-board ships its BOM inline as partno= tc-data-row
+          // rows (no pncHierCode / json-vin-bom-detail secondary fetch), same
+          // shape the PSA parser handles.
+          parts = this.parsePsaBomParts(html);
+        }
       } else if (isImageBoard) {
         // Try Hyundai/Kia/Nissan format: pnc= rows with json-bom-detail.action
         const pncRows = this.parseHyundaiPncRows(html);
@@ -3119,8 +3179,17 @@ export class PL24FordLegacyService {
     serviceName: string,
     isFullUrl = false,
     account: "tr" | "de" = "tr",
+    retried = false,
   ): Promise<string | null> {
-    const fullUrl = isFullUrl ? url : `${this.baseUrl}${url}`;
+    // Some catalogs (Volvo vin-group.action) store hrefs relative to the catalog
+    // directory (e.g. "vin-group.action?group1=…"). Prefix the service basePath
+    // so `${baseUrl}${path}` doesn't collapse into "partslink24.comvin-group…".
+    let path = url;
+    if (!isFullUrl && !path.startsWith("/")) {
+      const cfg = getServiceConfig(serviceName);
+      if (cfg) path = `${cfg.basePath}/${serviceName}/${path}`;
+    }
+    const fullUrl = isFullUrl ? url : `${this.baseUrl}${path}`;
     const headers = await this.authService.buildFordLegacyHeadersForAccount(serviceName, account);
     const dispatcher = await this.authService.getProxyAgent4Account(account);
 
@@ -3161,7 +3230,21 @@ export class PL24FordLegacyService {
         return JSON.stringify(json);
       }
 
-      return await response.text();
+      const html = await response.text();
+      // Demo mode: PL24 serves a stripped NOT_LOGGED_IN_DEMO page (no real
+      // groups/parts) when the service token is stale. decodeVinForService
+      // retries on this, but drill paths reach upstream only through here —
+      // so retry once with fresh auth before parsing an empty page.
+      if (!retried && html.includes("PL24_SUPPORT")) {
+        const support = this.extractScriptVariable<FordPL24Support>(html, "PL24_SUPPORT");
+        if (support?.demo || support?.role === "NOT_LOGGED_IN_DEMO") {
+          this.logger.warn(`Ford legacy: demo page for ${serviceName}, re-authing + retry`);
+          this.authService.clearTokensForAccount(account);
+          await this.authService.authorizeServiceForAccount(serviceName, account);
+          return this.fetchP4Page(url, serviceName, isFullUrl, account, true);
+        }
+      }
+      return html;
     } catch (error) {
       const err = error as Error;
       this.logger.error(
@@ -4012,7 +4095,13 @@ export class PL24FordLegacyService {
       if (
         href.includes("portal.action") ||
         href.includes("vehicle.action") ||
-        href.includes("vin-group.action") ||
+        // vin-group.action is the VIN breadcrumb crumb — EXCEPT when it carries
+        // group1=. Volvo (and other VIN-indexed legacy catalogs) ship their real
+        // top groups as vin-group.action?group1=… in the HTML. PL24's
+        // json-vin-main-group.action now 404s, so decode falls back to this
+        // scrape; the blanket vin-group.action exclude then drops every real
+        // Volvo group → 0 categories. Keep the group1= links.
+        (href.includes("vin-group.action") && !href.includes("group1=")) ||
         href.includes("logout.action") ||
         href.includes("login")
       ) {

apps/api/src/jobs/prefetch-utils.ts

@@ -65,14 +65,18 @@ function currentIstanbulHour(): number {
 
 /**
  * Whether `source` may be scraped right now.
- * PL24 → 09:00–18:00, parts-catalogs → 09:00–19:00 (Europe/Istanbul).
- * EMEX and everything else have no window (always true).
+ * PL24 → 09:00–18:00 (Europe/Istanbul); upstream rate-limit is still
+ * tighter outside that window so user-facing requests benefit from the
+ * guard. EMEX, parts-catalogs, and everything else have no window
+ * (always true). parts-catalogs used to be 09:00–19:00 too, but the
+ * Redis-persisted warm JWT pool (commits aa4d055 + dcf7e06) now keeps
+ * captures alive 24/7 so backfill can sweep through the night when the
+ * upstream is most idle.
  */
 export function isWithinTimeWindow(source: string): boolean {
-  if (source !== "pl24" && source !== "parts-catalogs") return true;
+  if (source !== "pl24") return true;
   const h = currentIstanbulHour();
-  const endHour = source === "parts-catalogs" ? 19 : 18;
-  return h >= 9 && h < endHour;
+  return h >= 9 && h < 18;
 }
 
 /**

apps/api/src/jobs/prefetch-worker.service.ts

@@ -27,8 +27,13 @@ import { CATALOG_PREFETCH_QUEUE } from "./queues/catalog-prefetch.queue";
 const MAX_DEPTH = 5;
 
 // ── Backfill scan (hourly cron) tuning ──
-/** Vehicles queued per scan wave. */
-const BACKFILL_BATCH_SIZE = 20;
+/**
+ * Vehicles queued per scan wave. Was 20 while pcat needed business-hours
+ * gating; doubled to 40 once the warm JWT pool + Redis hydration made the
+ * scrape stack 24/7. The MAX_BACKLOG guard below still gates run-away
+ * pileups, and per-source cooldown still yields to live users.
+ */
+const BACKFILL_BATCH_SIZE = 40;
 /** Skip the wave entirely if the queue already has more than this many jobs pending. */
 const BACKFILL_MAX_BACKLOG = 1000;
 /** In-flight guard TTL (seconds) — safety net if a run dies without clearing. */

apps/api/src/parts/parts.service.ts

@@ -76,7 +76,7 @@ export class PartsService {
         source: "pl24" as const,
       }));
 
-      dbParts = await this.db.insert(parts).values(insertData).returning();
+      dbParts = await this.db.insert(parts).values(insertData).onConflictDoNothing().returning();
     }
 
     // Store schema image if available

apps/web/src/components/schema/parts-panel.tsx

@@ -92,8 +92,8 @@ export function PartsPanel({ parts, vehicleId, categoryId, isLoading }: PartsPan
     return (
       <div className="flex h-full flex-col overflow-hidden">
         <div className="border-b border-border px-4 py-3">
-          <h3 className="text-sm font-semibold">Parcalar</h3>
-          <p className="text-xs text-muted-foreground">Yukleniyor...</p>
+          <h3 className="text-sm font-semibold">Parçalar</h3>
+          <p className="text-xs text-muted-foreground">Yükleniyor...</p>
         </div>
         <div className="flex-1 overflow-y-auto">
           <div className="space-y-2 p-3">
@@ -109,16 +109,16 @@ export function PartsPanel({ parts, vehicleId, categoryId, isLoading }: PartsPan
   return (
     <div className="flex h-full flex-col overflow-hidden">
       <div className="border-b border-border px-4 py-3">
-        <h3 className="text-sm font-semibold">Parcalar</h3>
-        <p className="text-xs text-muted-foreground">{parts.length} parca listeleniyor</p>
+        <h3 className="text-sm font-semibold">Parçalar</h3>
+        <p className="text-xs text-muted-foreground">{parts.length} parça listeleniyor</p>
       </div>
 
       <div className="flex-1 overflow-y-auto">
         {parts.length === 0 ? (
           <div className="flex flex-col items-center justify-center gap-4 py-12 text-center text-sm text-muted-foreground">
-            <p>Bu kategori icin parca bulunamadi.</p>
+            <p>Bu kategori için parça bulunamadı.</p>
             <Button type="button" variant="outline" onClick={() => window.history.back()}>
-              Geri don
+              Geri dön
             </Button>
           </div>
         ) : (
@@ -126,7 +126,7 @@ export function PartsPanel({ parts, vehicleId, categoryId, isLoading }: PartsPan
             <thead className="sticky top-0 z-10 bg-background">
               <tr className="border-b border-border text-left text-xs font-medium text-muted-foreground">
                 <th className="px-3 py-2 w-10">#</th>
-                <th className="px-3 py-2">Parca Adi</th>
+                <th className="px-3 py-2">Parça Adı</th>
                 <th className="px-3 py-2">OEM Kodu</th>
                 <th className="px-3 py-2 w-14 text-center">Adet</th>
                 <th className="px-3 py-2">Pozisyon</th>

apps/web/src/routes/index.tsx

@@ -3,6 +3,7 @@ import { CarBrandLogo } from "@/components/ui/car-brand-logo";
 import { SmartAnimateText } from "@/components/ui/smart-animate-text";
 import { VinBrandIcon } from "@/components/ui/vin-brand-icon";
 import { KEYS_17, dynamicKeys } from "@/lib/keys";
+import { cleanModelName } from "@/lib/vehicle";
 import { Button, Input, Separator } from "@sase/ui";
 import { Link, createFileRoute, useNavigate } from "@tanstack/react-router";
 import {
@@ -997,9 +998,11 @@ export function HomePage() {
                 <div className="flex flex-wrap items-center gap-3">
                   <Car className="size-5 text-brand" />
                   <span className="font-semibold text-foreground">
-                    {vinPreview.make} {vinPreview.model}
+                    {vinPreview.make} {cleanModelName(vinPreview.model) || vinPreview.model}
                   </span>
-                  <span className="text-sm text-muted-foreground">{vinPreview.year}</span>
+                  {vinPreview.year && vinPreview.year !== "—" && (
+                    <span className="text-sm text-muted-foreground">({vinPreview.year})</span>
+                  )}
                   {vinPreview.engine !== "—" && (
                     <span className="rounded-full bg-muted px-2.5 py-0.5 text-xs text-muted-foreground">
                       {vinPreview.engine}