refactor(emex): rewrite EMEX integration, add category fallback

Replace browser-based Puppeteer scraper with standalone scraper wrapper.
Add EMEX as fallback source for categories when PL24 is unavailable.
Update vehicle decoding to use new synchronous EMEX API.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

apps/api/src/categories/categories.module.ts

@@ -2,9 +2,10 @@ import { Module } from "@nestjs/common";
 import { CategoriesController } from "./categories.controller";
 import { CategoriesService } from "./categories.service";
 import { PL24Module } from "../integrations/pl24/pl24.module";
+import { EmexModule } from "../integrations/emex/emex.module";
 
 @Module({
-  imports: [PL24Module],
+  imports: [PL24Module, EmexModule],
   controllers: [CategoriesController],
   providers: [CategoriesService],
   exports: [CategoriesService],

apps/api/src/categories/categories.service.spec.ts

@@ -10,7 +10,15 @@ function createService(db: any) {
   const pl24Service = {
     getCategories: vi.fn().mockResolvedValue([]),
   };
-  const service = new CategoriesService(db as any, redis as any, pl24Service as any);
+  const emexService = {
+    isSupported: vi.fn().mockReturnValue(false),
+    decodeVin: vi.fn().mockResolvedValue(null),
+    fetchCategoryParts: vi.fn().mockResolvedValue([]),
+  };
+  const storage = {
+    upload: vi.fn().mockResolvedValue("https://storage.sase.tr/sase-schemas/test.png"),
+  };
+  const service = new CategoriesService(db as any, redis as any, pl24Service as any, emexService as any, storage as any);
   return { service, db, redis, pl24Service };
 }
 

apps/api/src/categories/categories.service.ts

@@ -4,6 +4,8 @@ import { DATABASE, Database } from "../database/database.provider";
 import { categories, vehicles, schemaPics, parts } from "../database/schema/core";
 import { RedisService } from "../redis/redis.service";
 import { PL24Service } from "../integrations/pl24/pl24.service";
+import { EmexService } from "../integrations/emex/emex.service";
+import { StorageService } from "../storage/storage.service";
 
 @Injectable()
 export class CategoriesService {
@@ -13,6 +15,8 @@ export class CategoriesService {
     @Inject(DATABASE) private db: Database,
     private redis: RedisService,
     private pl24Service: PL24Service,
+    private emexService: EmexService,
+    private storage: StorageService,
   ) {}
 
   async getCategoryTree(vehicleId: string) {
@@ -41,28 +45,138 @@ export class CategoriesService {
       const catalogInfo = rawData.catalogInfo;
 
       if (catalogInfo?.serviceName && catalogInfo?.mainGroupsPath) {
-        const pl24Categories = await this.pl24Service.fetchMainGroups(
-          catalogInfo.serviceName,
-          catalogInfo.mainGroupsPath,
-        );
+        try {
+          const pl24Categories = await this.pl24Service.fetchMainGroups(
+            catalogInfo.serviceName,
+            catalogInfo.mainGroupsPath,
+          );
 
-        if (pl24Categories.length > 0) {
-          const insertData = pl24Categories.map((c) => ({
-            vehicleId,
-            name: c.nameTr || c.nameEn,
-            nameOriginal: c.nameEn,
-            parentId: null as string | null,
-            externalId: c.code,
-            linkPath: c.linkPath || null,
-            linkWid: c.linkWid || null,
-            source: "pl24" as const,
-          }));
+          if (pl24Categories.length > 0) {
+            // Deduplicate by name
+            const seenNames = new Set<string>();
+            const uniquePl24 = pl24Categories.filter((c) => {
+              const name = c.nameTr || c.nameEn;
+              if (seenNames.has(name)) return false;
+              seenNames.add(name);
+              return true;
+            });
 
-          dbCategories = await this.db.insert(categories).values(insertData).returning();
+            const insertData = uniquePl24.map((c) => ({
+              vehicleId,
+              name: c.nameTr || c.nameEn,
+              nameOriginal: c.nameEn,
+              parentId: null as string | null,
+              externalId: c.code,
+              linkPath: c.linkPath || null,
+              linkWid: c.linkWid || null,
+              source: "pl24" as const,
+            }));
+
+            dbCategories = await this.db.insert(categories).values(insertData).returning();
+          }
+        } catch (err) {
+          this.logger.warn(`PL24 category fetch failed for ${vehicleId}: ${(err as Error).message}`);
         }
       }
     }
 
+    // If still no categories, try EMEX fallback
+    if (dbCategories.length === 0 && vehicle.vin && this.emexService.isSupported(vehicle.vin)) {
+      this.logger.log(`No PL24 categories for ${vehicle.vin}, trying EMEX fallback`);
+      try {
+        const emexResult = await this.emexService.decodeVin(vehicle.vin);
+        if (emexResult) {
+          const rawData = emexResult.raw as Record<string, unknown>;
+          const tree = rawData?.emexCategoryTree as Array<{
+            name: string;
+            gid: string | null;
+            url: string | null;
+            children: any[];
+          }> | undefined;
+
+          if (tree && tree.length > 0) {
+            // Recursive tree insertion from QuickGroups.aspx
+            this.logger.log(`Inserting ${tree.length} EMEX top-level category groups recursively`);
+
+            const insertNodes = async (
+              nodes: Array<{ name: string; gid: string | null; url: string | null; children?: any[] }>,
+              parentId: string | null,
+            ) => {
+              for (const node of nodes) {
+                if (!node.name) continue;
+                const isLeaf = !node.children?.length;
+
+                const [inserted] = await this.db
+                  .insert(categories)
+                  .values({
+                    vehicleId,
+                    name: node.name,
+                    nameOriginal: node.name,
+                    parentId,
+                    externalId: node.gid || null,
+                    linkPath: isLeaf ? (node.url || null) : null,
+                    linkWid: null as string | null,
+                    source: "emex" as const,
+                  })
+                  .onConflictDoNothing()
+                  .returning();
+
+                if (inserted) {
+                  dbCategories.push(inserted);
+                  if (node.children?.length) {
+                    await insertNodes(node.children, inserted.id);
+                  }
+                }
+              }
+            };
+
+            await insertNodes(tree, null);
+            this.logger.log(`Stored ${dbCategories.length} EMEX categories (tree) for ${vehicle.vin}`);
+          } else if (emexResult.categories.length > 0) {
+            // Flat fallback: insert all categories without hierarchy
+            const emexCats = (rawData?.emexCategories as Array<{ gid: string; name: string; url: string | null }>) || [];
+            const urlMap = new Map(emexCats.map((c) => [c.gid, c.url]));
+
+            const seenNames = new Set<string>();
+            const uniqueCategories = emexResult.categories.filter((c) => {
+              const name = c.nameTr || c.nameEn;
+              if (seenNames.has(name)) return false;
+              seenNames.add(name);
+              return true;
+            });
+
+            const insertData = uniqueCategories.map((c) => ({
+              vehicleId,
+              name: c.nameTr || c.nameEn,
+              nameOriginal: c.nameEn,
+              parentId: null as string | null,
+              externalId: c.code,
+              linkPath: urlMap.get(c.code) || null,
+              linkWid: null as string | null,
+              source: "emex" as const,
+            }));
+
+            dbCategories = await this.db.insert(categories).values(insertData).returning();
+            this.logger.log(`Stored ${dbCategories.length} EMEX categories (flat) for ${vehicle.vin}`);
+          }
+
+          // Update vehicle source/rawData if it was vin-api
+          if (vehicle.source === "vin-api") {
+            await this.db
+              .update(vehicles)
+              .set({
+                model: emexResult.model !== "Unknown" ? emexResult.model : vehicle.model,
+                rawData: emexResult.raw,
+                source: "emex",
+              })
+              .where(eq(vehicles.id, vehicleId));
+          }
+        }
+      } catch (emexErr) {
+        this.logger.warn(`EMEX category fallback failed for ${vehicle.vin}: ${(emexErr as Error).message}`);
+      }
+    }
+
     // Build tree
     const tree = this.buildTree(dbCategories);
     await this.redis.setJson(cacheKey, tree, 3600);
@@ -118,7 +232,15 @@ export class CategoriesService {
     );
 
     if (subGroups.length > 0) {
-      const insertData = subGroups.map((sg) => ({
+      // Deduplicate by name
+      const seenNames = new Set<string>();
+      const uniqueSubGroups = subGroups.filter((sg) => {
+        if (seenNames.has(sg.name)) return false;
+        seenNames.add(sg.name);
+        return true;
+      });
+
+      const insertData = uniqueSubGroups.map((sg) => ({
         vehicleId: category.vehicleId,
         name: sg.name,
         nameOriginal: sg.name,
@@ -181,7 +303,7 @@ export class CategoriesService {
       .from(schemaPics)
       .where(eq(schemaPics.categoryId, categoryId));
 
-    // If no parts in DB, fetch from PL24
+    // If no parts in DB, fetch from source
     if (dbParts.length === 0 && category.linkPath) {
       const [vehicle] = await this.db
         .select()
@@ -189,7 +311,84 @@ export class CategoriesService {
         .where(eq(vehicles.id, category.vehicleId))
         .limit(1);
 
-      if (vehicle) {
+      if (vehicle && category.source === "emex") {
+        // EMEX: fetch parts + schema image via Puppeteer from QuickDetails URL
+        try {
+          const emexResult = await this.emexService.fetchCategoryParts(category.linkPath);
+
+          // Build position code → sequential integer mapping for hotspot linking
+          const posCodeToIndex = new Map<string, number>();
+          let nextIndex = 1;
+          for (const hs of emexResult.hotspots) {
+            if (!posCodeToIndex.has(hs.key)) {
+              posCodeToIndex.set(hs.key, nextIndex++);
+            }
+          }
+
+          if (emexResult.parts.length > 0) {
+            const insertData = emexResult.parts.map((p) => ({
+              vehicleId: vehicle.id,
+              categoryId,
+              oemCode: p.oemCode || "N/A",
+              name: p.nameEn || "Unknown",
+              nameOriginal: p.nameEn || null,
+              description: null as string | null,
+              quantity: null as number | null,
+              position: p.positionCode || null,
+              hotspotIndex: p.positionCode ? (posCodeToIndex.get(p.positionCode) ?? null) : null,
+              source: "emex" as const,
+            }));
+
+            dbParts = await this.db.insert(parts).values(insertData).returning();
+            this.logger.log(`Stored ${dbParts.length} EMEX parts for category ${categoryId}`);
+          }
+
+          // Download schema image from img.laximo.net and upload to MinIO
+          if (emexResult.schemaImageUrl && pics.length === 0) {
+            try {
+              const imgResp = await fetch(emexResult.schemaImageUrl, {
+                signal: AbortSignal.timeout(15000),
+              });
+              if (imgResp.ok) {
+                const buf = Buffer.from(await imgResp.arrayBuffer());
+                const ext = emexResult.schemaImageUrl.includes('.gif') ? 'gif' : 'png';
+                const key = `schemas/emex-${categoryId}.${ext}`;
+                const minioUrl = await this.storage.upload(key, buf, `image/${ext}`);
+
+                // Use scraper dimensions (from naturalWidth/Height), fallback to buffer parsing
+                const dims = (emexResult.schemaWidth && emexResult.schemaHeight)
+                  ? { width: emexResult.schemaWidth, height: emexResult.schemaHeight }
+                  : this.getImageDimensions(buf, ext);
+
+                // Convert EMEX hotspots to storage format with sequential integer keys
+                const hotspotItems = emexResult.hotspots.map((hs) => ({
+                  key: String(posCodeToIndex.get(hs.key) ?? hs.key),
+                  label: hs.key,
+                  areas: hs.areas,
+                }));
+
+                const [inserted] = await this.db
+                  .insert(schemaPics)
+                  .values({
+                    categoryId,
+                    imageUrl: minioUrl,
+                    hotspots: JSON.stringify({ width: dims.width, height: dims.height, items: hotspotItems }),
+                    source: "emex",
+                  })
+                  .returning();
+
+                pics.push(inserted);
+                this.logger.log(`Stored EMEX schema image in MinIO for category ${categoryId}: ${minioUrl} (${dims.width}x${dims.height})`);
+              }
+            } catch (imgErr) {
+              this.logger.warn(`Failed to download EMEX schema image: ${(imgErr as Error).message}`);
+            }
+          }
+        } catch (err) {
+          this.logger.error(`Failed to fetch EMEX parts for category ${categoryId}: ${(err as Error).message}`);
+        }
+      } else if (vehicle) {
+        // PL24: fetch parts via PL24 API
         const rawData = vehicle.rawData as any;
         const catalogInfo = rawData?.catalogInfo;
 
@@ -280,16 +479,16 @@ export class CategoriesService {
       }
     }
 
-    // Transform raw PL24 hotspots {key, areas} to frontend format
+    // Transform raw hotspots {key, label?, areas} to frontend format
     const mappedHotspots = hotspots.flatMap(
-      (hs: { key: string; areas?: Array<{ left: number; top: number; width: number; height: number }> }) =>
+      (hs: { key: string; label?: string; areas?: Array<{ left: number; top: number; width: number; height: number }> }) =>
         (hs.areas || []).map((area, areaIdx) => ({
           id: `hs-${hs.key}-${areaIdx}`,
           key: hs.key,
           group: parseInt(hs.key, 10) || 0,
           shape: "rect" as const,
           coordinates: [area.left, area.top, area.width, area.height],
-          label: hs.key,
+          label: hs.label || hs.key,
         })),
     );
 
@@ -329,6 +528,22 @@ export class CategoriesService {
     return { ...category, schemaPics: pics };
   }
 
+  private getImageDimensions(buf: Buffer, ext: string): { width: number; height: number } {
+    try {
+      if (ext === 'gif' && buf.length >= 10) {
+        // GIF: width at bytes 6-7, height at bytes 8-9 (little-endian)
+        return { width: buf.readUInt16LE(6), height: buf.readUInt16LE(8) };
+      }
+      if (ext === 'png' && buf.length >= 24) {
+        // PNG: width at bytes 16-19, height at bytes 20-23 (big-endian)
+        return { width: buf.readUInt32BE(16), height: buf.readUInt32BE(20) };
+      }
+    } catch {
+      // fallback
+    }
+    return { width: 0, height: 0 };
+  }
+
   private buildTree(items: any[]): any[] {
     const map = new Map<string, any>();
     const roots: any[] = [];