/** * EMEX VIN Service * * NestJS service for emexdwc.ae VIN integration. * Primary path: pure HTTP fetch (no browser) — fast (~1-5s). * Fallback: Playwright scraper via EmexBrowserService. * * emexdwc.ae does NOT require authentication for Vehicles.aspx, * QuickGroups.aspx, or QuickDetails.aspx — plain HTTP GET works. */ import * as path from "node:path"; import { BadRequestException, Injectable, InternalServerErrorException, Logger, ServiceUnavailableException, } from "@nestjs/common"; import { ConfigService } from "@nestjs/config"; import { ProxyAgent } from "undici"; import { isBackfillContext } from "../../jobs/prefetch-context"; import { PostHogService } from "../../posthog/posthog.service"; import { RedisService } from "../../redis/redis.service"; import { ProxyHealthService } from "../proxy-telemetry/proxy-health.service"; import { ProxyTelemetryService, classifyTransportError, isProxyConnectFailure, } from "../proxy-telemetry/proxy-telemetry.service"; import { parseUnitLeaves, parseVehicleTree } from "./emex-tree.parser"; import { EmexBrowserService } from "./emex.browser"; import { createEmptyDecodedVehicle, mapEmexResponse } from "./emex.mapper"; import { CATALOG_MAP, type DecodedVehicle, type EmexCategoryTreeNode, type EmexHotspot, type EmexHotspotArea, type EmexPart, type EmexPartsResult, type EmexScraperResponse, type EmexVehicleTreeNode, } from "./emex.types"; const EMEX_BASE_URL = "https://emexdwc.ae"; const EMEX_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"; interface EmexHttpVehicle { label: string; model: string; yearFrom: number | null; catalogCode: string | null; vid: string | null; ssd: string | null; quickGroupsUrl: string | null; } interface EmexHttpCategory { gid: string | null; name: string; url: string; } /** * Candidate vehicle returned when EMEX finds multiple matches for a VIN. * Shape matches PcatCar so VehicleSelectModal can render without changes. */ export interface EmexCandidate { /** String index into the Vehicles.aspx list — "0", "1", etc. */ id: string; name: string; parameters?: Array<{ key: string; idx: string; value: string }>; catalogId: string; /** Internal: used by resolveEmexCarByIndex to fetch the specific vehicle */ _index: number; _quickGroupsUrl: string | null; _ssd: string | null; _vid: string | null; } // Type definition for the imported scraper module interface EmexScraperModule { EmexVinScraper: new (options?: { page?: unknown }) => EmexVinScraperInstance; getCatalogCode: (vin: string) => string | null; getYearFromVIN: (vin: string) => number | null; CONFIG: Record; } interface EmexCategoryResult { gid: string; name: string; url: string | null; } interface EmexVinScraperInstance { init(): Promise; close(): Promise; searchByVIN(vin: string): Promise; getCategories(quickGroupsUrl: string): Promise; getCategoryTree(quickGroupsUrl: string): Promise; getParts(detailsUrl: string): Promise; } @Injectable() export class EmexService { private readonly logger = new Logger(EmexService.name); private scraperModule: EmexScraperModule | null = null; private isInitialized = false; private initializationPromise: Promise | null = null; private readonly scraperPath: string; private readonly timeout: number; private readonly debug: boolean; private readonly proxyAgent: ProxyAgent | null; private readonly emexProxy: { host: string; user: string; pass: string; portStart: number; portEnd: number; } | null; // Floxy residential fallback — used when the DataImpulse pool throws transport // errors. `lifetime` > 0 → sticky session (one exit IP held that many seconds, // appended to the password as `_session-_lifetime-`); 0 → rotating. private readonly emexFloxy: { host: string; port: number; user: string; pass: string; lifetime: number; } | null; // Rolling Floxy sticky-session id: held for ~lifetime so a chained emex flow // (decode → tree → drill → parts) keeps ONE exit IP and emex's IP-bound ssd // tokens stay valid. Rotated when stale or after a transport failure. private floxySessionId = Math.random().toString(36).slice(2, 10); private floxySessionBornMs = Date.now(); private readonly emexDirectFallback: boolean; // Transport provider for EMEX egress (HTTP + image-dims). "floxy" residential // is primary by default (sticky session pins one exit IP across a chained flow), // DataImpulse is the last-ditch fallback. See constructor. private readonly emexProxyProvider: "floxy" | "dataimpulse" | "none"; constructor( private configService: ConfigService, private browserService: EmexBrowserService, private redis: RedisService, private posthog: PostHogService, private proxyTelemetry: ProxyTelemetryService, private proxyHealth: ProxyHealthService, ) { // __dirname is apps/api/src/integrations/emex/ or dist/integrations/emex/ // Scraper lives at /scripts/emex-vin-scraper.js const monorepoRoot = path.resolve(__dirname, "..", "..", "..", "..", ".."); const defaultPath = path.resolve(monorepoRoot, "scripts/emex-vin-scraper.js"); this.scraperPath = this.configService.get("EMEX_SCRAPER_PATH", defaultPath); this.timeout = this.configService.get("EMEX_TIMEOUT", 60000); this.debug = this.configService.get("EMEX_DEBUG", false); const useProxy = this.configService.get("EMEX_USE_PROXY", "true") === "true"; this.emexDirectFallback = this.configService.get("EMEX_DIRECT_FALLBACK", "false") === "true"; if (useProxy) { // ConfigService.get returns the raw env STRING, not a number. Left // un-coerced, the port-pick arithmetic does string concat — e.g. // 45 + "10001" = "4510001", an out-of-range port → undici `new URL` throws // "Invalid URL" and the whole app fails to boot. A single-port range (823) // happens to concat to a still-parseable "0823", which masked this for // months until the range was widened. Always coerce to a valid port number. const toPort = (key: string, def: number): number => { const n = Number(this.configService.get(key, def)); return Number.isInteger(n) && n >= 1 && n <= 65535 ? n : def; }; this.emexProxy = { host: this.configService.get("EMEX_PROXY_HOST", "74.81.81.81"), user: this.configService.get("EMEX_PROXY_USER", "1726bbe361918676d44e"), pass: this.configService.get("EMEX_PROXY_PASS", "f11c7b6128cc86c6"), portStart: toPort("EMEX_PROXY_PORT_START", 10001), portEnd: toPort("EMEX_PROXY_PORT_END", 10099), }; this.logger.log( `EMEX DataImpulse fallback enabled: ${this.emexProxy.host}:${this.emexProxy.portStart}-${this.emexProxy.portEnd}`, ); } else { this.emexProxy = null; } // Floxy residential fallback for when the DataImpulse pool flakes (connect // timeouts / resets). On by default; creds + endpoint overridable via env. const floxyEnabled = this.configService.get("EMEX_FLOXY_FALLBACK", "true") === "true"; if (floxyEnabled) { const fport = Number(this.configService.get("EMEX_FLOXY_PORT", 12321)); const flife = Number(this.configService.get("EMEX_FLOXY_LIFETIME", 300)); this.emexFloxy = { host: this.configService.get("EMEX_FLOXY_HOST", "residential.floxy.io"), port: Number.isInteger(fport) && fport >= 1 && fport <= 65535 ? fport : 12321, user: this.configService.get("EMEX_FLOXY_USER", "cac4b0d96a80"), pass: this.configService.get("EMEX_FLOXY_PASS", "dfc38fe467a3"), lifetime: Number.isInteger(flife) && flife >= 0 ? flife : 300, }; this.logger.log( `EMEX Floxy fallback enabled: ${this.emexFloxy.host}:${this.emexFloxy.port} (${this.emexFloxy.lifetime > 0 ? `sticky ${this.emexFloxy.lifetime}s` : "rotating"})`, ); } else { this.emexFloxy = null; } // Transport provider. EMEX_PROXY_PROVIDER: "floxy" (default) → Floxy residential // sticky primary + a single DataImpulse last-ditch fallback; "dataimpulse" → // legacy DataImpulse-first order; "none" → direct. ssd tokens are replayed from // emex's own HTML and aren't strictly IP-bound on the .aspx endpoints (the old // random-port-per-call primary proved that), but a sticky Floxy IP across the // chained flow is still the safest choice — and avoids DataImpulse's ~50% dead // ports that were eating the decode budget before reaching the Floxy fallback. // Default dataimpulse: Floxy retired 2026-07 (permanently down). Set // EMEX_PROXY_PROVIDER=floxy to re-enable it. this.emexProxyProvider = !useProxy ? "none" : (() => { const p = this.configService .get("EMEX_PROXY_PROVIDER", "dataimpulse") .toLowerCase(); return p === "floxy" || p === "none" ? p : "dataimpulse"; })(); // Default agent for the low-stakes image-dims path; fetchEmexHtml builds a // fresh agent per request so a flaky exit can't pin every call. this.proxyAgent = this.emexProxyProvider === "none" ? null : (this.newProxyAgent(this.emexProxyProvider === "dataimpulse" ? "dataimpulse" : "floxy") ?? this.newProxyAgent("dataimpulse")); this.logger.log(`EMEX transport provider: ${this.emexProxyProvider}`); this.logger.log(`EMEX Service initialized with scraper path: ${this.scraperPath}`); } /** Mark EMEX as actively used (5min TTL) to defer prefetch worker */ private async touchActivity(): Promise { // Worker-originated fetches must NOT register as user activity, or the // backfill worker throttles itself via checkCooldown. if (isBackfillContext()) return; try { await this.redis.set("prefetch:activity:emex", String(Date.now()), 90); } catch { // Non-critical — don't break the request } } /** * Lazily initialize the scraper module */ private async initializeScraper(): Promise { if (this.isInitialized) { return; } if (this.initializationPromise) { return this.initializationPromise; } this.initializationPromise = this.doInitialize(); return this.initializationPromise; } private async doInitialize(): Promise { try { this.logger.log(`Loading EMEX scraper module from: ${this.scraperPath}`); const fs = require("node:fs"); if (!fs.existsSync(this.scraperPath)) { this.logger.error(`Scraper file not found at: ${this.scraperPath}`); this.logger.error(`Current working directory: ${process.cwd()}`); throw new Error(`Scraper file not found: ${this.scraperPath}`); } // Clear require cache to always load the latest scraper version delete require.cache[require.resolve(this.scraperPath)]; // eslint-disable-next-line @typescript-eslint/no-var-requires this.scraperModule = require(this.scraperPath) as EmexScraperModule; this.logger.log("EMEX scraper module loaded successfully"); this.isInitialized = true; } catch (error) { const err = error as Error; this.logger.error(`Failed to load EMEX scraper module: ${err.message}`, err.stack); throw new InternalServerErrorException("EMEX servis modulu yuklenemedi"); } } /** * Creates a scraper instance bound to a pre-created page from the browser pool. * Returns { scraper, release } — caller MUST call release() in finally. */ private async createScraperInstance(): Promise<{ scraper: EmexVinScraperInstance; release: () => Promise; }> { await this.initializeScraper(); if (!this.scraperModule) { throw new InternalServerErrorException("EMEX scraper modulu yuklenemedi"); } const { page, release } = await this.browserService.acquirePage(); const scraper = new this.scraperModule.EmexVinScraper({ page }); // init() is a no-op in managed mode, but call it for consistency await scraper.init(); return { scraper, release }; } /** * Validates VIN format */ private validateVin(vin: string): void { if (!vin) { throw new BadRequestException("VIN numarasi gereklidir"); } const cleanVin = vin.toUpperCase().replace(/[^A-HJ-NPR-Z0-9]/g, ""); if (cleanVin.length !== 17) { throw new BadRequestException("VIN numarasi 17 karakter olmalidir"); } if (/[IOQ]/i.test(cleanVin)) { throw new BadRequestException( "VIN numarasi gecersiz karakterler iceriyor (I, O, Q kullanilamaz)", ); } } // ─── HTTP-based methods (no browser required) ──────────────── /** * Fetch a URL from emexdwc.ae without a browser session. * emexdwc.ae serves Vehicles.aspx, QuickGroups.aspx, and QuickDetails.aspx * without requiring authentication cookies. */ /** Build a fresh proxy agent on a random port from the pool (null if proxy off). */ /** Fresh Floxy sticky-session id (new exit IP). */ private rotateFloxySession(): void { this.floxySessionId = Math.random().toString(36).slice(2, 10); this.floxySessionBornMs = Date.now(); } /** * Floxy auth password. With lifetime > 0, appends `_session-_lifetime-` * to pin one exit IP across a chained emex flow; the session id rolls over * once it nears the lifetime so we never reuse an expired sticky slot. */ private floxyPassword(): string { if (!this.emexFloxy) return ""; const { pass, lifetime } = this.emexFloxy; if (lifetime <= 0) return pass; // rotating mode // Roll at ~80% of lifetime so a sticky IP is never reused past expiry. if (Date.now() - this.floxySessionBornMs > lifetime * 800) this.rotateFloxySession(); return `${pass}_session-${this.floxySessionId}_lifetime-${lifetime}`; } private newProxyAgent(provider: "dataimpulse" | "floxy" = "dataimpulse"): ProxyAgent | null { if (provider === "floxy") { if (!this.emexFloxy) return null; const { host, port, user } = this.emexFloxy; return new ProxyAgent({ uri: `http://${user}:${this.floxyPassword()}@${host}:${port}`, connect: { timeout: 30000 }, requestTls: { timeout: 30000 }, }); } if (!this.emexProxy) return null; const { host, user, pass, portStart, portEnd } = this.emexProxy; const port = Math.floor(Math.random() * (portEnd - portStart + 1)) + portStart; return new ProxyAgent({ uri: `http://${user}:${pass}@${host}:${port}`, connect: { timeout: 30000 }, requestTls: { timeout: 30000 }, }); } private async fetchEmexHtml(url: string): Promise { // Attempt schedule by provider. floxy (default): Floxy residential sticky // primary (one exit IP across the flow; rolls to a fresh IP on transport // failure), then a single DataImpulse last-ditch try — this avoids DataImpulse's // ~50% dead ports that used to burn the decode budget before reaching Floxy. // dataimpulse (legacy): DataImpulse-first (rotating port each try), Floxy after. // A definitive HTTP answer (e.g. 404) stops the schedule — it's a real result, // and a different proxy IP must not "retry" it away. const schedule: Array<"dataimpulse" | "floxy"> = []; if (this.emexProxyProvider === "floxy") { if (this.emexFloxy && this.proxyHealth.isFloxyDown() && this.emexProxy) { // Floxy gate tripped → DataImpulse-first (rotating ports cover its ~50% // dead ports), with one Floxy probe last so a recovery is still detected. schedule.push("dataimpulse", "dataimpulse", "floxy"); } else { if (this.emexFloxy) schedule.push("floxy", "floxy"); if (this.emexProxy) schedule.push("dataimpulse"); } } else if (this.emexProxyProvider === "dataimpulse") { if (this.emexProxy) schedule.push("dataimpulse", "dataimpulse", "dataimpulse"); if (this.emexFloxy) schedule.push("floxy", "floxy"); } const maxAttempts = schedule.length || 1; // 0 → single proxy-less attempt let lastErr: unknown; for (let attempt = 1; attempt <= maxAttempts; attempt++) { const provider = schedule[attempt - 1]; // undefined when no proxy → direct const agent = provider ? this.newProxyAgent(provider) : null; const attemptStart = Date.now(); // Sticky Floxy sessions are the attributable identity here; DataImpulse // rotates a random port per agent, so there is nothing stable to pin. const sessionKey = provider === "floxy" ? `s-${this.floxySessionId}` : null; try { const res = await fetch(url, { headers: { "User-Agent": EMEX_UA, Accept: "text/html,application/xhtml+xml" }, signal: AbortSignal.timeout(this.timeout), ...(agent ? { dispatcher: agent } : {}), } as RequestInit); this.proxyTelemetry.record({ service: "emex_http", provider: provider ?? "none", sessionKey, targetHost: new URL(url).hostname, statusCode: res.status, success: res.ok, durationMs: Date.now() - attemptStart, }); if (!res.ok) { throw new Error(`EMEX HTTP ${res.status} for ${url}`); } if (provider === "floxy") { this.proxyHealth.reportFloxySuccess(); // Floxy answered — clear cooldown this.logger.log(`EMEX fetch via Floxy fallback succeeded for ${url}`); } return await res.text(); } catch (err) { lastErr = err; const e = err as Error & { cause?: unknown }; // A real HTTP response ("EMEX HTTP 404") is a definitive answer — never retry. const httpAnswer = /^EMEX HTTP \d/.test(e.message); // HTTP answers were already recorded right after fetch; only transport // failures (no answer at all) still need a row. if (!httpAnswer) { this.proxyTelemetry.record({ service: "emex_http", provider: provider ?? "none", sessionKey, targetHost: new URL(url).hostname, errorKind: classifyTransportError(e), success: false, durationMs: Date.now() - attemptStart, }); } const transient = !httpAnswer && (e.name === "TimeoutError" || e.name === "AbortError" || e.name === "TypeError" || /fetch failed|ECONNRESET|ECONNREFUSED|EAI_AGAIN|socket hang up|other side closed|terminated|UND_ERR/i.test( `${e.message} ${String(e.cause ?? "")}`, )); // A Floxy connect failure (dead exit / tunnel refused) trips the shared // gate so EMEX + pcat fail over to DataImpulse for the cooldown window. if (provider === "floxy" && isProxyConnectFailure(e)) { this.proxyHealth.reportFloxyFailure(`emex http: ${classifyTransportError(e)}`); } if (!transient || attempt === maxAttempts) break; // A Floxy transport failure means the current sticky exit IP is dead — // roll to a fresh session so the next Floxy attempt gets a new IP. if (provider === "floxy") this.rotateFloxySession(); const next = schedule[attempt]; const via = next === "floxy" && provider !== "floxy" ? "via Floxy fallback" : "with fresh proxy"; this.logger.warn( `EMEX fetch transient error (attempt ${attempt}/${maxAttempts}, ${provider}) for ${url}: ${e.message} — retrying ${via}`, ); await new Promise((r) => setTimeout(r, 300 * attempt)); } } // Last resort (opt-in via EMEX_DIRECT_FALLBACK): one direct, proxy-less attempt // for when both proxy pools are down. Off by default — it exposes the origin IP. if ((this.emexProxy || this.emexFloxy) && this.emexDirectFallback) { try { const res = await fetch(url, { headers: { "User-Agent": EMEX_UA, Accept: "text/html,application/xhtml+xml" }, signal: AbortSignal.timeout(this.timeout), } as RequestInit); if (res.ok) return await res.text(); throw new Error(`EMEX HTTP ${res.status} for ${url}`); } catch (err) { lastErr = err; } } throw lastErr; } /** * Parse Vehicles.aspx HTML — extract vehicle list from "Found vehicles" table. */ private parseVehiclesList(html: string): EmexHttpVehicle[] { const linkRx = /href="(Vehicle\.aspx\?[^"]+)">([^<]+)<\/a>/g; const seen = new Set(); const vehicles: EmexHttpVehicle[] = []; for (const m of html.matchAll(linkRx)) { const href = m[1].replace(/&/g, "&"); if (seen.has(href)) continue; seen.add(href); const label = m[2].trim(); const params = new URLSearchParams(href.replace("Vehicle.aspx?", "")); const c = params.get("c"); const vid = params.get("vid"); const ssd = params.get("ssd"); const modelMatch = label.match(/^([^\[]+)/); const yearMatch = label.match(/\((\d{4})/); vehicles.push({ label, model: modelMatch ? modelMatch[1].trim() : label, yearFrom: yearMatch ? Number.parseInt(yearMatch[1], 10) : null, catalogCode: c, vid, ssd, quickGroupsUrl: c && vid != null && ssd ? `${EMEX_BASE_URL}/QuickGroups.aspx?c=${c}&vid=${vid}&ssd=${encodeURIComponent(ssd)}` : null, }); } return vehicles; } /** * Parse QuickGroups.aspx HTML — extract flat category list (QuickDetails links). */ private parseCategoryList(html: string): EmexHttpCategory[] { const catRx = /href="(QuickDetails\.aspx\?[^"]+)">([^<]+)<\/a>/g; const seen = new Set(); const cats: EmexHttpCategory[] = []; for (const m of html.matchAll(catRx)) { const href = m[1].replace(/&/g, "&"); const name = m[2].trim(); if (name.length < 2 || seen.has(href)) continue; seen.add(href); const params = new URLSearchParams(href.replace("QuickDetails.aspx?", "")); cats.push({ gid: params.get("gid"), name, url: `${EMEX_BASE_URL}/${href}` }); } return cats; } /** * Fetch + parse the hierarchical Vehicle.aspx tree (top groups → sub groups) * for a decoded vehicle. One plain GET of the catalog root; the persistent * sidebar carries the whole 2-level group tree (units are drilled lazily per * sub-group via {@link drillVehicleNode}). Returns [] on any failure so the * caller falls back to the flat QuickGroups categories. */ private async fetchVehicleTree( catalogCode: string, vid: string, ssd: string, ): Promise { const url = `${EMEX_BASE_URL}/Vehicle.aspx?c=${catalogCode}&vid=${vid}&ssd=${encodeURIComponent(ssd)}`; const html = await this.fetchEmexHtml(url); return parseVehicleTree(html); } /** * Drill a Vehicle.aspx group node (its verbatim href, ssd embedded) to its * Unit.aspx leaves. Used by the lazy category drill (categories.service * getChildren) when an emex group is first expanded. */ async drillVehicleNode(vehicleAspxUrl: string): Promise { const html = await this.fetchEmexHtml(vehicleAspxUrl); return parseUnitLeaves(html); } /** * Determine brand name from EMEX catalog code (e.g. "BMW202501" → "BMW"). */ private brandFromCatalogCode(c: string | null): string | null { if (!c) return null; const upper = c.toUpperCase(); const prefixes: [string, string][] = [ ["BMW", "BMW"], ["MB", "Mercedes-Benz"], ["AU", "Audi"], ["VW", "Volkswagen"], ["FFIAT", "Fiat"], ["RFIAT", "Alfa Romeo"], ["FORD", "Ford"], ["RENAULT", "Renault"], ["TOYOTA", "Toyota"], ["HONDA", "Honda"], ["KIA", "Kia"], ["HYUNDAI", "Hyundai"], ["PORSCHE", "Porsche"], ["SUBARU", "Subaru"], ["MAZDA", "Mazda"], ["CPSA", "Citroën/Peugeot"], ["VOLVO", "Volvo"], ["NISSAN", "Nissan"], ["OPEL", "Opel"], ]; for (const [prefix, brand] of prefixes) { if (upper.startsWith(prefix)) return brand; } return null; } /** * VIN decode via pure HTTP (primary path — no browser needed). * Returns null if VIN is not found in EMEX database. */ private async decodeVinHttp(vin: string): Promise { const vinUrl = `${EMEX_BASE_URL}/Vehicles.aspx?ft=findByVIN&c=&ssd=&vin=${vin}`; this.logger.log(`EMEX HTTP: fetching ${vinUrl}`); const vinHtml = await this.fetchEmexHtml(vinUrl); const vehicles = this.parseVehiclesList(vinHtml); if (vehicles.length === 0) { this.logger.log(`EMEX HTTP: no vehicles found for VIN ${vin}`); return null; } const v = vehicles[0]; this.logger.log(`EMEX HTTP: found "${v.label}" (c=${v.catalogCode})`); // Determine brand: prefer CATALOG_MAP lookup, then catalog code heuristic const wmi = vin.substring(0, 3).toUpperCase(); const catalogEntry = CATALOG_MAP[wmi]; const brand = catalogEntry?.brand || this.brandFromCatalogCode(v.catalogCode) || "Unknown"; // Fetch categories from QuickGroups.aspx (fast HTTP, no browser) let categories: EmexHttpCategory[] = []; if (v.quickGroupsUrl) { try { const qgHtml = await this.fetchEmexHtml(v.quickGroupsUrl); categories = this.parseCategoryList(qgHtml); this.logger.log(`EMEX HTTP: found ${categories.length} categories`); } catch (err) { this.logger.warn(`EMEX HTTP: category fetch failed: ${(err as Error).message}`); } } // Hierarchical Vehicle.aspx tree (primary nav); the flat `categories` // above is the fallback. One GET of the catalog root yields the full // 2-level group tree; units are drilled lazily per sub-group on expand. let categoryTree: EmexVehicleTreeNode[] = []; if (v.catalogCode && v.ssd) { try { categoryTree = await this.fetchVehicleTree(v.catalogCode, v.vid ?? "0", v.ssd); this.logger.log(`EMEX HTTP: parsed ${categoryTree.length} Vehicle.aspx top groups`); } catch (err) { this.logger.warn(`EMEX Vehicle.aspx tree fetch failed: ${(err as Error).message}`); } } // Build a response compatible with mapEmexResponse const response: EmexScraperResponse = { success: true, source: "emexdwc.ae", method: "vin_url", vin, catalogCode: v.catalogCode || "", ssd: v.ssd || undefined, vehicle: { brand, model: v.model, year: v.yearFrom, bodyType: null, engineCode: null, engineType: null, engineVolume: null, transmission: null, driveType: null, }, quickGroupsUrl: v.quickGroupsUrl || null, categories: categories.map((c) => ({ gid: c.gid || "", name: c.name, url: c.url })), categoryTree, timestamp: new Date().toISOString(), }; return mapEmexResponse(response); } /** * Single-fetch combined decode: fetches Vehicles.aspx once and returns either: * - `{ type: 'vehicle', vehicle }` — single match decoded as DecodedVehicle * - `{ type: 'candidates', candidates }` — multiple matches, caller shows selection UI * - `{ type: 'notFound' }` — VIN not in EMEX * - `{ type: 'error' }` — fetch failed */ async decodeVinOrCandidates( vin: string, ): Promise< | { type: "vehicle"; vehicle: DecodedVehicle } | { type: "candidates"; candidates: EmexCandidate[] } | { type: "notFound" } | { type: "error" } > { if (!(await this.posthog.isSourceLive("emex"))) { this.logger.warn("EMEX disabled by kill switch (kill-source-emex)"); return { type: "notFound" }; } try { const vinUrl = `${EMEX_BASE_URL}/Vehicles.aspx?ft=findByVIN&c=&ssd=&vin=${vin}`; const html = await this.fetchEmexHtml(vinUrl); const vehicleList = this.parseVehiclesList(html); if (vehicleList.length === 0) return { type: "notFound" }; if (vehicleList.length > 1) { const candidates: EmexCandidate[] = vehicleList.map((v, i) => { const params: Array<{ key: string; idx: string; value: string }> = []; if (v.yearFrom) params.push({ key: "year", idx: "0", value: String(v.yearFrom) }); if (v.catalogCode) params.push({ key: "catalog", idx: "1", value: v.catalogCode }); return { id: String(i), name: v.label, parameters: params, catalogId: v.catalogCode || "", _index: i, _quickGroupsUrl: v.quickGroupsUrl, _ssd: v.ssd, _vid: v.vid, }; }); return { type: "candidates", candidates }; } // Single result — decode directly const v = vehicleList[0]; const wmi = vin.substring(0, 3).toUpperCase(); const catalogEntry = CATALOG_MAP[wmi]; const brand = catalogEntry?.brand || this.brandFromCatalogCode(v.catalogCode) || "Unknown"; let categories: EmexHttpCategory[] = []; if (v.quickGroupsUrl) { try { const qgHtml = await this.fetchEmexHtml(v.quickGroupsUrl); categories = this.parseCategoryList(qgHtml); this.logger.log(`EMEX HTTP: found ${categories.length} categories`); } catch (err) { this.logger.warn(`EMEX HTTP category fetch failed: ${(err as Error).message}`); } } // Hierarchical Vehicle.aspx tree (primary nav); flat `categories` = fallback. let categoryTree: EmexVehicleTreeNode[] = []; if (v.catalogCode && v.ssd) { try { categoryTree = await this.fetchVehicleTree(v.catalogCode, v.vid ?? "0", v.ssd); this.logger.log(`EMEX HTTP: parsed ${categoryTree.length} Vehicle.aspx top groups`); } catch (err) { this.logger.warn(`EMEX Vehicle.aspx tree fetch failed: ${(err as Error).message}`); } } const response: EmexScraperResponse = { success: true, source: "emexdwc.ae", method: "vin_url", vin, catalogCode: v.catalogCode || "", ssd: v.ssd || undefined, vehicle: { brand, model: v.model, year: v.yearFrom, bodyType: null, engineCode: null, engineType: null, engineVolume: null, transmission: null, driveType: null, }, quickGroupsUrl: v.quickGroupsUrl || null, categories: categories.map((c) => ({ gid: c.gid || "", name: c.name, url: c.url })), categoryTree, timestamp: new Date().toISOString(), }; return { type: "vehicle", vehicle: mapEmexResponse(response) }; } catch (err) { this.logger.warn(`decodeVinOrCandidates failed: ${(err as Error).message}`); return { type: "error" }; } } /** * Decodes a specific EMEX candidate by its index in the Vehicles.aspx list. * Used after the user selects a vehicle from the multi-candidate modal. */ async decodeVinByIndex(vin: string, index: number): Promise { if (!(await this.posthog.isSourceLive("emex"))) { this.logger.warn("EMEX disabled by kill switch (kill-source-emex)"); return null; } try { const vinUrl = `${EMEX_BASE_URL}/Vehicles.aspx?ft=findByVIN&c=&ssd=&vin=${vin}`; const vinHtml = await this.fetchEmexHtml(vinUrl); const vehicleList = this.parseVehiclesList(vinHtml); if (index < 0 || index >= vehicleList.length) { this.logger.warn( `EMEX decodeVinByIndex: index ${index} out of range (${vehicleList.length} vehicles)`, ); return null; } const v = vehicleList[index]; this.logger.log(`EMEX decodeVinByIndex[${index}]: "${v.label}" (c=${v.catalogCode})`); const wmi = vin.substring(0, 3).toUpperCase(); const catalogEntry = CATALOG_MAP[wmi]; const brand = catalogEntry?.brand || this.brandFromCatalogCode(v.catalogCode) || "Unknown"; let categories: EmexHttpCategory[] = []; if (v.quickGroupsUrl) { try { const qgHtml = await this.fetchEmexHtml(v.quickGroupsUrl); categories = this.parseCategoryList(qgHtml); } catch (err) { this.logger.warn( `EMEX decodeVinByIndex category fetch failed: ${(err as Error).message}`, ); } } // Hierarchical Vehicle.aspx tree (primary nav); flat `categories` = fallback. let categoryTree: EmexVehicleTreeNode[] = []; if (v.catalogCode && v.ssd) { try { categoryTree = await this.fetchVehicleTree(v.catalogCode, v.vid ?? "0", v.ssd); this.logger.log(`EMEX HTTP: parsed ${categoryTree.length} Vehicle.aspx top groups`); } catch (err) { this.logger.warn(`EMEX Vehicle.aspx tree fetch failed: ${(err as Error).message}`); } } const response: EmexScraperResponse = { success: true, source: "emexdwc.ae", method: "vin_url", vin, catalogCode: v.catalogCode || "", ssd: v.ssd || undefined, vehicle: { brand, model: v.model, year: v.yearFrom, bodyType: null, engineCode: null, engineType: null, engineVolume: null, transmission: null, driveType: null, }, quickGroupsUrl: v.quickGroupsUrl || null, categories: categories.map((c) => ({ gid: c.gid || "", name: c.name, url: c.url })), categoryTree, timestamp: new Date().toISOString(), }; return mapEmexResponse(response); } catch (err) { this.logger.warn(`EMEX decodeVinByIndex failed: ${(err as Error).message}`); return null; } } /** * Decodes a VIN number. * Primary: pure HTTP fetch (fast, ~1-5s). * Fallback: Playwright browser scraper (slower, used if HTTP fails). */ async decodeVin(vin: string): Promise { const cleanVin = vin.toUpperCase().replace(/[^A-HJ-NPR-Z0-9]/g, ""); this.validateVin(cleanVin); const supported = this.isSupported(cleanVin); this.logger.log(`Decoding VIN: ${cleanVin} (catalog supported: ${supported})`); await this.touchActivity(); // ── Primary: HTTP (no browser) ────────────────────────────── try { const result = await this.decodeVinHttp(cleanVin); if (result) { this.logger.log(`EMEX HTTP decode OK: ${result.brand} ${result.model} (${result.year})`); return result; } // VIN not in EMEX — return empty rather than hitting browser return createEmptyDecodedVehicle(cleanVin, "Vehicle not found in EMEX database"); } catch (httpErr) { const err = httpErr as Error; this.logger.warn(`EMEX HTTP decode failed (${err.message}), falling back to browser`); } // ── Fallback: Playwright browser scraper ──────────────────── let release: (() => Promise) | null = null; try { const instance = await this.createScraperInstance(); const scraper = instance.scraper; release = instance.release; const response = await this.executeWithTimeout(scraper.searchByVIN(cleanVin), this.timeout); if (this.debug) { this.logger.debug(`EMEX browser raw response: ${JSON.stringify(response, null, 2)}`); } if (!response.success) { this.logger.warn(`EMEX browser search unsuccessful: ${response.message || response.error}`); if (response.vehicle?.brand) { return mapEmexResponse(response); } return createEmptyDecodedVehicle(cleanVin, response.message || response.error); } const decodedVehicle = mapEmexResponse(response); this.logger.log( `EMEX browser decode OK: ${decodedVehicle.brand} ${decodedVehicle.model} (${decodedVehicle.year})`, ); return decodedVehicle; } catch (error) { const err = error as Error; // Floxy tunnel dead mid-scrape → trip failover so the next acquirePage // re-establishes the session on DataImpulse. if (isProxyConnectFailure(err)) { void this.browserService.tripFloxyFailover(`emex decode: ${classifyTransportError(err)}`); } if ( err instanceof BadRequestException || err instanceof ServiceUnavailableException || err instanceof InternalServerErrorException ) { throw err; } if (err.message?.includes("timeout") || err.name === "TimeoutError") { this.logger.error(`VIN decode timeout for: ${cleanVin}`); throw new ServiceUnavailableException( "EMEX servisi zaman asimina ugradi. Lutfen tekrar deneyin.", ); } this.logger.error(`VIN decode error: ${err.message}`, err.stack); throw new ServiceUnavailableException("VIN sorgulama sirasinda bir hata olustu"); } finally { if (release) { try { await release(); } catch (closeError) { const err = closeError as Error; this.logger.warn(`Error releasing page: ${err.message}`); } } } } /** * Executes a promise with timeout */ private async executeWithTimeout(promise: Promise, timeoutMs: number): Promise { let timeoutId: NodeJS.Timeout | undefined; const timeoutPromise = new Promise((_, reject) => { timeoutId = setTimeout(() => { const error = new Error(`Operation timed out after ${timeoutMs}ms`); error.name = "TimeoutError"; reject(error); }, timeoutMs); }); try { const result = await Promise.race([promise, timeoutPromise]); if (timeoutId) clearTimeout(timeoutId); return result; } catch (error) { if (timeoutId) clearTimeout(timeoutId); throw error; } } /** * Gets the catalog code for a VIN */ getCatalogCode(vin: string): string | null { const wmi = vin.substring(0, 3).toUpperCase(); return CATALOG_MAP[wmi]?.code || null; } /** * Checks if a VIN's manufacturer is supported */ isSupported(vin: string): boolean { if (!vin || vin.length < 3) { return false; } const wmi = vin.substring(0, 3).toUpperCase(); return wmi in CATALOG_MAP; } /** * Gets list of supported manufacturers */ getSupportedBrands(): string[] { const brands = new Set(); for (const entry of Object.values(CATALOG_MAP)) { brands.add(entry.brand); } return Array.from(brands).sort(); } /** * Fetches parts + schema image for a specific category (on-demand). * * Strategy: plain-HTTP fast path first, browser fallback on parse failure. * The probe in scripts/dev (FN-* perf work) showed Unit.aspx is fully * server-rendered for the data we need — parts come from `` rows * with `td[name=c_oem|c_pnc|c_name]`, hotspots from inline-styled * `
`, * and image dims from the first 24 bytes of the GIF/PNG itself. Sidesteps * the ~1-2s browser-launch cost AND the 3-page semaphore in EmexBrowserService, * dropping a cold leaf from ~6-7s (Tier 1) to ~5s and removing the * concurrency cap (prefetch can fan out beyond 3 simultaneous fetches). * Falls back to the Playwright scraper if the HTML doesn't yield parts — * keeps us honest when emexdwc.ae changes layout or returns a JS-gated page. */ async fetchCategoryParts(categoryUrl: string): Promise { if (!categoryUrl) { this.logger.warn("fetchCategoryParts called with empty URL"); return { parts: [], schemaImageUrl: null, hotspots: [], schemaWidth: 0, schemaHeight: 0 }; } this.logger.log(`Fetching parts from category URL: ${categoryUrl}`); await this.touchActivity(); // Fast path: plain HTTP + HTML parse. Browser-free. try { const httpResult = await this.fetchCategoryPartsViaHttp(categoryUrl); if (httpResult.parts.length > 0) { this.logger.log( `Fetched ${httpResult.parts.length} parts from category (http path${httpResult.schemaImageUrl ? ", with schema" : ""})`, ); return httpResult; } this.logger.log("Plain-HTTP path returned 0 parts; falling back to Playwright scraper"); } catch (err) { this.logger.warn( `Plain-HTTP path failed: ${(err as Error).message}; falling back to Playwright`, ); } // Slow path: existing Playwright scraper. let release: (() => Promise) | null = null; try { const instance = await this.createScraperInstance(); const scraper = instance.scraper; release = instance.release; const result = await this.executeWithTimeout(scraper.getParts(categoryUrl), this.timeout); if (result && result.parts.length > 0) { this.logger.log(`Fetched ${result.parts.length} parts from category (browser path)`); if (result.schemaImageUrl) { this.logger.log(`Schema image found: ${result.schemaImageUrl}`); } return result; } return { parts: [], schemaImageUrl: null, hotspots: [], schemaWidth: 0, schemaHeight: 0 }; } catch (error) { const err = error as Error; // Floxy tunnel dead mid-scrape → trip failover for the next acquirePage. if (isProxyConnectFailure(err)) { void this.browserService.tripFloxyFailover(`emex parts: ${classifyTransportError(err)}`); } this.logger.error(`Failed to fetch category parts: ${err.message}`); return { parts: [], schemaImageUrl: null, hotspots: [], schemaWidth: 0, schemaHeight: 0 }; } finally { if (release) { try { await release(); } catch (closeError) { const err = closeError as Error; this.logger.warn(`Error releasing page: ${err.message}`); } } } } /** * Plain-HTTP equivalent of the Playwright getParts() flow. Two sequential * GETs (QuickDetails → Unit) + one Range GET for image dims. Throws if it * can't resolve the Unit.aspx URL or if the image is fetched but headers * are unparseable. Returns an empty result (no throw) if the page has no * parts — caller treats that as "fall back to browser". */ private async fetchCategoryPartsViaHttp(categoryUrl: string): Promise { // Vehicle.aspx-tree leaves already point straight at Unit.aspx — skip the // QuickDetails→Unit hop and extract parts directly. if (/Unit\.aspx/i.test(categoryUrl)) { const unitHtml = await this.fetchEmexHtml(categoryUrl); const extracted = this.extractEmexPartsFromHtml(unitHtml); if ( extracted.schemaImageUrl && (extracted.schemaWidth === 0 || extracted.schemaHeight === 0) ) { try { const dims = await this.fetchImageDims(extracted.schemaImageUrl); if (dims.width > 0 && dims.height > 0) { extracted.schemaWidth = dims.width; extracted.schemaHeight = dims.height; } } catch (err) { this.logger.debug(`Image dims fetch failed: ${(err as Error).message}`); } } return extracted; } // 1) QuickDetails.aspx → Unit.aspx anchor const qdHtml = await this.fetchEmexHtml(categoryUrl); const unitMatch = qdHtml.match(/href="([^"]*Unit\.aspx[^"]*)"/i); if (!unitMatch) { // QuickDetails returned an Error.aspx-style page or has parts inline — // try extracting parts directly; if none, signal fallback. const direct = this.extractEmexPartsFromHtml(qdHtml); if (direct.parts.length > 0) return direct; throw new Error("No Unit.aspx link in QuickDetails response"); } const unitRel = unitMatch[1].replace(/&/g, "&"); const unitUrl = unitRel.startsWith("http") ? unitRel : new URL(unitRel, categoryUrl).toString(); // 2) Unit.aspx — main extraction target const unitHtml = await this.fetchEmexHtml(unitUrl); const extracted = this.extractEmexPartsFromHtml(unitHtml); if (extracted.parts.length === 0) { // Empty parts table: structural change or session-gated page. Let the // caller try Playwright (which sometimes succeeds where plain HTTP // doesn't, e.g. if a JS redirect refreshes the ssd token). return extracted; } // 3) Image dims — read GIF/PNG header from a 128-byte Range GET if a // schema URL was found. Don't block parts extraction on image failure. if (extracted.schemaImageUrl && (extracted.schemaWidth === 0 || extracted.schemaHeight === 0)) { try { const dims = await this.fetchImageDims(extracted.schemaImageUrl); if (dims.width > 0 && dims.height > 0) { extracted.schemaWidth = dims.width; extracted.schemaHeight = dims.height; } } catch (err) { this.logger.debug(`Image dims fetch failed: ${(err as Error).message}`); } } if (extracted.schemaImageUrl) { this.logger.log(`Schema image found: ${extracted.schemaImageUrl}`); } return extracted; } /** * Parse Unit.aspx (or fallback QuickDetails.aspx) HTML into the same shape * the Playwright scraper returns. Pure regex/string-based — no DOM, no * browser. Mirrors the page.evaluate() in scripts/emex-vin-scraper.js so * upstream consumers don't care which path produced the result. */ private extractEmexPartsFromHtml(html: string): EmexPartsResult { // Parts: with , , . const parts: EmexPart[] = []; const trRx = /]*\bname="[^"]+"[^>]*>([\s\S]*?)<\/tr>/g; const stripTags = (s: string) => s .replace(/<[^>]+>/g, "") .replace(/ /g, " ") .replace(/&/g, "&") .replace(/"/g, '"') .replace(/'/g, "'") .replace(/</g, "<") .replace(/>/g, ">") .trim(); for (const trMatch of html.matchAll(trRx)) { const body = trMatch[1]; const oem = stripTags( body.match(/]*\bname="c_oem"[^>]*>([\s\S]*?)<\/td>/)?.[1] ?? "", ); if (!oem) continue; const pnc = stripTags( body.match(/]*\bname="c_pnc"[^>]*>([\s\S]*?)<\/td>/)?.[1] ?? "", ); const name = stripTags( body.match(/]*\bname="c_name"[^>]*>([\s\S]*?)<\/td>/)?.[1] ?? "", ); parts.push({ oemCode: oem, nameEn: name, positionCode: pnc }); } // Hotspots:
. // Inline style is in image-natural pixel coordinates (probe confirmed // max-right and max-bottom always sit within the image's natural dims), // so we use the values directly — no rect arithmetic or scaling. const hotspotMap = new Map(); const divRx = /]*\bname="([^"]+)"[^>]*\bclass="[^"]*\bdragger\b[^"]*\bg_highlight\b[^"]*"[^>]*\bstyle="([^"]+)"/g; for (const m of html.matchAll(divRx)) { const key = m[1]; const style = m[2]; const w = Number.parseInt(style.match(/width:\s*(\d+)px/)?.[1] ?? "0", 10); const h = Number.parseInt(style.match(/height:\s*(\d+)px/)?.[1] ?? "0", 10); const top = Number.parseInt(style.match(/margin-top:\s*(\d+)px/)?.[1] ?? "0", 10); const left = Number.parseInt(style.match(/margin-left:\s*(\d+)px/)?.[1] ?? "0", 10); const area: EmexHotspotArea = { left, top, width: w, height: h }; const entry = hotspotMap.get(key); if (entry) entry.areas.push(area); else hotspotMap.set(key, { key, areas: [area] }); } const hotspots = Array.from(hotspotMap.values()); // Schema image: . // Fallback to any img.laximo.net URL, normalising the /NNN/ path to // /source/ the same way the browser scraper did (some catalog pages link // to a thumbnail variant that doesn't carry full-resolution coords). let schemaImageUrl: string | null = null; const draggerMatch = html.match( /]*\bclass="[^"]*\bdragger\b[^"]*"[^>]*\bsrc="([^"]+laximo[^"]+)"/, ); if (draggerMatch) schemaImageUrl = draggerMatch[1].replace(/&/g, "&"); if (!schemaImageUrl) { const fallback = html.match(/src="(https?:\/\/img\.laximo\.net[^"]+)"/); if (fallback) { schemaImageUrl = fallback[1].replace(/&/g, "&").replace(/\/\d+\//, "/source/"); } } return { parts, schemaImageUrl, hotspots, schemaWidth: 0, schemaHeight: 0, }; } /** * Read the natural image dimensions from the first ~24 bytes of a GIF or * PNG. Uses an HTTP Range request so we never download the whole image * just to read its size. Returns 0×0 on unknown formats — the schema-image * downloader downstream uses its own dim-parsing fallback as a safety net. */ private async fetchImageDims(url: string): Promise<{ width: number; height: number }> { const res = await fetch(url, { headers: { "User-Agent": EMEX_UA, Range: "bytes=0-127" }, signal: AbortSignal.timeout(10000), ...(this.proxyAgent ? { dispatcher: this.proxyAgent } : {}), } as RequestInit); if (!res.ok && res.status !== 206) { throw new Error(`Image HTTP ${res.status}`); } const buf = Buffer.from(await res.arrayBuffer()); // GIF87a / GIF89a — width at byte 6 LE, height at byte 8 LE. if (buf.length >= 10 && buf.slice(0, 3).toString("ascii") === "GIF") { return { width: buf.readUInt16LE(6), height: buf.readUInt16LE(8) }; } // PNG — width at byte 16 BE, height at byte 20 BE. if (buf.length >= 24 && buf[0] === 0x89 && buf.slice(1, 4).toString("ascii") === "PNG") { return { width: buf.readUInt32BE(16), height: buf.readUInt32BE(20) }; } return { width: 0, height: 0 }; } }