Semih Yesilyurt
bf834b8f41
Some checks failed
QA Gate (P0/P1) / Test affected app (pull_request) Has been cancelled
Floxy kalıcı olarak devre dışı (bakiye/tünel ölü — pcat+emex loglarında sürekli "Floxy unhealthy → DataImpulse failover"). Floxy-primary her çağrıda önce bir başarısız deneme yakıp sonra failover ediyordu. DataImpulse zaten kanıtlanmış çalışıyor (call-leg %97,6). PCAT_PROXY_PROVIDER ve EMEX_PROXY_PROVIDER default'ları floxy→dataimpulse (pcat auth + emex browser + emex HTTP ayağı). Floxy hâlâ env ile seçilebilir (PCAT_PROXY_PROVIDER=floxy / EMEX_PROXY_PROVIDER=floxy); failover makinesi dormant kalıyor. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1279 lines
50 KiB
TypeScript
1279 lines
50 KiB
TypeScript
/**
|
||
* EMEX VIN Service
|
||
*
|
||
* NestJS service for emexdwc.ae VIN integration.
|
||
* Primary path: pure HTTP fetch (no browser) — fast (~1-5s).
|
||
* Fallback: Playwright scraper via EmexBrowserService.
|
||
*
|
||
* emexdwc.ae does NOT require authentication for Vehicles.aspx,
|
||
* QuickGroups.aspx, or QuickDetails.aspx — plain HTTP GET works.
|
||
*/
|
||
|
||
import * as path from "node:path";
|
||
import {
|
||
BadRequestException,
|
||
Injectable,
|
||
InternalServerErrorException,
|
||
Logger,
|
||
ServiceUnavailableException,
|
||
} from "@nestjs/common";
|
||
import { ConfigService } from "@nestjs/config";
|
||
|
||
import { ProxyAgent } from "undici";
|
||
import { isBackfillContext } from "../../jobs/prefetch-context";
|
||
import { PostHogService } from "../../posthog/posthog.service";
|
||
import { RedisService } from "../../redis/redis.service";
|
||
import { ProxyHealthService } from "../proxy-telemetry/proxy-health.service";
|
||
import {
|
||
ProxyTelemetryService,
|
||
classifyTransportError,
|
||
isProxyConnectFailure,
|
||
} from "../proxy-telemetry/proxy-telemetry.service";
|
||
import { parseUnitLeaves, parseVehicleTree } from "./emex-tree.parser";
|
||
import { EmexBrowserService } from "./emex.browser";
|
||
import { createEmptyDecodedVehicle, mapEmexResponse } from "./emex.mapper";
|
||
import {
|
||
CATALOG_MAP,
|
||
type DecodedVehicle,
|
||
type EmexCategoryTreeNode,
|
||
type EmexHotspot,
|
||
type EmexHotspotArea,
|
||
type EmexPart,
|
||
type EmexPartsResult,
|
||
type EmexScraperResponse,
|
||
type EmexVehicleTreeNode,
|
||
} from "./emex.types";
|
||
|
||
const EMEX_BASE_URL = "https://emexdwc.ae";
|
||
const EMEX_UA =
|
||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36";
|
||
|
||
interface EmexHttpVehicle {
|
||
label: string;
|
||
model: string;
|
||
yearFrom: number | null;
|
||
catalogCode: string | null;
|
||
vid: string | null;
|
||
ssd: string | null;
|
||
quickGroupsUrl: string | null;
|
||
}
|
||
|
||
interface EmexHttpCategory {
|
||
gid: string | null;
|
||
name: string;
|
||
url: string;
|
||
}
|
||
|
||
/**
|
||
* Candidate vehicle returned when EMEX finds multiple matches for a VIN.
|
||
* Shape matches PcatCar so VehicleSelectModal can render without changes.
|
||
*/
|
||
export interface EmexCandidate {
|
||
/** String index into the Vehicles.aspx list — "0", "1", etc. */
|
||
id: string;
|
||
name: string;
|
||
parameters?: Array<{ key: string; idx: string; value: string }>;
|
||
catalogId: string;
|
||
/** Internal: used by resolveEmexCarByIndex to fetch the specific vehicle */
|
||
_index: number;
|
||
_quickGroupsUrl: string | null;
|
||
_ssd: string | null;
|
||
_vid: string | null;
|
||
}
|
||
|
||
// Type definition for the imported scraper module
|
||
interface EmexScraperModule {
|
||
EmexVinScraper: new (options?: { page?: unknown }) => EmexVinScraperInstance;
|
||
getCatalogCode: (vin: string) => string | null;
|
||
getYearFromVIN: (vin: string) => number | null;
|
||
CONFIG: Record<string, unknown>;
|
||
}
|
||
|
||
interface EmexCategoryResult {
|
||
gid: string;
|
||
name: string;
|
||
url: string | null;
|
||
}
|
||
|
||
interface EmexVinScraperInstance {
|
||
init(): Promise<void>;
|
||
close(): Promise<void>;
|
||
searchByVIN(vin: string): Promise<EmexScraperResponse>;
|
||
getCategories(quickGroupsUrl: string): Promise<EmexCategoryResult[]>;
|
||
getCategoryTree(quickGroupsUrl: string): Promise<EmexCategoryTreeNode[]>;
|
||
getParts(detailsUrl: string): Promise<EmexPartsResult>;
|
||
}
|
||
|
||
@Injectable()
|
||
export class EmexService {
|
||
private readonly logger = new Logger(EmexService.name);
|
||
private scraperModule: EmexScraperModule | null = null;
|
||
private isInitialized = false;
|
||
private initializationPromise: Promise<void> | null = null;
|
||
|
||
private readonly scraperPath: string;
|
||
private readonly timeout: number;
|
||
private readonly debug: boolean;
|
||
private readonly proxyAgent: ProxyAgent | null;
|
||
private readonly emexProxy: {
|
||
host: string;
|
||
user: string;
|
||
pass: string;
|
||
portStart: number;
|
||
portEnd: number;
|
||
} | null;
|
||
// Floxy residential fallback — used when the DataImpulse pool throws transport
|
||
// errors. `lifetime` > 0 → sticky session (one exit IP held that many seconds,
|
||
// appended to the password as `_session-<id>_lifetime-<n>`); 0 → rotating.
|
||
private readonly emexFloxy: {
|
||
host: string;
|
||
port: number;
|
||
user: string;
|
||
pass: string;
|
||
lifetime: number;
|
||
} | null;
|
||
// Rolling Floxy sticky-session id: held for ~lifetime so a chained emex flow
|
||
// (decode → tree → drill → parts) keeps ONE exit IP and emex's IP-bound ssd
|
||
// tokens stay valid. Rotated when stale or after a transport failure.
|
||
private floxySessionId = Math.random().toString(36).slice(2, 10);
|
||
private floxySessionBornMs = Date.now();
|
||
private readonly emexDirectFallback: boolean;
|
||
// Transport provider for EMEX egress (HTTP + image-dims). "floxy" residential
|
||
// is primary by default (sticky session pins one exit IP across a chained flow),
|
||
// DataImpulse is the last-ditch fallback. See constructor.
|
||
private readonly emexProxyProvider: "floxy" | "dataimpulse" | "none";
|
||
|
||
constructor(
|
||
private configService: ConfigService,
|
||
private browserService: EmexBrowserService,
|
||
private redis: RedisService,
|
||
private posthog: PostHogService,
|
||
private proxyTelemetry: ProxyTelemetryService,
|
||
private proxyHealth: ProxyHealthService,
|
||
) {
|
||
// __dirname is apps/api/src/integrations/emex/ or dist/integrations/emex/
|
||
// Scraper lives at <monorepo-root>/scripts/emex-vin-scraper.js
|
||
const monorepoRoot = path.resolve(__dirname, "..", "..", "..", "..", "..");
|
||
const defaultPath = path.resolve(monorepoRoot, "scripts/emex-vin-scraper.js");
|
||
this.scraperPath = this.configService.get<string>("EMEX_SCRAPER_PATH", defaultPath);
|
||
|
||
this.timeout = this.configService.get<number>("EMEX_TIMEOUT", 60000);
|
||
this.debug = this.configService.get<boolean>("EMEX_DEBUG", false);
|
||
|
||
const useProxy = this.configService.get<string>("EMEX_USE_PROXY", "true") === "true";
|
||
this.emexDirectFallback =
|
||
this.configService.get<string>("EMEX_DIRECT_FALLBACK", "false") === "true";
|
||
if (useProxy) {
|
||
// ConfigService.get<number> returns the raw env STRING, not a number. Left
|
||
// un-coerced, the port-pick arithmetic does string concat — e.g.
|
||
// 45 + "10001" = "4510001", an out-of-range port → undici `new URL` throws
|
||
// "Invalid URL" and the whole app fails to boot. A single-port range (823)
|
||
// happens to concat to a still-parseable "0823", which masked this for
|
||
// months until the range was widened. Always coerce to a valid port number.
|
||
const toPort = (key: string, def: number): number => {
|
||
const n = Number(this.configService.get(key, def));
|
||
return Number.isInteger(n) && n >= 1 && n <= 65535 ? n : def;
|
||
};
|
||
this.emexProxy = {
|
||
host: this.configService.get<string>("EMEX_PROXY_HOST", "74.81.81.81"),
|
||
user: this.configService.get<string>("EMEX_PROXY_USER", "1726bbe361918676d44e"),
|
||
pass: this.configService.get<string>("EMEX_PROXY_PASS", "f11c7b6128cc86c6"),
|
||
portStart: toPort("EMEX_PROXY_PORT_START", 10001),
|
||
portEnd: toPort("EMEX_PROXY_PORT_END", 10099),
|
||
};
|
||
this.logger.log(
|
||
`EMEX DataImpulse fallback enabled: ${this.emexProxy.host}:${this.emexProxy.portStart}-${this.emexProxy.portEnd}`,
|
||
);
|
||
} else {
|
||
this.emexProxy = null;
|
||
}
|
||
|
||
// Floxy residential fallback for when the DataImpulse pool flakes (connect
|
||
// timeouts / resets). On by default; creds + endpoint overridable via env.
|
||
const floxyEnabled = this.configService.get<string>("EMEX_FLOXY_FALLBACK", "true") === "true";
|
||
if (floxyEnabled) {
|
||
const fport = Number(this.configService.get("EMEX_FLOXY_PORT", 12321));
|
||
const flife = Number(this.configService.get("EMEX_FLOXY_LIFETIME", 300));
|
||
this.emexFloxy = {
|
||
host: this.configService.get<string>("EMEX_FLOXY_HOST", "residential.floxy.io"),
|
||
port: Number.isInteger(fport) && fport >= 1 && fport <= 65535 ? fport : 12321,
|
||
user: this.configService.get<string>("EMEX_FLOXY_USER", "cac4b0d96a80"),
|
||
pass: this.configService.get<string>("EMEX_FLOXY_PASS", "dfc38fe467a3"),
|
||
lifetime: Number.isInteger(flife) && flife >= 0 ? flife : 300,
|
||
};
|
||
this.logger.log(
|
||
`EMEX Floxy fallback enabled: ${this.emexFloxy.host}:${this.emexFloxy.port} (${this.emexFloxy.lifetime > 0 ? `sticky ${this.emexFloxy.lifetime}s` : "rotating"})`,
|
||
);
|
||
} else {
|
||
this.emexFloxy = null;
|
||
}
|
||
|
||
// Transport provider. EMEX_PROXY_PROVIDER: "floxy" (default) → Floxy residential
|
||
// sticky primary + a single DataImpulse last-ditch fallback; "dataimpulse" →
|
||
// legacy DataImpulse-first order; "none" → direct. ssd tokens are replayed from
|
||
// emex's own HTML and aren't strictly IP-bound on the .aspx endpoints (the old
|
||
// random-port-per-call primary proved that), but a sticky Floxy IP across the
|
||
// chained flow is still the safest choice — and avoids DataImpulse's ~50% dead
|
||
// ports that were eating the decode budget before reaching the Floxy fallback.
|
||
// Default dataimpulse: Floxy retired 2026-07 (permanently down). Set
|
||
// EMEX_PROXY_PROVIDER=floxy to re-enable it.
|
||
this.emexProxyProvider = !useProxy
|
||
? "none"
|
||
: (() => {
|
||
const p = this.configService
|
||
.get<string>("EMEX_PROXY_PROVIDER", "dataimpulse")
|
||
.toLowerCase();
|
||
return p === "floxy" || p === "none" ? p : "dataimpulse";
|
||
})();
|
||
// Default agent for the low-stakes image-dims path; fetchEmexHtml builds a
|
||
// fresh agent per request so a flaky exit can't pin every call.
|
||
this.proxyAgent =
|
||
this.emexProxyProvider === "none"
|
||
? null
|
||
: (this.newProxyAgent(this.emexProxyProvider === "dataimpulse" ? "dataimpulse" : "floxy") ??
|
||
this.newProxyAgent("dataimpulse"));
|
||
this.logger.log(`EMEX transport provider: ${this.emexProxyProvider}`);
|
||
|
||
this.logger.log(`EMEX Service initialized with scraper path: ${this.scraperPath}`);
|
||
}
|
||
|
||
/** Mark EMEX as actively used (5min TTL) to defer prefetch worker */
|
||
private async touchActivity(): Promise<void> {
|
||
// Worker-originated fetches must NOT register as user activity, or the
|
||
// backfill worker throttles itself via checkCooldown.
|
||
if (isBackfillContext()) return;
|
||
try {
|
||
await this.redis.set("prefetch:activity:emex", String(Date.now()), 90);
|
||
} catch {
|
||
// Non-critical — don't break the request
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Lazily initialize the scraper module
|
||
*/
|
||
private async initializeScraper(): Promise<void> {
|
||
if (this.isInitialized) {
|
||
return;
|
||
}
|
||
|
||
if (this.initializationPromise) {
|
||
return this.initializationPromise;
|
||
}
|
||
|
||
this.initializationPromise = this.doInitialize();
|
||
return this.initializationPromise;
|
||
}
|
||
|
||
private async doInitialize(): Promise<void> {
|
||
try {
|
||
this.logger.log(`Loading EMEX scraper module from: ${this.scraperPath}`);
|
||
|
||
const fs = require("node:fs");
|
||
if (!fs.existsSync(this.scraperPath)) {
|
||
this.logger.error(`Scraper file not found at: ${this.scraperPath}`);
|
||
this.logger.error(`Current working directory: ${process.cwd()}`);
|
||
throw new Error(`Scraper file not found: ${this.scraperPath}`);
|
||
}
|
||
|
||
// Clear require cache to always load the latest scraper version
|
||
delete require.cache[require.resolve(this.scraperPath)];
|
||
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
||
this.scraperModule = require(this.scraperPath) as EmexScraperModule;
|
||
|
||
this.logger.log("EMEX scraper module loaded successfully");
|
||
this.isInitialized = true;
|
||
} catch (error) {
|
||
const err = error as Error;
|
||
this.logger.error(`Failed to load EMEX scraper module: ${err.message}`, err.stack);
|
||
throw new InternalServerErrorException("EMEX servis modulu yuklenemedi");
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Creates a scraper instance bound to a pre-created page from the browser pool.
|
||
* Returns { scraper, release } — caller MUST call release() in finally.
|
||
*/
|
||
private async createScraperInstance(): Promise<{
|
||
scraper: EmexVinScraperInstance;
|
||
release: () => Promise<void>;
|
||
}> {
|
||
await this.initializeScraper();
|
||
|
||
if (!this.scraperModule) {
|
||
throw new InternalServerErrorException("EMEX scraper modulu yuklenemedi");
|
||
}
|
||
|
||
const { page, release } = await this.browserService.acquirePage();
|
||
const scraper = new this.scraperModule.EmexVinScraper({ page });
|
||
// init() is a no-op in managed mode, but call it for consistency
|
||
await scraper.init();
|
||
return { scraper, release };
|
||
}
|
||
|
||
/**
|
||
* Validates VIN format
|
||
*/
|
||
private validateVin(vin: string): void {
|
||
if (!vin) {
|
||
throw new BadRequestException("VIN numarasi gereklidir");
|
||
}
|
||
|
||
const cleanVin = vin.toUpperCase().replace(/[^A-HJ-NPR-Z0-9]/g, "");
|
||
|
||
if (cleanVin.length !== 17) {
|
||
throw new BadRequestException("VIN numarasi 17 karakter olmalidir");
|
||
}
|
||
|
||
if (/[IOQ]/i.test(cleanVin)) {
|
||
throw new BadRequestException(
|
||
"VIN numarasi gecersiz karakterler iceriyor (I, O, Q kullanilamaz)",
|
||
);
|
||
}
|
||
}
|
||
|
||
// ─── HTTP-based methods (no browser required) ────────────────
|
||
|
||
/**
|
||
* Fetch a URL from emexdwc.ae without a browser session.
|
||
* emexdwc.ae serves Vehicles.aspx, QuickGroups.aspx, and QuickDetails.aspx
|
||
* without requiring authentication cookies.
|
||
*/
|
||
/** Build a fresh proxy agent on a random port from the pool (null if proxy off). */
|
||
/** Fresh Floxy sticky-session id (new exit IP). */
|
||
private rotateFloxySession(): void {
|
||
this.floxySessionId = Math.random().toString(36).slice(2, 10);
|
||
this.floxySessionBornMs = Date.now();
|
||
}
|
||
|
||
/**
|
||
* Floxy auth password. With lifetime > 0, appends `_session-<id>_lifetime-<n>`
|
||
* to pin one exit IP across a chained emex flow; the session id rolls over
|
||
* once it nears the lifetime so we never reuse an expired sticky slot.
|
||
*/
|
||
private floxyPassword(): string {
|
||
if (!this.emexFloxy) return "";
|
||
const { pass, lifetime } = this.emexFloxy;
|
||
if (lifetime <= 0) return pass; // rotating mode
|
||
// Roll at ~80% of lifetime so a sticky IP is never reused past expiry.
|
||
if (Date.now() - this.floxySessionBornMs > lifetime * 800) this.rotateFloxySession();
|
||
return `${pass}_session-${this.floxySessionId}_lifetime-${lifetime}`;
|
||
}
|
||
|
||
private newProxyAgent(provider: "dataimpulse" | "floxy" = "dataimpulse"): ProxyAgent | null {
|
||
if (provider === "floxy") {
|
||
if (!this.emexFloxy) return null;
|
||
const { host, port, user } = this.emexFloxy;
|
||
return new ProxyAgent({
|
||
uri: `http://${user}:${this.floxyPassword()}@${host}:${port}`,
|
||
connect: { timeout: 30000 },
|
||
requestTls: { timeout: 30000 },
|
||
});
|
||
}
|
||
if (!this.emexProxy) return null;
|
||
const { host, user, pass, portStart, portEnd } = this.emexProxy;
|
||
const port = Math.floor(Math.random() * (portEnd - portStart + 1)) + portStart;
|
||
return new ProxyAgent({
|
||
uri: `http://${user}:${pass}@${host}:${port}`,
|
||
connect: { timeout: 30000 },
|
||
requestTls: { timeout: 30000 },
|
||
});
|
||
}
|
||
|
||
private async fetchEmexHtml(url: string): Promise<string> {
|
||
// Attempt schedule by provider. floxy (default): Floxy residential sticky
|
||
// primary (one exit IP across the flow; rolls to a fresh IP on transport
|
||
// failure), then a single DataImpulse last-ditch try — this avoids DataImpulse's
|
||
// ~50% dead ports that used to burn the decode budget before reaching Floxy.
|
||
// dataimpulse (legacy): DataImpulse-first (rotating port each try), Floxy after.
|
||
// A definitive HTTP answer (e.g. 404) stops the schedule — it's a real result,
|
||
// and a different proxy IP must not "retry" it away.
|
||
const schedule: Array<"dataimpulse" | "floxy"> = [];
|
||
if (this.emexProxyProvider === "floxy") {
|
||
if (this.emexFloxy && this.proxyHealth.isFloxyDown() && this.emexProxy) {
|
||
// Floxy gate tripped → DataImpulse-first (rotating ports cover its ~50%
|
||
// dead ports), with one Floxy probe last so a recovery is still detected.
|
||
schedule.push("dataimpulse", "dataimpulse", "floxy");
|
||
} else {
|
||
if (this.emexFloxy) schedule.push("floxy", "floxy");
|
||
if (this.emexProxy) schedule.push("dataimpulse");
|
||
}
|
||
} else if (this.emexProxyProvider === "dataimpulse") {
|
||
if (this.emexProxy) schedule.push("dataimpulse", "dataimpulse", "dataimpulse");
|
||
if (this.emexFloxy) schedule.push("floxy", "floxy");
|
||
}
|
||
const maxAttempts = schedule.length || 1; // 0 → single proxy-less attempt
|
||
let lastErr: unknown;
|
||
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
||
const provider = schedule[attempt - 1]; // undefined when no proxy → direct
|
||
const agent = provider ? this.newProxyAgent(provider) : null;
|
||
const attemptStart = Date.now();
|
||
// Sticky Floxy sessions are the attributable identity here; DataImpulse
|
||
// rotates a random port per agent, so there is nothing stable to pin.
|
||
const sessionKey = provider === "floxy" ? `s-${this.floxySessionId}` : null;
|
||
try {
|
||
const res = await fetch(url, {
|
||
headers: { "User-Agent": EMEX_UA, Accept: "text/html,application/xhtml+xml" },
|
||
signal: AbortSignal.timeout(this.timeout),
|
||
...(agent ? { dispatcher: agent } : {}),
|
||
} as RequestInit);
|
||
this.proxyTelemetry.record({
|
||
service: "emex_http",
|
||
provider: provider ?? "none",
|
||
sessionKey,
|
||
targetHost: new URL(url).hostname,
|
||
statusCode: res.status,
|
||
success: res.ok,
|
||
durationMs: Date.now() - attemptStart,
|
||
});
|
||
if (!res.ok) {
|
||
throw new Error(`EMEX HTTP ${res.status} for ${url}`);
|
||
}
|
||
if (provider === "floxy") {
|
||
this.proxyHealth.reportFloxySuccess(); // Floxy answered — clear cooldown
|
||
this.logger.log(`EMEX fetch via Floxy fallback succeeded for ${url}`);
|
||
}
|
||
return await res.text();
|
||
} catch (err) {
|
||
lastErr = err;
|
||
const e = err as Error & { cause?: unknown };
|
||
// A real HTTP response ("EMEX HTTP 404") is a definitive answer — never retry.
|
||
const httpAnswer = /^EMEX HTTP \d/.test(e.message);
|
||
// HTTP answers were already recorded right after fetch; only transport
|
||
// failures (no answer at all) still need a row.
|
||
if (!httpAnswer) {
|
||
this.proxyTelemetry.record({
|
||
service: "emex_http",
|
||
provider: provider ?? "none",
|
||
sessionKey,
|
||
targetHost: new URL(url).hostname,
|
||
errorKind: classifyTransportError(e),
|
||
success: false,
|
||
durationMs: Date.now() - attemptStart,
|
||
});
|
||
}
|
||
const transient =
|
||
!httpAnswer &&
|
||
(e.name === "TimeoutError" ||
|
||
e.name === "AbortError" ||
|
||
e.name === "TypeError" ||
|
||
/fetch failed|ECONNRESET|ECONNREFUSED|EAI_AGAIN|socket hang up|other side closed|terminated|UND_ERR/i.test(
|
||
`${e.message} ${String(e.cause ?? "")}`,
|
||
));
|
||
// A Floxy connect failure (dead exit / tunnel refused) trips the shared
|
||
// gate so EMEX + pcat fail over to DataImpulse for the cooldown window.
|
||
if (provider === "floxy" && isProxyConnectFailure(e)) {
|
||
this.proxyHealth.reportFloxyFailure(`emex http: ${classifyTransportError(e)}`);
|
||
}
|
||
if (!transient || attempt === maxAttempts) break;
|
||
// A Floxy transport failure means the current sticky exit IP is dead —
|
||
// roll to a fresh session so the next Floxy attempt gets a new IP.
|
||
if (provider === "floxy") this.rotateFloxySession();
|
||
const next = schedule[attempt];
|
||
const via =
|
||
next === "floxy" && provider !== "floxy" ? "via Floxy fallback" : "with fresh proxy";
|
||
this.logger.warn(
|
||
`EMEX fetch transient error (attempt ${attempt}/${maxAttempts}, ${provider}) for ${url}: ${e.message} — retrying ${via}`,
|
||
);
|
||
await new Promise((r) => setTimeout(r, 300 * attempt));
|
||
}
|
||
}
|
||
// Last resort (opt-in via EMEX_DIRECT_FALLBACK): one direct, proxy-less attempt
|
||
// for when both proxy pools are down. Off by default — it exposes the origin IP.
|
||
if ((this.emexProxy || this.emexFloxy) && this.emexDirectFallback) {
|
||
try {
|
||
const res = await fetch(url, {
|
||
headers: { "User-Agent": EMEX_UA, Accept: "text/html,application/xhtml+xml" },
|
||
signal: AbortSignal.timeout(this.timeout),
|
||
} as RequestInit);
|
||
if (res.ok) return await res.text();
|
||
throw new Error(`EMEX HTTP ${res.status} for ${url}`);
|
||
} catch (err) {
|
||
lastErr = err;
|
||
}
|
||
}
|
||
throw lastErr;
|
||
}
|
||
|
||
/**
|
||
* Parse Vehicles.aspx HTML — extract vehicle list from "Found vehicles" table.
|
||
*/
|
||
private parseVehiclesList(html: string): EmexHttpVehicle[] {
|
||
const linkRx = /href="(Vehicle\.aspx\?[^"]+)">([^<]+)<\/a>/g;
|
||
const seen = new Set<string>();
|
||
const vehicles: EmexHttpVehicle[] = [];
|
||
for (const m of html.matchAll(linkRx)) {
|
||
const href = m[1].replace(/&/g, "&");
|
||
if (seen.has(href)) continue;
|
||
seen.add(href);
|
||
const label = m[2].trim();
|
||
const params = new URLSearchParams(href.replace("Vehicle.aspx?", ""));
|
||
const c = params.get("c");
|
||
const vid = params.get("vid");
|
||
const ssd = params.get("ssd");
|
||
const modelMatch = label.match(/^([^\[]+)/);
|
||
const yearMatch = label.match(/\((\d{4})/);
|
||
vehicles.push({
|
||
label,
|
||
model: modelMatch ? modelMatch[1].trim() : label,
|
||
yearFrom: yearMatch ? Number.parseInt(yearMatch[1], 10) : null,
|
||
catalogCode: c,
|
||
vid,
|
||
ssd,
|
||
quickGroupsUrl:
|
||
c && vid != null && ssd
|
||
? `${EMEX_BASE_URL}/QuickGroups.aspx?c=${c}&vid=${vid}&ssd=${encodeURIComponent(ssd)}`
|
||
: null,
|
||
});
|
||
}
|
||
return vehicles;
|
||
}
|
||
|
||
/**
|
||
* Parse QuickGroups.aspx HTML — extract flat category list (QuickDetails links).
|
||
*/
|
||
private parseCategoryList(html: string): EmexHttpCategory[] {
|
||
const catRx = /href="(QuickDetails\.aspx\?[^"]+)">([^<]+)<\/a>/g;
|
||
const seen = new Set<string>();
|
||
const cats: EmexHttpCategory[] = [];
|
||
for (const m of html.matchAll(catRx)) {
|
||
const href = m[1].replace(/&/g, "&");
|
||
const name = m[2].trim();
|
||
if (name.length < 2 || seen.has(href)) continue;
|
||
seen.add(href);
|
||
const params = new URLSearchParams(href.replace("QuickDetails.aspx?", ""));
|
||
cats.push({ gid: params.get("gid"), name, url: `${EMEX_BASE_URL}/${href}` });
|
||
}
|
||
return cats;
|
||
}
|
||
|
||
/**
|
||
* Fetch + parse the hierarchical Vehicle.aspx tree (top groups → sub groups)
|
||
* for a decoded vehicle. One plain GET of the catalog root; the persistent
|
||
* sidebar carries the whole 2-level group tree (units are drilled lazily per
|
||
* sub-group via {@link drillVehicleNode}). Returns [] on any failure so the
|
||
* caller falls back to the flat QuickGroups categories.
|
||
*/
|
||
private async fetchVehicleTree(
|
||
catalogCode: string,
|
||
vid: string,
|
||
ssd: string,
|
||
): Promise<EmexVehicleTreeNode[]> {
|
||
const url = `${EMEX_BASE_URL}/Vehicle.aspx?c=${catalogCode}&vid=${vid}&ssd=${encodeURIComponent(ssd)}`;
|
||
const html = await this.fetchEmexHtml(url);
|
||
return parseVehicleTree(html);
|
||
}
|
||
|
||
/**
|
||
* Drill a Vehicle.aspx group node (its verbatim href, ssd embedded) to its
|
||
* Unit.aspx leaves. Used by the lazy category drill (categories.service
|
||
* getChildren) when an emex group is first expanded.
|
||
*/
|
||
async drillVehicleNode(vehicleAspxUrl: string): Promise<EmexVehicleTreeNode[]> {
|
||
const html = await this.fetchEmexHtml(vehicleAspxUrl);
|
||
return parseUnitLeaves(html);
|
||
}
|
||
|
||
/**
|
||
* Determine brand name from EMEX catalog code (e.g. "BMW202501" → "BMW").
|
||
*/
|
||
private brandFromCatalogCode(c: string | null): string | null {
|
||
if (!c) return null;
|
||
const upper = c.toUpperCase();
|
||
const prefixes: [string, string][] = [
|
||
["BMW", "BMW"],
|
||
["MB", "Mercedes-Benz"],
|
||
["AU", "Audi"],
|
||
["VW", "Volkswagen"],
|
||
["FFIAT", "Fiat"],
|
||
["RFIAT", "Alfa Romeo"],
|
||
["FORD", "Ford"],
|
||
["RENAULT", "Renault"],
|
||
["TOYOTA", "Toyota"],
|
||
["HONDA", "Honda"],
|
||
["KIA", "Kia"],
|
||
["HYUNDAI", "Hyundai"],
|
||
["PORSCHE", "Porsche"],
|
||
["SUBARU", "Subaru"],
|
||
["MAZDA", "Mazda"],
|
||
["CPSA", "Citroën/Peugeot"],
|
||
["VOLVO", "Volvo"],
|
||
["NISSAN", "Nissan"],
|
||
["OPEL", "Opel"],
|
||
];
|
||
for (const [prefix, brand] of prefixes) {
|
||
if (upper.startsWith(prefix)) return brand;
|
||
}
|
||
return null;
|
||
}
|
||
|
||
/**
|
||
* VIN decode via pure HTTP (primary path — no browser needed).
|
||
* Returns null if VIN is not found in EMEX database.
|
||
*/
|
||
private async decodeVinHttp(vin: string): Promise<DecodedVehicle | null> {
|
||
const vinUrl = `${EMEX_BASE_URL}/Vehicles.aspx?ft=findByVIN&c=&ssd=&vin=${vin}`;
|
||
this.logger.log(`EMEX HTTP: fetching ${vinUrl}`);
|
||
|
||
const vinHtml = await this.fetchEmexHtml(vinUrl);
|
||
const vehicles = this.parseVehiclesList(vinHtml);
|
||
|
||
if (vehicles.length === 0) {
|
||
this.logger.log(`EMEX HTTP: no vehicles found for VIN ${vin}`);
|
||
return null;
|
||
}
|
||
|
||
const v = vehicles[0];
|
||
this.logger.log(`EMEX HTTP: found "${v.label}" (c=${v.catalogCode})`);
|
||
|
||
// Determine brand: prefer CATALOG_MAP lookup, then catalog code heuristic
|
||
const wmi = vin.substring(0, 3).toUpperCase();
|
||
const catalogEntry = CATALOG_MAP[wmi];
|
||
const brand = catalogEntry?.brand || this.brandFromCatalogCode(v.catalogCode) || "Unknown";
|
||
|
||
// Fetch categories from QuickGroups.aspx (fast HTTP, no browser)
|
||
let categories: EmexHttpCategory[] = [];
|
||
if (v.quickGroupsUrl) {
|
||
try {
|
||
const qgHtml = await this.fetchEmexHtml(v.quickGroupsUrl);
|
||
categories = this.parseCategoryList(qgHtml);
|
||
this.logger.log(`EMEX HTTP: found ${categories.length} categories`);
|
||
} catch (err) {
|
||
this.logger.warn(`EMEX HTTP: category fetch failed: ${(err as Error).message}`);
|
||
}
|
||
}
|
||
|
||
// Hierarchical Vehicle.aspx tree (primary nav); the flat `categories`
|
||
// above is the fallback. One GET of the catalog root yields the full
|
||
// 2-level group tree; units are drilled lazily per sub-group on expand.
|
||
let categoryTree: EmexVehicleTreeNode[] = [];
|
||
if (v.catalogCode && v.ssd) {
|
||
try {
|
||
categoryTree = await this.fetchVehicleTree(v.catalogCode, v.vid ?? "0", v.ssd);
|
||
this.logger.log(`EMEX HTTP: parsed ${categoryTree.length} Vehicle.aspx top groups`);
|
||
} catch (err) {
|
||
this.logger.warn(`EMEX Vehicle.aspx tree fetch failed: ${(err as Error).message}`);
|
||
}
|
||
}
|
||
|
||
// Build a response compatible with mapEmexResponse
|
||
const response: EmexScraperResponse = {
|
||
success: true,
|
||
source: "emexdwc.ae",
|
||
method: "vin_url",
|
||
vin,
|
||
catalogCode: v.catalogCode || "",
|
||
ssd: v.ssd || undefined,
|
||
vehicle: {
|
||
brand,
|
||
model: v.model,
|
||
year: v.yearFrom,
|
||
bodyType: null,
|
||
engineCode: null,
|
||
engineType: null,
|
||
engineVolume: null,
|
||
transmission: null,
|
||
driveType: null,
|
||
},
|
||
quickGroupsUrl: v.quickGroupsUrl || null,
|
||
categories: categories.map((c) => ({ gid: c.gid || "", name: c.name, url: c.url })),
|
||
categoryTree,
|
||
timestamp: new Date().toISOString(),
|
||
};
|
||
|
||
return mapEmexResponse(response);
|
||
}
|
||
|
||
/**
|
||
* Single-fetch combined decode: fetches Vehicles.aspx once and returns either:
|
||
* - `{ type: 'vehicle', vehicle }` — single match decoded as DecodedVehicle
|
||
* - `{ type: 'candidates', candidates }` — multiple matches, caller shows selection UI
|
||
* - `{ type: 'notFound' }` — VIN not in EMEX
|
||
* - `{ type: 'error' }` — fetch failed
|
||
*/
|
||
async decodeVinOrCandidates(
|
||
vin: string,
|
||
): Promise<
|
||
| { type: "vehicle"; vehicle: DecodedVehicle }
|
||
| { type: "candidates"; candidates: EmexCandidate[] }
|
||
| { type: "notFound" }
|
||
| { type: "error" }
|
||
> {
|
||
if (!(await this.posthog.isSourceLive("emex"))) {
|
||
this.logger.warn("EMEX disabled by kill switch (kill-source-emex)");
|
||
return { type: "notFound" };
|
||
}
|
||
try {
|
||
const vinUrl = `${EMEX_BASE_URL}/Vehicles.aspx?ft=findByVIN&c=&ssd=&vin=${vin}`;
|
||
const html = await this.fetchEmexHtml(vinUrl);
|
||
const vehicleList = this.parseVehiclesList(html);
|
||
|
||
if (vehicleList.length === 0) return { type: "notFound" };
|
||
|
||
if (vehicleList.length > 1) {
|
||
const candidates: EmexCandidate[] = vehicleList.map((v, i) => {
|
||
const params: Array<{ key: string; idx: string; value: string }> = [];
|
||
if (v.yearFrom) params.push({ key: "year", idx: "0", value: String(v.yearFrom) });
|
||
if (v.catalogCode) params.push({ key: "catalog", idx: "1", value: v.catalogCode });
|
||
return {
|
||
id: String(i),
|
||
name: v.label,
|
||
parameters: params,
|
||
catalogId: v.catalogCode || "",
|
||
_index: i,
|
||
_quickGroupsUrl: v.quickGroupsUrl,
|
||
_ssd: v.ssd,
|
||
_vid: v.vid,
|
||
};
|
||
});
|
||
return { type: "candidates", candidates };
|
||
}
|
||
|
||
// Single result — decode directly
|
||
const v = vehicleList[0];
|
||
const wmi = vin.substring(0, 3).toUpperCase();
|
||
const catalogEntry = CATALOG_MAP[wmi];
|
||
const brand = catalogEntry?.brand || this.brandFromCatalogCode(v.catalogCode) || "Unknown";
|
||
|
||
let categories: EmexHttpCategory[] = [];
|
||
if (v.quickGroupsUrl) {
|
||
try {
|
||
const qgHtml = await this.fetchEmexHtml(v.quickGroupsUrl);
|
||
categories = this.parseCategoryList(qgHtml);
|
||
this.logger.log(`EMEX HTTP: found ${categories.length} categories`);
|
||
} catch (err) {
|
||
this.logger.warn(`EMEX HTTP category fetch failed: ${(err as Error).message}`);
|
||
}
|
||
}
|
||
|
||
// Hierarchical Vehicle.aspx tree (primary nav); flat `categories` = fallback.
|
||
let categoryTree: EmexVehicleTreeNode[] = [];
|
||
if (v.catalogCode && v.ssd) {
|
||
try {
|
||
categoryTree = await this.fetchVehicleTree(v.catalogCode, v.vid ?? "0", v.ssd);
|
||
this.logger.log(`EMEX HTTP: parsed ${categoryTree.length} Vehicle.aspx top groups`);
|
||
} catch (err) {
|
||
this.logger.warn(`EMEX Vehicle.aspx tree fetch failed: ${(err as Error).message}`);
|
||
}
|
||
}
|
||
|
||
const response: EmexScraperResponse = {
|
||
success: true,
|
||
source: "emexdwc.ae",
|
||
method: "vin_url",
|
||
vin,
|
||
catalogCode: v.catalogCode || "",
|
||
ssd: v.ssd || undefined,
|
||
vehicle: {
|
||
brand,
|
||
model: v.model,
|
||
year: v.yearFrom,
|
||
bodyType: null,
|
||
engineCode: null,
|
||
engineType: null,
|
||
engineVolume: null,
|
||
transmission: null,
|
||
driveType: null,
|
||
},
|
||
quickGroupsUrl: v.quickGroupsUrl || null,
|
||
categories: categories.map((c) => ({ gid: c.gid || "", name: c.name, url: c.url })),
|
||
categoryTree,
|
||
timestamp: new Date().toISOString(),
|
||
};
|
||
|
||
return { type: "vehicle", vehicle: mapEmexResponse(response) };
|
||
} catch (err) {
|
||
this.logger.warn(`decodeVinOrCandidates failed: ${(err as Error).message}`);
|
||
return { type: "error" };
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Decodes a specific EMEX candidate by its index in the Vehicles.aspx list.
|
||
* Used after the user selects a vehicle from the multi-candidate modal.
|
||
*/
|
||
async decodeVinByIndex(vin: string, index: number): Promise<DecodedVehicle | null> {
|
||
if (!(await this.posthog.isSourceLive("emex"))) {
|
||
this.logger.warn("EMEX disabled by kill switch (kill-source-emex)");
|
||
return null;
|
||
}
|
||
try {
|
||
const vinUrl = `${EMEX_BASE_URL}/Vehicles.aspx?ft=findByVIN&c=&ssd=&vin=${vin}`;
|
||
const vinHtml = await this.fetchEmexHtml(vinUrl);
|
||
const vehicleList = this.parseVehiclesList(vinHtml);
|
||
|
||
if (index < 0 || index >= vehicleList.length) {
|
||
this.logger.warn(
|
||
`EMEX decodeVinByIndex: index ${index} out of range (${vehicleList.length} vehicles)`,
|
||
);
|
||
return null;
|
||
}
|
||
|
||
const v = vehicleList[index];
|
||
this.logger.log(`EMEX decodeVinByIndex[${index}]: "${v.label}" (c=${v.catalogCode})`);
|
||
|
||
const wmi = vin.substring(0, 3).toUpperCase();
|
||
const catalogEntry = CATALOG_MAP[wmi];
|
||
const brand = catalogEntry?.brand || this.brandFromCatalogCode(v.catalogCode) || "Unknown";
|
||
|
||
let categories: EmexHttpCategory[] = [];
|
||
if (v.quickGroupsUrl) {
|
||
try {
|
||
const qgHtml = await this.fetchEmexHtml(v.quickGroupsUrl);
|
||
categories = this.parseCategoryList(qgHtml);
|
||
} catch (err) {
|
||
this.logger.warn(
|
||
`EMEX decodeVinByIndex category fetch failed: ${(err as Error).message}`,
|
||
);
|
||
}
|
||
}
|
||
|
||
// Hierarchical Vehicle.aspx tree (primary nav); flat `categories` = fallback.
|
||
let categoryTree: EmexVehicleTreeNode[] = [];
|
||
if (v.catalogCode && v.ssd) {
|
||
try {
|
||
categoryTree = await this.fetchVehicleTree(v.catalogCode, v.vid ?? "0", v.ssd);
|
||
this.logger.log(`EMEX HTTP: parsed ${categoryTree.length} Vehicle.aspx top groups`);
|
||
} catch (err) {
|
||
this.logger.warn(`EMEX Vehicle.aspx tree fetch failed: ${(err as Error).message}`);
|
||
}
|
||
}
|
||
|
||
const response: EmexScraperResponse = {
|
||
success: true,
|
||
source: "emexdwc.ae",
|
||
method: "vin_url",
|
||
vin,
|
||
catalogCode: v.catalogCode || "",
|
||
ssd: v.ssd || undefined,
|
||
vehicle: {
|
||
brand,
|
||
model: v.model,
|
||
year: v.yearFrom,
|
||
bodyType: null,
|
||
engineCode: null,
|
||
engineType: null,
|
||
engineVolume: null,
|
||
transmission: null,
|
||
driveType: null,
|
||
},
|
||
quickGroupsUrl: v.quickGroupsUrl || null,
|
||
categories: categories.map((c) => ({ gid: c.gid || "", name: c.name, url: c.url })),
|
||
categoryTree,
|
||
timestamp: new Date().toISOString(),
|
||
};
|
||
|
||
return mapEmexResponse(response);
|
||
} catch (err) {
|
||
this.logger.warn(`EMEX decodeVinByIndex failed: ${(err as Error).message}`);
|
||
return null;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Decodes a VIN number.
|
||
* Primary: pure HTTP fetch (fast, ~1-5s).
|
||
* Fallback: Playwright browser scraper (slower, used if HTTP fails).
|
||
*/
|
||
async decodeVin(vin: string): Promise<DecodedVehicle> {
|
||
const cleanVin = vin.toUpperCase().replace(/[^A-HJ-NPR-Z0-9]/g, "");
|
||
|
||
this.validateVin(cleanVin);
|
||
|
||
const supported = this.isSupported(cleanVin);
|
||
this.logger.log(`Decoding VIN: ${cleanVin} (catalog supported: ${supported})`);
|
||
await this.touchActivity();
|
||
|
||
// ── Primary: HTTP (no browser) ──────────────────────────────
|
||
try {
|
||
const result = await this.decodeVinHttp(cleanVin);
|
||
if (result) {
|
||
this.logger.log(`EMEX HTTP decode OK: ${result.brand} ${result.model} (${result.year})`);
|
||
return result;
|
||
}
|
||
// VIN not in EMEX — return empty rather than hitting browser
|
||
return createEmptyDecodedVehicle(cleanVin, "Vehicle not found in EMEX database");
|
||
} catch (httpErr) {
|
||
const err = httpErr as Error;
|
||
this.logger.warn(`EMEX HTTP decode failed (${err.message}), falling back to browser`);
|
||
}
|
||
|
||
// ── Fallback: Playwright browser scraper ────────────────────
|
||
let release: (() => Promise<void>) | null = null;
|
||
|
||
try {
|
||
const instance = await this.createScraperInstance();
|
||
const scraper = instance.scraper;
|
||
release = instance.release;
|
||
|
||
const response = await this.executeWithTimeout(scraper.searchByVIN(cleanVin), this.timeout);
|
||
|
||
if (this.debug) {
|
||
this.logger.debug(`EMEX browser raw response: ${JSON.stringify(response, null, 2)}`);
|
||
}
|
||
|
||
if (!response.success) {
|
||
this.logger.warn(`EMEX browser search unsuccessful: ${response.message || response.error}`);
|
||
if (response.vehicle?.brand) {
|
||
return mapEmexResponse(response);
|
||
}
|
||
return createEmptyDecodedVehicle(cleanVin, response.message || response.error);
|
||
}
|
||
|
||
const decodedVehicle = mapEmexResponse(response);
|
||
this.logger.log(
|
||
`EMEX browser decode OK: ${decodedVehicle.brand} ${decodedVehicle.model} (${decodedVehicle.year})`,
|
||
);
|
||
return decodedVehicle;
|
||
} catch (error) {
|
||
const err = error as Error;
|
||
|
||
// Floxy tunnel dead mid-scrape → trip failover so the next acquirePage
|
||
// re-establishes the session on DataImpulse.
|
||
if (isProxyConnectFailure(err)) {
|
||
void this.browserService.tripFloxyFailover(`emex decode: ${classifyTransportError(err)}`);
|
||
}
|
||
|
||
if (
|
||
err instanceof BadRequestException ||
|
||
err instanceof ServiceUnavailableException ||
|
||
err instanceof InternalServerErrorException
|
||
) {
|
||
throw err;
|
||
}
|
||
|
||
if (err.message?.includes("timeout") || err.name === "TimeoutError") {
|
||
this.logger.error(`VIN decode timeout for: ${cleanVin}`);
|
||
throw new ServiceUnavailableException(
|
||
"EMEX servisi zaman asimina ugradi. Lutfen tekrar deneyin.",
|
||
);
|
||
}
|
||
|
||
this.logger.error(`VIN decode error: ${err.message}`, err.stack);
|
||
throw new ServiceUnavailableException("VIN sorgulama sirasinda bir hata olustu");
|
||
} finally {
|
||
if (release) {
|
||
try {
|
||
await release();
|
||
} catch (closeError) {
|
||
const err = closeError as Error;
|
||
this.logger.warn(`Error releasing page: ${err.message}`);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Executes a promise with timeout
|
||
*/
|
||
private async executeWithTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T> {
|
||
let timeoutId: NodeJS.Timeout | undefined;
|
||
|
||
const timeoutPromise = new Promise<never>((_, reject) => {
|
||
timeoutId = setTimeout(() => {
|
||
const error = new Error(`Operation timed out after ${timeoutMs}ms`);
|
||
error.name = "TimeoutError";
|
||
reject(error);
|
||
}, timeoutMs);
|
||
});
|
||
|
||
try {
|
||
const result = await Promise.race([promise, timeoutPromise]);
|
||
if (timeoutId) clearTimeout(timeoutId);
|
||
return result;
|
||
} catch (error) {
|
||
if (timeoutId) clearTimeout(timeoutId);
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Gets the catalog code for a VIN
|
||
*/
|
||
getCatalogCode(vin: string): string | null {
|
||
const wmi = vin.substring(0, 3).toUpperCase();
|
||
return CATALOG_MAP[wmi]?.code || null;
|
||
}
|
||
|
||
/**
|
||
* Checks if a VIN's manufacturer is supported
|
||
*/
|
||
isSupported(vin: string): boolean {
|
||
if (!vin || vin.length < 3) {
|
||
return false;
|
||
}
|
||
const wmi = vin.substring(0, 3).toUpperCase();
|
||
return wmi in CATALOG_MAP;
|
||
}
|
||
|
||
/**
|
||
* Gets list of supported manufacturers
|
||
*/
|
||
getSupportedBrands(): string[] {
|
||
const brands = new Set<string>();
|
||
for (const entry of Object.values(CATALOG_MAP)) {
|
||
brands.add(entry.brand);
|
||
}
|
||
return Array.from(brands).sort();
|
||
}
|
||
|
||
/**
|
||
* Fetches parts + schema image for a specific category (on-demand).
|
||
*
|
||
* Strategy: plain-HTTP fast path first, browser fallback on parse failure.
|
||
* The probe in scripts/dev (FN-* perf work) showed Unit.aspx is fully
|
||
* server-rendered for the data we need — parts come from `<tr name>` rows
|
||
* with `td[name=c_oem|c_pnc|c_name]`, hotspots from inline-styled
|
||
* `<div class="dragger g_highlight" name=N style="margin-top:Ypx; margin-left:Xpx; width:Wpx; height:Hpx">`,
|
||
* and image dims from the first 24 bytes of the GIF/PNG itself. Sidesteps
|
||
* the ~1-2s browser-launch cost AND the 3-page semaphore in EmexBrowserService,
|
||
* dropping a cold leaf from ~6-7s (Tier 1) to ~5s and removing the
|
||
* concurrency cap (prefetch can fan out beyond 3 simultaneous fetches).
|
||
* Falls back to the Playwright scraper if the HTML doesn't yield parts —
|
||
* keeps us honest when emexdwc.ae changes layout or returns a JS-gated page.
|
||
*/
|
||
async fetchCategoryParts(categoryUrl: string): Promise<EmexPartsResult> {
|
||
if (!categoryUrl) {
|
||
this.logger.warn("fetchCategoryParts called with empty URL");
|
||
return { parts: [], schemaImageUrl: null, hotspots: [], schemaWidth: 0, schemaHeight: 0 };
|
||
}
|
||
|
||
this.logger.log(`Fetching parts from category URL: ${categoryUrl}`);
|
||
await this.touchActivity();
|
||
|
||
// Fast path: plain HTTP + HTML parse. Browser-free.
|
||
try {
|
||
const httpResult = await this.fetchCategoryPartsViaHttp(categoryUrl);
|
||
if (httpResult.parts.length > 0) {
|
||
this.logger.log(
|
||
`Fetched ${httpResult.parts.length} parts from category (http path${httpResult.schemaImageUrl ? ", with schema" : ""})`,
|
||
);
|
||
return httpResult;
|
||
}
|
||
this.logger.log("Plain-HTTP path returned 0 parts; falling back to Playwright scraper");
|
||
} catch (err) {
|
||
this.logger.warn(
|
||
`Plain-HTTP path failed: ${(err as Error).message}; falling back to Playwright`,
|
||
);
|
||
}
|
||
|
||
// Slow path: existing Playwright scraper.
|
||
let release: (() => Promise<void>) | null = null;
|
||
try {
|
||
const instance = await this.createScraperInstance();
|
||
const scraper = instance.scraper;
|
||
release = instance.release;
|
||
|
||
const result = await this.executeWithTimeout(scraper.getParts(categoryUrl), this.timeout);
|
||
|
||
if (result && result.parts.length > 0) {
|
||
this.logger.log(`Fetched ${result.parts.length} parts from category (browser path)`);
|
||
if (result.schemaImageUrl) {
|
||
this.logger.log(`Schema image found: ${result.schemaImageUrl}`);
|
||
}
|
||
return result;
|
||
}
|
||
|
||
return { parts: [], schemaImageUrl: null, hotspots: [], schemaWidth: 0, schemaHeight: 0 };
|
||
} catch (error) {
|
||
const err = error as Error;
|
||
// Floxy tunnel dead mid-scrape → trip failover for the next acquirePage.
|
||
if (isProxyConnectFailure(err)) {
|
||
void this.browserService.tripFloxyFailover(`emex parts: ${classifyTransportError(err)}`);
|
||
}
|
||
this.logger.error(`Failed to fetch category parts: ${err.message}`);
|
||
return { parts: [], schemaImageUrl: null, hotspots: [], schemaWidth: 0, schemaHeight: 0 };
|
||
} finally {
|
||
if (release) {
|
||
try {
|
||
await release();
|
||
} catch (closeError) {
|
||
const err = closeError as Error;
|
||
this.logger.warn(`Error releasing page: ${err.message}`);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Plain-HTTP equivalent of the Playwright getParts() flow. Two sequential
|
||
* GETs (QuickDetails → Unit) + one Range GET for image dims. Throws if it
|
||
* can't resolve the Unit.aspx URL or if the image is fetched but headers
|
||
* are unparseable. Returns an empty result (no throw) if the page has no
|
||
* parts — caller treats that as "fall back to browser".
|
||
*/
|
||
private async fetchCategoryPartsViaHttp(categoryUrl: string): Promise<EmexPartsResult> {
|
||
// Vehicle.aspx-tree leaves already point straight at Unit.aspx — skip the
|
||
// QuickDetails→Unit hop and extract parts directly.
|
||
if (/Unit\.aspx/i.test(categoryUrl)) {
|
||
const unitHtml = await this.fetchEmexHtml(categoryUrl);
|
||
const extracted = this.extractEmexPartsFromHtml(unitHtml);
|
||
if (
|
||
extracted.schemaImageUrl &&
|
||
(extracted.schemaWidth === 0 || extracted.schemaHeight === 0)
|
||
) {
|
||
try {
|
||
const dims = await this.fetchImageDims(extracted.schemaImageUrl);
|
||
if (dims.width > 0 && dims.height > 0) {
|
||
extracted.schemaWidth = dims.width;
|
||
extracted.schemaHeight = dims.height;
|
||
}
|
||
} catch (err) {
|
||
this.logger.debug(`Image dims fetch failed: ${(err as Error).message}`);
|
||
}
|
||
}
|
||
return extracted;
|
||
}
|
||
|
||
// 1) QuickDetails.aspx → Unit.aspx anchor
|
||
const qdHtml = await this.fetchEmexHtml(categoryUrl);
|
||
const unitMatch = qdHtml.match(/href="([^"]*Unit\.aspx[^"]*)"/i);
|
||
if (!unitMatch) {
|
||
// QuickDetails returned an Error.aspx-style page or has parts inline —
|
||
// try extracting parts directly; if none, signal fallback.
|
||
const direct = this.extractEmexPartsFromHtml(qdHtml);
|
||
if (direct.parts.length > 0) return direct;
|
||
throw new Error("No Unit.aspx link in QuickDetails response");
|
||
}
|
||
const unitRel = unitMatch[1].replace(/&/g, "&");
|
||
const unitUrl = unitRel.startsWith("http") ? unitRel : new URL(unitRel, categoryUrl).toString();
|
||
|
||
// 2) Unit.aspx — main extraction target
|
||
const unitHtml = await this.fetchEmexHtml(unitUrl);
|
||
const extracted = this.extractEmexPartsFromHtml(unitHtml);
|
||
if (extracted.parts.length === 0) {
|
||
// Empty parts table: structural change or session-gated page. Let the
|
||
// caller try Playwright (which sometimes succeeds where plain HTTP
|
||
// doesn't, e.g. if a JS redirect refreshes the ssd token).
|
||
return extracted;
|
||
}
|
||
|
||
// 3) Image dims — read GIF/PNG header from a 128-byte Range GET if a
|
||
// schema URL was found. Don't block parts extraction on image failure.
|
||
if (extracted.schemaImageUrl && (extracted.schemaWidth === 0 || extracted.schemaHeight === 0)) {
|
||
try {
|
||
const dims = await this.fetchImageDims(extracted.schemaImageUrl);
|
||
if (dims.width > 0 && dims.height > 0) {
|
||
extracted.schemaWidth = dims.width;
|
||
extracted.schemaHeight = dims.height;
|
||
}
|
||
} catch (err) {
|
||
this.logger.debug(`Image dims fetch failed: ${(err as Error).message}`);
|
||
}
|
||
}
|
||
|
||
if (extracted.schemaImageUrl) {
|
||
this.logger.log(`Schema image found: ${extracted.schemaImageUrl}`);
|
||
}
|
||
return extracted;
|
||
}
|
||
|
||
/**
|
||
* Parse Unit.aspx (or fallback QuickDetails.aspx) HTML into the same shape
|
||
* the Playwright scraper returns. Pure regex/string-based — no DOM, no
|
||
* browser. Mirrors the page.evaluate() in scripts/emex-vin-scraper.js so
|
||
* upstream consumers don't care which path produced the result.
|
||
*/
|
||
private extractEmexPartsFromHtml(html: string): EmexPartsResult {
|
||
// Parts: <tr name="..."> with <td name="c_oem">, <td name="c_pnc">, <td name="c_name">.
|
||
const parts: EmexPart[] = [];
|
||
const trRx = /<tr\b[^>]*\bname="[^"]+"[^>]*>([\s\S]*?)<\/tr>/g;
|
||
const stripTags = (s: string) =>
|
||
s
|
||
.replace(/<[^>]+>/g, "")
|
||
.replace(/ /g, " ")
|
||
.replace(/&/g, "&")
|
||
.replace(/"/g, '"')
|
||
.replace(/'/g, "'")
|
||
.replace(/</g, "<")
|
||
.replace(/>/g, ">")
|
||
.trim();
|
||
for (const trMatch of html.matchAll(trRx)) {
|
||
const body = trMatch[1];
|
||
const oem = stripTags(
|
||
body.match(/<td\b[^>]*\bname="c_oem"[^>]*>([\s\S]*?)<\/td>/)?.[1] ?? "",
|
||
);
|
||
if (!oem) continue;
|
||
const pnc = stripTags(
|
||
body.match(/<td\b[^>]*\bname="c_pnc"[^>]*>([\s\S]*?)<\/td>/)?.[1] ?? "",
|
||
);
|
||
const name = stripTags(
|
||
body.match(/<td\b[^>]*\bname="c_name"[^>]*>([\s\S]*?)<\/td>/)?.[1] ?? "",
|
||
);
|
||
parts.push({ oemCode: oem, nameEn: name, positionCode: pnc });
|
||
}
|
||
|
||
// Hotspots: <div name="N" class="... dragger ... g_highlight ..." style="…margin-top:Ypx; margin-left:Xpx; width:Wpx; height:Hpx…">.
|
||
// Inline style is in image-natural pixel coordinates (probe confirmed
|
||
// max-right and max-bottom always sit within the image's natural dims),
|
||
// so we use the values directly — no rect arithmetic or scaling.
|
||
const hotspotMap = new Map<string, EmexHotspot>();
|
||
const divRx =
|
||
/<div\b[^>]*\bname="([^"]+)"[^>]*\bclass="[^"]*\bdragger\b[^"]*\bg_highlight\b[^"]*"[^>]*\bstyle="([^"]+)"/g;
|
||
for (const m of html.matchAll(divRx)) {
|
||
const key = m[1];
|
||
const style = m[2];
|
||
const w = Number.parseInt(style.match(/width:\s*(\d+)px/)?.[1] ?? "0", 10);
|
||
const h = Number.parseInt(style.match(/height:\s*(\d+)px/)?.[1] ?? "0", 10);
|
||
const top = Number.parseInt(style.match(/margin-top:\s*(\d+)px/)?.[1] ?? "0", 10);
|
||
const left = Number.parseInt(style.match(/margin-left:\s*(\d+)px/)?.[1] ?? "0", 10);
|
||
const area: EmexHotspotArea = { left, top, width: w, height: h };
|
||
const entry = hotspotMap.get(key);
|
||
if (entry) entry.areas.push(area);
|
||
else hotspotMap.set(key, { key, areas: [area] });
|
||
}
|
||
const hotspots = Array.from(hotspotMap.values());
|
||
|
||
// Schema image: <img class="dragger" src="...img.laximo.net/.../*.gif">.
|
||
// Fallback to any img.laximo.net URL, normalising the /NNN/ path to
|
||
// /source/ the same way the browser scraper did (some catalog pages link
|
||
// to a thumbnail variant that doesn't carry full-resolution coords).
|
||
let schemaImageUrl: string | null = null;
|
||
const draggerMatch = html.match(
|
||
/<img\b[^>]*\bclass="[^"]*\bdragger\b[^"]*"[^>]*\bsrc="([^"]+laximo[^"]+)"/,
|
||
);
|
||
if (draggerMatch) schemaImageUrl = draggerMatch[1].replace(/&/g, "&");
|
||
if (!schemaImageUrl) {
|
||
const fallback = html.match(/src="(https?:\/\/img\.laximo\.net[^"]+)"/);
|
||
if (fallback) {
|
||
schemaImageUrl = fallback[1].replace(/&/g, "&").replace(/\/\d+\//, "/source/");
|
||
}
|
||
}
|
||
|
||
return {
|
||
parts,
|
||
schemaImageUrl,
|
||
hotspots,
|
||
schemaWidth: 0,
|
||
schemaHeight: 0,
|
||
};
|
||
}
|
||
|
||
/**
|
||
* Read the natural image dimensions from the first ~24 bytes of a GIF or
|
||
* PNG. Uses an HTTP Range request so we never download the whole image
|
||
* just to read its size. Returns 0×0 on unknown formats — the schema-image
|
||
* downloader downstream uses its own dim-parsing fallback as a safety net.
|
||
*/
|
||
private async fetchImageDims(url: string): Promise<{ width: number; height: number }> {
|
||
const res = await fetch(url, {
|
||
headers: { "User-Agent": EMEX_UA, Range: "bytes=0-127" },
|
||
signal: AbortSignal.timeout(10000),
|
||
...(this.proxyAgent ? { dispatcher: this.proxyAgent } : {}),
|
||
} as RequestInit);
|
||
if (!res.ok && res.status !== 206) {
|
||
throw new Error(`Image HTTP ${res.status}`);
|
||
}
|
||
const buf = Buffer.from(await res.arrayBuffer());
|
||
// GIF87a / GIF89a — width at byte 6 LE, height at byte 8 LE.
|
||
if (buf.length >= 10 && buf.slice(0, 3).toString("ascii") === "GIF") {
|
||
return { width: buf.readUInt16LE(6), height: buf.readUInt16LE(8) };
|
||
}
|
||
// PNG — width at byte 16 BE, height at byte 20 BE.
|
||
if (buf.length >= 24 && buf[0] === 0x89 && buf.slice(1, 4).toString("ascii") === "PNG") {
|
||
return { width: buf.readUInt32BE(16), height: buf.readUInt32BE(20) };
|
||
}
|
||
return { width: 0, height: 0 };
|
||
}
|
||
}
|