root/sase.tr
1
0
Fork 0
Code Issues 5 Pull Requests 3 Actions 7 Packages Projects Releases Wiki Activity

perf(emex): plain-HTTP fast path with Playwright fallback (Tier 3-A)

Browse Source 
Probe (scripts/dev) confirmed that everything PartsPanel reads from an
emexdwc.ae leaf — parts table, hotspot coordinates, and the schema image
URL — is fully server-rendered in Unit.aspx HTML. Parts arrive as
`<tr name>` rows with `td[name=c_oem|c_pnc|c_name]`; hotspots are
`<div class="dragger g_highlight" name=N style="margin-top:Ypx; margin-left:Xpx; ...">`
with the coords already in image-natural pixel space; the image URL is
in `<img class="dragger" src=...laximo...>` and its native dims can be
read from the first 24 bytes of the GIF/PNG via a Range GET.

Behaviour:
1. fetchCategoryParts now tries fetchCategoryPartsViaHttp first — two
   sequential GETs (QuickDetails → Unit) + a Range GET for image dims.
2. If the HTML yields ≥1 part, we return it.
3. If the HTML returns no Unit.aspx anchor, or 0 parts, we fall back to
   the existing Playwright scraper (same code path as before).

The plain-HTTP path skips the ~1-2s browser launch, sidesteps the
3-page semaphore in EmexBrowserService (concurrency cap was throttling
prefetch fan-out), and uses no chromium memory. Measured on dev with 5
fresh-ssd Renault Espace IV leaves: 4.9-5.5s wall per leaf (vs 6-7s on
the Tier 1 browser path, vs 12-14s pre-Tier-1). The 6th sample
(stale-ssd Fren Kaliyeri) failed both paths identically — confirms the
plain-HTTP path doesn't introduce new failure modes.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Semih Yesilyurt
2026-05-18 09:56:53 +03:00
parent 37a5e8f0f7
commit d2fb64484c
1 changed files with 197 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

200
apps/api/src/integrations/emex/emex.service.ts
View File

@@ -27,6 +27,9 @@ import {
CATALOG_MAP,
type DecodedVehicle,
type EmexCategoryTreeNode,
type EmexHotspot,
type EmexHotspotArea,
type EmexPart,
type EmexPartsResult,
type EmexScraperResponse,
} from "./emex.types";
@@ -696,7 +699,19 @@ export class EmexService {
}
/**
* Fetches parts + schema image for a specific category (on-demand)
* Fetches parts + schema image for a specific category (on-demand).
*
* Strategy: plain-HTTP fast path first, browser fallback on parse failure.
* The probe in scripts/dev (FN-* perf work) showed Unit.aspx is fully
* server-rendered for the data we need — parts come from `<tr name>` rows
* with `td[name=c_oem|c_pnc|c_name]`, hotspots from inline-styled
* `<div class="dragger g_highlight" name=N style="margin-top:Ypx; margin-left:Xpx; width:Wpx; height:Hpx">`,
* and image dims from the first 24 bytes of the GIF/PNG itself. Sidesteps
* the ~1-2s browser-launch cost AND the 3-page semaphore in EmexBrowserService,
* dropping a cold leaf from ~6-7s (Tier 1) to ~5s and removing the
* concurrency cap (prefetch can fan out beyond 3 simultaneous fetches).
* Falls back to the Playwright scraper if the HTML doesn't yield parts —
* keeps us honest when emexdwc.ae changes layout or returns a JS-gated page.
*/
async fetchCategoryParts(categoryUrl: string): Promise<EmexPartsResult> {
if (!categoryUrl) {
@@ -707,8 +722,26 @@ export class EmexService {
this.logger.log(`Fetching parts from category URL: ${categoryUrl}`);
await this.touchActivity();
let release: (() => Promise<void>) | null = null;
// Fast path: plain HTTP + HTML parse. Browser-free.
try {
const httpResult = await this.fetchCategoryPartsViaHttp(categoryUrl);
if (httpResult.parts.length > 0) {
this.logger.log(
`Fetched ${httpResult.parts.length} parts from category (http path${httpResult.schemaImageUrl ? ", with schema" : ""})`,
);
return httpResult;
}
this.logger.log(
"Plain-HTTP path returned 0 parts; falling back to Playwright scraper",
);
} catch (err) {
this.logger.warn(
`Plain-HTTP path failed: ${(err as Error).message}; falling back to Playwright`,
);
}
// Slow path: existing Playwright scraper.
let release: (() => Promise<void>) | null = null;
try {
const instance = await this.createScraperInstance();
const scraper = instance.scraper;
@@ -717,7 +750,7 @@ export class EmexService {
const result = await this.executeWithTimeout(scraper.getParts(categoryUrl), this.timeout);
if (result && result.parts.length > 0) {
this.logger.log(`Fetched ${result.parts.length} parts from category`);
this.logger.log(`Fetched ${result.parts.length} parts from category (browser path)`);
if (result.schemaImageUrl) {
this.logger.log(`Schema image found: ${result.schemaImageUrl}`);
}
@@ -741,6 +774,167 @@ export class EmexService {
}
}
/**
* Plain-HTTP equivalent of the Playwright getParts() flow. Two sequential
* GETs (QuickDetails → Unit) + one Range GET for image dims. Throws if it
* can't resolve the Unit.aspx URL or if the image is fetched but headers
* are unparseable. Returns an empty result (no throw) if the page has no
* parts — caller treats that as "fall back to browser".
*/
private async fetchCategoryPartsViaHttp(categoryUrl: string): Promise<EmexPartsResult> {
// 1) QuickDetails.aspx → Unit.aspx anchor
const qdHtml = await this.fetchEmexHtml(categoryUrl);
const unitMatch = qdHtml.match(/href="([^"]*Unit\.aspx[^"]*)"/i);
if (!unitMatch) {
// QuickDetails returned an Error.aspx-style page or has parts inline —
// try extracting parts directly; if none, signal fallback.
const direct = this.extractEmexPartsFromHtml(qdHtml);
if (direct.parts.length > 0) return direct;
throw new Error("No Unit.aspx link in QuickDetails response");
}
const unitRel = unitMatch[1].replace(/&amp;/g, "&");
const unitUrl = unitRel.startsWith("http")
? unitRel
: new URL(unitRel, categoryUrl).toString();
// 2) Unit.aspx — main extraction target
const unitHtml = await this.fetchEmexHtml(unitUrl);
const extracted = this.extractEmexPartsFromHtml(unitHtml);
if (extracted.parts.length === 0) {
// Empty parts table: structural change or session-gated page. Let the
// caller try Playwright (which sometimes succeeds where plain HTTP
// doesn't, e.g. if a JS redirect refreshes the ssd token).
return extracted;
}
// 3) Image dims — read GIF/PNG header from a 128-byte Range GET if a
// schema URL was found. Don't block parts extraction on image failure.
if (extracted.schemaImageUrl && (extracted.schemaWidth === 0 || extracted.schemaHeight === 0)) {
try {
const dims = await this.fetchImageDims(extracted.schemaImageUrl);
if (dims.width > 0 && dims.height > 0) {
extracted.schemaWidth = dims.width;
extracted.schemaHeight = dims.height;
}
} catch (err) {
this.logger.debug(`Image dims fetch failed: ${(err as Error).message}`);
}
}
if (extracted.schemaImageUrl) {
this.logger.log(`Schema image found: ${extracted.schemaImageUrl}`);
}
return extracted;
}
/**
* Parse Unit.aspx (or fallback QuickDetails.aspx) HTML into the same shape
* the Playwright scraper returns. Pure regex/string-based — no DOM, no
* browser. Mirrors the page.evaluate() in scripts/emex-vin-scraper.js so
* upstream consumers don't care which path produced the result.
*/
private extractEmexPartsFromHtml(html: string): EmexPartsResult {
// Parts: <tr name="..."> with <td name="c_oem">, <td name="c_pnc">, <td name="c_name">.
const parts: EmexPart[] = [];
const trRx = /<tr\b[^>]*\bname="[^"]+"[^>]*>([\s\S]*?)<\/tr>/g;
const stripTags = (s: string) =>
s
.replace(/<[^>]+>/g, "")
.replace(/&nbsp;/g, " ")
.replace(/&amp;/g, "&")
.replace(/&quot;/g, '"')
.replace(/&#39;/g, "'")
.replace(/&lt;/g, "<")
.replace(/&gt;/g, ">")
.trim();
for (const trMatch of html.matchAll(trRx)) {
const body = trMatch[1];
const oem = stripTags(body.match(/<td\b[^>]*\bname="c_oem"[^>]*>([\s\S]*?)<\/td>/)?.[1] ?? "");
if (!oem) continue;
const pnc = stripTags(body.match(/<td\b[^>]*\bname="c_pnc"[^>]*>([\s\S]*?)<\/td>/)?.[1] ?? "");
const name = stripTags(
body.match(/<td\b[^>]*\bname="c_name"[^>]*>([\s\S]*?)<\/td>/)?.[1] ?? "",
);
parts.push({ oemCode: oem, nameEn: name, positionCode: pnc });
}
// Hotspots: <div name="N" class="... dragger ... g_highlight ..." style="…margin-top:Ypx; margin-left:Xpx; width:Wpx; height:Hpx…">.
// Inline style is in image-natural pixel coordinates (probe confirmed
// max-right and max-bottom always sit within the image's natural dims),
// so we use the values directly — no rect arithmetic or scaling.
const hotspotMap = new Map<string, EmexHotspot>();
const divRx =
/<div\b[^>]*\bname="([^"]+)"[^>]*\bclass="[^"]*\bdragger\b[^"]*\bg_highlight\b[^"]*"[^>]*\bstyle="([^"]+)"/g;
for (const m of html.matchAll(divRx)) {
const key = m[1];
const style = m[2];
const w = Number.parseInt(style.match(/width:\s*(\d+)px/)?.[1] ?? "0", 10);
const h = Number.parseInt(style.match(/height:\s*(\d+)px/)?.[1] ?? "0", 10);
const top = Number.parseInt(style.match(/margin-top:\s*(\d+)px/)?.[1] ?? "0", 10);
const left = Number.parseInt(style.match(/margin-left:\s*(\d+)px/)?.[1] ?? "0", 10);
const area: EmexHotspotArea = { left, top, width: w, height: h };
const entry = hotspotMap.get(key);
if (entry) entry.areas.push(area);
else hotspotMap.set(key, { key, areas: [area] });
}
const hotspots = Array.from(hotspotMap.values());
// Schema image: <img class="dragger" src="...img.laximo.net/.../*.gif">.
// Fallback to any img.laximo.net URL, normalising the /NNN/ path to
// /source/ the same way the browser scraper did (some catalog pages link
// to a thumbnail variant that doesn't carry full-resolution coords).
let schemaImageUrl: string | null = null;
const draggerMatch = html.match(
/<img\b[^>]*\bclass="[^"]*\bdragger\b[^"]*"[^>]*\bsrc="([^"]+laximo[^"]+)"/,
);
if (draggerMatch) schemaImageUrl = draggerMatch[1].replace(/&amp;/g, "&");
if (!schemaImageUrl) {
const fallback = html.match(/src="(https?:\/\/img\.laximo\.net[^"]+)"/);
if (fallback) {
schemaImageUrl = fallback[1].replace(/&amp;/g, "&").replace(/\/\d+\//, "/source/");
}
}
return {
parts,
schemaImageUrl,
hotspots,
schemaWidth: 0,
schemaHeight: 0,
};
}
/**
* Read the natural image dimensions from the first ~24 bytes of a GIF or
* PNG. Uses an HTTP Range request so we never download the whole image
* just to read its size. Returns 0×0 on unknown formats — the schema-image
* downloader downstream uses its own dim-parsing fallback as a safety net.
*/
private async fetchImageDims(url: string): Promise<{ width: number; height: number }> {
const res = await fetch(url, {
headers: { "User-Agent": EMEX_UA, Range: "bytes=0-127" },
signal: AbortSignal.timeout(10000),
...(this.proxyAgent ? { dispatcher: this.proxyAgent } : {}),
} as RequestInit);
if (!res.ok && res.status !== 206) {
throw new Error(`Image HTTP ${res.status}`);
}
const buf = Buffer.from(await res.arrayBuffer());
// GIF87a / GIF89a — width at byte 6 LE, height at byte 8 LE.
if (buf.length >= 10 && buf.slice(0, 3).toString("ascii") === "GIF") {
return { width: buf.readUInt16LE(6), height: buf.readUInt16LE(8) };
}
// PNG — width at byte 16 BE, height at byte 20 BE.
if (
buf.length >= 24 &&
buf[0] === 0x89 &&
buf.slice(1, 4).toString("ascii") === "PNG"
) {
return { width: buf.readUInt32BE(16), height: buf.readUInt32BE(20) };
}
return { width: 0, height: 0 };
}
/**
* Extracts year from VIN (10th character)
*/