feat(emex): seed catalogs from hierarchical Vehicle.aspx tree

emex catalogs were seeded from the flat QuickGroups taxonomy (~351
unsearchable leaves on one page). Switch the seeder to the Vehicle.aspx
OEM-microfiche tree (top groups -> sub groups -> Unit.aspx leaves), which
also reaches units that have no QuickGroups gid (e.g. the torque
converter, OEM 5205547 on the Fiesta).

- emex-tree.parser.ts: parseVehicleTree / parseUnitLeaves — pure,
  GET-based (per-href ssd, no browser/postback). Unit-tested vs fixtures.
- emex.service: fetchVehicleTree + drillVehicleNode; all 3 decode methods
  now populate categoryTree; Unit.aspx leaves fetch parts directly.
- categories.service: revive the dormant tree-insert branch with group
  markers (linkWid="emex-group" / hasSubgroups), add an emex getChildren
  lazy-drill branch + emex-group parent guard, tighten leaf detection,
  disambiguate colliding sub-group names.
- prefetch: emex Vehicle.aspx = parent to drill; Unit/QuickDetails = leaf.

No DDL and no frontend changes — the generic tree UI renders it like
pl24/pcat. Existing flat emex categories re-seed lazily once deleted.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-06-05 15:28:26 +03:00
parent 9772ce1ef4
commit 26efef22d4
9 changed files with 6009 additions and 25 deletions

View File

@@ -258,21 +258,24 @@ export class CategoriesService {
const emexResult = await this.emexService.decodeVin(vehicle.vin);
if (emexResult) {
const rawData = emexResult.raw as Record<string, unknown>;
// Hierarchical Vehicle.aspx tree (top groups → sub groups). Units are
// NOT seeded here — they're drilled lazily per sub-group by
// getChildren on first expand (mirrors the pl24/pcat lazy model).
const tree = rawData?.emexCategoryTree as
| Array<{
cid?: string;
name: string;
gid: string | null;
code?: string | null;
url: string | null;
children: any[];
kind?: "group" | "leaf";
children?: any[];
}>
| undefined;
if (tree && tree.length > 0) {
// Recursive tree insertion from QuickGroups.aspx
this.logger.log(`Inserting ${tree.length} EMEX top-level category groups recursively`);
this.logger.log(`Inserting EMEX Vehicle.aspx tree: ${tree.length} top groups`);
// Phase 1: walk the tree, collect every unique English name so we
// can bulk-translate before touching the DB.
// Bulk-translate every node name before touching the DB.
const uniqueNames = new Set<string>();
const collect = (nodes: Array<{ name: string; children?: any[] }>) => {
for (const node of nodes) {
@@ -282,33 +285,57 @@ export class CategoriesService {
};
collect(tree);
const trMap = await this.translationsService.translateMany([...uniqueNames]);
const tr = (n: string) => trMap.get(n) ?? n;
// The categories unique index is (vehicle, catalogVehicle, name,
// source), so two sub-groups that share a display name across
// different top groups (e.g. "Speakers" under both Electrical and
// Accessories) would silently collide and orphan a branch. Detect
// such names and disambiguate them with their top-group name.
const subNameCounts = new Map<string, number>();
for (const top of tree) {
for (const sub of top.children ?? []) {
const dn = tr(sub.name);
subNameCounts.set(dn, (subNameCounts.get(dn) ?? 0) + 1);
}
}
const insertNodes = async (
nodes: Array<{
cid?: string;
name: string;
gid: string | null;
code?: string | null;
url: string | null;
kind?: "group" | "leaf";
children?: any[];
}>,
parentId: string | null,
topName: string | null,
) => {
for (const node of nodes) {
if (!node.name) continue;
const isLeaf = !node.children?.length;
// URL'si olmayan leaf node'lar dead-end — kaydetme
if (isLeaf && !node.url) continue;
if (!node.name || !node.url) continue;
const isLeaf = node.kind === "leaf";
let displayName = tr(node.name);
// Disambiguate colliding sub-group names with their top group.
if (parentId !== null && topName && (subNameCounts.get(displayName) ?? 0) > 1) {
displayName = `${displayName} (${topName})`;
}
const [inserted] = await this.db
.insert(categories)
.values({
vehicleId,
catalogVehicleId: null as string | null,
name: trMap.get(node.name) ?? node.name,
name: displayName,
nameOriginal: node.name,
parentId,
externalId: node.gid || null,
linkPath: isLeaf ? node.url || null : null,
linkWid: null as string | null,
externalId: isLeaf ? (node.code ?? null) : (node.cid ?? null),
// Groups keep their Vehicle.aspx drill URL but are marked
// non-leaf via linkWid="emex-group" + hasSubgroups so the
// tree UI / lazy-drill never treat them as parts leaves.
linkPath: node.url,
linkWid: isLeaf ? (null as string | null) : "emex-group",
hasSubgroups: !isLeaf,
hasParts: isLeaf,
source: "emex" as const,
})
.onConflictDoNothing()
@@ -316,16 +343,19 @@ export class CategoriesService {
if (inserted) {
dbCategories.push(inserted);
// A top group passes its own display name down as the
// disambiguation context for its sub-groups.
const ctx = parentId === null ? displayName : topName;
if (node.children?.length) {
await insertNodes(node.children, inserted.id);
await insertNodes(node.children, inserted.id, ctx);
}
}
}
};
await insertNodes(tree, null);
await insertNodes(tree, null, null);
this.logger.log(
`Stored ${dbCategories.length} EMEX categories (tree) for ${vehicle.vin}`,
`Stored ${dbCategories.length} EMEX categories (Vehicle.aspx tree) for ${vehicle.vin}`,
);
} else if (emexResult.categories.length > 0) {
// Flat fallback: insert all categories without hierarchy
@@ -524,6 +554,65 @@ export class CategoriesService {
return children;
}
// EMEX Vehicle.aspx group node — drill its href to Unit.aspx leaves on
// demand. Top groups already have their sub-groups seeded (returned from
// the DB-children check above), so this only fires for sub-groups whose
// unit leaves haven't been fetched yet. emex carries no catalogInfo, so
// this must run before the pl24 serviceName guard below.
if (category.source === "emex" && category.linkWid === "emex-group" && linkPath) {
try {
const leaves = await this.emexService.drillVehicleNode(linkPath);
if (leaves.length > 0) {
const trMap = await this.translationsService.translateMany(
leaves.map((l) => l.name).filter(Boolean),
);
const seenNames = new Set<string>();
const insertData: (typeof categories.$inferInsert)[] = [];
for (const l of leaves) {
const name = trMap.get(l.name) ?? l.name;
if (seenNames.has(name)) continue; // unique index is per-name
seenNames.add(name);
insertData.push({
vehicleId: category.vehicleId,
catalogVehicleId: category.catalogVehicleId,
name,
nameOriginal: l.name,
parentId: categoryId,
externalId: l.code ?? l.cid ?? null,
linkPath: l.url, // Unit.aspx → direct parts fetch
linkWid: null as string | null,
hasSubgroups: false,
hasParts: true,
source: "emex" as const,
});
}
children = await this.db
.insert(categories)
.values(insertData)
.onConflictDoNothing()
.returning();
if (children.length < insertData.length) {
children = await this.db
.select()
.from(categories)
.where(eq(categories.parentId, categoryId));
}
await this.redis.del(`cat:tree:${category.vehicleId}`);
}
} catch (err) {
this.logger.error(
`EMEX node drill failed for ${categoryId} (${linkPath}): ${(err as Error).message}`,
);
}
// Unit leaves carry a Unit.aspx linkPath + no DB children, so
// enrichWithSchemaImages already marks them as leaves (children: []).
return children.length > 0 ? this.enrichWithSchemaImages(children) : children;
}
if (!catalogInfo?.serviceName || !linkPath) {
return [];
}
@@ -930,6 +1019,27 @@ export class CategoriesService {
// No subgroups → leaf; fall through to the parts path below.
}
// EMEX Vehicle.aspx group node (linkWid="emex-group") — a parent, never a
// parts leaf. Return its children (seeded sub-groups, or Unit leaves drilled
// on demand by getChildren). Mirrors the pl24 group-node branch above; an
// emex group is never legitimately empty, so surface a load error if drill
// comes back empty rather than a misleading "no parts" leaf.
if (category.source === "emex" && category.linkWid === "emex-group" && category.vehicleId) {
const emexChildren = await this.getChildren(categoryId);
const base = {
id: category.id,
name: category.name,
description: category.nameOriginal || null,
parentId: category.parentId || null,
parts: [],
schemaPics: [],
hotspots: [],
};
return emexChildren.length > 0
? { ...base, children: emexChildren }
: { ...base, loadError: true };
}
// Leaf category — get or fetch parts
let loadError = false;
let discoveredChildren: any[] = [];
@@ -1674,7 +1784,9 @@ export class CategoriesService {
// rendered as empty leaves.
const isLeaf =
c.source === "emex"
? !!c.linkPath && dbChildCount === 0
? // emex-group = Vehicle.aspx parent node (never a leaf); a unit leaf
// has a Unit.aspx linkPath and no DB children.
c.linkWid !== "emex-group" && !!c.linkPath && dbChildCount === 0
: c.source === "parts-catalogs"
? (() => {
if (dbChildCount > 0) return false; // already has children → parent

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,111 @@
import { readFileSync } from "node:fs";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import { parseUnitLeaves, parseVehicleTree } from "./emex-tree.parser";
const fixture = (name: string) => readFileSync(join(__dirname, "__fixtures__", name), "utf-8");
// Captured 2026-06-05 from emexdwc.ae catalog FORD202201 (VIN WF0JXXGAJJBJ88319, vid=0):
// vehicle-root.html — Vehicle.aspx root (nothing selected)
// vehicle-powertrain.html — Vehicle.aspx?cid=3 (Powertrain selected; 162 raw / 81 unique units)
// vehicle-engblock.html — Vehicle.aspx?cid=8289 (Eng Block sub-group; 22 raw / 11 unique units)
const ROOT = fixture("vehicle-root.html");
const POWERTRAIN = fixture("vehicle-powertrain.html");
const ENGBLOCK = fixture("vehicle-engblock.html");
const TOP_LABELS = ["Chassis", "Powertrain", "Electrical", "Body And Paint"];
describe("parseVehicleTree — 2-level group tree from the Vehicle.aspx sidebar", () => {
it("extracts exactly the 7 FCSD top groups from the root page", () => {
const tree = parseVehicleTree(ROOT);
expect(tree).toHaveLength(7);
expect(tree.every((n) => n.kind === "group")).toBe(true);
// The 7 section roots carry small cids.
expect(tree.map((n) => n.cid).sort((a, b) => Number(a) - Number(b))).toEqual([
"1",
"2",
"3",
"4",
"5",
"8",
"9",
]);
const labels = tree.map((n) => n.name);
for (const l of TOP_LABELS) {
expect(labels.some((x) => x.startsWith(l))).toBe(true);
}
});
it("yields the same 7 top groups from a drilled node page (menu is identical everywhere)", () => {
// Powertrain is the SELECTED node here (rendered as bare text, no <a>) —
// must still be captured via its onclick URL.
const tree = parseVehicleTree(POWERTRAIN);
expect(tree).toHaveLength(7);
const powertrain = tree.find((n) => n.cid === "3");
expect(powertrain?.name).toBe("Powertrain");
});
it("nests Powertrain sub-groups under the Powertrain top group", () => {
const tree = parseVehicleTree(ROOT);
const powertrain = tree.find((n) => n.cid === "3");
expect(powertrain).toBeDefined();
const subCids = powertrain!.children.map((c) => c.cid);
const subNames = powertrain!.children.map((c) => c.name);
// Automatic Transmission is cid=4004 — the group whose Unit leaves include
// the torque converter. Confirms the converter's home is under Powertrain.
expect(subCids).toContain("4004");
expect(subNames).toContain("Automatic Transmission");
expect(subNames).toContain("Clutch");
expect(subNames).toContain("Manual Transmission");
expect(powertrain!.children.every((c) => c.kind === "group")).toBe(true);
});
it("gives every group a verbatim Vehicle.aspx drill URL with cid + ssd", () => {
const tree = parseVehicleTree(ROOT);
const all = tree.flatMap((t) => [t, ...t.children]);
expect(all.length).toBeGreaterThan(120); // ~136 total group nodes
for (const n of all) {
expect(n.url).toMatch(/^https:\/\/emexdwc\.ae\/Vehicle\.aspx\?/);
expect(n.url).toMatch(/[?&]cid=\d+/);
expect(n.url).toMatch(/[?&]ssd=\$\*/); // opaque per-node ssd present
}
});
});
describe("parseUnitLeaves — Unit.aspx leaves from a drilled group page", () => {
it("extracts all Powertrain unit leaves incl. the torque converter (closes the taxonomy gap)", () => {
const leaves = parseUnitLeaves(POWERTRAIN);
// 81 distinct unit hrefs (the raw HTML lists each twice — image + text
// anchor — so dedup-by-href halves the ~162 raw matches).
expect(leaves.length).toBeGreaterThan(60);
expect(leaves.every((l) => l.kind === "leaf")).toBe(true);
const converter = leaves.find((l) => l.code === "3070160A");
expect(converter).toBeDefined();
expect(converter!.name).toMatch(/Converter/i);
expect(converter!.name).toMatch(/Automatic Trans/i);
// Leaf URL is a self-contained Unit.aspx fetch URL (the existing Unit
// parts-parser can GET it directly).
expect(converter!.url).toMatch(/^https:\/\/emexdwc\.ae\/Unit\.aspx\?/);
expect(converter!.url).toMatch(/[?&]ssd=\$\*/);
});
it("parses section codes off the 'CODE: name' label", () => {
const leaves = parseUnitLeaves(POWERTRAIN);
// Service Engine And Short Block is 3030105A.
const engine = leaves.find((l) => l.code === "3030105A");
expect(engine).toBeDefined();
expect(engine!.name).toMatch(/Service Engine And Short Block/i);
// Every leaf got a non-empty code + name (the label always has the prefix).
expect(leaves.every((l) => l.code && l.code.length >= 4)).toBe(true);
expect(leaves.every((l) => l.name.length > 0)).toBe(true);
});
it("narrows to fewer leaves when a SUB-group is drilled (Eng Block ⊂ Powertrain)", () => {
const sub = parseUnitLeaves(ENGBLOCK);
const top = parseUnitLeaves(POWERTRAIN);
expect(sub.length).toBeGreaterThan(0);
expect(sub.length).toBeLessThan(top.length); // 11 < 81 (unique)
expect(sub.every((l) => l.url.includes("Unit.aspx"))).toBe(true);
});
});

View File

@@ -0,0 +1,185 @@
/**
* emex Vehicle.aspx hierarchical-tree parsers.
*
* Background: emex exposes the SAME catalog through two navigation trees.
* • QuickGroups.aspx — a FLAT list of ~351 functional Laximo categories
* (gid 10xxx). This is what the legacy seeder (`parseCategoryList` in
* emex.service.ts) scrapes, producing the unsearchable "everything on one
* page" experience.
* • Vehicle.aspx — the OEM-microfiche tree: a small set of TOP groups
* (Chassis, Powertrain, Electrical, …) → SUB groups (Automatic
* Transmission, Clutch, …) → Unit.aspx LEAVES carrying section codes
* (e.g. "3070160A: Converter, 4 Speed Automatic Trans") and parts.
*
* The Vehicle.aspx tree is fully traversable by plain HTTP GET: every node
* link carries its own opaque `ssd` in the href, copied verbatim hop-to-hop
* (no cookies, no __VIEWSTATE postback — verified 2026-06-05). These parsers
* turn the raw HTML into a tree we can seed into the generic `categories`
* table (parentId chains + hasSubgroups/hasParts), so the existing tree UI
* renders it exactly like pl24/pcat — no frontend changes.
*
* Pure functions, no DI — unit-tested against captured fixtures in
* __fixtures__/. Regex-based (no cheerio) to match the existing emex parsers.
*/
import type { EmexVehicleTreeNode } from "./emex.types";
const EMEX_BASE_URL = "https://emexdwc.ae";
/**
* Section roots (top groups) carry a small `cid`. For FORD202201 the 7 roots
* are cid 1,2,3,4,5,8,9 (Information, Chassis, Powertrain, Electrical, Body,
* Fluids, Suspension/Accessories); every other menu entry is a sub-group. If a
* future catalog numbers differently and NO node has cid ≤ this max, the tree
* degrades gracefully to a single flat group level (see parseVehicleTree).
*/
const TOP_GROUP_CID_MAX = 9;
function decodeEntities(s: string): string {
return s
.replace(/&#39;/g, "'")
.replace(/&amp;/g, "&")
.replace(/&quot;/g, '"')
.replace(/&lt;/g, "<")
.replace(/&gt;/g, ">")
.replace(/&nbsp;/g, " ");
}
function stripTags(s: string): string {
return s.replace(/<[^>]+>/g, " ");
}
function cidOf(url: string): string | null {
const m = url.match(/[?&]cid=(\d+)/);
return m ? m[1] : null;
}
/** A raw category entry from the persistent Vehicle.aspx sidebar menu. */
interface MenuNode {
cid: string;
name: string;
url: string; // absolute Vehicle.aspx drill URL (ssd embedded)
}
/**
* Extract the persistent category sidebar (identical on every Vehicle.aspx
* page). Each item is:
* <div class="guayaquil_categoryitem"
* onclick="window.location='Vehicle.aspx?...cid=N&ssd=...'"
* onmouseout=... onmouseover=... style="margin-left:0px">
* <a href="Vehicle.aspx?...">LABEL</a> // normal
* </div>
* The CURRENTLY-SELECTED node renders with class `guayaquil_categoryitem_selected`
* and its label as bare text (no <a>), but still carries the onclick URL — so
* we read cid+url+ssd uniformly from the onclick, and the label from the inner
* text (anchor or bare). Document order is preserved (top group, then its
* sub-groups, then the next top group …).
*/
function parseMenuNodes(html: string): MenuNode[] {
const itemRx =
/<div class="guayaquil_categoryitem(?:_selected)?"\s+onclick="window\.location=&#39;(Vehicle\.aspx\?[\s\S]*?)&#39;"[^>]*>([\s\S]*?)<\/div>/g;
const nodes: MenuNode[] = [];
const seen = new Set<string>();
for (const m of html.matchAll(itemRx)) {
const url = decodeEntities(m[1]);
const cid = cidOf(url);
if (!cid) continue;
const name = decodeEntities(stripTags(m[2])).replace(/\s+/g, " ").trim();
if (!name) continue;
// Dedup on cid: the menu can repeat an entry; first occurrence wins (keeps
// document order for parent inference).
if (seen.has(cid)) continue;
seen.add(cid);
nodes.push({ cid, name, url: `${EMEX_BASE_URL}/${url}` });
}
return nodes;
}
/**
* Build the 2-level GROUP tree (top groups → sub-groups) from a Vehicle.aspx
* page's sidebar. Works on ANY page (root or a drilled node) since the menu is
* identical everywhere. Leaves (Unit.aspx) are NOT included here — they're
* fetched lazily per sub-group via {@link parseUnitLeaves}.
*
* Classification: cid ≤ TOP_GROUP_CID_MAX → top group; otherwise a sub-group
* attached to the most recent top group seen in document order. If the catalog
* exposes no small-cid roots, every node becomes a flat top-level group.
*/
export function parseVehicleTree(html: string): EmexVehicleTreeNode[] {
const menu = parseMenuNodes(html);
const hasRoots = menu.some((n) => Number(n.cid) <= TOP_GROUP_CID_MAX);
const tops: EmexVehicleTreeNode[] = [];
let current: EmexVehicleTreeNode | null = null;
for (const n of menu) {
const isTop = !hasRoots || Number(n.cid) <= TOP_GROUP_CID_MAX;
const node: EmexVehicleTreeNode = {
cid: n.cid,
name: n.name,
code: null,
url: n.url,
kind: "group",
imageUrl: null,
children: [],
};
if (isTop) {
tops.push(node);
current = node;
} else if (current) {
current.children.push(node);
} else {
// Sub-group seen before any top group (shouldn't happen for Ford, but be
// safe): promote it to a top-level group.
tops.push(node);
}
}
return tops;
}
/** title → img.laximo.net url, harvested from the unit illustration thumbnails. */
function parseUnitImages(html: string): Map<string, string> {
const map = new Map<string, string>();
const rx =
/<div class="guayaquil_zoom"\s+full="(https:\/\/img\.laximo\.net[^"]+)"\s+title="([^"]*)"/g;
for (const m of html.matchAll(rx)) {
const url = decodeEntities(m[1]);
const title = decodeEntities(m[2]).replace(/\s+/g, " ").trim();
if (title && !map.has(title)) map.set(title, url);
}
return map;
}
/**
* Parse the Unit.aspx LEAVES from a drilled node page's content area. The
* sidebar contains only Vehicle.aspx links, so every Unit.aspx anchor on the
* page belongs to the currently-drilled group. Each leaf label is
* "<sectionCode>: <name>" (e.g. "3070160A: Converter, 4 Speed Automatic Trans")
* and the href is a self-contained Unit.aspx fetch URL (cid + ssd + path_id).
*/
export function parseUnitLeaves(html: string): EmexVehicleTreeNode[] {
const images = parseUnitImages(html);
const anchorRx = /<a href="(Unit\.aspx\?[^"]+)"[^>]*>([\s\S]*?)<\/a>/g;
const leaves: EmexVehicleTreeNode[] = [];
const seen = new Set<string>();
for (const m of html.matchAll(anchorRx)) {
const url = decodeEntities(m[1]);
if (seen.has(url)) continue;
seen.add(url);
const label = decodeEntities(stripTags(m[2])).replace(/\s+/g, " ").trim();
if (!label) continue;
const codeMatch = label.match(/^([0-9A-Za-z]+):\s*(.*)$/);
const code = codeMatch ? codeMatch[1] : null;
const name = codeMatch ? codeMatch[2].trim() : label;
leaves.push({
cid: cidOf(url) ?? "",
name,
code,
url: `${EMEX_BASE_URL}/${url}`,
kind: "leaf",
imageUrl: images.get(label) ?? null,
children: [],
});
}
return leaves;
}

View File

@@ -22,6 +22,7 @@ import { ConfigService } from "@nestjs/config";
import { ProxyAgent } from "undici";
import { isBackfillContext } from "../../jobs/prefetch-context";
import { RedisService } from "../../redis/redis.service";
import { parseUnitLeaves, parseVehicleTree } from "./emex-tree.parser";
import { EmexBrowserService } from "./emex.browser";
import { createEmptyDecodedVehicle, mapEmexResponse } from "./emex.mapper";
import {
@@ -33,6 +34,7 @@ import {
type EmexPart,
type EmexPartsResult,
type EmexScraperResponse,
type EmexVehicleTreeNode,
} from "./emex.types";
const EMEX_BASE_URL = "https://emexdwc.ae";
@@ -386,6 +388,33 @@ export class EmexService {
return cats;
}
/**
* Fetch + parse the hierarchical Vehicle.aspx tree (top groups → sub groups)
* for a decoded vehicle. One plain GET of the catalog root; the persistent
* sidebar carries the whole 2-level group tree (units are drilled lazily per
* sub-group via {@link drillVehicleNode}). Returns [] on any failure so the
* caller falls back to the flat QuickGroups categories.
*/
private async fetchVehicleTree(
catalogCode: string,
vid: string,
ssd: string,
): Promise<EmexVehicleTreeNode[]> {
const url = `${EMEX_BASE_URL}/Vehicle.aspx?c=${catalogCode}&vid=${vid}&ssd=${encodeURIComponent(ssd)}`;
const html = await this.fetchEmexHtml(url);
return parseVehicleTree(html);
}
/**
* Drill a Vehicle.aspx group node (its verbatim href, ssd embedded) to its
* Unit.aspx leaves. Used by the lazy category drill (categories.service
* getChildren) when an emex group is first expanded.
*/
async drillVehicleNode(vehicleAspxUrl: string): Promise<EmexVehicleTreeNode[]> {
const html = await this.fetchEmexHtml(vehicleAspxUrl);
return parseUnitLeaves(html);
}
/**
* Determine brand name from EMEX catalog code (e.g. "BMW202501" → "BMW").
*/
@@ -455,6 +484,19 @@ export class EmexService {
}
}
// Hierarchical Vehicle.aspx tree (primary nav); the flat `categories`
// above is the fallback. One GET of the catalog root yields the full
// 2-level group tree; units are drilled lazily per sub-group on expand.
let categoryTree: EmexVehicleTreeNode[] = [];
if (v.catalogCode && v.ssd) {
try {
categoryTree = await this.fetchVehicleTree(v.catalogCode, v.vid ?? "0", v.ssd);
this.logger.log(`EMEX HTTP: parsed ${categoryTree.length} Vehicle.aspx top groups`);
} catch (err) {
this.logger.warn(`EMEX Vehicle.aspx tree fetch failed: ${(err as Error).message}`);
}
}
// Build a response compatible with mapEmexResponse
const response: EmexScraperResponse = {
success: true,
@@ -476,7 +518,7 @@ export class EmexService {
},
quickGroupsUrl: v.quickGroupsUrl || null,
categories: categories.map((c) => ({ gid: c.gid || "", name: c.name, url: c.url })),
categoryTree: [],
categoryTree,
timestamp: new Date().toISOString(),
};
@@ -541,6 +583,17 @@ export class EmexService {
}
}
// Hierarchical Vehicle.aspx tree (primary nav); flat `categories` = fallback.
let categoryTree: EmexVehicleTreeNode[] = [];
if (v.catalogCode && v.ssd) {
try {
categoryTree = await this.fetchVehicleTree(v.catalogCode, v.vid ?? "0", v.ssd);
this.logger.log(`EMEX HTTP: parsed ${categoryTree.length} Vehicle.aspx top groups`);
} catch (err) {
this.logger.warn(`EMEX Vehicle.aspx tree fetch failed: ${(err as Error).message}`);
}
}
const response: EmexScraperResponse = {
success: true,
source: "emexdwc.ae",
@@ -561,7 +614,7 @@ export class EmexService {
},
quickGroupsUrl: v.quickGroupsUrl || null,
categories: categories.map((c) => ({ gid: c.gid || "", name: c.name, url: c.url })),
categoryTree: [],
categoryTree,
timestamp: new Date().toISOString(),
};
@@ -608,6 +661,17 @@ export class EmexService {
}
}
// Hierarchical Vehicle.aspx tree (primary nav); flat `categories` = fallback.
let categoryTree: EmexVehicleTreeNode[] = [];
if (v.catalogCode && v.ssd) {
try {
categoryTree = await this.fetchVehicleTree(v.catalogCode, v.vid ?? "0", v.ssd);
this.logger.log(`EMEX HTTP: parsed ${categoryTree.length} Vehicle.aspx top groups`);
} catch (err) {
this.logger.warn(`EMEX Vehicle.aspx tree fetch failed: ${(err as Error).message}`);
}
}
const response: EmexScraperResponse = {
success: true,
source: "emexdwc.ae",
@@ -628,7 +692,7 @@ export class EmexService {
},
quickGroupsUrl: v.quickGroupsUrl || null,
categories: categories.map((c) => ({ gid: c.gid || "", name: c.name, url: c.url })),
categoryTree: [],
categoryTree,
timestamp: new Date().toISOString(),
};
@@ -862,6 +926,28 @@ export class EmexService {
* parts — caller treats that as "fall back to browser".
*/
private async fetchCategoryPartsViaHttp(categoryUrl: string): Promise<EmexPartsResult> {
// Vehicle.aspx-tree leaves already point straight at Unit.aspx — skip the
// QuickDetails→Unit hop and extract parts directly.
if (/Unit\.aspx/i.test(categoryUrl)) {
const unitHtml = await this.fetchEmexHtml(categoryUrl);
const extracted = this.extractEmexPartsFromHtml(unitHtml);
if (
extracted.schemaImageUrl &&
(extracted.schemaWidth === 0 || extracted.schemaHeight === 0)
) {
try {
const dims = await this.fetchImageDims(extracted.schemaImageUrl);
if (dims.width > 0 && dims.height > 0) {
extracted.schemaWidth = dims.width;
extracted.schemaHeight = dims.height;
}
} catch (err) {
this.logger.debug(`Image dims fetch failed: ${(err as Error).message}`);
}
}
return extracted;
}
// 1) QuickDetails.aspx → Unit.aspx anchor
const qdHtml = await this.fetchEmexHtml(categoryUrl);
const unitMatch = qdHtml.match(/href="([^"]*Unit\.aspx[^"]*)"/i);

View File

@@ -63,6 +63,29 @@ export interface EmexCategoryTreeNode {
children: EmexCategoryTreeNode[];
}
/**
* Node in the hierarchical Vehicle.aspx tree (the OEM-microfiche taxonomy):
* top groups → sub groups → Unit.aspx leaves. Produced by emex-tree.parser.ts
* and seeded into the generic `categories` table (parentId chains +
* hasSubgroups/hasParts), so the same tree UI renders it as pl24/pcat.
*/
export type EmexTreeNodeKind = "group" | "leaf";
export interface EmexVehicleTreeNode {
/** emex category id (`cid`) for groups; the parent group's cid for leaves. */
cid: string;
/** Display name (for leaves the "NNNN:" section-code prefix is stripped). */
name: string;
/** Section code for a leaf (e.g. "3070160A"); null for groups. */
code: string | null;
/** Absolute, verbatim URL: Vehicle.aspx (group, to drill) or Unit.aspx (leaf, to fetch parts). */
url: string;
kind: EmexTreeNodeKind;
/** Schema illustration (img.laximo.net gif) for a leaf, when listed; else null. */
imageUrl: string | null;
children: EmexVehicleTreeNode[];
}
/**
* Part data from EMEX
*/
@@ -142,7 +165,7 @@ export interface EmexScraperResponse {
quickGroupsUrl?: string | null;
parsedOptions?: EmexParsedOptions;
categories?: EmexCategory[];
categoryTree?: EmexCategoryTreeNode[];
categoryTree?: EmexVehicleTreeNode[];
sampleParts?: EmexPart[];
timestamp: string;
}

View File

@@ -528,7 +528,9 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy {
private isLeafLinkPath(linkPath: string | null, source: string): boolean {
if (!linkPath) return false;
if (source === "emex") return true; // EMEX leaves always have a URL
// EMEX: Vehicle.aspx group nodes are parents to drill; Unit.aspx (hierarchical
// tree) and QuickDetails.aspx (legacy flat) leaves carry parts.
if (source === "emex") return !linkPath.includes("Vehicle.aspx");
if (source === "parts-catalogs") return linkPath.startsWith("pcat:"); // pcat: prefix = leaf
// PL24 leaf indicators
return (