feat(emex): seed catalogs from hierarchical Vehicle.aspx tree
emex catalogs were seeded from the flat QuickGroups taxonomy (~351 unsearchable leaves on one page). Switch the seeder to the Vehicle.aspx OEM-microfiche tree (top groups -> sub groups -> Unit.aspx leaves), which also reaches units that have no QuickGroups gid (e.g. the torque converter, OEM 5205547 on the Fiesta). - emex-tree.parser.ts: parseVehicleTree / parseUnitLeaves — pure, GET-based (per-href ssd, no browser/postback). Unit-tested vs fixtures. - emex.service: fetchVehicleTree + drillVehicleNode; all 3 decode methods now populate categoryTree; Unit.aspx leaves fetch parts directly. - categories.service: revive the dormant tree-insert branch with group markers (linkWid="emex-group" / hasSubgroups), add an emex getChildren lazy-drill branch + emex-group parent guard, tighten leaf detection, disambiguate colliding sub-group names. - prefetch: emex Vehicle.aspx = parent to drill; Unit/QuickDetails = leaf. No DDL and no frontend changes — the generic tree UI renders it like pl24/pcat. Existing flat emex categories re-seed lazily once deleted. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -258,21 +258,24 @@ export class CategoriesService {
|
||||
const emexResult = await this.emexService.decodeVin(vehicle.vin);
|
||||
if (emexResult) {
|
||||
const rawData = emexResult.raw as Record<string, unknown>;
|
||||
// Hierarchical Vehicle.aspx tree (top groups → sub groups). Units are
|
||||
// NOT seeded here — they're drilled lazily per sub-group by
|
||||
// getChildren on first expand (mirrors the pl24/pcat lazy model).
|
||||
const tree = rawData?.emexCategoryTree as
|
||||
| Array<{
|
||||
cid?: string;
|
||||
name: string;
|
||||
gid: string | null;
|
||||
code?: string | null;
|
||||
url: string | null;
|
||||
children: any[];
|
||||
kind?: "group" | "leaf";
|
||||
children?: any[];
|
||||
}>
|
||||
| undefined;
|
||||
|
||||
if (tree && tree.length > 0) {
|
||||
// Recursive tree insertion from QuickGroups.aspx
|
||||
this.logger.log(`Inserting ${tree.length} EMEX top-level category groups recursively`);
|
||||
this.logger.log(`Inserting EMEX Vehicle.aspx tree: ${tree.length} top groups`);
|
||||
|
||||
// Phase 1: walk the tree, collect every unique English name so we
|
||||
// can bulk-translate before touching the DB.
|
||||
// Bulk-translate every node name before touching the DB.
|
||||
const uniqueNames = new Set<string>();
|
||||
const collect = (nodes: Array<{ name: string; children?: any[] }>) => {
|
||||
for (const node of nodes) {
|
||||
@@ -282,33 +285,57 @@ export class CategoriesService {
|
||||
};
|
||||
collect(tree);
|
||||
const trMap = await this.translationsService.translateMany([...uniqueNames]);
|
||||
const tr = (n: string) => trMap.get(n) ?? n;
|
||||
|
||||
// The categories unique index is (vehicle, catalogVehicle, name,
|
||||
// source), so two sub-groups that share a display name across
|
||||
// different top groups (e.g. "Speakers" under both Electrical and
|
||||
// Accessories) would silently collide and orphan a branch. Detect
|
||||
// such names and disambiguate them with their top-group name.
|
||||
const subNameCounts = new Map<string, number>();
|
||||
for (const top of tree) {
|
||||
for (const sub of top.children ?? []) {
|
||||
const dn = tr(sub.name);
|
||||
subNameCounts.set(dn, (subNameCounts.get(dn) ?? 0) + 1);
|
||||
}
|
||||
}
|
||||
|
||||
const insertNodes = async (
|
||||
nodes: Array<{
|
||||
cid?: string;
|
||||
name: string;
|
||||
gid: string | null;
|
||||
code?: string | null;
|
||||
url: string | null;
|
||||
kind?: "group" | "leaf";
|
||||
children?: any[];
|
||||
}>,
|
||||
parentId: string | null,
|
||||
topName: string | null,
|
||||
) => {
|
||||
for (const node of nodes) {
|
||||
if (!node.name) continue;
|
||||
const isLeaf = !node.children?.length;
|
||||
// URL'si olmayan leaf node'lar dead-end — kaydetme
|
||||
if (isLeaf && !node.url) continue;
|
||||
|
||||
if (!node.name || !node.url) continue;
|
||||
const isLeaf = node.kind === "leaf";
|
||||
let displayName = tr(node.name);
|
||||
// Disambiguate colliding sub-group names with their top group.
|
||||
if (parentId !== null && topName && (subNameCounts.get(displayName) ?? 0) > 1) {
|
||||
displayName = `${displayName} (${topName})`;
|
||||
}
|
||||
const [inserted] = await this.db
|
||||
.insert(categories)
|
||||
.values({
|
||||
vehicleId,
|
||||
catalogVehicleId: null as string | null,
|
||||
name: trMap.get(node.name) ?? node.name,
|
||||
name: displayName,
|
||||
nameOriginal: node.name,
|
||||
parentId,
|
||||
externalId: node.gid || null,
|
||||
linkPath: isLeaf ? node.url || null : null,
|
||||
linkWid: null as string | null,
|
||||
externalId: isLeaf ? (node.code ?? null) : (node.cid ?? null),
|
||||
// Groups keep their Vehicle.aspx drill URL but are marked
|
||||
// non-leaf via linkWid="emex-group" + hasSubgroups so the
|
||||
// tree UI / lazy-drill never treat them as parts leaves.
|
||||
linkPath: node.url,
|
||||
linkWid: isLeaf ? (null as string | null) : "emex-group",
|
||||
hasSubgroups: !isLeaf,
|
||||
hasParts: isLeaf,
|
||||
source: "emex" as const,
|
||||
})
|
||||
.onConflictDoNothing()
|
||||
@@ -316,16 +343,19 @@ export class CategoriesService {
|
||||
|
||||
if (inserted) {
|
||||
dbCategories.push(inserted);
|
||||
// A top group passes its own display name down as the
|
||||
// disambiguation context for its sub-groups.
|
||||
const ctx = parentId === null ? displayName : topName;
|
||||
if (node.children?.length) {
|
||||
await insertNodes(node.children, inserted.id);
|
||||
await insertNodes(node.children, inserted.id, ctx);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
await insertNodes(tree, null);
|
||||
await insertNodes(tree, null, null);
|
||||
this.logger.log(
|
||||
`Stored ${dbCategories.length} EMEX categories (tree) for ${vehicle.vin}`,
|
||||
`Stored ${dbCategories.length} EMEX categories (Vehicle.aspx tree) for ${vehicle.vin}`,
|
||||
);
|
||||
} else if (emexResult.categories.length > 0) {
|
||||
// Flat fallback: insert all categories without hierarchy
|
||||
@@ -524,6 +554,65 @@ export class CategoriesService {
|
||||
return children;
|
||||
}
|
||||
|
||||
// EMEX Vehicle.aspx group node — drill its href to Unit.aspx leaves on
|
||||
// demand. Top groups already have their sub-groups seeded (returned from
|
||||
// the DB-children check above), so this only fires for sub-groups whose
|
||||
// unit leaves haven't been fetched yet. emex carries no catalogInfo, so
|
||||
// this must run before the pl24 serviceName guard below.
|
||||
if (category.source === "emex" && category.linkWid === "emex-group" && linkPath) {
|
||||
try {
|
||||
const leaves = await this.emexService.drillVehicleNode(linkPath);
|
||||
if (leaves.length > 0) {
|
||||
const trMap = await this.translationsService.translateMany(
|
||||
leaves.map((l) => l.name).filter(Boolean),
|
||||
);
|
||||
const seenNames = new Set<string>();
|
||||
const insertData: (typeof categories.$inferInsert)[] = [];
|
||||
for (const l of leaves) {
|
||||
const name = trMap.get(l.name) ?? l.name;
|
||||
if (seenNames.has(name)) continue; // unique index is per-name
|
||||
seenNames.add(name);
|
||||
insertData.push({
|
||||
vehicleId: category.vehicleId,
|
||||
catalogVehicleId: category.catalogVehicleId,
|
||||
name,
|
||||
nameOriginal: l.name,
|
||||
parentId: categoryId,
|
||||
externalId: l.code ?? l.cid ?? null,
|
||||
linkPath: l.url, // Unit.aspx → direct parts fetch
|
||||
linkWid: null as string | null,
|
||||
hasSubgroups: false,
|
||||
hasParts: true,
|
||||
source: "emex" as const,
|
||||
});
|
||||
}
|
||||
|
||||
children = await this.db
|
||||
.insert(categories)
|
||||
.values(insertData)
|
||||
.onConflictDoNothing()
|
||||
.returning();
|
||||
|
||||
if (children.length < insertData.length) {
|
||||
children = await this.db
|
||||
.select()
|
||||
.from(categories)
|
||||
.where(eq(categories.parentId, categoryId));
|
||||
}
|
||||
|
||||
await this.redis.del(`cat:tree:${category.vehicleId}`);
|
||||
}
|
||||
} catch (err) {
|
||||
this.logger.error(
|
||||
`EMEX node drill failed for ${categoryId} (${linkPath}): ${(err as Error).message}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Unit leaves carry a Unit.aspx linkPath + no DB children, so
|
||||
// enrichWithSchemaImages already marks them as leaves (children: []).
|
||||
return children.length > 0 ? this.enrichWithSchemaImages(children) : children;
|
||||
}
|
||||
|
||||
if (!catalogInfo?.serviceName || !linkPath) {
|
||||
return [];
|
||||
}
|
||||
@@ -930,6 +1019,27 @@ export class CategoriesService {
|
||||
// No subgroups → leaf; fall through to the parts path below.
|
||||
}
|
||||
|
||||
// EMEX Vehicle.aspx group node (linkWid="emex-group") — a parent, never a
|
||||
// parts leaf. Return its children (seeded sub-groups, or Unit leaves drilled
|
||||
// on demand by getChildren). Mirrors the pl24 group-node branch above; an
|
||||
// emex group is never legitimately empty, so surface a load error if drill
|
||||
// comes back empty rather than a misleading "no parts" leaf.
|
||||
if (category.source === "emex" && category.linkWid === "emex-group" && category.vehicleId) {
|
||||
const emexChildren = await this.getChildren(categoryId);
|
||||
const base = {
|
||||
id: category.id,
|
||||
name: category.name,
|
||||
description: category.nameOriginal || null,
|
||||
parentId: category.parentId || null,
|
||||
parts: [],
|
||||
schemaPics: [],
|
||||
hotspots: [],
|
||||
};
|
||||
return emexChildren.length > 0
|
||||
? { ...base, children: emexChildren }
|
||||
: { ...base, loadError: true };
|
||||
}
|
||||
|
||||
// Leaf category — get or fetch parts
|
||||
let loadError = false;
|
||||
let discoveredChildren: any[] = [];
|
||||
@@ -1674,7 +1784,9 @@ export class CategoriesService {
|
||||
// rendered as empty leaves.
|
||||
const isLeaf =
|
||||
c.source === "emex"
|
||||
? !!c.linkPath && dbChildCount === 0
|
||||
? // emex-group = Vehicle.aspx parent node (never a leaf); a unit leaf
|
||||
// has a Unit.aspx linkPath and no DB children.
|
||||
c.linkWid !== "emex-group" && !!c.linkPath && dbChildCount === 0
|
||||
: c.source === "parts-catalogs"
|
||||
? (() => {
|
||||
if (dbChildCount > 0) return false; // already has children → parent
|
||||
|
||||
1395
apps/api/src/integrations/emex/__fixtures__/vehicle-engblock.html
Normal file
1395
apps/api/src/integrations/emex/__fixtures__/vehicle-engblock.html
Normal file
File diff suppressed because it is too large
Load Diff
2795
apps/api/src/integrations/emex/__fixtures__/vehicle-powertrain.html
Normal file
2795
apps/api/src/integrations/emex/__fixtures__/vehicle-powertrain.html
Normal file
File diff suppressed because it is too large
Load Diff
1275
apps/api/src/integrations/emex/__fixtures__/vehicle-root.html
Normal file
1275
apps/api/src/integrations/emex/__fixtures__/vehicle-root.html
Normal file
File diff suppressed because it is too large
Load Diff
111
apps/api/src/integrations/emex/emex-tree.parser.spec.ts
Normal file
111
apps/api/src/integrations/emex/emex-tree.parser.spec.ts
Normal file
@@ -0,0 +1,111 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { parseUnitLeaves, parseVehicleTree } from "./emex-tree.parser";
|
||||
|
||||
const fixture = (name: string) => readFileSync(join(__dirname, "__fixtures__", name), "utf-8");
|
||||
|
||||
// Captured 2026-06-05 from emexdwc.ae catalog FORD202201 (VIN WF0JXXGAJJBJ88319, vid=0):
|
||||
// vehicle-root.html — Vehicle.aspx root (nothing selected)
|
||||
// vehicle-powertrain.html — Vehicle.aspx?cid=3 (Powertrain selected; 162 raw / 81 unique units)
|
||||
// vehicle-engblock.html — Vehicle.aspx?cid=8289 (Eng Block sub-group; 22 raw / 11 unique units)
|
||||
const ROOT = fixture("vehicle-root.html");
|
||||
const POWERTRAIN = fixture("vehicle-powertrain.html");
|
||||
const ENGBLOCK = fixture("vehicle-engblock.html");
|
||||
|
||||
const TOP_LABELS = ["Chassis", "Powertrain", "Electrical", "Body And Paint"];
|
||||
|
||||
describe("parseVehicleTree — 2-level group tree from the Vehicle.aspx sidebar", () => {
|
||||
it("extracts exactly the 7 FCSD top groups from the root page", () => {
|
||||
const tree = parseVehicleTree(ROOT);
|
||||
expect(tree).toHaveLength(7);
|
||||
expect(tree.every((n) => n.kind === "group")).toBe(true);
|
||||
// The 7 section roots carry small cids.
|
||||
expect(tree.map((n) => n.cid).sort((a, b) => Number(a) - Number(b))).toEqual([
|
||||
"1",
|
||||
"2",
|
||||
"3",
|
||||
"4",
|
||||
"5",
|
||||
"8",
|
||||
"9",
|
||||
]);
|
||||
const labels = tree.map((n) => n.name);
|
||||
for (const l of TOP_LABELS) {
|
||||
expect(labels.some((x) => x.startsWith(l))).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("yields the same 7 top groups from a drilled node page (menu is identical everywhere)", () => {
|
||||
// Powertrain is the SELECTED node here (rendered as bare text, no <a>) —
|
||||
// must still be captured via its onclick URL.
|
||||
const tree = parseVehicleTree(POWERTRAIN);
|
||||
expect(tree).toHaveLength(7);
|
||||
const powertrain = tree.find((n) => n.cid === "3");
|
||||
expect(powertrain?.name).toBe("Powertrain");
|
||||
});
|
||||
|
||||
it("nests Powertrain sub-groups under the Powertrain top group", () => {
|
||||
const tree = parseVehicleTree(ROOT);
|
||||
const powertrain = tree.find((n) => n.cid === "3");
|
||||
expect(powertrain).toBeDefined();
|
||||
const subCids = powertrain!.children.map((c) => c.cid);
|
||||
const subNames = powertrain!.children.map((c) => c.name);
|
||||
// Automatic Transmission is cid=4004 — the group whose Unit leaves include
|
||||
// the torque converter. Confirms the converter's home is under Powertrain.
|
||||
expect(subCids).toContain("4004");
|
||||
expect(subNames).toContain("Automatic Transmission");
|
||||
expect(subNames).toContain("Clutch");
|
||||
expect(subNames).toContain("Manual Transmission");
|
||||
expect(powertrain!.children.every((c) => c.kind === "group")).toBe(true);
|
||||
});
|
||||
|
||||
it("gives every group a verbatim Vehicle.aspx drill URL with cid + ssd", () => {
|
||||
const tree = parseVehicleTree(ROOT);
|
||||
const all = tree.flatMap((t) => [t, ...t.children]);
|
||||
expect(all.length).toBeGreaterThan(120); // ~136 total group nodes
|
||||
for (const n of all) {
|
||||
expect(n.url).toMatch(/^https:\/\/emexdwc\.ae\/Vehicle\.aspx\?/);
|
||||
expect(n.url).toMatch(/[?&]cid=\d+/);
|
||||
expect(n.url).toMatch(/[?&]ssd=\$\*/); // opaque per-node ssd present
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseUnitLeaves — Unit.aspx leaves from a drilled group page", () => {
|
||||
it("extracts all Powertrain unit leaves incl. the torque converter (closes the taxonomy gap)", () => {
|
||||
const leaves = parseUnitLeaves(POWERTRAIN);
|
||||
// 81 distinct unit hrefs (the raw HTML lists each twice — image + text
|
||||
// anchor — so dedup-by-href halves the ~162 raw matches).
|
||||
expect(leaves.length).toBeGreaterThan(60);
|
||||
expect(leaves.every((l) => l.kind === "leaf")).toBe(true);
|
||||
|
||||
const converter = leaves.find((l) => l.code === "3070160A");
|
||||
expect(converter).toBeDefined();
|
||||
expect(converter!.name).toMatch(/Converter/i);
|
||||
expect(converter!.name).toMatch(/Automatic Trans/i);
|
||||
// Leaf URL is a self-contained Unit.aspx fetch URL (the existing Unit
|
||||
// parts-parser can GET it directly).
|
||||
expect(converter!.url).toMatch(/^https:\/\/emexdwc\.ae\/Unit\.aspx\?/);
|
||||
expect(converter!.url).toMatch(/[?&]ssd=\$\*/);
|
||||
});
|
||||
|
||||
it("parses section codes off the 'CODE: name' label", () => {
|
||||
const leaves = parseUnitLeaves(POWERTRAIN);
|
||||
// Service Engine And Short Block is 3030105A.
|
||||
const engine = leaves.find((l) => l.code === "3030105A");
|
||||
expect(engine).toBeDefined();
|
||||
expect(engine!.name).toMatch(/Service Engine And Short Block/i);
|
||||
// Every leaf got a non-empty code + name (the label always has the prefix).
|
||||
expect(leaves.every((l) => l.code && l.code.length >= 4)).toBe(true);
|
||||
expect(leaves.every((l) => l.name.length > 0)).toBe(true);
|
||||
});
|
||||
|
||||
it("narrows to fewer leaves when a SUB-group is drilled (Eng Block ⊂ Powertrain)", () => {
|
||||
const sub = parseUnitLeaves(ENGBLOCK);
|
||||
const top = parseUnitLeaves(POWERTRAIN);
|
||||
expect(sub.length).toBeGreaterThan(0);
|
||||
expect(sub.length).toBeLessThan(top.length); // 11 < 81 (unique)
|
||||
expect(sub.every((l) => l.url.includes("Unit.aspx"))).toBe(true);
|
||||
});
|
||||
});
|
||||
185
apps/api/src/integrations/emex/emex-tree.parser.ts
Normal file
185
apps/api/src/integrations/emex/emex-tree.parser.ts
Normal file
@@ -0,0 +1,185 @@
|
||||
/**
|
||||
* emex Vehicle.aspx hierarchical-tree parsers.
|
||||
*
|
||||
* Background: emex exposes the SAME catalog through two navigation trees.
|
||||
* • QuickGroups.aspx — a FLAT list of ~351 functional Laximo categories
|
||||
* (gid 10xxx). This is what the legacy seeder (`parseCategoryList` in
|
||||
* emex.service.ts) scrapes, producing the unsearchable "everything on one
|
||||
* page" experience.
|
||||
* • Vehicle.aspx — the OEM-microfiche tree: a small set of TOP groups
|
||||
* (Chassis, Powertrain, Electrical, …) → SUB groups (Automatic
|
||||
* Transmission, Clutch, …) → Unit.aspx LEAVES carrying section codes
|
||||
* (e.g. "3070160A: Converter, 4 Speed Automatic Trans") and parts.
|
||||
*
|
||||
* The Vehicle.aspx tree is fully traversable by plain HTTP GET: every node
|
||||
* link carries its own opaque `ssd` in the href, copied verbatim hop-to-hop
|
||||
* (no cookies, no __VIEWSTATE postback — verified 2026-06-05). These parsers
|
||||
* turn the raw HTML into a tree we can seed into the generic `categories`
|
||||
* table (parentId chains + hasSubgroups/hasParts), so the existing tree UI
|
||||
* renders it exactly like pl24/pcat — no frontend changes.
|
||||
*
|
||||
* Pure functions, no DI — unit-tested against captured fixtures in
|
||||
* __fixtures__/. Regex-based (no cheerio) to match the existing emex parsers.
|
||||
*/
|
||||
|
||||
import type { EmexVehicleTreeNode } from "./emex.types";
|
||||
|
||||
const EMEX_BASE_URL = "https://emexdwc.ae";
|
||||
|
||||
/**
|
||||
* Section roots (top groups) carry a small `cid`. For FORD202201 the 7 roots
|
||||
* are cid 1,2,3,4,5,8,9 (Information, Chassis, Powertrain, Electrical, Body,
|
||||
* Fluids, Suspension/Accessories); every other menu entry is a sub-group. If a
|
||||
* future catalog numbers differently and NO node has cid ≤ this max, the tree
|
||||
* degrades gracefully to a single flat group level (see parseVehicleTree).
|
||||
*/
|
||||
const TOP_GROUP_CID_MAX = 9;
|
||||
|
||||
function decodeEntities(s: string): string {
|
||||
return s
|
||||
.replace(/'/g, "'")
|
||||
.replace(/&/g, "&")
|
||||
.replace(/"/g, '"')
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/ /g, " ");
|
||||
}
|
||||
|
||||
function stripTags(s: string): string {
|
||||
return s.replace(/<[^>]+>/g, " ");
|
||||
}
|
||||
|
||||
function cidOf(url: string): string | null {
|
||||
const m = url.match(/[?&]cid=(\d+)/);
|
||||
return m ? m[1] : null;
|
||||
}
|
||||
|
||||
/** A raw category entry from the persistent Vehicle.aspx sidebar menu. */
|
||||
interface MenuNode {
|
||||
cid: string;
|
||||
name: string;
|
||||
url: string; // absolute Vehicle.aspx drill URL (ssd embedded)
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract the persistent category sidebar (identical on every Vehicle.aspx
|
||||
* page). Each item is:
|
||||
* <div class="guayaquil_categoryitem"
|
||||
* onclick="window.location='Vehicle.aspx?...cid=N&ssd=...'"
|
||||
* onmouseout=... onmouseover=... style="margin-left:0px">
|
||||
* <a href="Vehicle.aspx?...">LABEL</a> // normal
|
||||
* </div>
|
||||
* The CURRENTLY-SELECTED node renders with class `guayaquil_categoryitem_selected`
|
||||
* and its label as bare text (no <a>), but still carries the onclick URL — so
|
||||
* we read cid+url+ssd uniformly from the onclick, and the label from the inner
|
||||
* text (anchor or bare). Document order is preserved (top group, then its
|
||||
* sub-groups, then the next top group …).
|
||||
*/
|
||||
function parseMenuNodes(html: string): MenuNode[] {
|
||||
const itemRx =
|
||||
/<div class="guayaquil_categoryitem(?:_selected)?"\s+onclick="window\.location='(Vehicle\.aspx\?[\s\S]*?)'"[^>]*>([\s\S]*?)<\/div>/g;
|
||||
const nodes: MenuNode[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const m of html.matchAll(itemRx)) {
|
||||
const url = decodeEntities(m[1]);
|
||||
const cid = cidOf(url);
|
||||
if (!cid) continue;
|
||||
const name = decodeEntities(stripTags(m[2])).replace(/\s+/g, " ").trim();
|
||||
if (!name) continue;
|
||||
// Dedup on cid: the menu can repeat an entry; first occurrence wins (keeps
|
||||
// document order for parent inference).
|
||||
if (seen.has(cid)) continue;
|
||||
seen.add(cid);
|
||||
nodes.push({ cid, name, url: `${EMEX_BASE_URL}/${url}` });
|
||||
}
|
||||
return nodes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the 2-level GROUP tree (top groups → sub-groups) from a Vehicle.aspx
|
||||
* page's sidebar. Works on ANY page (root or a drilled node) since the menu is
|
||||
* identical everywhere. Leaves (Unit.aspx) are NOT included here — they're
|
||||
* fetched lazily per sub-group via {@link parseUnitLeaves}.
|
||||
*
|
||||
* Classification: cid ≤ TOP_GROUP_CID_MAX → top group; otherwise a sub-group
|
||||
* attached to the most recent top group seen in document order. If the catalog
|
||||
* exposes no small-cid roots, every node becomes a flat top-level group.
|
||||
*/
|
||||
export function parseVehicleTree(html: string): EmexVehicleTreeNode[] {
|
||||
const menu = parseMenuNodes(html);
|
||||
const hasRoots = menu.some((n) => Number(n.cid) <= TOP_GROUP_CID_MAX);
|
||||
|
||||
const tops: EmexVehicleTreeNode[] = [];
|
||||
let current: EmexVehicleTreeNode | null = null;
|
||||
|
||||
for (const n of menu) {
|
||||
const isTop = !hasRoots || Number(n.cid) <= TOP_GROUP_CID_MAX;
|
||||
const node: EmexVehicleTreeNode = {
|
||||
cid: n.cid,
|
||||
name: n.name,
|
||||
code: null,
|
||||
url: n.url,
|
||||
kind: "group",
|
||||
imageUrl: null,
|
||||
children: [],
|
||||
};
|
||||
if (isTop) {
|
||||
tops.push(node);
|
||||
current = node;
|
||||
} else if (current) {
|
||||
current.children.push(node);
|
||||
} else {
|
||||
// Sub-group seen before any top group (shouldn't happen for Ford, but be
|
||||
// safe): promote it to a top-level group.
|
||||
tops.push(node);
|
||||
}
|
||||
}
|
||||
return tops;
|
||||
}
|
||||
|
||||
/** title → img.laximo.net url, harvested from the unit illustration thumbnails. */
|
||||
function parseUnitImages(html: string): Map<string, string> {
|
||||
const map = new Map<string, string>();
|
||||
const rx =
|
||||
/<div class="guayaquil_zoom"\s+full="(https:\/\/img\.laximo\.net[^"]+)"\s+title="([^"]*)"/g;
|
||||
for (const m of html.matchAll(rx)) {
|
||||
const url = decodeEntities(m[1]);
|
||||
const title = decodeEntities(m[2]).replace(/\s+/g, " ").trim();
|
||||
if (title && !map.has(title)) map.set(title, url);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the Unit.aspx LEAVES from a drilled node page's content area. The
|
||||
* sidebar contains only Vehicle.aspx links, so every Unit.aspx anchor on the
|
||||
* page belongs to the currently-drilled group. Each leaf label is
|
||||
* "<sectionCode>: <name>" (e.g. "3070160A: Converter, 4 Speed Automatic Trans")
|
||||
* and the href is a self-contained Unit.aspx fetch URL (cid + ssd + path_id).
|
||||
*/
|
||||
export function parseUnitLeaves(html: string): EmexVehicleTreeNode[] {
|
||||
const images = parseUnitImages(html);
|
||||
const anchorRx = /<a href="(Unit\.aspx\?[^"]+)"[^>]*>([\s\S]*?)<\/a>/g;
|
||||
const leaves: EmexVehicleTreeNode[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const m of html.matchAll(anchorRx)) {
|
||||
const url = decodeEntities(m[1]);
|
||||
if (seen.has(url)) continue;
|
||||
seen.add(url);
|
||||
const label = decodeEntities(stripTags(m[2])).replace(/\s+/g, " ").trim();
|
||||
if (!label) continue;
|
||||
const codeMatch = label.match(/^([0-9A-Za-z]+):\s*(.*)$/);
|
||||
const code = codeMatch ? codeMatch[1] : null;
|
||||
const name = codeMatch ? codeMatch[2].trim() : label;
|
||||
leaves.push({
|
||||
cid: cidOf(url) ?? "",
|
||||
name,
|
||||
code,
|
||||
url: `${EMEX_BASE_URL}/${url}`,
|
||||
kind: "leaf",
|
||||
imageUrl: images.get(label) ?? null,
|
||||
children: [],
|
||||
});
|
||||
}
|
||||
return leaves;
|
||||
}
|
||||
@@ -22,6 +22,7 @@ import { ConfigService } from "@nestjs/config";
|
||||
import { ProxyAgent } from "undici";
|
||||
import { isBackfillContext } from "../../jobs/prefetch-context";
|
||||
import { RedisService } from "../../redis/redis.service";
|
||||
import { parseUnitLeaves, parseVehicleTree } from "./emex-tree.parser";
|
||||
import { EmexBrowserService } from "./emex.browser";
|
||||
import { createEmptyDecodedVehicle, mapEmexResponse } from "./emex.mapper";
|
||||
import {
|
||||
@@ -33,6 +34,7 @@ import {
|
||||
type EmexPart,
|
||||
type EmexPartsResult,
|
||||
type EmexScraperResponse,
|
||||
type EmexVehicleTreeNode,
|
||||
} from "./emex.types";
|
||||
|
||||
const EMEX_BASE_URL = "https://emexdwc.ae";
|
||||
@@ -386,6 +388,33 @@ export class EmexService {
|
||||
return cats;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch + parse the hierarchical Vehicle.aspx tree (top groups → sub groups)
|
||||
* for a decoded vehicle. One plain GET of the catalog root; the persistent
|
||||
* sidebar carries the whole 2-level group tree (units are drilled lazily per
|
||||
* sub-group via {@link drillVehicleNode}). Returns [] on any failure so the
|
||||
* caller falls back to the flat QuickGroups categories.
|
||||
*/
|
||||
private async fetchVehicleTree(
|
||||
catalogCode: string,
|
||||
vid: string,
|
||||
ssd: string,
|
||||
): Promise<EmexVehicleTreeNode[]> {
|
||||
const url = `${EMEX_BASE_URL}/Vehicle.aspx?c=${catalogCode}&vid=${vid}&ssd=${encodeURIComponent(ssd)}`;
|
||||
const html = await this.fetchEmexHtml(url);
|
||||
return parseVehicleTree(html);
|
||||
}
|
||||
|
||||
/**
|
||||
* Drill a Vehicle.aspx group node (its verbatim href, ssd embedded) to its
|
||||
* Unit.aspx leaves. Used by the lazy category drill (categories.service
|
||||
* getChildren) when an emex group is first expanded.
|
||||
*/
|
||||
async drillVehicleNode(vehicleAspxUrl: string): Promise<EmexVehicleTreeNode[]> {
|
||||
const html = await this.fetchEmexHtml(vehicleAspxUrl);
|
||||
return parseUnitLeaves(html);
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine brand name from EMEX catalog code (e.g. "BMW202501" → "BMW").
|
||||
*/
|
||||
@@ -455,6 +484,19 @@ export class EmexService {
|
||||
}
|
||||
}
|
||||
|
||||
// Hierarchical Vehicle.aspx tree (primary nav); the flat `categories`
|
||||
// above is the fallback. One GET of the catalog root yields the full
|
||||
// 2-level group tree; units are drilled lazily per sub-group on expand.
|
||||
let categoryTree: EmexVehicleTreeNode[] = [];
|
||||
if (v.catalogCode && v.ssd) {
|
||||
try {
|
||||
categoryTree = await this.fetchVehicleTree(v.catalogCode, v.vid ?? "0", v.ssd);
|
||||
this.logger.log(`EMEX HTTP: parsed ${categoryTree.length} Vehicle.aspx top groups`);
|
||||
} catch (err) {
|
||||
this.logger.warn(`EMEX Vehicle.aspx tree fetch failed: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Build a response compatible with mapEmexResponse
|
||||
const response: EmexScraperResponse = {
|
||||
success: true,
|
||||
@@ -476,7 +518,7 @@ export class EmexService {
|
||||
},
|
||||
quickGroupsUrl: v.quickGroupsUrl || null,
|
||||
categories: categories.map((c) => ({ gid: c.gid || "", name: c.name, url: c.url })),
|
||||
categoryTree: [],
|
||||
categoryTree,
|
||||
timestamp: new Date().toISOString(),
|
||||
};
|
||||
|
||||
@@ -541,6 +583,17 @@ export class EmexService {
|
||||
}
|
||||
}
|
||||
|
||||
// Hierarchical Vehicle.aspx tree (primary nav); flat `categories` = fallback.
|
||||
let categoryTree: EmexVehicleTreeNode[] = [];
|
||||
if (v.catalogCode && v.ssd) {
|
||||
try {
|
||||
categoryTree = await this.fetchVehicleTree(v.catalogCode, v.vid ?? "0", v.ssd);
|
||||
this.logger.log(`EMEX HTTP: parsed ${categoryTree.length} Vehicle.aspx top groups`);
|
||||
} catch (err) {
|
||||
this.logger.warn(`EMEX Vehicle.aspx tree fetch failed: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
const response: EmexScraperResponse = {
|
||||
success: true,
|
||||
source: "emexdwc.ae",
|
||||
@@ -561,7 +614,7 @@ export class EmexService {
|
||||
},
|
||||
quickGroupsUrl: v.quickGroupsUrl || null,
|
||||
categories: categories.map((c) => ({ gid: c.gid || "", name: c.name, url: c.url })),
|
||||
categoryTree: [],
|
||||
categoryTree,
|
||||
timestamp: new Date().toISOString(),
|
||||
};
|
||||
|
||||
@@ -608,6 +661,17 @@ export class EmexService {
|
||||
}
|
||||
}
|
||||
|
||||
// Hierarchical Vehicle.aspx tree (primary nav); flat `categories` = fallback.
|
||||
let categoryTree: EmexVehicleTreeNode[] = [];
|
||||
if (v.catalogCode && v.ssd) {
|
||||
try {
|
||||
categoryTree = await this.fetchVehicleTree(v.catalogCode, v.vid ?? "0", v.ssd);
|
||||
this.logger.log(`EMEX HTTP: parsed ${categoryTree.length} Vehicle.aspx top groups`);
|
||||
} catch (err) {
|
||||
this.logger.warn(`EMEX Vehicle.aspx tree fetch failed: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
const response: EmexScraperResponse = {
|
||||
success: true,
|
||||
source: "emexdwc.ae",
|
||||
@@ -628,7 +692,7 @@ export class EmexService {
|
||||
},
|
||||
quickGroupsUrl: v.quickGroupsUrl || null,
|
||||
categories: categories.map((c) => ({ gid: c.gid || "", name: c.name, url: c.url })),
|
||||
categoryTree: [],
|
||||
categoryTree,
|
||||
timestamp: new Date().toISOString(),
|
||||
};
|
||||
|
||||
@@ -862,6 +926,28 @@ export class EmexService {
|
||||
* parts — caller treats that as "fall back to browser".
|
||||
*/
|
||||
private async fetchCategoryPartsViaHttp(categoryUrl: string): Promise<EmexPartsResult> {
|
||||
// Vehicle.aspx-tree leaves already point straight at Unit.aspx — skip the
|
||||
// QuickDetails→Unit hop and extract parts directly.
|
||||
if (/Unit\.aspx/i.test(categoryUrl)) {
|
||||
const unitHtml = await this.fetchEmexHtml(categoryUrl);
|
||||
const extracted = this.extractEmexPartsFromHtml(unitHtml);
|
||||
if (
|
||||
extracted.schemaImageUrl &&
|
||||
(extracted.schemaWidth === 0 || extracted.schemaHeight === 0)
|
||||
) {
|
||||
try {
|
||||
const dims = await this.fetchImageDims(extracted.schemaImageUrl);
|
||||
if (dims.width > 0 && dims.height > 0) {
|
||||
extracted.schemaWidth = dims.width;
|
||||
extracted.schemaHeight = dims.height;
|
||||
}
|
||||
} catch (err) {
|
||||
this.logger.debug(`Image dims fetch failed: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
return extracted;
|
||||
}
|
||||
|
||||
// 1) QuickDetails.aspx → Unit.aspx anchor
|
||||
const qdHtml = await this.fetchEmexHtml(categoryUrl);
|
||||
const unitMatch = qdHtml.match(/href="([^"]*Unit\.aspx[^"]*)"/i);
|
||||
|
||||
@@ -63,6 +63,29 @@ export interface EmexCategoryTreeNode {
|
||||
children: EmexCategoryTreeNode[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Node in the hierarchical Vehicle.aspx tree (the OEM-microfiche taxonomy):
|
||||
* top groups → sub groups → Unit.aspx leaves. Produced by emex-tree.parser.ts
|
||||
* and seeded into the generic `categories` table (parentId chains +
|
||||
* hasSubgroups/hasParts), so the same tree UI renders it as pl24/pcat.
|
||||
*/
|
||||
export type EmexTreeNodeKind = "group" | "leaf";
|
||||
|
||||
export interface EmexVehicleTreeNode {
|
||||
/** emex category id (`cid`) for groups; the parent group's cid for leaves. */
|
||||
cid: string;
|
||||
/** Display name (for leaves the "NNNN:" section-code prefix is stripped). */
|
||||
name: string;
|
||||
/** Section code for a leaf (e.g. "3070160A"); null for groups. */
|
||||
code: string | null;
|
||||
/** Absolute, verbatim URL: Vehicle.aspx (group, to drill) or Unit.aspx (leaf, to fetch parts). */
|
||||
url: string;
|
||||
kind: EmexTreeNodeKind;
|
||||
/** Schema illustration (img.laximo.net gif) for a leaf, when listed; else null. */
|
||||
imageUrl: string | null;
|
||||
children: EmexVehicleTreeNode[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Part data from EMEX
|
||||
*/
|
||||
@@ -142,7 +165,7 @@ export interface EmexScraperResponse {
|
||||
quickGroupsUrl?: string | null;
|
||||
parsedOptions?: EmexParsedOptions;
|
||||
categories?: EmexCategory[];
|
||||
categoryTree?: EmexCategoryTreeNode[];
|
||||
categoryTree?: EmexVehicleTreeNode[];
|
||||
sampleParts?: EmexPart[];
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
@@ -528,7 +528,9 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy {
|
||||
|
||||
private isLeafLinkPath(linkPath: string | null, source: string): boolean {
|
||||
if (!linkPath) return false;
|
||||
if (source === "emex") return true; // EMEX leaves always have a URL
|
||||
// EMEX: Vehicle.aspx group nodes are parents to drill; Unit.aspx (hierarchical
|
||||
// tree) and QuickDetails.aspx (legacy flat) leaves carry parts.
|
||||
if (source === "emex") return !linkPath.includes("Vehicle.aspx");
|
||||
if (source === "parts-catalogs") return linkPath.startsWith("pcat:"); // pcat: prefix = leaf
|
||||
// PL24 leaf indicators
|
||||
return (
|
||||
|
||||
Reference in New Issue
Block a user