feat(emex): seed catalogs from hierarchical Vehicle.aspx tree

emex catalogs were seeded from the flat QuickGroups taxonomy (~351
unsearchable leaves on one page). Switch the seeder to the Vehicle.aspx
OEM-microfiche tree (top groups -> sub groups -> Unit.aspx leaves), which
also reaches units that have no QuickGroups gid (e.g. the torque
converter, OEM 5205547 on the Fiesta).

- emex-tree.parser.ts: parseVehicleTree / parseUnitLeaves — pure,
  GET-based (per-href ssd, no browser/postback). Unit-tested vs fixtures.
- emex.service: fetchVehicleTree + drillVehicleNode; all 3 decode methods
  now populate categoryTree; Unit.aspx leaves fetch parts directly.
- categories.service: revive the dormant tree-insert branch with group
  markers (linkWid="emex-group" / hasSubgroups), add an emex getChildren
  lazy-drill branch + emex-group parent guard, tighten leaf detection,
  disambiguate colliding sub-group names.
- prefetch: emex Vehicle.aspx = parent to drill; Unit/QuickDetails = leaf.

No DDL and no frontend changes — the generic tree UI renders it like
pl24/pcat. Existing flat emex categories re-seed lazily once deleted.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

apps/api/src/integrations/emex/emex-tree.parser.spec.ts

@@ -0,0 +1,111 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { describe, expect, it } from "vitest";
+import { parseUnitLeaves, parseVehicleTree } from "./emex-tree.parser";
+
+const fixture = (name: string) => readFileSync(join(__dirname, "__fixtures__", name), "utf-8");
+
+// Captured 2026-06-05 from emexdwc.ae catalog FORD202201 (VIN WF0JXXGAJJBJ88319, vid=0):
+//   vehicle-root.html       — Vehicle.aspx root (nothing selected)
+//   vehicle-powertrain.html — Vehicle.aspx?cid=3 (Powertrain selected; 162 raw / 81 unique units)
+//   vehicle-engblock.html   — Vehicle.aspx?cid=8289 (Eng Block sub-group; 22 raw / 11 unique units)
+const ROOT = fixture("vehicle-root.html");
+const POWERTRAIN = fixture("vehicle-powertrain.html");
+const ENGBLOCK = fixture("vehicle-engblock.html");
+
+const TOP_LABELS = ["Chassis", "Powertrain", "Electrical", "Body And Paint"];
+
+describe("parseVehicleTree — 2-level group tree from the Vehicle.aspx sidebar", () => {
+  it("extracts exactly the 7 FCSD top groups from the root page", () => {
+    const tree = parseVehicleTree(ROOT);
+    expect(tree).toHaveLength(7);
+    expect(tree.every((n) => n.kind === "group")).toBe(true);
+    // The 7 section roots carry small cids.
+    expect(tree.map((n) => n.cid).sort((a, b) => Number(a) - Number(b))).toEqual([
+      "1",
+      "2",
+      "3",
+      "4",
+      "5",
+      "8",
+      "9",
+    ]);
+    const labels = tree.map((n) => n.name);
+    for (const l of TOP_LABELS) {
+      expect(labels.some((x) => x.startsWith(l))).toBe(true);
+    }
+  });
+
+  it("yields the same 7 top groups from a drilled node page (menu is identical everywhere)", () => {
+    // Powertrain is the SELECTED node here (rendered as bare text, no <a>) —
+    // must still be captured via its onclick URL.
+    const tree = parseVehicleTree(POWERTRAIN);
+    expect(tree).toHaveLength(7);
+    const powertrain = tree.find((n) => n.cid === "3");
+    expect(powertrain?.name).toBe("Powertrain");
+  });
+
+  it("nests Powertrain sub-groups under the Powertrain top group", () => {
+    const tree = parseVehicleTree(ROOT);
+    const powertrain = tree.find((n) => n.cid === "3");
+    expect(powertrain).toBeDefined();
+    const subCids = powertrain!.children.map((c) => c.cid);
+    const subNames = powertrain!.children.map((c) => c.name);
+    // Automatic Transmission is cid=4004 — the group whose Unit leaves include
+    // the torque converter. Confirms the converter's home is under Powertrain.
+    expect(subCids).toContain("4004");
+    expect(subNames).toContain("Automatic Transmission");
+    expect(subNames).toContain("Clutch");
+    expect(subNames).toContain("Manual Transmission");
+    expect(powertrain!.children.every((c) => c.kind === "group")).toBe(true);
+  });
+
+  it("gives every group a verbatim Vehicle.aspx drill URL with cid + ssd", () => {
+    const tree = parseVehicleTree(ROOT);
+    const all = tree.flatMap((t) => [t, ...t.children]);
+    expect(all.length).toBeGreaterThan(120); // ~136 total group nodes
+    for (const n of all) {
+      expect(n.url).toMatch(/^https:\/\/emexdwc\.ae\/Vehicle\.aspx\?/);
+      expect(n.url).toMatch(/[?&]cid=\d+/);
+      expect(n.url).toMatch(/[?&]ssd=\$\*/); // opaque per-node ssd present
+    }
+  });
+});
+
+describe("parseUnitLeaves — Unit.aspx leaves from a drilled group page", () => {
+  it("extracts all Powertrain unit leaves incl. the torque converter (closes the taxonomy gap)", () => {
+    const leaves = parseUnitLeaves(POWERTRAIN);
+    // 81 distinct unit hrefs (the raw HTML lists each twice — image + text
+    // anchor — so dedup-by-href halves the ~162 raw matches).
+    expect(leaves.length).toBeGreaterThan(60);
+    expect(leaves.every((l) => l.kind === "leaf")).toBe(true);
+
+    const converter = leaves.find((l) => l.code === "3070160A");
+    expect(converter).toBeDefined();
+    expect(converter!.name).toMatch(/Converter/i);
+    expect(converter!.name).toMatch(/Automatic Trans/i);
+    // Leaf URL is a self-contained Unit.aspx fetch URL (the existing Unit
+    // parts-parser can GET it directly).
+    expect(converter!.url).toMatch(/^https:\/\/emexdwc\.ae\/Unit\.aspx\?/);
+    expect(converter!.url).toMatch(/[?&]ssd=\$\*/);
+  });
+
+  it("parses section codes off the 'CODE: name' label", () => {
+    const leaves = parseUnitLeaves(POWERTRAIN);
+    // Service Engine And Short Block is 3030105A.
+    const engine = leaves.find((l) => l.code === "3030105A");
+    expect(engine).toBeDefined();
+    expect(engine!.name).toMatch(/Service Engine And Short Block/i);
+    // Every leaf got a non-empty code + name (the label always has the prefix).
+    expect(leaves.every((l) => l.code && l.code.length >= 4)).toBe(true);
+    expect(leaves.every((l) => l.name.length > 0)).toBe(true);
+  });
+
+  it("narrows to fewer leaves when a SUB-group is drilled (Eng Block ⊂ Powertrain)", () => {
+    const sub = parseUnitLeaves(ENGBLOCK);
+    const top = parseUnitLeaves(POWERTRAIN);
+    expect(sub.length).toBeGreaterThan(0);
+    expect(sub.length).toBeLessThan(top.length); // 11 < 81 (unique)
+    expect(sub.every((l) => l.url.includes("Unit.aspx"))).toBe(true);
+  });
+});

apps/api/src/integrations/emex/emex-tree.parser.ts

@@ -0,0 +1,185 @@
+/**
+ * emex Vehicle.aspx hierarchical-tree parsers.
+ *
+ * Background: emex exposes the SAME catalog through two navigation trees.
+ *   • QuickGroups.aspx — a FLAT list of ~351 functional Laximo categories
+ *     (gid 10xxx). This is what the legacy seeder (`parseCategoryList` in
+ *     emex.service.ts) scrapes, producing the unsearchable "everything on one
+ *     page" experience.
+ *   • Vehicle.aspx — the OEM-microfiche tree: a small set of TOP groups
+ *     (Chassis, Powertrain, Electrical, …) → SUB groups (Automatic
+ *     Transmission, Clutch, …) → Unit.aspx LEAVES carrying section codes
+ *     (e.g. "3070160A: Converter, 4 Speed Automatic Trans") and parts.
+ *
+ * The Vehicle.aspx tree is fully traversable by plain HTTP GET: every node
+ * link carries its own opaque `ssd` in the href, copied verbatim hop-to-hop
+ * (no cookies, no __VIEWSTATE postback — verified 2026-06-05). These parsers
+ * turn the raw HTML into a tree we can seed into the generic `categories`
+ * table (parentId chains + hasSubgroups/hasParts), so the existing tree UI
+ * renders it exactly like pl24/pcat — no frontend changes.
+ *
+ * Pure functions, no DI — unit-tested against captured fixtures in
+ * __fixtures__/. Regex-based (no cheerio) to match the existing emex parsers.
+ */
+
+import type { EmexVehicleTreeNode } from "./emex.types";
+
+const EMEX_BASE_URL = "https://emexdwc.ae";
+
+/**
+ * Section roots (top groups) carry a small `cid`. For FORD202201 the 7 roots
+ * are cid 1,2,3,4,5,8,9 (Information, Chassis, Powertrain, Electrical, Body,
+ * Fluids, Suspension/Accessories); every other menu entry is a sub-group. If a
+ * future catalog numbers differently and NO node has cid ≤ this max, the tree
+ * degrades gracefully to a single flat group level (see parseVehicleTree).
+ */
+const TOP_GROUP_CID_MAX = 9;
+
+function decodeEntities(s: string): string {
+  return s
+    .replace(/'/g, "'")
+    .replace(/&/g, "&")
+    .replace(/"/g, '"')
+    .replace(/&lt;/g, "<")
+    .replace(/&gt;/g, ">")
+    .replace(/&nbsp;/g, " ");
+}
+
+function stripTags(s: string): string {
+  return s.replace(/<[^>]+>/g, " ");
+}
+
+function cidOf(url: string): string | null {
+  const m = url.match(/[?&]cid=(\d+)/);
+  return m ? m[1] : null;
+}
+
+/** A raw category entry from the persistent Vehicle.aspx sidebar menu. */
+interface MenuNode {
+  cid: string;
+  name: string;
+  url: string; // absolute Vehicle.aspx drill URL (ssd embedded)
+}
+
+/**
+ * Extract the persistent category sidebar (identical on every Vehicle.aspx
+ * page). Each item is:
+ *   <div class="guayaquil_categoryitem"
+ *        onclick="window.location='Vehicle.aspx?...cid=N&ssd=...'"
+ *        onmouseout=... onmouseover=... style="margin-left:0px">
+ *     <a href="Vehicle.aspx?...">LABEL</a>        // normal
+ *   </div>
+ * The CURRENTLY-SELECTED node renders with class `guayaquil_categoryitem_selected`
+ * and its label as bare text (no <a>), but still carries the onclick URL — so
+ * we read cid+url+ssd uniformly from the onclick, and the label from the inner
+ * text (anchor or bare). Document order is preserved (top group, then its
+ * sub-groups, then the next top group …).
+ */
+function parseMenuNodes(html: string): MenuNode[] {
+  const itemRx =
+    /<div class="guayaquil_categoryitem(?:_selected)?"\s+onclick="window\.location=&#39;(Vehicle\.aspx\?[\s\S]*?)&#39;"[^>]*>([\s\S]*?)<\/div>/g;
+  const nodes: MenuNode[] = [];
+  const seen = new Set<string>();
+  for (const m of html.matchAll(itemRx)) {
+    const url = decodeEntities(m[1]);
+    const cid = cidOf(url);
+    if (!cid) continue;
+    const name = decodeEntities(stripTags(m[2])).replace(/\s+/g, " ").trim();
+    if (!name) continue;
+    // Dedup on cid: the menu can repeat an entry; first occurrence wins (keeps
+    // document order for parent inference).
+    if (seen.has(cid)) continue;
+    seen.add(cid);
+    nodes.push({ cid, name, url: `${EMEX_BASE_URL}/${url}` });
+  }
+  return nodes;
+}
+
+/**
+ * Build the 2-level GROUP tree (top groups → sub-groups) from a Vehicle.aspx
+ * page's sidebar. Works on ANY page (root or a drilled node) since the menu is
+ * identical everywhere. Leaves (Unit.aspx) are NOT included here — they're
+ * fetched lazily per sub-group via {@link parseUnitLeaves}.
+ *
+ * Classification: cid ≤ TOP_GROUP_CID_MAX → top group; otherwise a sub-group
+ * attached to the most recent top group seen in document order. If the catalog
+ * exposes no small-cid roots, every node becomes a flat top-level group.
+ */
+export function parseVehicleTree(html: string): EmexVehicleTreeNode[] {
+  const menu = parseMenuNodes(html);
+  const hasRoots = menu.some((n) => Number(n.cid) <= TOP_GROUP_CID_MAX);
+
+  const tops: EmexVehicleTreeNode[] = [];
+  let current: EmexVehicleTreeNode | null = null;
+
+  for (const n of menu) {
+    const isTop = !hasRoots || Number(n.cid) <= TOP_GROUP_CID_MAX;
+    const node: EmexVehicleTreeNode = {
+      cid: n.cid,
+      name: n.name,
+      code: null,
+      url: n.url,
+      kind: "group",
+      imageUrl: null,
+      children: [],
+    };
+    if (isTop) {
+      tops.push(node);
+      current = node;
+    } else if (current) {
+      current.children.push(node);
+    } else {
+      // Sub-group seen before any top group (shouldn't happen for Ford, but be
+      // safe): promote it to a top-level group.
+      tops.push(node);
+    }
+  }
+  return tops;
+}
+
+/** title → img.laximo.net url, harvested from the unit illustration thumbnails. */
+function parseUnitImages(html: string): Map<string, string> {
+  const map = new Map<string, string>();
+  const rx =
+    /<div class="guayaquil_zoom"\s+full="(https:\/\/img\.laximo\.net[^"]+)"\s+title="([^"]*)"/g;
+  for (const m of html.matchAll(rx)) {
+    const url = decodeEntities(m[1]);
+    const title = decodeEntities(m[2]).replace(/\s+/g, " ").trim();
+    if (title && !map.has(title)) map.set(title, url);
+  }
+  return map;
+}
+
+/**
+ * Parse the Unit.aspx LEAVES from a drilled node page's content area. The
+ * sidebar contains only Vehicle.aspx links, so every Unit.aspx anchor on the
+ * page belongs to the currently-drilled group. Each leaf label is
+ * "<sectionCode>: <name>" (e.g. "3070160A: Converter, 4 Speed Automatic Trans")
+ * and the href is a self-contained Unit.aspx fetch URL (cid + ssd + path_id).
+ */
+export function parseUnitLeaves(html: string): EmexVehicleTreeNode[] {
+  const images = parseUnitImages(html);
+  const anchorRx = /<a href="(Unit\.aspx\?[^"]+)"[^>]*>([\s\S]*?)<\/a>/g;
+  const leaves: EmexVehicleTreeNode[] = [];
+  const seen = new Set<string>();
+  for (const m of html.matchAll(anchorRx)) {
+    const url = decodeEntities(m[1]);
+    if (seen.has(url)) continue;
+    seen.add(url);
+    const label = decodeEntities(stripTags(m[2])).replace(/\s+/g, " ").trim();
+    if (!label) continue;
+    const codeMatch = label.match(/^([0-9A-Za-z]+):\s*(.*)$/);
+    const code = codeMatch ? codeMatch[1] : null;
+    const name = codeMatch ? codeMatch[2].trim() : label;
+    leaves.push({
+      cid: cidOf(url) ?? "",
+      name,
+      code,
+      url: `${EMEX_BASE_URL}/${url}`,
+      kind: "leaf",
+      imageUrl: images.get(label) ?? null,
+      children: [],
+    });
+  }
+  return leaves;
+}

apps/api/src/integrations/emex/emex.service.ts

@@ -22,6 +22,7 @@ import { ConfigService } from "@nestjs/config";
 import { ProxyAgent } from "undici";
 import { isBackfillContext } from "../../jobs/prefetch-context";
 import { RedisService } from "../../redis/redis.service";
+import { parseUnitLeaves, parseVehicleTree } from "./emex-tree.parser";
 import { EmexBrowserService } from "./emex.browser";
 import { createEmptyDecodedVehicle, mapEmexResponse } from "./emex.mapper";
 import {
@@ -33,6 +34,7 @@ import {
   type EmexPart,
   type EmexPartsResult,
   type EmexScraperResponse,
+  type EmexVehicleTreeNode,
 } from "./emex.types";
 
 const EMEX_BASE_URL = "https://emexdwc.ae";
@@ -386,6 +388,33 @@ export class EmexService {
     return cats;
   }
 
+  /**
+   * Fetch + parse the hierarchical Vehicle.aspx tree (top groups → sub groups)
+   * for a decoded vehicle. One plain GET of the catalog root; the persistent
+   * sidebar carries the whole 2-level group tree (units are drilled lazily per
+   * sub-group via {@link drillVehicleNode}). Returns [] on any failure so the
+   * caller falls back to the flat QuickGroups categories.
+   */
+  private async fetchVehicleTree(
+    catalogCode: string,
+    vid: string,
+    ssd: string,
+  ): Promise<EmexVehicleTreeNode[]> {
+    const url = `${EMEX_BASE_URL}/Vehicle.aspx?c=${catalogCode}&vid=${vid}&ssd=${encodeURIComponent(ssd)}`;
+    const html = await this.fetchEmexHtml(url);
+    return parseVehicleTree(html);
+  }
+
+  /**
+   * Drill a Vehicle.aspx group node (its verbatim href, ssd embedded) to its
+   * Unit.aspx leaves. Used by the lazy category drill (categories.service
+   * getChildren) when an emex group is first expanded.
+   */
+  async drillVehicleNode(vehicleAspxUrl: string): Promise<EmexVehicleTreeNode[]> {
+    const html = await this.fetchEmexHtml(vehicleAspxUrl);
+    return parseUnitLeaves(html);
+  }
+
   /**
    * Determine brand name from EMEX catalog code (e.g. "BMW202501" → "BMW").
    */
@@ -455,6 +484,19 @@ export class EmexService {
       }
     }
 
+    // Hierarchical Vehicle.aspx tree (primary nav); the flat `categories`
+    // above is the fallback. One GET of the catalog root yields the full
+    // 2-level group tree; units are drilled lazily per sub-group on expand.
+    let categoryTree: EmexVehicleTreeNode[] = [];
+    if (v.catalogCode && v.ssd) {
+      try {
+        categoryTree = await this.fetchVehicleTree(v.catalogCode, v.vid ?? "0", v.ssd);
+        this.logger.log(`EMEX HTTP: parsed ${categoryTree.length} Vehicle.aspx top groups`);
+      } catch (err) {
+        this.logger.warn(`EMEX Vehicle.aspx tree fetch failed: ${(err as Error).message}`);
+      }
+    }
+
     // Build a response compatible with mapEmexResponse
     const response: EmexScraperResponse = {
       success: true,
@@ -476,7 +518,7 @@ export class EmexService {
       },
       quickGroupsUrl: v.quickGroupsUrl || null,
       categories: categories.map((c) => ({ gid: c.gid || "", name: c.name, url: c.url })),
-      categoryTree: [],
+      categoryTree,
       timestamp: new Date().toISOString(),
     };
 
@@ -541,6 +583,17 @@ export class EmexService {
         }
       }
 
+      // Hierarchical Vehicle.aspx tree (primary nav); flat `categories` = fallback.
+      let categoryTree: EmexVehicleTreeNode[] = [];
+      if (v.catalogCode && v.ssd) {
+        try {
+          categoryTree = await this.fetchVehicleTree(v.catalogCode, v.vid ?? "0", v.ssd);
+          this.logger.log(`EMEX HTTP: parsed ${categoryTree.length} Vehicle.aspx top groups`);
+        } catch (err) {
+          this.logger.warn(`EMEX Vehicle.aspx tree fetch failed: ${(err as Error).message}`);
+        }
+      }
+
       const response: EmexScraperResponse = {
         success: true,
         source: "emexdwc.ae",
@@ -561,7 +614,7 @@ export class EmexService {
         },
         quickGroupsUrl: v.quickGroupsUrl || null,
         categories: categories.map((c) => ({ gid: c.gid || "", name: c.name, url: c.url })),
-        categoryTree: [],
+        categoryTree,
         timestamp: new Date().toISOString(),
       };
 
@@ -608,6 +661,17 @@ export class EmexService {
         }
       }
 
+      // Hierarchical Vehicle.aspx tree (primary nav); flat `categories` = fallback.
+      let categoryTree: EmexVehicleTreeNode[] = [];
+      if (v.catalogCode && v.ssd) {
+        try {
+          categoryTree = await this.fetchVehicleTree(v.catalogCode, v.vid ?? "0", v.ssd);
+          this.logger.log(`EMEX HTTP: parsed ${categoryTree.length} Vehicle.aspx top groups`);
+        } catch (err) {
+          this.logger.warn(`EMEX Vehicle.aspx tree fetch failed: ${(err as Error).message}`);
+        }
+      }
+
       const response: EmexScraperResponse = {
         success: true,
         source: "emexdwc.ae",
@@ -628,7 +692,7 @@ export class EmexService {
         },
         quickGroupsUrl: v.quickGroupsUrl || null,
         categories: categories.map((c) => ({ gid: c.gid || "", name: c.name, url: c.url })),
-        categoryTree: [],
+        categoryTree,
         timestamp: new Date().toISOString(),
       };
 
@@ -862,6 +926,28 @@ export class EmexService {
    * parts — caller treats that as "fall back to browser".
    */
   private async fetchCategoryPartsViaHttp(categoryUrl: string): Promise<EmexPartsResult> {
+    // Vehicle.aspx-tree leaves already point straight at Unit.aspx — skip the
+    // QuickDetails→Unit hop and extract parts directly.
+    if (/Unit\.aspx/i.test(categoryUrl)) {
+      const unitHtml = await this.fetchEmexHtml(categoryUrl);
+      const extracted = this.extractEmexPartsFromHtml(unitHtml);
+      if (
+        extracted.schemaImageUrl &&
+        (extracted.schemaWidth === 0 || extracted.schemaHeight === 0)
+      ) {
+        try {
+          const dims = await this.fetchImageDims(extracted.schemaImageUrl);
+          if (dims.width > 0 && dims.height > 0) {
+            extracted.schemaWidth = dims.width;
+            extracted.schemaHeight = dims.height;
+          }
+        } catch (err) {
+          this.logger.debug(`Image dims fetch failed: ${(err as Error).message}`);
+        }
+      }
+      return extracted;
+    }
+
     // 1) QuickDetails.aspx → Unit.aspx anchor
     const qdHtml = await this.fetchEmexHtml(categoryUrl);
     const unitMatch = qdHtml.match(/href="([^"]*Unit\.aspx[^"]*)"/i);

apps/api/src/integrations/emex/emex.types.ts

@@ -63,6 +63,29 @@ export interface EmexCategoryTreeNode {
   children: EmexCategoryTreeNode[];
 }
 
+/**
+ * Node in the hierarchical Vehicle.aspx tree (the OEM-microfiche taxonomy):
+ * top groups → sub groups → Unit.aspx leaves. Produced by emex-tree.parser.ts
+ * and seeded into the generic `categories` table (parentId chains +
+ * hasSubgroups/hasParts), so the same tree UI renders it as pl24/pcat.
+ */
+export type EmexTreeNodeKind = "group" | "leaf";
+
+export interface EmexVehicleTreeNode {
+  /** emex category id (`cid`) for groups; the parent group's cid for leaves. */
+  cid: string;
+  /** Display name (for leaves the "NNNN:" section-code prefix is stripped). */
+  name: string;
+  /** Section code for a leaf (e.g. "3070160A"); null for groups. */
+  code: string | null;
+  /** Absolute, verbatim URL: Vehicle.aspx (group, to drill) or Unit.aspx (leaf, to fetch parts). */
+  url: string;
+  kind: EmexTreeNodeKind;
+  /** Schema illustration (img.laximo.net gif) for a leaf, when listed; else null. */
+  imageUrl: string | null;
+  children: EmexVehicleTreeNode[];
+}
+
 /**
  * Part data from EMEX
  */
@@ -142,7 +165,7 @@ export interface EmexScraperResponse {
   quickGroupsUrl?: string | null;
   parsedOptions?: EmexParsedOptions;
   categories?: EmexCategory[];
-  categoryTree?: EmexCategoryTreeNode[];
+  categoryTree?: EmexVehicleTreeNode[];
   sampleParts?: EmexPart[];
   timestamp: string;
 }