dev #111

Merged
root merged 37 commits from dev into main 2026-06-05 21:12:55 +03:00
16 changed files with 6583 additions and 161 deletions

View File

@@ -13,6 +13,7 @@ import {
} from "../database/schema/core";
import { PL24Service } from "../integrations/pl24/pl24.service";
import {
type PL24DecodedCategory,
PL24_SERVICE_CATALOGS,
SERVICE_DISPLAY_NAMES,
SERVICE_TO_BRAND,
@@ -497,58 +498,12 @@ export class CatalogService {
);
if (pl24Categories.length > 0) {
let dbCategories = await this.db
.select()
.from(categories)
.where(eq(categories.catalogVehicleId, catalogVehicleId));
if (dbCategories.length === 0) {
const insertData = pl24Categories.map((c) => ({
catalogVehicleId,
vehicleId: null as string | null,
name: c.nameTr || c.nameEn,
nameOriginal: c.nameEn,
parentId: null as string | null,
externalId: c.code,
linkPath: c.linkPath || null,
linkWid: c.linkWid || null,
source: "pl24" as const,
}));
const inserted = await this.db
.insert(categories)
.values(insertData)
.onConflictDoNothing()
.returning();
dbCategories =
inserted.length > 0
? inserted
: await this.db
.select()
.from(categories)
.where(eq(categories.catalogVehicleId, catalogVehicleId));
await this.db
.update(catalogVehicles)
.set({ categoriesFetched: true, updatedAt: new Date() })
.where(eq(catalogVehicles.id, catalogVehicleId));
}
const externalIdToUuid = new Map(dbCategories.map((c) => [c.externalId, c.id]));
const tree = this.buildTree(
pl24Categories.map((c) => ({
id: externalIdToUuid.get(c.code) ?? c.code,
catalogVehicleId,
vehicleId: null,
name: c.nameTr || c.nameEn,
nameOriginal: c.nameEn,
parentId: null,
externalId: c.code,
linkPath: c.linkPath || null,
linkWid: c.linkWid || null,
source: "pl24",
})),
const tree = await this.persistAndBuildLegacyTree(
catalogVehicleId,
pl24Categories,
cacheKey,
);
await this.redis.setJson(cacheKey, tree, 7200);
return tree;
if (tree.length > 0) return tree;
}
}
} catch (err) {
@@ -600,59 +555,12 @@ export class CatalogService {
}
if (pl24Categories.length > 0) {
// Persist _all_ variant to DB as canonical records
let dbCategories = await this.db
.select()
.from(categories)
.where(eq(categories.catalogVehicleId, catalogVehicleId));
if (dbCategories.length === 0) {
const insertData = pl24Categories.map((c) => ({
catalogVehicleId,
vehicleId: null as string | null,
name: c.nameTr || c.nameEn,
nameOriginal: c.nameEn,
parentId: null as string | null,
externalId: c.code,
linkPath: c.linkPath || null,
linkWid: c.linkWid || null,
source: "pl24" as const,
}));
const inserted = await this.db
.insert(categories)
.values(insertData)
.onConflictDoNothing()
.returning();
dbCategories =
inserted.length > 0
? inserted
: await this.db
.select()
.from(categories)
.where(eq(categories.catalogVehicleId, catalogVehicleId));
await this.db
.update(catalogVehicles)
.set({ categoriesFetched: true, updatedAt: new Date() })
.where(eq(catalogVehicles.id, catalogVehicleId));
}
const externalIdToUuid = new Map(dbCategories.map((c) => [c.externalId, c.id]));
const tree = this.buildTree(
pl24Categories.map((c) => ({
id: externalIdToUuid.get(c.code) ?? c.code,
catalogVehicleId,
vehicleId: null,
name: c.nameTr || c.nameEn,
nameOriginal: c.nameEn,
parentId: null,
externalId: c.code,
linkPath: c.linkPath || null,
linkWid: c.linkWid || null,
source: "pl24",
})),
const tree = await this.persistAndBuildLegacyTree(
catalogVehicleId,
pl24Categories,
cacheKey,
);
await this.redis.setJson(cacheKey, tree, 7200);
return tree;
if (tree.length > 0) return tree;
}
}
} catch (err) {
@@ -758,7 +666,9 @@ export class CatalogService {
and(eq(categories.catalogVehicleId, catalogVehicleId), sql`${categories.parentId} IS NULL`),
);
if (dbCategories.length === 0) return [];
const tree = this.buildTree(dbCategories);
// Same nav-crumb read filter as persistAndBuildLegacyTree (self-heals
// previously-persisted Hyundai "Portal" / Nissan eu.nissan.biz junk rows).
const tree = this.buildTree(dbCategories.filter((c) => !this.isNavCrumbLink(c.linkPath)));
await this.redis.setJson(cacheKey, tree, 3600); // 1h cache for DB fallback
return tree;
}
@@ -828,6 +738,20 @@ export class CatalogService {
.from(categories)
.where(eq(categories.parentId, categoryId));
// Self-healing: drop nav-crumb junk children (e.g. "Portal"/eu.nissan.biz rows
// persisted by the old parser) and delete them so this node re-fetches the real
// subgroups instead of returning a junk-only child set.
const navCrumbChildren = children.filter((c) => this.isNavCrumbLink(c.linkPath));
if (navCrumbChildren.length > 0) {
await this.db.delete(categories).where(
inArray(
categories.id,
navCrumbChildren.map((c) => c.id),
),
);
children = children.filter((c) => !this.isNavCrumbLink(c.linkPath));
}
// Self-healing: if the linkPath is a leaf path but DB has children, those are
// stale records created by the previous case-insensitive bug. Delete and re-fetch.
if (children.length > 0 && linkPath && this.isLeafPath(linkPath)) {
@@ -861,14 +785,19 @@ export class CatalogService {
gearbox,
);
if (subGroups.length > 0) {
const seenNames = new Set<string>();
const unique = subGroups.filter((sg) => {
if (seenNames.has(sg.name)) return false;
seenNames.add(sg.name);
return true;
});
// After dropping nav-crumb junk the subgroup set can be empty (upstream
// returned only a "Portal"/external stub for this node). Guard the insert —
// drizzle .values([]) throws — and fall through to the leaf/parts path,
// which yields a clean empty node instead of a 500.
const seenNames = new Set<string>();
const unique = subGroups.filter((sg) => {
if (this.isNavCrumbLink(sg.linkPath)) return false;
if (seenNames.has(sg.name)) return false;
seenNames.add(sg.name);
return true;
});
if (unique.length > 0) {
const insertData = unique.map((sg) => ({
catalogVehicleId,
vehicleId: null as string | null,
@@ -1321,6 +1250,93 @@ export class CatalogService {
});
}
/**
* Header/breadcrumb nav links that PL24's group.action HTML leaks into the
* category table as pseudo-categories. They are not real part groups and drill
* to nothing. Mirrors the filter in categories.service (VIN-decode path).
* Keeps Volvo's `vin-group.action?...group1=...` real categories.
*/
private isNavCrumbLink(linkPath?: string | null): boolean {
if (!linkPath) return false;
if (/(portal|logout)\.action/i.test(linkPath)) return true;
if (/vehicle\.action/i.test(linkPath)) return true;
if (linkPath.includes("vin-group.action") && !linkPath.includes("group1=")) return true;
// External absolute URL embedded anywhere that is not a partslink .action
// endpoint. Nissan's "Repair & Maintenance Information" comes back as
// `/nissan/nissan_parts/https://eu.nissan.biz/` — the basePath prefix means an
// anchored ^https check misses it, so match `://` anywhere. Real PL24 linkPaths
// are relative (group.action / json-*.action / pcat: / emex:) and never contain `://`.
if (linkPath.includes("://") && !/\.action(\?|$)/i.test(linkPath)) return true;
return false;
}
/**
* Persist freshly-fetched PL24 legacy main groups (Ford/Volvo/Opel/Hyundai-Kia/
* Nissan) and return the tree built FROM the DB rows — so every node id is a real
* UUID, never a raw upstream linkPath (which would 404 on drill). Filters nav-crumb
* junk and dedupes by name first (matching the PSA block's discipline), so
* onConflictDoNothing can't silently drop a colliding row and leak a raw-id node.
*/
private async persistAndBuildLegacyTree(
catalogVehicleId: string,
pl24Categories: PL24DecodedCategory[],
cacheKey: string,
): Promise<any[]> {
const seenNames = new Set<string>();
const clean = pl24Categories.filter((c) => {
if (this.isNavCrumbLink(c.linkPath)) return false;
const name = c.nameTr || c.nameEn;
if (!name || seenNames.has(name)) return false;
seenNames.add(name);
return true;
});
if (clean.length === 0) return [];
let dbCategories = await this.db
.select()
.from(categories)
.where(eq(categories.catalogVehicleId, catalogVehicleId))
.orderBy(categories.createdAt, categories.id);
if (dbCategories.length === 0) {
await this.db
.insert(categories)
.values(
clean.map((c) => ({
catalogVehicleId,
vehicleId: null as string | null,
name: c.nameTr || c.nameEn,
nameOriginal: c.nameEn,
parentId: null as string | null,
externalId: c.code,
linkPath: c.linkPath || null,
linkWid: c.linkWid || null,
source: "pl24" as const,
})),
)
.onConflictDoNothing();
// Always re-select (never trust .returning(): onConflictDoNothing drops
// name-collision rows, which would otherwise leave gaps that fall back to
// raw-path ids). Deterministic order so the tree is stable across rebuilds.
dbCategories = await this.db
.select()
.from(categories)
.where(eq(categories.catalogVehicleId, catalogVehicleId))
.orderBy(categories.createdAt, categories.id);
await this.db
.update(catalogVehicles)
.set({ categoriesFetched: true, updatedAt: new Date() })
.where(eq(catalogVehicles.id, catalogVehicleId));
}
// Filter on read too — drops nav-crumb junk that was persisted before this
// fix (e.g. existing Hyundai "Portal" / Nissan eu.nissan.biz rows), so the
// response self-heals without a DB migration.
const tree = this.buildTree(dbCategories.filter((c) => !this.isNavCrumbLink(c.linkPath)));
await this.redis.setJson(cacheKey, tree, 7200);
return tree;
}
private buildTree(items: any[]): any[] {
const map = new Map<string, any>();
const roots: any[] = [];

View File

@@ -258,21 +258,24 @@ export class CategoriesService {
const emexResult = await this.emexService.decodeVin(vehicle.vin);
if (emexResult) {
const rawData = emexResult.raw as Record<string, unknown>;
// Hierarchical Vehicle.aspx tree (top groups → sub groups). Units are
// NOT seeded here — they're drilled lazily per sub-group by
// getChildren on first expand (mirrors the pl24/pcat lazy model).
const tree = rawData?.emexCategoryTree as
| Array<{
cid?: string;
name: string;
gid: string | null;
code?: string | null;
url: string | null;
children: any[];
kind?: "group" | "leaf";
children?: any[];
}>
| undefined;
if (tree && tree.length > 0) {
// Recursive tree insertion from QuickGroups.aspx
this.logger.log(`Inserting ${tree.length} EMEX top-level category groups recursively`);
this.logger.log(`Inserting EMEX Vehicle.aspx tree: ${tree.length} top groups`);
// Phase 1: walk the tree, collect every unique English name so we
// can bulk-translate before touching the DB.
// Bulk-translate every node name before touching the DB.
const uniqueNames = new Set<string>();
const collect = (nodes: Array<{ name: string; children?: any[] }>) => {
for (const node of nodes) {
@@ -282,33 +285,57 @@ export class CategoriesService {
};
collect(tree);
const trMap = await this.translationsService.translateMany([...uniqueNames]);
const tr = (n: string) => trMap.get(n) ?? n;
// The categories unique index is (vehicle, catalogVehicle, name,
// source), so two sub-groups that share a display name across
// different top groups (e.g. "Speakers" under both Electrical and
// Accessories) would silently collide and orphan a branch. Detect
// such names and disambiguate them with their top-group name.
const subNameCounts = new Map<string, number>();
for (const top of tree) {
for (const sub of top.children ?? []) {
const dn = tr(sub.name);
subNameCounts.set(dn, (subNameCounts.get(dn) ?? 0) + 1);
}
}
const insertNodes = async (
nodes: Array<{
cid?: string;
name: string;
gid: string | null;
code?: string | null;
url: string | null;
kind?: "group" | "leaf";
children?: any[];
}>,
parentId: string | null,
topName: string | null,
) => {
for (const node of nodes) {
if (!node.name) continue;
const isLeaf = !node.children?.length;
// URL'si olmayan leaf node'lar dead-end — kaydetme
if (isLeaf && !node.url) continue;
if (!node.name || !node.url) continue;
const isLeaf = node.kind === "leaf";
let displayName = tr(node.name);
// Disambiguate colliding sub-group names with their top group.
if (parentId !== null && topName && (subNameCounts.get(displayName) ?? 0) > 1) {
displayName = `${displayName} (${topName})`;
}
const [inserted] = await this.db
.insert(categories)
.values({
vehicleId,
catalogVehicleId: null as string | null,
name: trMap.get(node.name) ?? node.name,
name: displayName,
nameOriginal: node.name,
parentId,
externalId: node.gid || null,
linkPath: isLeaf ? node.url || null : null,
linkWid: null as string | null,
externalId: isLeaf ? (node.code ?? null) : (node.cid ?? null),
// Groups keep their Vehicle.aspx drill URL but are marked
// non-leaf via linkWid="emex-group" + hasSubgroups so the
// tree UI / lazy-drill never treat them as parts leaves.
linkPath: node.url,
linkWid: isLeaf ? (null as string | null) : "emex-group",
hasSubgroups: !isLeaf,
hasParts: isLeaf,
source: "emex" as const,
})
.onConflictDoNothing()
@@ -316,16 +343,19 @@ export class CategoriesService {
if (inserted) {
dbCategories.push(inserted);
// A top group passes its own display name down as the
// disambiguation context for its sub-groups.
const ctx = parentId === null ? displayName : topName;
if (node.children?.length) {
await insertNodes(node.children, inserted.id);
await insertNodes(node.children, inserted.id, ctx);
}
}
}
};
await insertNodes(tree, null);
await insertNodes(tree, null, null);
this.logger.log(
`Stored ${dbCategories.length} EMEX categories (tree) for ${vehicle.vin}`,
`Stored ${dbCategories.length} EMEX categories (Vehicle.aspx tree) for ${vehicle.vin}`,
);
} else if (emexResult.categories.length > 0) {
// Flat fallback: insert all categories without hierarchy
@@ -390,7 +420,11 @@ export class CategoriesService {
// Build tree
const tree = this.buildTree(dbCategories);
await this.redis.setJson(cacheKey, tree, 3600);
// Cache a populated tree for an hour; an EMPTY tree (transient decode/proxy
// failure) only for 60s so a blip doesn't poison the catalog for an hour —
// it self-heals on the next request after the source recovers, while still
// throttling re-decode attempts during a real outage.
await this.redis.setJson(cacheKey, tree, tree.length > 0 ? 3600 : 60);
return tree;
}
@@ -524,6 +558,65 @@ export class CategoriesService {
return children;
}
// EMEX Vehicle.aspx group node — drill its href to Unit.aspx leaves on
// demand. Top groups already have their sub-groups seeded (returned from
// the DB-children check above), so this only fires for sub-groups whose
// unit leaves haven't been fetched yet. emex carries no catalogInfo, so
// this must run before the pl24 serviceName guard below.
if (category.source === "emex" && category.linkWid === "emex-group" && linkPath) {
try {
const leaves = await this.emexService.drillVehicleNode(linkPath);
if (leaves.length > 0) {
const trMap = await this.translationsService.translateMany(
leaves.map((l) => l.name).filter(Boolean),
);
const seenNames = new Set<string>();
const insertData: (typeof categories.$inferInsert)[] = [];
for (const l of leaves) {
const name = trMap.get(l.name) ?? l.name;
if (seenNames.has(name)) continue; // unique index is per-name
seenNames.add(name);
insertData.push({
vehicleId: category.vehicleId,
catalogVehicleId: category.catalogVehicleId,
name,
nameOriginal: l.name,
parentId: categoryId,
externalId: l.code ?? l.cid ?? null,
linkPath: l.url, // Unit.aspx → direct parts fetch
linkWid: null as string | null,
hasSubgroups: false,
hasParts: true,
source: "emex" as const,
});
}
children = await this.db
.insert(categories)
.values(insertData)
.onConflictDoNothing()
.returning();
if (children.length < insertData.length) {
children = await this.db
.select()
.from(categories)
.where(eq(categories.parentId, categoryId));
}
await this.redis.del(`cat:tree:${category.vehicleId}`);
}
} catch (err) {
this.logger.error(
`EMEX node drill failed for ${categoryId} (${linkPath}): ${(err as Error).message}`,
);
}
// Unit leaves carry a Unit.aspx linkPath + no DB children, so
// enrichWithSchemaImages already marks them as leaves (children: []).
return children.length > 0 ? this.enrichWithSchemaImages(children) : children;
}
if (!catalogInfo?.serviceName || !linkPath) {
return [];
}
@@ -930,6 +1023,27 @@ export class CategoriesService {
// No subgroups → leaf; fall through to the parts path below.
}
// EMEX Vehicle.aspx group node (linkWid="emex-group") — a parent, never a
// parts leaf. Return its children (seeded sub-groups, or Unit leaves drilled
// on demand by getChildren). Mirrors the pl24 group-node branch above; an
// emex group is never legitimately empty, so surface a load error if drill
// comes back empty rather than a misleading "no parts" leaf.
if (category.source === "emex" && category.linkWid === "emex-group" && category.vehicleId) {
const emexChildren = await this.getChildren(categoryId);
const base = {
id: category.id,
name: category.name,
description: category.nameOriginal || null,
parentId: category.parentId || null,
parts: [],
schemaPics: [],
hotspots: [],
};
return emexChildren.length > 0
? { ...base, children: emexChildren }
: { ...base, loadError: true };
}
// Leaf category — get or fetch parts
let loadError = false;
let discoveredChildren: any[] = [];
@@ -1674,7 +1788,9 @@ export class CategoriesService {
// rendered as empty leaves.
const isLeaf =
c.source === "emex"
? !!c.linkPath && dbChildCount === 0
? // emex-group = Vehicle.aspx parent node (never a leaf); a unit leaf
// has a Unit.aspx linkPath and no DB children.
c.linkWid !== "emex-group" && !!c.linkPath && dbChildCount === 0
: c.source === "parts-catalogs"
? (() => {
if (dbChildCount > 0) return false; // already has children → parent

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,111 @@
import { readFileSync } from "node:fs";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import { parseUnitLeaves, parseVehicleTree } from "./emex-tree.parser";
const fixture = (name: string) => readFileSync(join(__dirname, "__fixtures__", name), "utf-8");
// Captured 2026-06-05 from emexdwc.ae catalog FORD202201 (VIN WF0JXXGAJJBJ88319, vid=0):
// vehicle-root.html — Vehicle.aspx root (nothing selected)
// vehicle-powertrain.html — Vehicle.aspx?cid=3 (Powertrain selected; 162 raw / 81 unique units)
// vehicle-engblock.html — Vehicle.aspx?cid=8289 (Eng Block sub-group; 22 raw / 11 unique units)
const ROOT = fixture("vehicle-root.html");
const POWERTRAIN = fixture("vehicle-powertrain.html");
const ENGBLOCK = fixture("vehicle-engblock.html");
const TOP_LABELS = ["Chassis", "Powertrain", "Electrical", "Body And Paint"];
describe("parseVehicleTree — 2-level group tree from the Vehicle.aspx sidebar", () => {
it("extracts exactly the 7 FCSD top groups from the root page", () => {
const tree = parseVehicleTree(ROOT);
expect(tree).toHaveLength(7);
expect(tree.every((n) => n.kind === "group")).toBe(true);
// The 7 section roots carry small cids.
expect(tree.map((n) => n.cid).sort((a, b) => Number(a) - Number(b))).toEqual([
"1",
"2",
"3",
"4",
"5",
"8",
"9",
]);
const labels = tree.map((n) => n.name);
for (const l of TOP_LABELS) {
expect(labels.some((x) => x.startsWith(l))).toBe(true);
}
});
it("yields the same 7 top groups from a drilled node page (menu is identical everywhere)", () => {
// Powertrain is the SELECTED node here (rendered as bare text, no <a>) —
// must still be captured via its onclick URL.
const tree = parseVehicleTree(POWERTRAIN);
expect(tree).toHaveLength(7);
const powertrain = tree.find((n) => n.cid === "3");
expect(powertrain?.name).toBe("Powertrain");
});
it("nests Powertrain sub-groups under the Powertrain top group", () => {
const tree = parseVehicleTree(ROOT);
const powertrain = tree.find((n) => n.cid === "3");
expect(powertrain).toBeDefined();
const subCids = powertrain!.children.map((c) => c.cid);
const subNames = powertrain!.children.map((c) => c.name);
// Automatic Transmission is cid=4004 — the group whose Unit leaves include
// the torque converter. Confirms the converter's home is under Powertrain.
expect(subCids).toContain("4004");
expect(subNames).toContain("Automatic Transmission");
expect(subNames).toContain("Clutch");
expect(subNames).toContain("Manual Transmission");
expect(powertrain!.children.every((c) => c.kind === "group")).toBe(true);
});
it("gives every group a verbatim Vehicle.aspx drill URL with cid + ssd", () => {
const tree = parseVehicleTree(ROOT);
const all = tree.flatMap((t) => [t, ...t.children]);
expect(all.length).toBeGreaterThan(120); // ~136 total group nodes
for (const n of all) {
expect(n.url).toMatch(/^https:\/\/emexdwc\.ae\/Vehicle\.aspx\?/);
expect(n.url).toMatch(/[?&]cid=\d+/);
expect(n.url).toMatch(/[?&]ssd=\$\*/); // opaque per-node ssd present
}
});
});
describe("parseUnitLeaves — Unit.aspx leaves from a drilled group page", () => {
it("extracts all Powertrain unit leaves incl. the torque converter (closes the taxonomy gap)", () => {
const leaves = parseUnitLeaves(POWERTRAIN);
// 81 distinct unit hrefs (the raw HTML lists each twice — image + text
// anchor — so dedup-by-href halves the ~162 raw matches).
expect(leaves.length).toBeGreaterThan(60);
expect(leaves.every((l) => l.kind === "leaf")).toBe(true);
const converter = leaves.find((l) => l.code === "3070160A");
expect(converter).toBeDefined();
expect(converter!.name).toMatch(/Converter/i);
expect(converter!.name).toMatch(/Automatic Trans/i);
// Leaf URL is a self-contained Unit.aspx fetch URL (the existing Unit
// parts-parser can GET it directly).
expect(converter!.url).toMatch(/^https:\/\/emexdwc\.ae\/Unit\.aspx\?/);
expect(converter!.url).toMatch(/[?&]ssd=\$\*/);
});
it("parses section codes off the 'CODE: name' label", () => {
const leaves = parseUnitLeaves(POWERTRAIN);
// Service Engine And Short Block is 3030105A.
const engine = leaves.find((l) => l.code === "3030105A");
expect(engine).toBeDefined();
expect(engine!.name).toMatch(/Service Engine And Short Block/i);
// Every leaf got a non-empty code + name (the label always has the prefix).
expect(leaves.every((l) => l.code && l.code.length >= 4)).toBe(true);
expect(leaves.every((l) => l.name.length > 0)).toBe(true);
});
it("narrows to fewer leaves when a SUB-group is drilled (Eng Block ⊂ Powertrain)", () => {
const sub = parseUnitLeaves(ENGBLOCK);
const top = parseUnitLeaves(POWERTRAIN);
expect(sub.length).toBeGreaterThan(0);
expect(sub.length).toBeLessThan(top.length); // 11 < 81 (unique)
expect(sub.every((l) => l.url.includes("Unit.aspx"))).toBe(true);
});
});

View File

@@ -0,0 +1,185 @@
/**
* emex Vehicle.aspx hierarchical-tree parsers.
*
* Background: emex exposes the SAME catalog through two navigation trees.
* • QuickGroups.aspx — a FLAT list of ~351 functional Laximo categories
* (gid 10xxx). This is what the legacy seeder (`parseCategoryList` in
* emex.service.ts) scrapes, producing the unsearchable "everything on one
* page" experience.
* • Vehicle.aspx — the OEM-microfiche tree: a small set of TOP groups
* (Chassis, Powertrain, Electrical, …) → SUB groups (Automatic
* Transmission, Clutch, …) → Unit.aspx LEAVES carrying section codes
* (e.g. "3070160A: Converter, 4 Speed Automatic Trans") and parts.
*
* The Vehicle.aspx tree is fully traversable by plain HTTP GET: every node
* link carries its own opaque `ssd` in the href, copied verbatim hop-to-hop
* (no cookies, no __VIEWSTATE postback — verified 2026-06-05). These parsers
* turn the raw HTML into a tree we can seed into the generic `categories`
* table (parentId chains + hasSubgroups/hasParts), so the existing tree UI
* renders it exactly like pl24/pcat — no frontend changes.
*
* Pure functions, no DI — unit-tested against captured fixtures in
* __fixtures__/. Regex-based (no cheerio) to match the existing emex parsers.
*/
import type { EmexVehicleTreeNode } from "./emex.types";
const EMEX_BASE_URL = "https://emexdwc.ae";
/**
* Section roots (top groups) carry a small `cid`. For FORD202201 the 7 roots
* are cid 1,2,3,4,5,8,9 (Information, Chassis, Powertrain, Electrical, Body,
* Fluids, Suspension/Accessories); every other menu entry is a sub-group. If a
* future catalog numbers differently and NO node has cid ≤ this max, the tree
* degrades gracefully to a single flat group level (see parseVehicleTree).
*/
const TOP_GROUP_CID_MAX = 9;
function decodeEntities(s: string): string {
return s
.replace(/&#39;/g, "'")
.replace(/&amp;/g, "&")
.replace(/&quot;/g, '"')
.replace(/&lt;/g, "<")
.replace(/&gt;/g, ">")
.replace(/&nbsp;/g, " ");
}
function stripTags(s: string): string {
return s.replace(/<[^>]+>/g, " ");
}
function cidOf(url: string): string | null {
const m = url.match(/[?&]cid=(\d+)/);
return m ? m[1] : null;
}
/** A raw category entry from the persistent Vehicle.aspx sidebar menu. */
interface MenuNode {
cid: string;
name: string;
url: string; // absolute Vehicle.aspx drill URL (ssd embedded)
}
/**
* Extract the persistent category sidebar (identical on every Vehicle.aspx
* page). Each item is:
* <div class="guayaquil_categoryitem"
* onclick="window.location='Vehicle.aspx?...cid=N&ssd=...'"
* onmouseout=... onmouseover=... style="margin-left:0px">
* <a href="Vehicle.aspx?...">LABEL</a> // normal
* </div>
* The CURRENTLY-SELECTED node renders with class `guayaquil_categoryitem_selected`
* and its label as bare text (no <a>), but still carries the onclick URL — so
* we read cid+url+ssd uniformly from the onclick, and the label from the inner
* text (anchor or bare). Document order is preserved (top group, then its
* sub-groups, then the next top group …).
*/
function parseMenuNodes(html: string): MenuNode[] {
const itemRx =
/<div class="guayaquil_categoryitem(?:_selected)?"\s+onclick="window\.location=&#39;(Vehicle\.aspx\?[\s\S]*?)&#39;"[^>]*>([\s\S]*?)<\/div>/g;
const nodes: MenuNode[] = [];
const seen = new Set<string>();
for (const m of html.matchAll(itemRx)) {
const url = decodeEntities(m[1]);
const cid = cidOf(url);
if (!cid) continue;
const name = decodeEntities(stripTags(m[2])).replace(/\s+/g, " ").trim();
if (!name) continue;
// Dedup on cid: the menu can repeat an entry; first occurrence wins (keeps
// document order for parent inference).
if (seen.has(cid)) continue;
seen.add(cid);
nodes.push({ cid, name, url: `${EMEX_BASE_URL}/${url}` });
}
return nodes;
}
/**
* Build the 2-level GROUP tree (top groups → sub-groups) from a Vehicle.aspx
* page's sidebar. Works on ANY page (root or a drilled node) since the menu is
* identical everywhere. Leaves (Unit.aspx) are NOT included here — they're
* fetched lazily per sub-group via {@link parseUnitLeaves}.
*
* Classification: cid ≤ TOP_GROUP_CID_MAX → top group; otherwise a sub-group
* attached to the most recent top group seen in document order. If the catalog
* exposes no small-cid roots, every node becomes a flat top-level group.
*/
export function parseVehicleTree(html: string): EmexVehicleTreeNode[] {
const menu = parseMenuNodes(html);
const hasRoots = menu.some((n) => Number(n.cid) <= TOP_GROUP_CID_MAX);
const tops: EmexVehicleTreeNode[] = [];
let current: EmexVehicleTreeNode | null = null;
for (const n of menu) {
const isTop = !hasRoots || Number(n.cid) <= TOP_GROUP_CID_MAX;
const node: EmexVehicleTreeNode = {
cid: n.cid,
name: n.name,
code: null,
url: n.url,
kind: "group",
imageUrl: null,
children: [],
};
if (isTop) {
tops.push(node);
current = node;
} else if (current) {
current.children.push(node);
} else {
// Sub-group seen before any top group (shouldn't happen for Ford, but be
// safe): promote it to a top-level group.
tops.push(node);
}
}
return tops;
}
/** title → img.laximo.net url, harvested from the unit illustration thumbnails. */
function parseUnitImages(html: string): Map<string, string> {
const map = new Map<string, string>();
const rx =
/<div class="guayaquil_zoom"\s+full="(https:\/\/img\.laximo\.net[^"]+)"\s+title="([^"]*)"/g;
for (const m of html.matchAll(rx)) {
const url = decodeEntities(m[1]);
const title = decodeEntities(m[2]).replace(/\s+/g, " ").trim();
if (title && !map.has(title)) map.set(title, url);
}
return map;
}
/**
* Parse the Unit.aspx LEAVES from a drilled node page's content area. The
* sidebar contains only Vehicle.aspx links, so every Unit.aspx anchor on the
* page belongs to the currently-drilled group. Each leaf label is
* "<sectionCode>: <name>" (e.g. "3070160A: Converter, 4 Speed Automatic Trans")
* and the href is a self-contained Unit.aspx fetch URL (cid + ssd + path_id).
*/
export function parseUnitLeaves(html: string): EmexVehicleTreeNode[] {
const images = parseUnitImages(html);
const anchorRx = /<a href="(Unit\.aspx\?[^"]+)"[^>]*>([\s\S]*?)<\/a>/g;
const leaves: EmexVehicleTreeNode[] = [];
const seen = new Set<string>();
for (const m of html.matchAll(anchorRx)) {
const url = decodeEntities(m[1]);
if (seen.has(url)) continue;
seen.add(url);
const label = decodeEntities(stripTags(m[2])).replace(/\s+/g, " ").trim();
if (!label) continue;
const codeMatch = label.match(/^([0-9A-Za-z]+):\s*(.*)$/);
const code = codeMatch ? codeMatch[1] : null;
const name = codeMatch ? codeMatch[2].trim() : label;
leaves.push({
cid: cidOf(url) ?? "",
name,
code,
url: `${EMEX_BASE_URL}/${url}`,
kind: "leaf",
imageUrl: images.get(label) ?? null,
children: [],
});
}
return leaves;
}

View File

@@ -22,6 +22,7 @@ import { ConfigService } from "@nestjs/config";
import { ProxyAgent } from "undici";
import { isBackfillContext } from "../../jobs/prefetch-context";
import { RedisService } from "../../redis/redis.service";
import { parseUnitLeaves, parseVehicleTree } from "./emex-tree.parser";
import { EmexBrowserService } from "./emex.browser";
import { createEmptyDecodedVehicle, mapEmexResponse } from "./emex.mapper";
import {
@@ -33,6 +34,7 @@ import {
type EmexPart,
type EmexPartsResult,
type EmexScraperResponse,
type EmexVehicleTreeNode,
} from "./emex.types";
const EMEX_BASE_URL = "https://emexdwc.ae";
@@ -113,6 +115,21 @@ export class EmexService {
portStart: number;
portEnd: number;
} | null;
// Floxy residential fallback — used when the DataImpulse pool throws transport
// errors. `lifetime` > 0 → sticky session (one exit IP held that many seconds,
// appended to the password as `_session-<id>_lifetime-<n>`); 0 → rotating.
private readonly emexFloxy: {
host: string;
port: number;
user: string;
pass: string;
lifetime: number;
} | null;
// Rolling Floxy sticky-session id: held for ~lifetime so a chained emex flow
// (decode → tree → drill → parts) keeps ONE exit IP and emex's IP-bound ssd
// tokens stay valid. Rotated when stale or after a transport failure.
private floxySessionId = Math.random().toString(36).slice(2, 10);
private floxySessionBornMs = Date.now();
private readonly emexDirectFallback: boolean;
constructor(
@@ -161,6 +178,26 @@ export class EmexService {
this.proxyAgent = null;
}
// Floxy residential fallback for when the DataImpulse pool flakes (connect
// timeouts / resets). On by default; creds + endpoint overridable via env.
const floxyEnabled = this.configService.get<string>("EMEX_FLOXY_FALLBACK", "true") === "true";
if (floxyEnabled) {
const fport = Number(this.configService.get("EMEX_FLOXY_PORT", 12321));
const flife = Number(this.configService.get("EMEX_FLOXY_LIFETIME", 300));
this.emexFloxy = {
host: this.configService.get<string>("EMEX_FLOXY_HOST", "residential.floxy.io"),
port: Number.isInteger(fport) && fport >= 1 && fport <= 65535 ? fport : 12321,
user: this.configService.get<string>("EMEX_FLOXY_USER", "d739255e819b"),
pass: this.configService.get<string>("EMEX_FLOXY_PASS", "9092873ba4e0"),
lifetime: Number.isInteger(flife) && flife >= 0 ? flife : 300,
};
this.logger.log(
`EMEX Floxy fallback enabled: ${this.emexFloxy.host}:${this.emexFloxy.port} (${this.emexFloxy.lifetime > 0 ? `sticky ${this.emexFloxy.lifetime}s` : "rotating"})`,
);
} else {
this.emexFloxy = null;
}
this.logger.log(`EMEX Service initialized with scraper path: ${this.scraperPath}`);
}
@@ -267,7 +304,36 @@ export class EmexService {
* without requiring authentication cookies.
*/
/** Build a fresh proxy agent on a random port from the pool (null if proxy off). */
private newProxyAgent(): ProxyAgent | null {
/** Fresh Floxy sticky-session id (new exit IP). */
private rotateFloxySession(): void {
this.floxySessionId = Math.random().toString(36).slice(2, 10);
this.floxySessionBornMs = Date.now();
}
/**
* Floxy auth password. With lifetime > 0, appends `_session-<id>_lifetime-<n>`
* to pin one exit IP across a chained emex flow; the session id rolls over
* once it nears the lifetime so we never reuse an expired sticky slot.
*/
private floxyPassword(): string {
if (!this.emexFloxy) return "";
const { pass, lifetime } = this.emexFloxy;
if (lifetime <= 0) return pass; // rotating mode
// Roll at ~80% of lifetime so a sticky IP is never reused past expiry.
if (Date.now() - this.floxySessionBornMs > lifetime * 800) this.rotateFloxySession();
return `${pass}_session-${this.floxySessionId}_lifetime-${lifetime}`;
}
private newProxyAgent(provider: "dataimpulse" | "floxy" = "dataimpulse"): ProxyAgent | null {
if (provider === "floxy") {
if (!this.emexFloxy) return null;
const { host, port, user } = this.emexFloxy;
return new ProxyAgent({
uri: `http://${user}:${this.floxyPassword()}@${host}:${port}`,
connect: { timeout: 30000 },
requestTls: { timeout: 30000 },
});
}
if (!this.emexProxy) return null;
const { host, user, pass, portStart, portEnd } = this.emexProxy;
const port = Math.floor(Math.random() * (portEnd - portStart + 1)) + portStart;
@@ -279,14 +345,19 @@ export class EmexService {
}
private async fetchEmexHtml(url: string): Promise<string> {
const maxAttempts = this.emexProxy ? 3 : 1;
// Attempt schedule: DataImpulse (rotating port, fresh agent each try to dodge
// a flaky sticky port — ~42% blip rate, undecoded-vin-rca.md EMEX #1) first,
// then the Floxy residential fallback when the DataImpulse pool throws
// transport errors. A definitive HTTP answer (e.g. 404) stops the schedule —
// it's a real result, and a different proxy IP must not "retry" it away.
const schedule: Array<"dataimpulse" | "floxy"> = [];
if (this.emexProxy) schedule.push("dataimpulse", "dataimpulse", "dataimpulse");
if (this.emexFloxy) schedule.push("floxy", "floxy");
const maxAttempts = schedule.length || 1; // 0 → single proxy-less attempt
let lastErr: unknown;
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
// Fresh agent every attempt: a single sticky DataImpulse port flakes
// intermittently ("fetch failed"/reset), silently dropping real emex hits
// (~42% blip rate observed — undecoded-vin-rca.md, EMEX #1). A new agent
// rotates the port and forces a new socket even on a single-port pool.
const agent = this.newProxyAgent();
const provider = schedule[attempt - 1]; // undefined when no proxy → direct
const agent = provider ? this.newProxyAgent(provider) : null;
try {
const res = await fetch(url, {
headers: { "User-Agent": EMEX_UA, Accept: "text/html,application/xhtml+xml" },
@@ -296,6 +367,9 @@ export class EmexService {
if (!res.ok) {
throw new Error(`EMEX HTTP ${res.status} for ${url}`);
}
if (provider === "floxy") {
this.logger.log(`EMEX fetch via Floxy fallback succeeded for ${url}`);
}
return await res.text();
} catch (err) {
lastErr = err;
@@ -311,15 +385,21 @@ export class EmexService {
`${e.message} ${String(e.cause ?? "")}`,
));
if (!transient || attempt === maxAttempts) break;
// A Floxy transport failure means the current sticky exit IP is dead —
// roll to a fresh session so the next Floxy attempt gets a new IP.
if (provider === "floxy") this.rotateFloxySession();
const next = schedule[attempt];
const via =
next === "floxy" && provider !== "floxy" ? "via Floxy fallback" : "with fresh proxy";
this.logger.warn(
`EMEX fetch transient error (attempt ${attempt}/${maxAttempts}) for ${url}: ${e.message} — retrying with fresh proxy`,
`EMEX fetch transient error (attempt ${attempt}/${maxAttempts}, ${provider}) for ${url}: ${e.message} — retrying ${via}`,
);
await new Promise((r) => setTimeout(r, 300 * attempt));
}
}
// Last resort (opt-in via EMEX_DIRECT_FALLBACK): one direct, proxy-less attempt
// for when the whole pool is down. Off by default — it exposes the origin IP.
if (this.emexProxy && this.emexDirectFallback) {
// for when both proxy pools are down. Off by default — it exposes the origin IP.
if ((this.emexProxy || this.emexFloxy) && this.emexDirectFallback) {
try {
const res = await fetch(url, {
headers: { "User-Agent": EMEX_UA, Accept: "text/html,application/xhtml+xml" },
@@ -386,6 +466,33 @@ export class EmexService {
return cats;
}
/**
* Fetch + parse the hierarchical Vehicle.aspx tree (top groups → sub groups)
* for a decoded vehicle. One plain GET of the catalog root; the persistent
* sidebar carries the whole 2-level group tree (units are drilled lazily per
* sub-group via {@link drillVehicleNode}). Returns [] on any failure so the
* caller falls back to the flat QuickGroups categories.
*/
private async fetchVehicleTree(
catalogCode: string,
vid: string,
ssd: string,
): Promise<EmexVehicleTreeNode[]> {
const url = `${EMEX_BASE_URL}/Vehicle.aspx?c=${catalogCode}&vid=${vid}&ssd=${encodeURIComponent(ssd)}`;
const html = await this.fetchEmexHtml(url);
return parseVehicleTree(html);
}
/**
* Drill a Vehicle.aspx group node (its verbatim href, ssd embedded) to its
* Unit.aspx leaves. Used by the lazy category drill (categories.service
* getChildren) when an emex group is first expanded.
*/
async drillVehicleNode(vehicleAspxUrl: string): Promise<EmexVehicleTreeNode[]> {
const html = await this.fetchEmexHtml(vehicleAspxUrl);
return parseUnitLeaves(html);
}
/**
* Determine brand name from EMEX catalog code (e.g. "BMW202501" → "BMW").
*/
@@ -455,6 +562,19 @@ export class EmexService {
}
}
// Hierarchical Vehicle.aspx tree (primary nav); the flat `categories`
// above is the fallback. One GET of the catalog root yields the full
// 2-level group tree; units are drilled lazily per sub-group on expand.
let categoryTree: EmexVehicleTreeNode[] = [];
if (v.catalogCode && v.ssd) {
try {
categoryTree = await this.fetchVehicleTree(v.catalogCode, v.vid ?? "0", v.ssd);
this.logger.log(`EMEX HTTP: parsed ${categoryTree.length} Vehicle.aspx top groups`);
} catch (err) {
this.logger.warn(`EMEX Vehicle.aspx tree fetch failed: ${(err as Error).message}`);
}
}
// Build a response compatible with mapEmexResponse
const response: EmexScraperResponse = {
success: true,
@@ -476,7 +596,7 @@ export class EmexService {
},
quickGroupsUrl: v.quickGroupsUrl || null,
categories: categories.map((c) => ({ gid: c.gid || "", name: c.name, url: c.url })),
categoryTree: [],
categoryTree,
timestamp: new Date().toISOString(),
};
@@ -541,6 +661,17 @@ export class EmexService {
}
}
// Hierarchical Vehicle.aspx tree (primary nav); flat `categories` = fallback.
let categoryTree: EmexVehicleTreeNode[] = [];
if (v.catalogCode && v.ssd) {
try {
categoryTree = await this.fetchVehicleTree(v.catalogCode, v.vid ?? "0", v.ssd);
this.logger.log(`EMEX HTTP: parsed ${categoryTree.length} Vehicle.aspx top groups`);
} catch (err) {
this.logger.warn(`EMEX Vehicle.aspx tree fetch failed: ${(err as Error).message}`);
}
}
const response: EmexScraperResponse = {
success: true,
source: "emexdwc.ae",
@@ -561,7 +692,7 @@ export class EmexService {
},
quickGroupsUrl: v.quickGroupsUrl || null,
categories: categories.map((c) => ({ gid: c.gid || "", name: c.name, url: c.url })),
categoryTree: [],
categoryTree,
timestamp: new Date().toISOString(),
};
@@ -608,6 +739,17 @@ export class EmexService {
}
}
// Hierarchical Vehicle.aspx tree (primary nav); flat `categories` = fallback.
let categoryTree: EmexVehicleTreeNode[] = [];
if (v.catalogCode && v.ssd) {
try {
categoryTree = await this.fetchVehicleTree(v.catalogCode, v.vid ?? "0", v.ssd);
this.logger.log(`EMEX HTTP: parsed ${categoryTree.length} Vehicle.aspx top groups`);
} catch (err) {
this.logger.warn(`EMEX Vehicle.aspx tree fetch failed: ${(err as Error).message}`);
}
}
const response: EmexScraperResponse = {
success: true,
source: "emexdwc.ae",
@@ -628,7 +770,7 @@ export class EmexService {
},
quickGroupsUrl: v.quickGroupsUrl || null,
categories: categories.map((c) => ({ gid: c.gid || "", name: c.name, url: c.url })),
categoryTree: [],
categoryTree,
timestamp: new Date().toISOString(),
};
@@ -862,6 +1004,28 @@ export class EmexService {
* parts — caller treats that as "fall back to browser".
*/
private async fetchCategoryPartsViaHttp(categoryUrl: string): Promise<EmexPartsResult> {
// Vehicle.aspx-tree leaves already point straight at Unit.aspx — skip the
// QuickDetails→Unit hop and extract parts directly.
if (/Unit\.aspx/i.test(categoryUrl)) {
const unitHtml = await this.fetchEmexHtml(categoryUrl);
const extracted = this.extractEmexPartsFromHtml(unitHtml);
if (
extracted.schemaImageUrl &&
(extracted.schemaWidth === 0 || extracted.schemaHeight === 0)
) {
try {
const dims = await this.fetchImageDims(extracted.schemaImageUrl);
if (dims.width > 0 && dims.height > 0) {
extracted.schemaWidth = dims.width;
extracted.schemaHeight = dims.height;
}
} catch (err) {
this.logger.debug(`Image dims fetch failed: ${(err as Error).message}`);
}
}
return extracted;
}
// 1) QuickDetails.aspx → Unit.aspx anchor
const qdHtml = await this.fetchEmexHtml(categoryUrl);
const unitMatch = qdHtml.match(/href="([^"]*Unit\.aspx[^"]*)"/i);

View File

@@ -63,6 +63,29 @@ export interface EmexCategoryTreeNode {
children: EmexCategoryTreeNode[];
}
/**
* Node in the hierarchical Vehicle.aspx tree (the OEM-microfiche taxonomy):
* top groups → sub groups → Unit.aspx leaves. Produced by emex-tree.parser.ts
* and seeded into the generic `categories` table (parentId chains +
* hasSubgroups/hasParts), so the same tree UI renders it as pl24/pcat.
*/
export type EmexTreeNodeKind = "group" | "leaf";
export interface EmexVehicleTreeNode {
/** emex category id (`cid`) for groups; the parent group's cid for leaves. */
cid: string;
/** Display name (for leaves the "NNNN:" section-code prefix is stripped). */
name: string;
/** Section code for a leaf (e.g. "3070160A"); null for groups. */
code: string | null;
/** Absolute, verbatim URL: Vehicle.aspx (group, to drill) or Unit.aspx (leaf, to fetch parts). */
url: string;
kind: EmexTreeNodeKind;
/** Schema illustration (img.laximo.net gif) for a leaf, when listed; else null. */
imageUrl: string | null;
children: EmexVehicleTreeNode[];
}
/**
* Part data from EMEX
*/
@@ -142,7 +165,7 @@ export interface EmexScraperResponse {
quickGroupsUrl?: string | null;
parsedOptions?: EmexParsedOptions;
categories?: EmexCategory[];
categoryTree?: EmexCategoryTreeNode[];
categoryTree?: EmexVehicleTreeNode[];
sampleParts?: EmexPart[];
timestamp: string;
}

View File

@@ -60,3 +60,45 @@ describe("PL24FordLegacyService.parseP4VehicleResponse — model-selection page
expect(r).toBeNull();
});
});
// parseFordGroupsFromHtml leaks header/breadcrumb nav links into the category list
// (Hyundai/Kia "Portal", logout, Nissan "Repair & Maintenance Information" → eu.nissan.biz).
// These must be filtered; real categories (group.action / group-detail.action /
// json-(main|sub)-group.action, and Volvo vin-group.action?...group1=...) must survive.
const pg = svc as unknown as {
parseFordGroupsFromHtml(
html: string,
serviceName: string,
familyId: string,
): Array<{ code: string; nameTr: string; nameEn: string; linkPath: string }>;
};
describe("PL24FordLegacyService.parseFordGroupsFromHtml — nav-crumb junk filter", () => {
it("drops Portal / logout / external nav-crumbs, keeps real categories", () => {
const html = [
"<table>",
'<tr class="tc-data-row" url="group-detail.action?catId=20&lang=tr" caption="MOTOR"><td>MOTOR</td></tr>',
'<tr class="tc-data-row" url="portal.action?lang=tr" caption="Portal"><td>Portal</td></tr>',
'<tr class="tc-data-row" url="logout.action" caption="Logout"><td>Logout</td></tr>',
'<tr class="tc-data-row" url="https://eu.nissan.biz/" caption="Repair &amp; Maintenance Information"><td>Repair</td></tr>',
"</table>",
].join("");
const groups = pg.parseFordGroupsFromHtml(html, "hyundai_parts", "");
const names = groups.map((g) => g.nameTr);
expect(names).toContain("MOTOR");
expect(names).not.toContain("Portal");
expect(names).not.toContain("Logout");
expect(groups.some((g) => g.linkPath.includes("eu.nissan.biz"))).toBe(false);
});
it("keeps Volvo vin-group.action?...group1=... (real category) but drops the bare breadcrumb", () => {
const html = [
'<tr class="tc-data-row" url="vin-group.action?group1=2&lang=tr" caption="Motor"><td>Motor</td></tr>',
'<tr class="tc-data-row" url="vin-group.action?lang=tr" caption="Geri"><td>Geri</td></tr>',
].join("");
const groups = pg.parseFordGroupsFromHtml(html, "volvo_parts", "");
const names = groups.map((g) => g.nameTr);
expect(names).toContain("Motor");
expect(names).not.toContain("Geri");
});
});

View File

@@ -46,6 +46,19 @@ export interface P4BrandHooks {
parseVehicleInfo?(html: string, vin: string): P4VehicleInfo | null;
}
// Brands whose parts are licensed only on the de (account 2) PL24 subscription —
// the tr account serves a NOT_LOGGED_IN_DEMO page for these. All P4 fetches for
// these services are routed to de (see fetchP4Page + resolveAccount).
const LEGACY_DE_SERVICES = new Set([
"hyundai_parts",
"kia_parts",
"nissan_parts",
"fordp_parts",
"fordt_parts",
]);
// Polestar tested on de → 0 categories (data gap). Fiat stays on de via
// resolveAccount but its model-list needs launchCatalog.do.
@Injectable()
export class PL24FordLegacyService {
private readonly logger = new Logger(PL24FordLegacyService.name);
@@ -250,13 +263,18 @@ export class PL24FordLegacyService {
// Nissan/Infiniti/Opel: json-sub-group.action
if (linkPath.includes("json-sub-group.action")) {
const basePath = linkPath.substring(0, linkPath.lastIndexOf("/") + 1);
// Nissan format: { subgroups: [...] }
// Nissan format: { subgroups: [{caption, code, url}] }
// Opel format: may use same { subgroups } or { vCfgData }
let groups = this.parseNissanSubGroupJson(html, basePath);
if (groups.length === 0) {
// Try Opel vCfgData format
groups = this.parseOpelVCfgDataJson(html, basePath);
}
if (groups.length === 0) {
// Ford de format: { subgroups: [{caption, id, subheader, jsonUrl:null}] } →
// construct image-board.action leaves (bomId = mainGroupId+subheaderId+leafId).
groups = this.parseFordSubGroupsJson(html, linkPath, basePath);
}
if (groups.length === 0) {
// Debug: log the response
this.logger.log(
@@ -1885,10 +1903,13 @@ export class PL24FordLegacyService {
* Fiat services always use 'de'; others use round-robin via Redis.
*/
private async resolveAccount(userId?: string, serviceName?: string): Promise<"tr" | "de"> {
// Fiat always needs de account
// Fiat always needs de account; Hyundai/Kia/Nissan parts are licensed only on de.
if (serviceName && ["fiatp_parts", "fiatt_parts"].includes(serviceName)) {
return "de";
}
if (serviceName && LEGACY_DE_SERVICES.has(serviceName)) {
return "de";
}
// No userId → tr (catalog browser / prefetch)
if (!userId) return "tr";
@@ -2625,6 +2646,67 @@ export class PL24FordLegacyService {
}
}
/**
* Parse Ford (de account) json-sub-group.action response into leaf illustrations.
*
* Ford de subgroups are a flat list of { caption, id, subheader, jsonUrl:null, url:null }.
* `subheader:true` rows are section labels (skipped); the leaves below them are the
* actual part illustrations but carry NO url. The illustration (BOM) URL is
* constructed: subGroupId = mainGroupId + subheaderId, bomId = subGroupId + leafId,
* leaf → image-board.action?bomId=… (same parts endpoint as Nissan/Hyundai). Verified
* on dev: the selected-subgroup response exposes exactly this image-board url.
*/
private parseFordSubGroupsJson(
jsonStr: string,
linkPath: string,
basePath: string,
): PL24MainGroup[] {
try {
const data = JSON.parse(jsonStr) as {
subgroups?: {
caption: string;
id: string;
subheader?: boolean;
gray?: boolean;
jsonUrl?: string | null;
url?: string | null;
}[];
};
const subs = data.subgroups ?? [];
const q = (k: string) => linkPath.match(new RegExp(`[?&]${k}=([^&]+)`))?.[1] ?? "";
const catCode = q("catCode");
const mainGroupId = q("mainGroupId");
const sharedCatCode = q("sharedCatCode");
const modelFamily = q("modelFamily");
const mode = q("mode");
const upds = q("upds");
if (!mainGroupId) return [];
const out: PL24MainGroup[] = [];
const seen = new Set<string>();
let currentSubheaderId = "";
for (const sg of subs) {
if (sg.gray) continue;
if (sg.subheader) {
currentSubheaderId = sg.id || currentSubheaderId;
continue;
}
if (!sg.id) continue;
const subGroupId = `${mainGroupId}${currentSubheaderId}`;
const bomId = `${subGroupId}${sg.id}`;
if (seen.has(bomId)) continue;
seen.add(bomId);
const leafLink =
`${basePath}image-board.action?bomId=${bomId}&catCode=${catCode}&lang=${this.language}` +
`&mainGroupId=${mainGroupId}&modelFamily=${modelFamily}&sharedCatCode=${sharedCatCode}` +
`&startup=false&subGroupId=${subGroupId}&mode=${mode}&upds=${upds}`;
out.push({ id: bomId, code: bomId, name: sg.caption, linkPath: leafLink });
}
return out;
} catch {
return [];
}
}
/**
* Parse Ford json-main-group.action response.
* Ford uses { maingroups: [{ caption, databaseKey, identifier, gray, subheader, jsonUrl, url }] } format.
@@ -3205,6 +3287,15 @@ export class PL24FordLegacyService {
account: "tr" | "de" = "tr",
retried = false,
): Promise<string | null> {
// Hyundai/Kia/Nissan parts are licensed ONLY on the de account — the tr
// account returns a NOT_LOGGED_IN_DEMO page (browse-only). Route every P4
// fetch for these brands to de (which carries the Korean/Japanese license),
// and ensure the de service token is authorized. Verified on dev: de yields
// 266 Hyundai / 169 Kia models, non-demo, with real drill content.
if (LEGACY_DE_SERVICES.has(serviceName)) {
account = "de";
await this.authService.authorizeServiceForAccount(serviceName, "de");
}
// Some catalogs (Volvo vin-group.action) store hrefs relative to the catalog
// directory (e.g. "vin-group.action?group1=…"). Prefix the service basePath
// so `${baseUrl}${path}` doesn't collapse into "partslink24.comvin-group…".
@@ -3259,6 +3350,10 @@ export class PL24FordLegacyService {
// groups/parts) when the service token is stale. decodeVinForService
// retries on this, but drill paths reach upstream only through here —
// so retry once with fresh auth before parsing an empty page.
// NOTE: Hyundai/Kia/Nissan parts are demo-gated on the tr account
// (not licensed); the de account is licensed but lacks the EUR/TR-region
// catalogs, so a de fallback returns non-demo-but-empty — a PL24 commercial
// licensing/region issue, not fixable here.
if (!retried && html.includes("PL24_SUPPORT")) {
const support = this.extractScriptVariable<FordPL24Support>(html, "PL24_SUPPORT");
if (support?.demo || support?.role === "NOT_LOGGED_IN_DEMO") {
@@ -3526,6 +3621,14 @@ export class PL24FordLegacyService {
const url = urlMatch[1];
// Skip vehicle.action rows — those are sub-model selectors, not part group links
if (url.includes("vehicle.action")) continue;
// Skip header/breadcrumb navigation links that leak into the group table as
// pseudo-categories: Hyundai/Kia "Portal" (portal.action), logout, and Nissan's
// "Repair & Maintenance Information" external link (https://eu.nissan.biz/). Real
// categories use group.action / group-detail.action / json-(main|sub)-group.action,
// none of which match these. Volvo's vin-group.action?...group1=... is kept.
if (/(portal|logout)\.action/i.test(url)) continue;
if (url.includes("vin-group.action") && !url.includes("group1=")) continue;
if (/^https?:\/\//i.test(url) && !/\.action(\?|$)/i.test(url)) continue;
if (seen.has(url)) continue;
seen.add(url);
@@ -3588,6 +3691,8 @@ export class PL24FordLegacyService {
const url = match[1];
const name = match[2].replace(/<[^>]+>/g, "").trim();
if (!name || seen.has(url)) continue;
// Same nav-crumb guard as the primary parser (vin-group.action breadcrumb).
if (url.includes("vin-group.action") && !url.includes("group1=")) continue;
seen.add(url);
const linkPath = url.startsWith("/") ? url : `${basePath}/${url}`;
groups.push({

View File

@@ -325,3 +325,24 @@ describe("PL24Service.parseVehicleResponse — p5mitsubishi (model from 'Araç',
expect(r.year).toBe(2006);
});
});
describe("PL24Service.formatFiatYear — p5fiat model year ranges", () => {
// Fiat /extern/vehicle/models records carry values.year as "(2016,2020)".
const fy = svc as unknown as { formatFiatYear(raw: unknown): string | undefined };
it("turns '(2016,2020)' into '2016-2020'", () => {
expect(fy.formatFiatYear("(2016,2020)")).toBe("2016-2020");
});
it("handles a single / open-ended year", () => {
expect(fy.formatFiatYear("(1999,)")).toBe("1999");
expect(fy.formatFiatYear("(2007)")).toBe("2007");
});
it("returns undefined for empty / non-string input", () => {
expect(fy.formatFiatYear("")).toBeUndefined();
expect(fy.formatFiatYear("()")).toBeUndefined();
expect(fy.formatFiatYear(undefined)).toBeUndefined();
expect(fy.formatFiatYear(null)).toBeUndefined();
});
});

View File

@@ -629,16 +629,15 @@ export class PL24Service {
}> {
await this.touchActivity();
try {
await this.authService.authorizeService(serviceName);
const headers = await this.authService.buildAuthHeaders(serviceName);
// Account-aware (de + DE proxy for Fiat). resolveAccount(undefined, …) → "de"
// only for fiatp_parts/fiatt_parts; all other P5 brands stay on tr (no proxy).
const account = await this.resolveAccount(undefined, serviceName);
await this.authService.authorizeServiceForAccount(serviceName, account);
const headers = await this.authService.buildAuthHeadersForAccount(account, serviceName);
const localizedPath = restrictionPath.replace(/lang=\w+/, `lang=${this.language}`);
const url = `${this.baseUrl}${localizedPath}`;
const response = await fetch(url, {
method: "GET",
headers,
signal: AbortSignal.timeout(this.timeout),
});
const response = await this.fetchWithRetry(url, headers, serviceName, account);
if (!response.ok) {
this.logger.warn(`P5 restrictions fetch failed: HTTP ${response.status}`);
@@ -711,9 +710,20 @@ export class PL24Service {
}));
}
try {
await this.authService.authorizeService(serviceName);
const headers = await this.authService.buildAuthHeaders(serviceName);
return this.fetchMainGroupsByPath(mainGroupsPath, headers);
// Account-aware: tr for most P5 brands (no proxy), de + DE proxy for Fiat.
// resolveAccount(undefined, …) → "de" only for fiatp_parts/fiatt_parts.
const account = await this.resolveAccount(undefined, serviceName);
await this.authService.authorizeServiceForAccount(serviceName, account);
const headers = await this.authService.buildAuthHeadersForAccount(account, serviceName);
const localizedPath = mainGroupsPath.replace(/lang=\w+/, `lang=${this.language}`);
const response = await this.fetchWithRetry(
`${this.baseUrl}${localizedPath}`,
headers,
serviceName,
account,
);
const data = (await response.json()) as Record<string, any>;
return this.parseMainGroupsResponse(data);
} catch (error) {
const err = error as Error;
this.logger.error(`Fetch main groups error: ${err.message}`);
@@ -1982,6 +1992,13 @@ export class PL24Service {
return this.fordLegacyService.fetchVehicleListForVolvo(serviceName);
}
// Fiat (p5fiat) has a two-level model hierarchy (families → model codes) that the
// generic single-endpoint P5 flow below cannot express, and is licensed only on the
// de-708171 account. Handle it via a dedicated expander.
if (serviceName === "fiatp_parts" || serviceName === "fiatt_parts") {
return this.fetchFiatVehicleList(serviceName);
}
try {
// Try de account; fall back to main token (demo mode) if that also fails
let headers: Record<string, string>;
@@ -2058,6 +2075,41 @@ export class PL24Service {
}
}
// Fallback: the primary modelPath returned nothing (e.g. p5fiat is not mapped
// in BACKEND_MODEL_PATH yet). Try the other known P5 listing endpoints so an
// unmapped backend self-discovers its model list instead of silently seeding 0.
// Only runs when the primary yields nothing → no impact on mapped backends.
const candidatePaths = [
"/extern/vehicle/modelfamilies",
"/extern/vehicle/models",
"/extern/vehicle/modelFamilies",
"/extern/vehicle/catalogs",
"/extern/vehicle/scope",
"/extern/vehicles/vehiclesOverview",
"/extern/model/categories",
].filter((p) => p !== modelPath);
for (const candidate of candidatePaths) {
try {
const cu = `${this.baseUrl}${catalogBase}${candidate}?lang=${this.language}&serviceName=${serviceName}`;
const cr = await fetch(cu, {
method: "GET",
headers,
signal: AbortSignal.timeout(15000),
});
if (!cr.ok) continue;
const cd = (await cr.json()) as Record<string, any>;
const cv = this.parseVehicleListResponse(cd, serviceName);
if (cv.length > 0) {
this.logger.log(
`fetchVehicleList: ${serviceName} discovered via ${candidate} (${cv.length} models) — pin BACKEND_MODEL_PATH["${backendKey}"]="${candidate}"`,
);
return cv;
}
} catch {
// try next candidate
}
}
this.logger.log(`No vehicle list available for ${serviceName} (HTTP ${response.status})`);
return [];
} catch (err) {
@@ -2066,6 +2118,122 @@ export class PL24Service {
}
}
/**
* Fiat (p5fiat) browse model list.
*
* Unlike other P5 backends, Fiat exposes a two-level hierarchy:
* modelOverview → model FAMILIES (124 SPIDER, 500L, LINEA, TIPO …)
* models?modelFamily=N → the actual model codes (+ year range)
* Each model's link.path is the maingroups endpoint consumed by the generic P5
* drill (getCategoryTree → fetchMainGroups → subgroups → bomDetails parts).
*
* Requires the de-708171 account (Fiat parts are licensed only there) and the
* DataImpulse DE proxy — both applied via fetchWithRetry(account="de").
*/
private async fetchFiatVehicleList(serviceName: string): Promise<
Array<{
vehicleId: string;
model: string;
year?: string;
engine?: string;
bodyType?: string;
transmission?: string;
market?: string;
catalogPath?: string;
metadata?: Record<string, unknown>;
}>
> {
const account = "de" as const;
try {
await this.authService.authorizeServiceForAccount(serviceName, account);
const headers = await this.authService.buildAuthHeadersForAccount(account, serviceName);
const base = getServiceApiPath(serviceName); // "/p5fiat"
// 1) Model families
const ovUrl = `${this.baseUrl}${base}/extern/vehicle/modelOverview?lang=${this.language}&serviceName=${serviceName}`;
const ovResp = await this.fetchWithRetry(ovUrl, headers, serviceName, account);
const ovData = (await ovResp.json()) as Record<string, any>;
const families: any[] = (ovData.data?.records || ovData.records || []).filter(
(f: any) => !f.unavailable,
);
if (families.length === 0) {
this.logger.warn(`Fiat ${serviceName}: modelOverview returned 0 families`);
return [];
}
// 2) Expand each family → model codes (bounded concurrency to keep the one-time seed fast)
type FiatVehicle = {
vehicleId: string;
model: string;
year?: string;
catalogPath?: string;
metadata?: Record<string, unknown>;
};
const out: FiatVehicle[] = [];
const CONCURRENCY = 6;
for (let i = 0; i < families.length; i += CONCURRENCY) {
const batch = families.slice(i, i + CONCURRENCY);
const batchResults = await Promise.all(
batch.map(async (fam: any): Promise<FiatVehicle[]> => {
const famName = String(fam.values?.description || fam.description || fam.id).trim();
const modelsPath = fam.link?.path as string | undefined;
const modelsUrl = modelsPath
? modelsPath.startsWith("http")
? modelsPath
: `${this.baseUrl}${modelsPath}`
: `${this.baseUrl}${base}/extern/vehicle/models?lang=${this.language}&serviceName=${serviceName}&modelFamily=${fam.id}`;
try {
const mResp = await this.fetchWithRetry(modelsUrl, headers, serviceName, account);
const mData = (await mResp.json()) as Record<string, any>;
const models: any[] = (mData.data?.records || mData.records || []).filter(
(m: any) => !m.unavailable && m.link?.path,
);
return models.map((m: any): FiatVehicle => {
const modelCode = String(m.values?.modelCode || m.id || "");
const desc = String(m.values?.description || m.description || famName)
.replace(/\\-/g, "-") // PL24 escapes hyphens as "\-"
.replace(/\s+/g, " ")
.trim();
const year = this.formatFiatYear(m.values?.year);
return {
vehicleId: `${fam.id}:${modelCode}`,
model: year ? `${desc} (${year})` : desc,
year: year || undefined,
catalogPath: m.link.path as string,
metadata: { modelFamily: String(fam.id), modelCode, family: famName },
};
});
} catch (err) {
this.logger.warn(
`Fiat ${serviceName}: models fetch failed for family ${fam.id}: ${(err as Error).message}`,
);
return [];
}
}),
);
for (const r of batchResults) out.push(...r);
}
this.logger.log(`Fiat ${serviceName}: ${families.length} families → ${out.length} models`);
return out;
} catch (err) {
this.logger.warn(`fetchFiatVehicleList failed for ${serviceName}: ${(err as Error).message}`);
return [];
}
}
/** Fiat year values arrive as "(2016,2020)" → "2016-2020"; single/open ranges handled. */
private formatFiatYear(raw: unknown): string | undefined {
if (typeof raw !== "string") return undefined;
const trimmed = raw.replace(/[()]/g, "").trim();
if (!trimmed) return undefined;
const parts = trimmed
.split(",")
.map((s) => s.trim())
.filter(Boolean);
return parts.length >= 2 ? `${parts[0]}-${parts[parts.length - 1]}` : parts[0] || undefined;
}
private parseVehicleListResponse(
data: Record<string, any>,
serviceName: string,

View File

@@ -528,7 +528,9 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy {
private isLeafLinkPath(linkPath: string | null, source: string): boolean {
if (!linkPath) return false;
if (source === "emex") return true; // EMEX leaves always have a URL
// EMEX: Vehicle.aspx group nodes are parents to drill; Unit.aspx (hierarchical
// tree) and QuickDetails.aspx (legacy flat) leaves carry parts.
if (source === "emex") return !linkPath.includes("Vehicle.aspx");
if (source === "parts-catalogs") return linkPath.startsWith("pcat:"); // pcat: prefix = leaf
// PL24 leaf indicators
return (

View File

@@ -116,7 +116,9 @@ export function ModelListColumns({
const needsP5Restrictions =
arch === "P5_MODERN" &&
!!vehicle.catalogPath &&
!vehicle.catalogPath.includes("/mainGroup");
// Case-insensitive: Fiat's catalogPath is already a maingroups endpoint
// (/extern/groups/mdl/maingroups) → no restriction step, go straight to categories.
!/\/maingroup/i.test(vehicle.catalogPath);
setLoadingCol(colIdx);
try {

View File

@@ -72,7 +72,9 @@ function CatalogVehiclePage() {
const isP5WithRestrictions =
vehicle?.architecture === "P5_MODERN" &&
!!vehicle?.catalogPath &&
!vehicle.catalogPath.includes("/mainGroup");
// Case-insensitive: Fiat's catalogPath is already a maingroups endpoint
// (/extern/groups/mdl/maingroups) → no restriction step, categories load directly.
!/\/maingroup/i.test(vehicle.catalogPath);
const showPsaVariantSelector = isPsa && !hasVariant;
const showFordVariantSelector = isP4Legacy && !hasVariant;
const showP5RestrictionSelector = isP5WithRestrictions && !hasVariant;