dev #111
@@ -13,6 +13,7 @@ import {
|
||||
} from "../database/schema/core";
|
||||
import { PL24Service } from "../integrations/pl24/pl24.service";
|
||||
import {
|
||||
type PL24DecodedCategory,
|
||||
PL24_SERVICE_CATALOGS,
|
||||
SERVICE_DISPLAY_NAMES,
|
||||
SERVICE_TO_BRAND,
|
||||
@@ -497,58 +498,12 @@ export class CatalogService {
|
||||
);
|
||||
|
||||
if (pl24Categories.length > 0) {
|
||||
let dbCategories = await this.db
|
||||
.select()
|
||||
.from(categories)
|
||||
.where(eq(categories.catalogVehicleId, catalogVehicleId));
|
||||
|
||||
if (dbCategories.length === 0) {
|
||||
const insertData = pl24Categories.map((c) => ({
|
||||
catalogVehicleId,
|
||||
vehicleId: null as string | null,
|
||||
name: c.nameTr || c.nameEn,
|
||||
nameOriginal: c.nameEn,
|
||||
parentId: null as string | null,
|
||||
externalId: c.code,
|
||||
linkPath: c.linkPath || null,
|
||||
linkWid: c.linkWid || null,
|
||||
source: "pl24" as const,
|
||||
}));
|
||||
const inserted = await this.db
|
||||
.insert(categories)
|
||||
.values(insertData)
|
||||
.onConflictDoNothing()
|
||||
.returning();
|
||||
dbCategories =
|
||||
inserted.length > 0
|
||||
? inserted
|
||||
: await this.db
|
||||
.select()
|
||||
.from(categories)
|
||||
.where(eq(categories.catalogVehicleId, catalogVehicleId));
|
||||
await this.db
|
||||
.update(catalogVehicles)
|
||||
.set({ categoriesFetched: true, updatedAt: new Date() })
|
||||
.where(eq(catalogVehicles.id, catalogVehicleId));
|
||||
}
|
||||
|
||||
const externalIdToUuid = new Map(dbCategories.map((c) => [c.externalId, c.id]));
|
||||
const tree = this.buildTree(
|
||||
pl24Categories.map((c) => ({
|
||||
id: externalIdToUuid.get(c.code) ?? c.code,
|
||||
catalogVehicleId,
|
||||
vehicleId: null,
|
||||
name: c.nameTr || c.nameEn,
|
||||
nameOriginal: c.nameEn,
|
||||
parentId: null,
|
||||
externalId: c.code,
|
||||
linkPath: c.linkPath || null,
|
||||
linkWid: c.linkWid || null,
|
||||
source: "pl24",
|
||||
})),
|
||||
const tree = await this.persistAndBuildLegacyTree(
|
||||
catalogVehicleId,
|
||||
pl24Categories,
|
||||
cacheKey,
|
||||
);
|
||||
await this.redis.setJson(cacheKey, tree, 7200);
|
||||
return tree;
|
||||
if (tree.length > 0) return tree;
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
@@ -600,59 +555,12 @@ export class CatalogService {
|
||||
}
|
||||
|
||||
if (pl24Categories.length > 0) {
|
||||
// Persist _all_ variant to DB as canonical records
|
||||
let dbCategories = await this.db
|
||||
.select()
|
||||
.from(categories)
|
||||
.where(eq(categories.catalogVehicleId, catalogVehicleId));
|
||||
|
||||
if (dbCategories.length === 0) {
|
||||
const insertData = pl24Categories.map((c) => ({
|
||||
catalogVehicleId,
|
||||
vehicleId: null as string | null,
|
||||
name: c.nameTr || c.nameEn,
|
||||
nameOriginal: c.nameEn,
|
||||
parentId: null as string | null,
|
||||
externalId: c.code,
|
||||
linkPath: c.linkPath || null,
|
||||
linkWid: c.linkWid || null,
|
||||
source: "pl24" as const,
|
||||
}));
|
||||
const inserted = await this.db
|
||||
.insert(categories)
|
||||
.values(insertData)
|
||||
.onConflictDoNothing()
|
||||
.returning();
|
||||
dbCategories =
|
||||
inserted.length > 0
|
||||
? inserted
|
||||
: await this.db
|
||||
.select()
|
||||
.from(categories)
|
||||
.where(eq(categories.catalogVehicleId, catalogVehicleId));
|
||||
await this.db
|
||||
.update(catalogVehicles)
|
||||
.set({ categoriesFetched: true, updatedAt: new Date() })
|
||||
.where(eq(catalogVehicles.id, catalogVehicleId));
|
||||
}
|
||||
|
||||
const externalIdToUuid = new Map(dbCategories.map((c) => [c.externalId, c.id]));
|
||||
const tree = this.buildTree(
|
||||
pl24Categories.map((c) => ({
|
||||
id: externalIdToUuid.get(c.code) ?? c.code,
|
||||
catalogVehicleId,
|
||||
vehicleId: null,
|
||||
name: c.nameTr || c.nameEn,
|
||||
nameOriginal: c.nameEn,
|
||||
parentId: null,
|
||||
externalId: c.code,
|
||||
linkPath: c.linkPath || null,
|
||||
linkWid: c.linkWid || null,
|
||||
source: "pl24",
|
||||
})),
|
||||
const tree = await this.persistAndBuildLegacyTree(
|
||||
catalogVehicleId,
|
||||
pl24Categories,
|
||||
cacheKey,
|
||||
);
|
||||
await this.redis.setJson(cacheKey, tree, 7200);
|
||||
return tree;
|
||||
if (tree.length > 0) return tree;
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
@@ -758,7 +666,9 @@ export class CatalogService {
|
||||
and(eq(categories.catalogVehicleId, catalogVehicleId), sql`${categories.parentId} IS NULL`),
|
||||
);
|
||||
if (dbCategories.length === 0) return [];
|
||||
const tree = this.buildTree(dbCategories);
|
||||
// Same nav-crumb read filter as persistAndBuildLegacyTree (self-heals
|
||||
// previously-persisted Hyundai "Portal" / Nissan eu.nissan.biz junk rows).
|
||||
const tree = this.buildTree(dbCategories.filter((c) => !this.isNavCrumbLink(c.linkPath)));
|
||||
await this.redis.setJson(cacheKey, tree, 3600); // 1h cache for DB fallback
|
||||
return tree;
|
||||
}
|
||||
@@ -828,6 +738,20 @@ export class CatalogService {
|
||||
.from(categories)
|
||||
.where(eq(categories.parentId, categoryId));
|
||||
|
||||
// Self-healing: drop nav-crumb junk children (e.g. "Portal"/eu.nissan.biz rows
|
||||
// persisted by the old parser) and delete them so this node re-fetches the real
|
||||
// subgroups instead of returning a junk-only child set.
|
||||
const navCrumbChildren = children.filter((c) => this.isNavCrumbLink(c.linkPath));
|
||||
if (navCrumbChildren.length > 0) {
|
||||
await this.db.delete(categories).where(
|
||||
inArray(
|
||||
categories.id,
|
||||
navCrumbChildren.map((c) => c.id),
|
||||
),
|
||||
);
|
||||
children = children.filter((c) => !this.isNavCrumbLink(c.linkPath));
|
||||
}
|
||||
|
||||
// Self-healing: if the linkPath is a leaf path but DB has children, those are
|
||||
// stale records created by the previous case-insensitive bug. Delete and re-fetch.
|
||||
if (children.length > 0 && linkPath && this.isLeafPath(linkPath)) {
|
||||
@@ -861,14 +785,19 @@ export class CatalogService {
|
||||
gearbox,
|
||||
);
|
||||
|
||||
if (subGroups.length > 0) {
|
||||
const seenNames = new Set<string>();
|
||||
const unique = subGroups.filter((sg) => {
|
||||
if (seenNames.has(sg.name)) return false;
|
||||
seenNames.add(sg.name);
|
||||
return true;
|
||||
});
|
||||
// After dropping nav-crumb junk the subgroup set can be empty (upstream
|
||||
// returned only a "Portal"/external stub for this node). Guard the insert —
|
||||
// drizzle .values([]) throws — and fall through to the leaf/parts path,
|
||||
// which yields a clean empty node instead of a 500.
|
||||
const seenNames = new Set<string>();
|
||||
const unique = subGroups.filter((sg) => {
|
||||
if (this.isNavCrumbLink(sg.linkPath)) return false;
|
||||
if (seenNames.has(sg.name)) return false;
|
||||
seenNames.add(sg.name);
|
||||
return true;
|
||||
});
|
||||
|
||||
if (unique.length > 0) {
|
||||
const insertData = unique.map((sg) => ({
|
||||
catalogVehicleId,
|
||||
vehicleId: null as string | null,
|
||||
@@ -1321,6 +1250,93 @@ export class CatalogService {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Header/breadcrumb nav links that PL24's group.action HTML leaks into the
|
||||
* category table as pseudo-categories. They are not real part groups and drill
|
||||
* to nothing. Mirrors the filter in categories.service (VIN-decode path).
|
||||
* Keeps Volvo's `vin-group.action?...group1=...` real categories.
|
||||
*/
|
||||
private isNavCrumbLink(linkPath?: string | null): boolean {
|
||||
if (!linkPath) return false;
|
||||
if (/(portal|logout)\.action/i.test(linkPath)) return true;
|
||||
if (/vehicle\.action/i.test(linkPath)) return true;
|
||||
if (linkPath.includes("vin-group.action") && !linkPath.includes("group1=")) return true;
|
||||
// External absolute URL embedded anywhere that is not a partslink .action
|
||||
// endpoint. Nissan's "Repair & Maintenance Information" comes back as
|
||||
// `/nissan/nissan_parts/https://eu.nissan.biz/` — the basePath prefix means an
|
||||
// anchored ^https check misses it, so match `://` anywhere. Real PL24 linkPaths
|
||||
// are relative (group.action / json-*.action / pcat: / emex:) and never contain `://`.
|
||||
if (linkPath.includes("://") && !/\.action(\?|$)/i.test(linkPath)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Persist freshly-fetched PL24 legacy main groups (Ford/Volvo/Opel/Hyundai-Kia/
|
||||
* Nissan) and return the tree built FROM the DB rows — so every node id is a real
|
||||
* UUID, never a raw upstream linkPath (which would 404 on drill). Filters nav-crumb
|
||||
* junk and dedupes by name first (matching the PSA block's discipline), so
|
||||
* onConflictDoNothing can't silently drop a colliding row and leak a raw-id node.
|
||||
*/
|
||||
private async persistAndBuildLegacyTree(
|
||||
catalogVehicleId: string,
|
||||
pl24Categories: PL24DecodedCategory[],
|
||||
cacheKey: string,
|
||||
): Promise<any[]> {
|
||||
const seenNames = new Set<string>();
|
||||
const clean = pl24Categories.filter((c) => {
|
||||
if (this.isNavCrumbLink(c.linkPath)) return false;
|
||||
const name = c.nameTr || c.nameEn;
|
||||
if (!name || seenNames.has(name)) return false;
|
||||
seenNames.add(name);
|
||||
return true;
|
||||
});
|
||||
if (clean.length === 0) return [];
|
||||
|
||||
let dbCategories = await this.db
|
||||
.select()
|
||||
.from(categories)
|
||||
.where(eq(categories.catalogVehicleId, catalogVehicleId))
|
||||
.orderBy(categories.createdAt, categories.id);
|
||||
|
||||
if (dbCategories.length === 0) {
|
||||
await this.db
|
||||
.insert(categories)
|
||||
.values(
|
||||
clean.map((c) => ({
|
||||
catalogVehicleId,
|
||||
vehicleId: null as string | null,
|
||||
name: c.nameTr || c.nameEn,
|
||||
nameOriginal: c.nameEn,
|
||||
parentId: null as string | null,
|
||||
externalId: c.code,
|
||||
linkPath: c.linkPath || null,
|
||||
linkWid: c.linkWid || null,
|
||||
source: "pl24" as const,
|
||||
})),
|
||||
)
|
||||
.onConflictDoNothing();
|
||||
// Always re-select (never trust .returning(): onConflictDoNothing drops
|
||||
// name-collision rows, which would otherwise leave gaps that fall back to
|
||||
// raw-path ids). Deterministic order so the tree is stable across rebuilds.
|
||||
dbCategories = await this.db
|
||||
.select()
|
||||
.from(categories)
|
||||
.where(eq(categories.catalogVehicleId, catalogVehicleId))
|
||||
.orderBy(categories.createdAt, categories.id);
|
||||
await this.db
|
||||
.update(catalogVehicles)
|
||||
.set({ categoriesFetched: true, updatedAt: new Date() })
|
||||
.where(eq(catalogVehicles.id, catalogVehicleId));
|
||||
}
|
||||
|
||||
// Filter on read too — drops nav-crumb junk that was persisted before this
|
||||
// fix (e.g. existing Hyundai "Portal" / Nissan eu.nissan.biz rows), so the
|
||||
// response self-heals without a DB migration.
|
||||
const tree = this.buildTree(dbCategories.filter((c) => !this.isNavCrumbLink(c.linkPath)));
|
||||
await this.redis.setJson(cacheKey, tree, 7200);
|
||||
return tree;
|
||||
}
|
||||
|
||||
private buildTree(items: any[]): any[] {
|
||||
const map = new Map<string, any>();
|
||||
const roots: any[] = [];
|
||||
|
||||
@@ -258,21 +258,24 @@ export class CategoriesService {
|
||||
const emexResult = await this.emexService.decodeVin(vehicle.vin);
|
||||
if (emexResult) {
|
||||
const rawData = emexResult.raw as Record<string, unknown>;
|
||||
// Hierarchical Vehicle.aspx tree (top groups → sub groups). Units are
|
||||
// NOT seeded here — they're drilled lazily per sub-group by
|
||||
// getChildren on first expand (mirrors the pl24/pcat lazy model).
|
||||
const tree = rawData?.emexCategoryTree as
|
||||
| Array<{
|
||||
cid?: string;
|
||||
name: string;
|
||||
gid: string | null;
|
||||
code?: string | null;
|
||||
url: string | null;
|
||||
children: any[];
|
||||
kind?: "group" | "leaf";
|
||||
children?: any[];
|
||||
}>
|
||||
| undefined;
|
||||
|
||||
if (tree && tree.length > 0) {
|
||||
// Recursive tree insertion from QuickGroups.aspx
|
||||
this.logger.log(`Inserting ${tree.length} EMEX top-level category groups recursively`);
|
||||
this.logger.log(`Inserting EMEX Vehicle.aspx tree: ${tree.length} top groups`);
|
||||
|
||||
// Phase 1: walk the tree, collect every unique English name so we
|
||||
// can bulk-translate before touching the DB.
|
||||
// Bulk-translate every node name before touching the DB.
|
||||
const uniqueNames = new Set<string>();
|
||||
const collect = (nodes: Array<{ name: string; children?: any[] }>) => {
|
||||
for (const node of nodes) {
|
||||
@@ -282,33 +285,57 @@ export class CategoriesService {
|
||||
};
|
||||
collect(tree);
|
||||
const trMap = await this.translationsService.translateMany([...uniqueNames]);
|
||||
const tr = (n: string) => trMap.get(n) ?? n;
|
||||
|
||||
// The categories unique index is (vehicle, catalogVehicle, name,
|
||||
// source), so two sub-groups that share a display name across
|
||||
// different top groups (e.g. "Speakers" under both Electrical and
|
||||
// Accessories) would silently collide and orphan a branch. Detect
|
||||
// such names and disambiguate them with their top-group name.
|
||||
const subNameCounts = new Map<string, number>();
|
||||
for (const top of tree) {
|
||||
for (const sub of top.children ?? []) {
|
||||
const dn = tr(sub.name);
|
||||
subNameCounts.set(dn, (subNameCounts.get(dn) ?? 0) + 1);
|
||||
}
|
||||
}
|
||||
|
||||
const insertNodes = async (
|
||||
nodes: Array<{
|
||||
cid?: string;
|
||||
name: string;
|
||||
gid: string | null;
|
||||
code?: string | null;
|
||||
url: string | null;
|
||||
kind?: "group" | "leaf";
|
||||
children?: any[];
|
||||
}>,
|
||||
parentId: string | null,
|
||||
topName: string | null,
|
||||
) => {
|
||||
for (const node of nodes) {
|
||||
if (!node.name) continue;
|
||||
const isLeaf = !node.children?.length;
|
||||
// URL'si olmayan leaf node'lar dead-end — kaydetme
|
||||
if (isLeaf && !node.url) continue;
|
||||
|
||||
if (!node.name || !node.url) continue;
|
||||
const isLeaf = node.kind === "leaf";
|
||||
let displayName = tr(node.name);
|
||||
// Disambiguate colliding sub-group names with their top group.
|
||||
if (parentId !== null && topName && (subNameCounts.get(displayName) ?? 0) > 1) {
|
||||
displayName = `${displayName} (${topName})`;
|
||||
}
|
||||
const [inserted] = await this.db
|
||||
.insert(categories)
|
||||
.values({
|
||||
vehicleId,
|
||||
catalogVehicleId: null as string | null,
|
||||
name: trMap.get(node.name) ?? node.name,
|
||||
name: displayName,
|
||||
nameOriginal: node.name,
|
||||
parentId,
|
||||
externalId: node.gid || null,
|
||||
linkPath: isLeaf ? node.url || null : null,
|
||||
linkWid: null as string | null,
|
||||
externalId: isLeaf ? (node.code ?? null) : (node.cid ?? null),
|
||||
// Groups keep their Vehicle.aspx drill URL but are marked
|
||||
// non-leaf via linkWid="emex-group" + hasSubgroups so the
|
||||
// tree UI / lazy-drill never treat them as parts leaves.
|
||||
linkPath: node.url,
|
||||
linkWid: isLeaf ? (null as string | null) : "emex-group",
|
||||
hasSubgroups: !isLeaf,
|
||||
hasParts: isLeaf,
|
||||
source: "emex" as const,
|
||||
})
|
||||
.onConflictDoNothing()
|
||||
@@ -316,16 +343,19 @@ export class CategoriesService {
|
||||
|
||||
if (inserted) {
|
||||
dbCategories.push(inserted);
|
||||
// A top group passes its own display name down as the
|
||||
// disambiguation context for its sub-groups.
|
||||
const ctx = parentId === null ? displayName : topName;
|
||||
if (node.children?.length) {
|
||||
await insertNodes(node.children, inserted.id);
|
||||
await insertNodes(node.children, inserted.id, ctx);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
await insertNodes(tree, null);
|
||||
await insertNodes(tree, null, null);
|
||||
this.logger.log(
|
||||
`Stored ${dbCategories.length} EMEX categories (tree) for ${vehicle.vin}`,
|
||||
`Stored ${dbCategories.length} EMEX categories (Vehicle.aspx tree) for ${vehicle.vin}`,
|
||||
);
|
||||
} else if (emexResult.categories.length > 0) {
|
||||
// Flat fallback: insert all categories without hierarchy
|
||||
@@ -390,7 +420,11 @@ export class CategoriesService {
|
||||
|
||||
// Build tree
|
||||
const tree = this.buildTree(dbCategories);
|
||||
await this.redis.setJson(cacheKey, tree, 3600);
|
||||
// Cache a populated tree for an hour; an EMPTY tree (transient decode/proxy
|
||||
// failure) only for 60s so a blip doesn't poison the catalog for an hour —
|
||||
// it self-heals on the next request after the source recovers, while still
|
||||
// throttling re-decode attempts during a real outage.
|
||||
await this.redis.setJson(cacheKey, tree, tree.length > 0 ? 3600 : 60);
|
||||
return tree;
|
||||
}
|
||||
|
||||
@@ -524,6 +558,65 @@ export class CategoriesService {
|
||||
return children;
|
||||
}
|
||||
|
||||
// EMEX Vehicle.aspx group node — drill its href to Unit.aspx leaves on
|
||||
// demand. Top groups already have their sub-groups seeded (returned from
|
||||
// the DB-children check above), so this only fires for sub-groups whose
|
||||
// unit leaves haven't been fetched yet. emex carries no catalogInfo, so
|
||||
// this must run before the pl24 serviceName guard below.
|
||||
if (category.source === "emex" && category.linkWid === "emex-group" && linkPath) {
|
||||
try {
|
||||
const leaves = await this.emexService.drillVehicleNode(linkPath);
|
||||
if (leaves.length > 0) {
|
||||
const trMap = await this.translationsService.translateMany(
|
||||
leaves.map((l) => l.name).filter(Boolean),
|
||||
);
|
||||
const seenNames = new Set<string>();
|
||||
const insertData: (typeof categories.$inferInsert)[] = [];
|
||||
for (const l of leaves) {
|
||||
const name = trMap.get(l.name) ?? l.name;
|
||||
if (seenNames.has(name)) continue; // unique index is per-name
|
||||
seenNames.add(name);
|
||||
insertData.push({
|
||||
vehicleId: category.vehicleId,
|
||||
catalogVehicleId: category.catalogVehicleId,
|
||||
name,
|
||||
nameOriginal: l.name,
|
||||
parentId: categoryId,
|
||||
externalId: l.code ?? l.cid ?? null,
|
||||
linkPath: l.url, // Unit.aspx → direct parts fetch
|
||||
linkWid: null as string | null,
|
||||
hasSubgroups: false,
|
||||
hasParts: true,
|
||||
source: "emex" as const,
|
||||
});
|
||||
}
|
||||
|
||||
children = await this.db
|
||||
.insert(categories)
|
||||
.values(insertData)
|
||||
.onConflictDoNothing()
|
||||
.returning();
|
||||
|
||||
if (children.length < insertData.length) {
|
||||
children = await this.db
|
||||
.select()
|
||||
.from(categories)
|
||||
.where(eq(categories.parentId, categoryId));
|
||||
}
|
||||
|
||||
await this.redis.del(`cat:tree:${category.vehicleId}`);
|
||||
}
|
||||
} catch (err) {
|
||||
this.logger.error(
|
||||
`EMEX node drill failed for ${categoryId} (${linkPath}): ${(err as Error).message}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Unit leaves carry a Unit.aspx linkPath + no DB children, so
|
||||
// enrichWithSchemaImages already marks them as leaves (children: []).
|
||||
return children.length > 0 ? this.enrichWithSchemaImages(children) : children;
|
||||
}
|
||||
|
||||
if (!catalogInfo?.serviceName || !linkPath) {
|
||||
return [];
|
||||
}
|
||||
@@ -930,6 +1023,27 @@ export class CategoriesService {
|
||||
// No subgroups → leaf; fall through to the parts path below.
|
||||
}
|
||||
|
||||
// EMEX Vehicle.aspx group node (linkWid="emex-group") — a parent, never a
|
||||
// parts leaf. Return its children (seeded sub-groups, or Unit leaves drilled
|
||||
// on demand by getChildren). Mirrors the pl24 group-node branch above; an
|
||||
// emex group is never legitimately empty, so surface a load error if drill
|
||||
// comes back empty rather than a misleading "no parts" leaf.
|
||||
if (category.source === "emex" && category.linkWid === "emex-group" && category.vehicleId) {
|
||||
const emexChildren = await this.getChildren(categoryId);
|
||||
const base = {
|
||||
id: category.id,
|
||||
name: category.name,
|
||||
description: category.nameOriginal || null,
|
||||
parentId: category.parentId || null,
|
||||
parts: [],
|
||||
schemaPics: [],
|
||||
hotspots: [],
|
||||
};
|
||||
return emexChildren.length > 0
|
||||
? { ...base, children: emexChildren }
|
||||
: { ...base, loadError: true };
|
||||
}
|
||||
|
||||
// Leaf category — get or fetch parts
|
||||
let loadError = false;
|
||||
let discoveredChildren: any[] = [];
|
||||
@@ -1674,7 +1788,9 @@ export class CategoriesService {
|
||||
// rendered as empty leaves.
|
||||
const isLeaf =
|
||||
c.source === "emex"
|
||||
? !!c.linkPath && dbChildCount === 0
|
||||
? // emex-group = Vehicle.aspx parent node (never a leaf); a unit leaf
|
||||
// has a Unit.aspx linkPath and no DB children.
|
||||
c.linkWid !== "emex-group" && !!c.linkPath && dbChildCount === 0
|
||||
: c.source === "parts-catalogs"
|
||||
? (() => {
|
||||
if (dbChildCount > 0) return false; // already has children → parent
|
||||
|
||||
1395
apps/api/src/integrations/emex/__fixtures__/vehicle-engblock.html
Normal file
1395
apps/api/src/integrations/emex/__fixtures__/vehicle-engblock.html
Normal file
File diff suppressed because it is too large
Load Diff
2795
apps/api/src/integrations/emex/__fixtures__/vehicle-powertrain.html
Normal file
2795
apps/api/src/integrations/emex/__fixtures__/vehicle-powertrain.html
Normal file
File diff suppressed because it is too large
Load Diff
1275
apps/api/src/integrations/emex/__fixtures__/vehicle-root.html
Normal file
1275
apps/api/src/integrations/emex/__fixtures__/vehicle-root.html
Normal file
File diff suppressed because it is too large
Load Diff
111
apps/api/src/integrations/emex/emex-tree.parser.spec.ts
Normal file
111
apps/api/src/integrations/emex/emex-tree.parser.spec.ts
Normal file
@@ -0,0 +1,111 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { parseUnitLeaves, parseVehicleTree } from "./emex-tree.parser";
|
||||
|
||||
const fixture = (name: string) => readFileSync(join(__dirname, "__fixtures__", name), "utf-8");
|
||||
|
||||
// Captured 2026-06-05 from emexdwc.ae catalog FORD202201 (VIN WF0JXXGAJJBJ88319, vid=0):
|
||||
// vehicle-root.html — Vehicle.aspx root (nothing selected)
|
||||
// vehicle-powertrain.html — Vehicle.aspx?cid=3 (Powertrain selected; 162 raw / 81 unique units)
|
||||
// vehicle-engblock.html — Vehicle.aspx?cid=8289 (Eng Block sub-group; 22 raw / 11 unique units)
|
||||
const ROOT = fixture("vehicle-root.html");
|
||||
const POWERTRAIN = fixture("vehicle-powertrain.html");
|
||||
const ENGBLOCK = fixture("vehicle-engblock.html");
|
||||
|
||||
const TOP_LABELS = ["Chassis", "Powertrain", "Electrical", "Body And Paint"];
|
||||
|
||||
describe("parseVehicleTree — 2-level group tree from the Vehicle.aspx sidebar", () => {
|
||||
it("extracts exactly the 7 FCSD top groups from the root page", () => {
|
||||
const tree = parseVehicleTree(ROOT);
|
||||
expect(tree).toHaveLength(7);
|
||||
expect(tree.every((n) => n.kind === "group")).toBe(true);
|
||||
// The 7 section roots carry small cids.
|
||||
expect(tree.map((n) => n.cid).sort((a, b) => Number(a) - Number(b))).toEqual([
|
||||
"1",
|
||||
"2",
|
||||
"3",
|
||||
"4",
|
||||
"5",
|
||||
"8",
|
||||
"9",
|
||||
]);
|
||||
const labels = tree.map((n) => n.name);
|
||||
for (const l of TOP_LABELS) {
|
||||
expect(labels.some((x) => x.startsWith(l))).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("yields the same 7 top groups from a drilled node page (menu is identical everywhere)", () => {
|
||||
// Powertrain is the SELECTED node here (rendered as bare text, no <a>) —
|
||||
// must still be captured via its onclick URL.
|
||||
const tree = parseVehicleTree(POWERTRAIN);
|
||||
expect(tree).toHaveLength(7);
|
||||
const powertrain = tree.find((n) => n.cid === "3");
|
||||
expect(powertrain?.name).toBe("Powertrain");
|
||||
});
|
||||
|
||||
it("nests Powertrain sub-groups under the Powertrain top group", () => {
|
||||
const tree = parseVehicleTree(ROOT);
|
||||
const powertrain = tree.find((n) => n.cid === "3");
|
||||
expect(powertrain).toBeDefined();
|
||||
const subCids = powertrain!.children.map((c) => c.cid);
|
||||
const subNames = powertrain!.children.map((c) => c.name);
|
||||
// Automatic Transmission is cid=4004 — the group whose Unit leaves include
|
||||
// the torque converter. Confirms the converter's home is under Powertrain.
|
||||
expect(subCids).toContain("4004");
|
||||
expect(subNames).toContain("Automatic Transmission");
|
||||
expect(subNames).toContain("Clutch");
|
||||
expect(subNames).toContain("Manual Transmission");
|
||||
expect(powertrain!.children.every((c) => c.kind === "group")).toBe(true);
|
||||
});
|
||||
|
||||
it("gives every group a verbatim Vehicle.aspx drill URL with cid + ssd", () => {
|
||||
const tree = parseVehicleTree(ROOT);
|
||||
const all = tree.flatMap((t) => [t, ...t.children]);
|
||||
expect(all.length).toBeGreaterThan(120); // ~136 total group nodes
|
||||
for (const n of all) {
|
||||
expect(n.url).toMatch(/^https:\/\/emexdwc\.ae\/Vehicle\.aspx\?/);
|
||||
expect(n.url).toMatch(/[?&]cid=\d+/);
|
||||
expect(n.url).toMatch(/[?&]ssd=\$\*/); // opaque per-node ssd present
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseUnitLeaves — Unit.aspx leaves from a drilled group page", () => {
|
||||
it("extracts all Powertrain unit leaves incl. the torque converter (closes the taxonomy gap)", () => {
|
||||
const leaves = parseUnitLeaves(POWERTRAIN);
|
||||
// 81 distinct unit hrefs (the raw HTML lists each twice — image + text
|
||||
// anchor — so dedup-by-href halves the ~162 raw matches).
|
||||
expect(leaves.length).toBeGreaterThan(60);
|
||||
expect(leaves.every((l) => l.kind === "leaf")).toBe(true);
|
||||
|
||||
const converter = leaves.find((l) => l.code === "3070160A");
|
||||
expect(converter).toBeDefined();
|
||||
expect(converter!.name).toMatch(/Converter/i);
|
||||
expect(converter!.name).toMatch(/Automatic Trans/i);
|
||||
// Leaf URL is a self-contained Unit.aspx fetch URL (the existing Unit
|
||||
// parts-parser can GET it directly).
|
||||
expect(converter!.url).toMatch(/^https:\/\/emexdwc\.ae\/Unit\.aspx\?/);
|
||||
expect(converter!.url).toMatch(/[?&]ssd=\$\*/);
|
||||
});
|
||||
|
||||
it("parses section codes off the 'CODE: name' label", () => {
|
||||
const leaves = parseUnitLeaves(POWERTRAIN);
|
||||
// Service Engine And Short Block is 3030105A.
|
||||
const engine = leaves.find((l) => l.code === "3030105A");
|
||||
expect(engine).toBeDefined();
|
||||
expect(engine!.name).toMatch(/Service Engine And Short Block/i);
|
||||
// Every leaf got a non-empty code + name (the label always has the prefix).
|
||||
expect(leaves.every((l) => l.code && l.code.length >= 4)).toBe(true);
|
||||
expect(leaves.every((l) => l.name.length > 0)).toBe(true);
|
||||
});
|
||||
|
||||
it("narrows to fewer leaves when a SUB-group is drilled (Eng Block ⊂ Powertrain)", () => {
|
||||
const sub = parseUnitLeaves(ENGBLOCK);
|
||||
const top = parseUnitLeaves(POWERTRAIN);
|
||||
expect(sub.length).toBeGreaterThan(0);
|
||||
expect(sub.length).toBeLessThan(top.length); // 11 < 81 (unique)
|
||||
expect(sub.every((l) => l.url.includes("Unit.aspx"))).toBe(true);
|
||||
});
|
||||
});
|
||||
185
apps/api/src/integrations/emex/emex-tree.parser.ts
Normal file
185
apps/api/src/integrations/emex/emex-tree.parser.ts
Normal file
@@ -0,0 +1,185 @@
|
||||
/**
|
||||
* emex Vehicle.aspx hierarchical-tree parsers.
|
||||
*
|
||||
* Background: emex exposes the SAME catalog through two navigation trees.
|
||||
* • QuickGroups.aspx — a FLAT list of ~351 functional Laximo categories
|
||||
* (gid 10xxx). This is what the legacy seeder (`parseCategoryList` in
|
||||
* emex.service.ts) scrapes, producing the unsearchable "everything on one
|
||||
* page" experience.
|
||||
* • Vehicle.aspx — the OEM-microfiche tree: a small set of TOP groups
|
||||
* (Chassis, Powertrain, Electrical, …) → SUB groups (Automatic
|
||||
* Transmission, Clutch, …) → Unit.aspx LEAVES carrying section codes
|
||||
* (e.g. "3070160A: Converter, 4 Speed Automatic Trans") and parts.
|
||||
*
|
||||
* The Vehicle.aspx tree is fully traversable by plain HTTP GET: every node
|
||||
* link carries its own opaque `ssd` in the href, copied verbatim hop-to-hop
|
||||
* (no cookies, no __VIEWSTATE postback — verified 2026-06-05). These parsers
|
||||
* turn the raw HTML into a tree we can seed into the generic `categories`
|
||||
* table (parentId chains + hasSubgroups/hasParts), so the existing tree UI
|
||||
* renders it exactly like pl24/pcat — no frontend changes.
|
||||
*
|
||||
* Pure functions, no DI — unit-tested against captured fixtures in
|
||||
* __fixtures__/. Regex-based (no cheerio) to match the existing emex parsers.
|
||||
*/
|
||||
|
||||
import type { EmexVehicleTreeNode } from "./emex.types";
|
||||
|
||||
const EMEX_BASE_URL = "https://emexdwc.ae";
|
||||
|
||||
/**
|
||||
* Section roots (top groups) carry a small `cid`. For FORD202201 the 7 roots
|
||||
* are cid 1,2,3,4,5,8,9 (Information, Chassis, Powertrain, Electrical, Body,
|
||||
* Fluids, Suspension/Accessories); every other menu entry is a sub-group. If a
|
||||
* future catalog numbers differently and NO node has cid ≤ this max, the tree
|
||||
* degrades gracefully to a single flat group level (see parseVehicleTree).
|
||||
*/
|
||||
const TOP_GROUP_CID_MAX = 9;
|
||||
|
||||
function decodeEntities(s: string): string {
|
||||
return s
|
||||
.replace(/'/g, "'")
|
||||
.replace(/&/g, "&")
|
||||
.replace(/"/g, '"')
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/ /g, " ");
|
||||
}
|
||||
|
||||
function stripTags(s: string): string {
|
||||
return s.replace(/<[^>]+>/g, " ");
|
||||
}
|
||||
|
||||
function cidOf(url: string): string | null {
|
||||
const m = url.match(/[?&]cid=(\d+)/);
|
||||
return m ? m[1] : null;
|
||||
}
|
||||
|
||||
/** A raw category entry from the persistent Vehicle.aspx sidebar menu. */
|
||||
interface MenuNode {
|
||||
cid: string;
|
||||
name: string;
|
||||
url: string; // absolute Vehicle.aspx drill URL (ssd embedded)
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract the persistent category sidebar (identical on every Vehicle.aspx
|
||||
* page). Each item is:
|
||||
* <div class="guayaquil_categoryitem"
|
||||
* onclick="window.location='Vehicle.aspx?...cid=N&ssd=...'"
|
||||
* onmouseout=... onmouseover=... style="margin-left:0px">
|
||||
* <a href="Vehicle.aspx?...">LABEL</a> // normal
|
||||
* </div>
|
||||
* The CURRENTLY-SELECTED node renders with class `guayaquil_categoryitem_selected`
|
||||
* and its label as bare text (no <a>), but still carries the onclick URL — so
|
||||
* we read cid+url+ssd uniformly from the onclick, and the label from the inner
|
||||
* text (anchor or bare). Document order is preserved (top group, then its
|
||||
* sub-groups, then the next top group …).
|
||||
*/
|
||||
function parseMenuNodes(html: string): MenuNode[] {
|
||||
const itemRx =
|
||||
/<div class="guayaquil_categoryitem(?:_selected)?"\s+onclick="window\.location='(Vehicle\.aspx\?[\s\S]*?)'"[^>]*>([\s\S]*?)<\/div>/g;
|
||||
const nodes: MenuNode[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const m of html.matchAll(itemRx)) {
|
||||
const url = decodeEntities(m[1]);
|
||||
const cid = cidOf(url);
|
||||
if (!cid) continue;
|
||||
const name = decodeEntities(stripTags(m[2])).replace(/\s+/g, " ").trim();
|
||||
if (!name) continue;
|
||||
// Dedup on cid: the menu can repeat an entry; first occurrence wins (keeps
|
||||
// document order for parent inference).
|
||||
if (seen.has(cid)) continue;
|
||||
seen.add(cid);
|
||||
nodes.push({ cid, name, url: `${EMEX_BASE_URL}/${url}` });
|
||||
}
|
||||
return nodes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the 2-level GROUP tree (top groups → sub-groups) from a Vehicle.aspx
|
||||
* page's sidebar. Works on ANY page (root or a drilled node) since the menu is
|
||||
* identical everywhere. Leaves (Unit.aspx) are NOT included here — they're
|
||||
* fetched lazily per sub-group via {@link parseUnitLeaves}.
|
||||
*
|
||||
* Classification: cid ≤ TOP_GROUP_CID_MAX → top group; otherwise a sub-group
|
||||
* attached to the most recent top group seen in document order. If the catalog
|
||||
* exposes no small-cid roots, every node becomes a flat top-level group.
|
||||
*/
|
||||
export function parseVehicleTree(html: string): EmexVehicleTreeNode[] {
|
||||
const menu = parseMenuNodes(html);
|
||||
const hasRoots = menu.some((n) => Number(n.cid) <= TOP_GROUP_CID_MAX);
|
||||
|
||||
const tops: EmexVehicleTreeNode[] = [];
|
||||
let current: EmexVehicleTreeNode | null = null;
|
||||
|
||||
for (const n of menu) {
|
||||
const isTop = !hasRoots || Number(n.cid) <= TOP_GROUP_CID_MAX;
|
||||
const node: EmexVehicleTreeNode = {
|
||||
cid: n.cid,
|
||||
name: n.name,
|
||||
code: null,
|
||||
url: n.url,
|
||||
kind: "group",
|
||||
imageUrl: null,
|
||||
children: [],
|
||||
};
|
||||
if (isTop) {
|
||||
tops.push(node);
|
||||
current = node;
|
||||
} else if (current) {
|
||||
current.children.push(node);
|
||||
} else {
|
||||
// Sub-group seen before any top group (shouldn't happen for Ford, but be
|
||||
// safe): promote it to a top-level group.
|
||||
tops.push(node);
|
||||
}
|
||||
}
|
||||
return tops;
|
||||
}
|
||||
|
||||
/** title → img.laximo.net url, harvested from the unit illustration thumbnails. */
|
||||
function parseUnitImages(html: string): Map<string, string> {
|
||||
const map = new Map<string, string>();
|
||||
const rx =
|
||||
/<div class="guayaquil_zoom"\s+full="(https:\/\/img\.laximo\.net[^"]+)"\s+title="([^"]*)"/g;
|
||||
for (const m of html.matchAll(rx)) {
|
||||
const url = decodeEntities(m[1]);
|
||||
const title = decodeEntities(m[2]).replace(/\s+/g, " ").trim();
|
||||
if (title && !map.has(title)) map.set(title, url);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the Unit.aspx LEAVES from a drilled node page's content area. The
|
||||
* sidebar contains only Vehicle.aspx links, so every Unit.aspx anchor on the
|
||||
* page belongs to the currently-drilled group. Each leaf label is
|
||||
* "<sectionCode>: <name>" (e.g. "3070160A: Converter, 4 Speed Automatic Trans")
|
||||
* and the href is a self-contained Unit.aspx fetch URL (cid + ssd + path_id).
|
||||
*/
|
||||
export function parseUnitLeaves(html: string): EmexVehicleTreeNode[] {
|
||||
const images = parseUnitImages(html);
|
||||
const anchorRx = /<a href="(Unit\.aspx\?[^"]+)"[^>]*>([\s\S]*?)<\/a>/g;
|
||||
const leaves: EmexVehicleTreeNode[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const m of html.matchAll(anchorRx)) {
|
||||
const url = decodeEntities(m[1]);
|
||||
if (seen.has(url)) continue;
|
||||
seen.add(url);
|
||||
const label = decodeEntities(stripTags(m[2])).replace(/\s+/g, " ").trim();
|
||||
if (!label) continue;
|
||||
const codeMatch = label.match(/^([0-9A-Za-z]+):\s*(.*)$/);
|
||||
const code = codeMatch ? codeMatch[1] : null;
|
||||
const name = codeMatch ? codeMatch[2].trim() : label;
|
||||
leaves.push({
|
||||
cid: cidOf(url) ?? "",
|
||||
name,
|
||||
code,
|
||||
url: `${EMEX_BASE_URL}/${url}`,
|
||||
kind: "leaf",
|
||||
imageUrl: images.get(label) ?? null,
|
||||
children: [],
|
||||
});
|
||||
}
|
||||
return leaves;
|
||||
}
|
||||
@@ -22,6 +22,7 @@ import { ConfigService } from "@nestjs/config";
|
||||
import { ProxyAgent } from "undici";
|
||||
import { isBackfillContext } from "../../jobs/prefetch-context";
|
||||
import { RedisService } from "../../redis/redis.service";
|
||||
import { parseUnitLeaves, parseVehicleTree } from "./emex-tree.parser";
|
||||
import { EmexBrowserService } from "./emex.browser";
|
||||
import { createEmptyDecodedVehicle, mapEmexResponse } from "./emex.mapper";
|
||||
import {
|
||||
@@ -33,6 +34,7 @@ import {
|
||||
type EmexPart,
|
||||
type EmexPartsResult,
|
||||
type EmexScraperResponse,
|
||||
type EmexVehicleTreeNode,
|
||||
} from "./emex.types";
|
||||
|
||||
const EMEX_BASE_URL = "https://emexdwc.ae";
|
||||
@@ -113,6 +115,21 @@ export class EmexService {
|
||||
portStart: number;
|
||||
portEnd: number;
|
||||
} | null;
|
||||
// Floxy residential fallback — used when the DataImpulse pool throws transport
|
||||
// errors. `lifetime` > 0 → sticky session (one exit IP held that many seconds,
|
||||
// appended to the password as `_session-<id>_lifetime-<n>`); 0 → rotating.
|
||||
private readonly emexFloxy: {
|
||||
host: string;
|
||||
port: number;
|
||||
user: string;
|
||||
pass: string;
|
||||
lifetime: number;
|
||||
} | null;
|
||||
// Rolling Floxy sticky-session id: held for ~lifetime so a chained emex flow
|
||||
// (decode → tree → drill → parts) keeps ONE exit IP and emex's IP-bound ssd
|
||||
// tokens stay valid. Rotated when stale or after a transport failure.
|
||||
private floxySessionId = Math.random().toString(36).slice(2, 10);
|
||||
private floxySessionBornMs = Date.now();
|
||||
private readonly emexDirectFallback: boolean;
|
||||
|
||||
constructor(
|
||||
@@ -161,6 +178,26 @@ export class EmexService {
|
||||
this.proxyAgent = null;
|
||||
}
|
||||
|
||||
// Floxy residential fallback for when the DataImpulse pool flakes (connect
|
||||
// timeouts / resets). On by default; creds + endpoint overridable via env.
|
||||
const floxyEnabled = this.configService.get<string>("EMEX_FLOXY_FALLBACK", "true") === "true";
|
||||
if (floxyEnabled) {
|
||||
const fport = Number(this.configService.get("EMEX_FLOXY_PORT", 12321));
|
||||
const flife = Number(this.configService.get("EMEX_FLOXY_LIFETIME", 300));
|
||||
this.emexFloxy = {
|
||||
host: this.configService.get<string>("EMEX_FLOXY_HOST", "residential.floxy.io"),
|
||||
port: Number.isInteger(fport) && fport >= 1 && fport <= 65535 ? fport : 12321,
|
||||
user: this.configService.get<string>("EMEX_FLOXY_USER", "d739255e819b"),
|
||||
pass: this.configService.get<string>("EMEX_FLOXY_PASS", "9092873ba4e0"),
|
||||
lifetime: Number.isInteger(flife) && flife >= 0 ? flife : 300,
|
||||
};
|
||||
this.logger.log(
|
||||
`EMEX Floxy fallback enabled: ${this.emexFloxy.host}:${this.emexFloxy.port} (${this.emexFloxy.lifetime > 0 ? `sticky ${this.emexFloxy.lifetime}s` : "rotating"})`,
|
||||
);
|
||||
} else {
|
||||
this.emexFloxy = null;
|
||||
}
|
||||
|
||||
this.logger.log(`EMEX Service initialized with scraper path: ${this.scraperPath}`);
|
||||
}
|
||||
|
||||
@@ -267,7 +304,36 @@ export class EmexService {
|
||||
* without requiring authentication cookies.
|
||||
*/
|
||||
/** Build a fresh proxy agent on a random port from the pool (null if proxy off). */
|
||||
private newProxyAgent(): ProxyAgent | null {
|
||||
/** Fresh Floxy sticky-session id (new exit IP). */
|
||||
private rotateFloxySession(): void {
|
||||
this.floxySessionId = Math.random().toString(36).slice(2, 10);
|
||||
this.floxySessionBornMs = Date.now();
|
||||
}
|
||||
|
||||
/**
|
||||
* Floxy auth password. With lifetime > 0, appends `_session-<id>_lifetime-<n>`
|
||||
* to pin one exit IP across a chained emex flow; the session id rolls over
|
||||
* once it nears the lifetime so we never reuse an expired sticky slot.
|
||||
*/
|
||||
private floxyPassword(): string {
|
||||
if (!this.emexFloxy) return "";
|
||||
const { pass, lifetime } = this.emexFloxy;
|
||||
if (lifetime <= 0) return pass; // rotating mode
|
||||
// Roll at ~80% of lifetime so a sticky IP is never reused past expiry.
|
||||
if (Date.now() - this.floxySessionBornMs > lifetime * 800) this.rotateFloxySession();
|
||||
return `${pass}_session-${this.floxySessionId}_lifetime-${lifetime}`;
|
||||
}
|
||||
|
||||
private newProxyAgent(provider: "dataimpulse" | "floxy" = "dataimpulse"): ProxyAgent | null {
|
||||
if (provider === "floxy") {
|
||||
if (!this.emexFloxy) return null;
|
||||
const { host, port, user } = this.emexFloxy;
|
||||
return new ProxyAgent({
|
||||
uri: `http://${user}:${this.floxyPassword()}@${host}:${port}`,
|
||||
connect: { timeout: 30000 },
|
||||
requestTls: { timeout: 30000 },
|
||||
});
|
||||
}
|
||||
if (!this.emexProxy) return null;
|
||||
const { host, user, pass, portStart, portEnd } = this.emexProxy;
|
||||
const port = Math.floor(Math.random() * (portEnd - portStart + 1)) + portStart;
|
||||
@@ -279,14 +345,19 @@ export class EmexService {
|
||||
}
|
||||
|
||||
private async fetchEmexHtml(url: string): Promise<string> {
|
||||
const maxAttempts = this.emexProxy ? 3 : 1;
|
||||
// Attempt schedule: DataImpulse (rotating port, fresh agent each try to dodge
|
||||
// a flaky sticky port — ~42% blip rate, undecoded-vin-rca.md EMEX #1) first,
|
||||
// then the Floxy residential fallback when the DataImpulse pool throws
|
||||
// transport errors. A definitive HTTP answer (e.g. 404) stops the schedule —
|
||||
// it's a real result, and a different proxy IP must not "retry" it away.
|
||||
const schedule: Array<"dataimpulse" | "floxy"> = [];
|
||||
if (this.emexProxy) schedule.push("dataimpulse", "dataimpulse", "dataimpulse");
|
||||
if (this.emexFloxy) schedule.push("floxy", "floxy");
|
||||
const maxAttempts = schedule.length || 1; // 0 → single proxy-less attempt
|
||||
let lastErr: unknown;
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||
// Fresh agent every attempt: a single sticky DataImpulse port flakes
|
||||
// intermittently ("fetch failed"/reset), silently dropping real emex hits
|
||||
// (~42% blip rate observed — undecoded-vin-rca.md, EMEX #1). A new agent
|
||||
// rotates the port and forces a new socket even on a single-port pool.
|
||||
const agent = this.newProxyAgent();
|
||||
const provider = schedule[attempt - 1]; // undefined when no proxy → direct
|
||||
const agent = provider ? this.newProxyAgent(provider) : null;
|
||||
try {
|
||||
const res = await fetch(url, {
|
||||
headers: { "User-Agent": EMEX_UA, Accept: "text/html,application/xhtml+xml" },
|
||||
@@ -296,6 +367,9 @@ export class EmexService {
|
||||
if (!res.ok) {
|
||||
throw new Error(`EMEX HTTP ${res.status} for ${url}`);
|
||||
}
|
||||
if (provider === "floxy") {
|
||||
this.logger.log(`EMEX fetch via Floxy fallback succeeded for ${url}`);
|
||||
}
|
||||
return await res.text();
|
||||
} catch (err) {
|
||||
lastErr = err;
|
||||
@@ -311,15 +385,21 @@ export class EmexService {
|
||||
`${e.message} ${String(e.cause ?? "")}`,
|
||||
));
|
||||
if (!transient || attempt === maxAttempts) break;
|
||||
// A Floxy transport failure means the current sticky exit IP is dead —
|
||||
// roll to a fresh session so the next Floxy attempt gets a new IP.
|
||||
if (provider === "floxy") this.rotateFloxySession();
|
||||
const next = schedule[attempt];
|
||||
const via =
|
||||
next === "floxy" && provider !== "floxy" ? "via Floxy fallback" : "with fresh proxy";
|
||||
this.logger.warn(
|
||||
`EMEX fetch transient error (attempt ${attempt}/${maxAttempts}) for ${url}: ${e.message} — retrying with fresh proxy`,
|
||||
`EMEX fetch transient error (attempt ${attempt}/${maxAttempts}, ${provider}) for ${url}: ${e.message} — retrying ${via}`,
|
||||
);
|
||||
await new Promise((r) => setTimeout(r, 300 * attempt));
|
||||
}
|
||||
}
|
||||
// Last resort (opt-in via EMEX_DIRECT_FALLBACK): one direct, proxy-less attempt
|
||||
// for when the whole pool is down. Off by default — it exposes the origin IP.
|
||||
if (this.emexProxy && this.emexDirectFallback) {
|
||||
// for when both proxy pools are down. Off by default — it exposes the origin IP.
|
||||
if ((this.emexProxy || this.emexFloxy) && this.emexDirectFallback) {
|
||||
try {
|
||||
const res = await fetch(url, {
|
||||
headers: { "User-Agent": EMEX_UA, Accept: "text/html,application/xhtml+xml" },
|
||||
@@ -386,6 +466,33 @@ export class EmexService {
|
||||
return cats;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch + parse the hierarchical Vehicle.aspx tree (top groups → sub groups)
|
||||
* for a decoded vehicle. One plain GET of the catalog root; the persistent
|
||||
* sidebar carries the whole 2-level group tree (units are drilled lazily per
|
||||
* sub-group via {@link drillVehicleNode}). Returns [] on any failure so the
|
||||
* caller falls back to the flat QuickGroups categories.
|
||||
*/
|
||||
private async fetchVehicleTree(
|
||||
catalogCode: string,
|
||||
vid: string,
|
||||
ssd: string,
|
||||
): Promise<EmexVehicleTreeNode[]> {
|
||||
const url = `${EMEX_BASE_URL}/Vehicle.aspx?c=${catalogCode}&vid=${vid}&ssd=${encodeURIComponent(ssd)}`;
|
||||
const html = await this.fetchEmexHtml(url);
|
||||
return parseVehicleTree(html);
|
||||
}
|
||||
|
||||
/**
|
||||
* Drill a Vehicle.aspx group node (its verbatim href, ssd embedded) to its
|
||||
* Unit.aspx leaves. Used by the lazy category drill (categories.service
|
||||
* getChildren) when an emex group is first expanded.
|
||||
*/
|
||||
async drillVehicleNode(vehicleAspxUrl: string): Promise<EmexVehicleTreeNode[]> {
|
||||
const html = await this.fetchEmexHtml(vehicleAspxUrl);
|
||||
return parseUnitLeaves(html);
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine brand name from EMEX catalog code (e.g. "BMW202501" → "BMW").
|
||||
*/
|
||||
@@ -455,6 +562,19 @@ export class EmexService {
|
||||
}
|
||||
}
|
||||
|
||||
// Hierarchical Vehicle.aspx tree (primary nav); the flat `categories`
|
||||
// above is the fallback. One GET of the catalog root yields the full
|
||||
// 2-level group tree; units are drilled lazily per sub-group on expand.
|
||||
let categoryTree: EmexVehicleTreeNode[] = [];
|
||||
if (v.catalogCode && v.ssd) {
|
||||
try {
|
||||
categoryTree = await this.fetchVehicleTree(v.catalogCode, v.vid ?? "0", v.ssd);
|
||||
this.logger.log(`EMEX HTTP: parsed ${categoryTree.length} Vehicle.aspx top groups`);
|
||||
} catch (err) {
|
||||
this.logger.warn(`EMEX Vehicle.aspx tree fetch failed: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Build a response compatible with mapEmexResponse
|
||||
const response: EmexScraperResponse = {
|
||||
success: true,
|
||||
@@ -476,7 +596,7 @@ export class EmexService {
|
||||
},
|
||||
quickGroupsUrl: v.quickGroupsUrl || null,
|
||||
categories: categories.map((c) => ({ gid: c.gid || "", name: c.name, url: c.url })),
|
||||
categoryTree: [],
|
||||
categoryTree,
|
||||
timestamp: new Date().toISOString(),
|
||||
};
|
||||
|
||||
@@ -541,6 +661,17 @@ export class EmexService {
|
||||
}
|
||||
}
|
||||
|
||||
// Hierarchical Vehicle.aspx tree (primary nav); flat `categories` = fallback.
|
||||
let categoryTree: EmexVehicleTreeNode[] = [];
|
||||
if (v.catalogCode && v.ssd) {
|
||||
try {
|
||||
categoryTree = await this.fetchVehicleTree(v.catalogCode, v.vid ?? "0", v.ssd);
|
||||
this.logger.log(`EMEX HTTP: parsed ${categoryTree.length} Vehicle.aspx top groups`);
|
||||
} catch (err) {
|
||||
this.logger.warn(`EMEX Vehicle.aspx tree fetch failed: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
const response: EmexScraperResponse = {
|
||||
success: true,
|
||||
source: "emexdwc.ae",
|
||||
@@ -561,7 +692,7 @@ export class EmexService {
|
||||
},
|
||||
quickGroupsUrl: v.quickGroupsUrl || null,
|
||||
categories: categories.map((c) => ({ gid: c.gid || "", name: c.name, url: c.url })),
|
||||
categoryTree: [],
|
||||
categoryTree,
|
||||
timestamp: new Date().toISOString(),
|
||||
};
|
||||
|
||||
@@ -608,6 +739,17 @@ export class EmexService {
|
||||
}
|
||||
}
|
||||
|
||||
// Hierarchical Vehicle.aspx tree (primary nav); flat `categories` = fallback.
|
||||
let categoryTree: EmexVehicleTreeNode[] = [];
|
||||
if (v.catalogCode && v.ssd) {
|
||||
try {
|
||||
categoryTree = await this.fetchVehicleTree(v.catalogCode, v.vid ?? "0", v.ssd);
|
||||
this.logger.log(`EMEX HTTP: parsed ${categoryTree.length} Vehicle.aspx top groups`);
|
||||
} catch (err) {
|
||||
this.logger.warn(`EMEX Vehicle.aspx tree fetch failed: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
const response: EmexScraperResponse = {
|
||||
success: true,
|
||||
source: "emexdwc.ae",
|
||||
@@ -628,7 +770,7 @@ export class EmexService {
|
||||
},
|
||||
quickGroupsUrl: v.quickGroupsUrl || null,
|
||||
categories: categories.map((c) => ({ gid: c.gid || "", name: c.name, url: c.url })),
|
||||
categoryTree: [],
|
||||
categoryTree,
|
||||
timestamp: new Date().toISOString(),
|
||||
};
|
||||
|
||||
@@ -862,6 +1004,28 @@ export class EmexService {
|
||||
* parts — caller treats that as "fall back to browser".
|
||||
*/
|
||||
private async fetchCategoryPartsViaHttp(categoryUrl: string): Promise<EmexPartsResult> {
|
||||
// Vehicle.aspx-tree leaves already point straight at Unit.aspx — skip the
|
||||
// QuickDetails→Unit hop and extract parts directly.
|
||||
if (/Unit\.aspx/i.test(categoryUrl)) {
|
||||
const unitHtml = await this.fetchEmexHtml(categoryUrl);
|
||||
const extracted = this.extractEmexPartsFromHtml(unitHtml);
|
||||
if (
|
||||
extracted.schemaImageUrl &&
|
||||
(extracted.schemaWidth === 0 || extracted.schemaHeight === 0)
|
||||
) {
|
||||
try {
|
||||
const dims = await this.fetchImageDims(extracted.schemaImageUrl);
|
||||
if (dims.width > 0 && dims.height > 0) {
|
||||
extracted.schemaWidth = dims.width;
|
||||
extracted.schemaHeight = dims.height;
|
||||
}
|
||||
} catch (err) {
|
||||
this.logger.debug(`Image dims fetch failed: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
return extracted;
|
||||
}
|
||||
|
||||
// 1) QuickDetails.aspx → Unit.aspx anchor
|
||||
const qdHtml = await this.fetchEmexHtml(categoryUrl);
|
||||
const unitMatch = qdHtml.match(/href="([^"]*Unit\.aspx[^"]*)"/i);
|
||||
|
||||
@@ -63,6 +63,29 @@ export interface EmexCategoryTreeNode {
|
||||
children: EmexCategoryTreeNode[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Node in the hierarchical Vehicle.aspx tree (the OEM-microfiche taxonomy):
|
||||
* top groups → sub groups → Unit.aspx leaves. Produced by emex-tree.parser.ts
|
||||
* and seeded into the generic `categories` table (parentId chains +
|
||||
* hasSubgroups/hasParts), so the same tree UI renders it as pl24/pcat.
|
||||
*/
|
||||
export type EmexTreeNodeKind = "group" | "leaf";
|
||||
|
||||
export interface EmexVehicleTreeNode {
|
||||
/** emex category id (`cid`) for groups; the parent group's cid for leaves. */
|
||||
cid: string;
|
||||
/** Display name (for leaves the "NNNN:" section-code prefix is stripped). */
|
||||
name: string;
|
||||
/** Section code for a leaf (e.g. "3070160A"); null for groups. */
|
||||
code: string | null;
|
||||
/** Absolute, verbatim URL: Vehicle.aspx (group, to drill) or Unit.aspx (leaf, to fetch parts). */
|
||||
url: string;
|
||||
kind: EmexTreeNodeKind;
|
||||
/** Schema illustration (img.laximo.net gif) for a leaf, when listed; else null. */
|
||||
imageUrl: string | null;
|
||||
children: EmexVehicleTreeNode[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Part data from EMEX
|
||||
*/
|
||||
@@ -142,7 +165,7 @@ export interface EmexScraperResponse {
|
||||
quickGroupsUrl?: string | null;
|
||||
parsedOptions?: EmexParsedOptions;
|
||||
categories?: EmexCategory[];
|
||||
categoryTree?: EmexCategoryTreeNode[];
|
||||
categoryTree?: EmexVehicleTreeNode[];
|
||||
sampleParts?: EmexPart[];
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
@@ -60,3 +60,45 @@ describe("PL24FordLegacyService.parseP4VehicleResponse — model-selection page
|
||||
expect(r).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// parseFordGroupsFromHtml leaks header/breadcrumb nav links into the category list
|
||||
// (Hyundai/Kia "Portal", logout, Nissan "Repair & Maintenance Information" → eu.nissan.biz).
|
||||
// These must be filtered; real categories (group.action / group-detail.action /
|
||||
// json-(main|sub)-group.action, and Volvo vin-group.action?...group1=...) must survive.
|
||||
const pg = svc as unknown as {
|
||||
parseFordGroupsFromHtml(
|
||||
html: string,
|
||||
serviceName: string,
|
||||
familyId: string,
|
||||
): Array<{ code: string; nameTr: string; nameEn: string; linkPath: string }>;
|
||||
};
|
||||
|
||||
describe("PL24FordLegacyService.parseFordGroupsFromHtml — nav-crumb junk filter", () => {
|
||||
it("drops Portal / logout / external nav-crumbs, keeps real categories", () => {
|
||||
const html = [
|
||||
"<table>",
|
||||
'<tr class="tc-data-row" url="group-detail.action?catId=20&lang=tr" caption="MOTOR"><td>MOTOR</td></tr>',
|
||||
'<tr class="tc-data-row" url="portal.action?lang=tr" caption="Portal"><td>Portal</td></tr>',
|
||||
'<tr class="tc-data-row" url="logout.action" caption="Logout"><td>Logout</td></tr>',
|
||||
'<tr class="tc-data-row" url="https://eu.nissan.biz/" caption="Repair & Maintenance Information"><td>Repair</td></tr>',
|
||||
"</table>",
|
||||
].join("");
|
||||
const groups = pg.parseFordGroupsFromHtml(html, "hyundai_parts", "");
|
||||
const names = groups.map((g) => g.nameTr);
|
||||
expect(names).toContain("MOTOR");
|
||||
expect(names).not.toContain("Portal");
|
||||
expect(names).not.toContain("Logout");
|
||||
expect(groups.some((g) => g.linkPath.includes("eu.nissan.biz"))).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps Volvo vin-group.action?...group1=... (real category) but drops the bare breadcrumb", () => {
|
||||
const html = [
|
||||
'<tr class="tc-data-row" url="vin-group.action?group1=2&lang=tr" caption="Motor"><td>Motor</td></tr>',
|
||||
'<tr class="tc-data-row" url="vin-group.action?lang=tr" caption="Geri"><td>Geri</td></tr>',
|
||||
].join("");
|
||||
const groups = pg.parseFordGroupsFromHtml(html, "volvo_parts", "");
|
||||
const names = groups.map((g) => g.nameTr);
|
||||
expect(names).toContain("Motor");
|
||||
expect(names).not.toContain("Geri");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -46,6 +46,19 @@ export interface P4BrandHooks {
|
||||
parseVehicleInfo?(html: string, vin: string): P4VehicleInfo | null;
|
||||
}
|
||||
|
||||
// Brands whose parts are licensed only on the de (account 2) PL24 subscription —
|
||||
// the tr account serves a NOT_LOGGED_IN_DEMO page for these. All P4 fetches for
|
||||
// these services are routed to de (see fetchP4Page + resolveAccount).
|
||||
const LEGACY_DE_SERVICES = new Set([
|
||||
"hyundai_parts",
|
||||
"kia_parts",
|
||||
"nissan_parts",
|
||||
"fordp_parts",
|
||||
"fordt_parts",
|
||||
]);
|
||||
// Polestar tested on de → 0 categories (data gap). Fiat stays on de via
|
||||
// resolveAccount but its model-list needs launchCatalog.do.
|
||||
|
||||
@Injectable()
|
||||
export class PL24FordLegacyService {
|
||||
private readonly logger = new Logger(PL24FordLegacyService.name);
|
||||
@@ -250,13 +263,18 @@ export class PL24FordLegacyService {
|
||||
// Nissan/Infiniti/Opel: json-sub-group.action
|
||||
if (linkPath.includes("json-sub-group.action")) {
|
||||
const basePath = linkPath.substring(0, linkPath.lastIndexOf("/") + 1);
|
||||
// Nissan format: { subgroups: [...] }
|
||||
// Nissan format: { subgroups: [{caption, code, url}] }
|
||||
// Opel format: may use same { subgroups } or { vCfgData }
|
||||
let groups = this.parseNissanSubGroupJson(html, basePath);
|
||||
if (groups.length === 0) {
|
||||
// Try Opel vCfgData format
|
||||
groups = this.parseOpelVCfgDataJson(html, basePath);
|
||||
}
|
||||
if (groups.length === 0) {
|
||||
// Ford de format: { subgroups: [{caption, id, subheader, jsonUrl:null}] } →
|
||||
// construct image-board.action leaves (bomId = mainGroupId+subheaderId+leafId).
|
||||
groups = this.parseFordSubGroupsJson(html, linkPath, basePath);
|
||||
}
|
||||
if (groups.length === 0) {
|
||||
// Debug: log the response
|
||||
this.logger.log(
|
||||
@@ -1885,10 +1903,13 @@ export class PL24FordLegacyService {
|
||||
* Fiat services always use 'de'; others use round-robin via Redis.
|
||||
*/
|
||||
private async resolveAccount(userId?: string, serviceName?: string): Promise<"tr" | "de"> {
|
||||
// Fiat always needs de account
|
||||
// Fiat always needs de account; Hyundai/Kia/Nissan parts are licensed only on de.
|
||||
if (serviceName && ["fiatp_parts", "fiatt_parts"].includes(serviceName)) {
|
||||
return "de";
|
||||
}
|
||||
if (serviceName && LEGACY_DE_SERVICES.has(serviceName)) {
|
||||
return "de";
|
||||
}
|
||||
|
||||
// No userId → tr (catalog browser / prefetch)
|
||||
if (!userId) return "tr";
|
||||
@@ -2625,6 +2646,67 @@ export class PL24FordLegacyService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse Ford (de account) json-sub-group.action response into leaf illustrations.
|
||||
*
|
||||
* Ford de subgroups are a flat list of { caption, id, subheader, jsonUrl:null, url:null }.
|
||||
* `subheader:true` rows are section labels (skipped); the leaves below them are the
|
||||
* actual part illustrations but carry NO url. The illustration (BOM) URL is
|
||||
* constructed: subGroupId = mainGroupId + subheaderId, bomId = subGroupId + leafId,
|
||||
* leaf → image-board.action?bomId=… (same parts endpoint as Nissan/Hyundai). Verified
|
||||
* on dev: the selected-subgroup response exposes exactly this image-board url.
|
||||
*/
|
||||
private parseFordSubGroupsJson(
|
||||
jsonStr: string,
|
||||
linkPath: string,
|
||||
basePath: string,
|
||||
): PL24MainGroup[] {
|
||||
try {
|
||||
const data = JSON.parse(jsonStr) as {
|
||||
subgroups?: {
|
||||
caption: string;
|
||||
id: string;
|
||||
subheader?: boolean;
|
||||
gray?: boolean;
|
||||
jsonUrl?: string | null;
|
||||
url?: string | null;
|
||||
}[];
|
||||
};
|
||||
const subs = data.subgroups ?? [];
|
||||
const q = (k: string) => linkPath.match(new RegExp(`[?&]${k}=([^&]+)`))?.[1] ?? "";
|
||||
const catCode = q("catCode");
|
||||
const mainGroupId = q("mainGroupId");
|
||||
const sharedCatCode = q("sharedCatCode");
|
||||
const modelFamily = q("modelFamily");
|
||||
const mode = q("mode");
|
||||
const upds = q("upds");
|
||||
if (!mainGroupId) return [];
|
||||
const out: PL24MainGroup[] = [];
|
||||
const seen = new Set<string>();
|
||||
let currentSubheaderId = "";
|
||||
for (const sg of subs) {
|
||||
if (sg.gray) continue;
|
||||
if (sg.subheader) {
|
||||
currentSubheaderId = sg.id || currentSubheaderId;
|
||||
continue;
|
||||
}
|
||||
if (!sg.id) continue;
|
||||
const subGroupId = `${mainGroupId}${currentSubheaderId}`;
|
||||
const bomId = `${subGroupId}${sg.id}`;
|
||||
if (seen.has(bomId)) continue;
|
||||
seen.add(bomId);
|
||||
const leafLink =
|
||||
`${basePath}image-board.action?bomId=${bomId}&catCode=${catCode}&lang=${this.language}` +
|
||||
`&mainGroupId=${mainGroupId}&modelFamily=${modelFamily}&sharedCatCode=${sharedCatCode}` +
|
||||
`&startup=false&subGroupId=${subGroupId}&mode=${mode}&upds=${upds}`;
|
||||
out.push({ id: bomId, code: bomId, name: sg.caption, linkPath: leafLink });
|
||||
}
|
||||
return out;
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse Ford json-main-group.action response.
|
||||
* Ford uses { maingroups: [{ caption, databaseKey, identifier, gray, subheader, jsonUrl, url }] } format.
|
||||
@@ -3205,6 +3287,15 @@ export class PL24FordLegacyService {
|
||||
account: "tr" | "de" = "tr",
|
||||
retried = false,
|
||||
): Promise<string | null> {
|
||||
// Hyundai/Kia/Nissan parts are licensed ONLY on the de account — the tr
|
||||
// account returns a NOT_LOGGED_IN_DEMO page (browse-only). Route every P4
|
||||
// fetch for these brands to de (which carries the Korean/Japanese license),
|
||||
// and ensure the de service token is authorized. Verified on dev: de yields
|
||||
// 266 Hyundai / 169 Kia models, non-demo, with real drill content.
|
||||
if (LEGACY_DE_SERVICES.has(serviceName)) {
|
||||
account = "de";
|
||||
await this.authService.authorizeServiceForAccount(serviceName, "de");
|
||||
}
|
||||
// Some catalogs (Volvo vin-group.action) store hrefs relative to the catalog
|
||||
// directory (e.g. "vin-group.action?group1=…"). Prefix the service basePath
|
||||
// so `${baseUrl}${path}` doesn't collapse into "partslink24.comvin-group…".
|
||||
@@ -3259,6 +3350,10 @@ export class PL24FordLegacyService {
|
||||
// groups/parts) when the service token is stale. decodeVinForService
|
||||
// retries on this, but drill paths reach upstream only through here —
|
||||
// so retry once with fresh auth before parsing an empty page.
|
||||
// NOTE: Hyundai/Kia/Nissan parts are demo-gated on the tr account
|
||||
// (not licensed); the de account is licensed but lacks the EUR/TR-region
|
||||
// catalogs, so a de fallback returns non-demo-but-empty — a PL24 commercial
|
||||
// licensing/region issue, not fixable here.
|
||||
if (!retried && html.includes("PL24_SUPPORT")) {
|
||||
const support = this.extractScriptVariable<FordPL24Support>(html, "PL24_SUPPORT");
|
||||
if (support?.demo || support?.role === "NOT_LOGGED_IN_DEMO") {
|
||||
@@ -3526,6 +3621,14 @@ export class PL24FordLegacyService {
|
||||
const url = urlMatch[1];
|
||||
// Skip vehicle.action rows — those are sub-model selectors, not part group links
|
||||
if (url.includes("vehicle.action")) continue;
|
||||
// Skip header/breadcrumb navigation links that leak into the group table as
|
||||
// pseudo-categories: Hyundai/Kia "Portal" (portal.action), logout, and Nissan's
|
||||
// "Repair & Maintenance Information" external link (https://eu.nissan.biz/). Real
|
||||
// categories use group.action / group-detail.action / json-(main|sub)-group.action,
|
||||
// none of which match these. Volvo's vin-group.action?...group1=... is kept.
|
||||
if (/(portal|logout)\.action/i.test(url)) continue;
|
||||
if (url.includes("vin-group.action") && !url.includes("group1=")) continue;
|
||||
if (/^https?:\/\//i.test(url) && !/\.action(\?|$)/i.test(url)) continue;
|
||||
if (seen.has(url)) continue;
|
||||
seen.add(url);
|
||||
|
||||
@@ -3588,6 +3691,8 @@ export class PL24FordLegacyService {
|
||||
const url = match[1];
|
||||
const name = match[2].replace(/<[^>]+>/g, "").trim();
|
||||
if (!name || seen.has(url)) continue;
|
||||
// Same nav-crumb guard as the primary parser (vin-group.action breadcrumb).
|
||||
if (url.includes("vin-group.action") && !url.includes("group1=")) continue;
|
||||
seen.add(url);
|
||||
const linkPath = url.startsWith("/") ? url : `${basePath}/${url}`;
|
||||
groups.push({
|
||||
|
||||
@@ -325,3 +325,24 @@ describe("PL24Service.parseVehicleResponse — p5mitsubishi (model from 'Araç',
|
||||
expect(r.year).toBe(2006);
|
||||
});
|
||||
});
|
||||
|
||||
describe("PL24Service.formatFiatYear — p5fiat model year ranges", () => {
|
||||
// Fiat /extern/vehicle/models records carry values.year as "(2016,2020)".
|
||||
const fy = svc as unknown as { formatFiatYear(raw: unknown): string | undefined };
|
||||
|
||||
it("turns '(2016,2020)' into '2016-2020'", () => {
|
||||
expect(fy.formatFiatYear("(2016,2020)")).toBe("2016-2020");
|
||||
});
|
||||
|
||||
it("handles a single / open-ended year", () => {
|
||||
expect(fy.formatFiatYear("(1999,)")).toBe("1999");
|
||||
expect(fy.formatFiatYear("(2007)")).toBe("2007");
|
||||
});
|
||||
|
||||
it("returns undefined for empty / non-string input", () => {
|
||||
expect(fy.formatFiatYear("")).toBeUndefined();
|
||||
expect(fy.formatFiatYear("()")).toBeUndefined();
|
||||
expect(fy.formatFiatYear(undefined)).toBeUndefined();
|
||||
expect(fy.formatFiatYear(null)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -629,16 +629,15 @@ export class PL24Service {
|
||||
}> {
|
||||
await this.touchActivity();
|
||||
try {
|
||||
await this.authService.authorizeService(serviceName);
|
||||
const headers = await this.authService.buildAuthHeaders(serviceName);
|
||||
// Account-aware (de + DE proxy for Fiat). resolveAccount(undefined, …) → "de"
|
||||
// only for fiatp_parts/fiatt_parts; all other P5 brands stay on tr (no proxy).
|
||||
const account = await this.resolveAccount(undefined, serviceName);
|
||||
await this.authService.authorizeServiceForAccount(serviceName, account);
|
||||
const headers = await this.authService.buildAuthHeadersForAccount(account, serviceName);
|
||||
const localizedPath = restrictionPath.replace(/lang=\w+/, `lang=${this.language}`);
|
||||
const url = `${this.baseUrl}${localizedPath}`;
|
||||
|
||||
const response = await fetch(url, {
|
||||
method: "GET",
|
||||
headers,
|
||||
signal: AbortSignal.timeout(this.timeout),
|
||||
});
|
||||
const response = await this.fetchWithRetry(url, headers, serviceName, account);
|
||||
|
||||
if (!response.ok) {
|
||||
this.logger.warn(`P5 restrictions fetch failed: HTTP ${response.status}`);
|
||||
@@ -711,9 +710,20 @@ export class PL24Service {
|
||||
}));
|
||||
}
|
||||
try {
|
||||
await this.authService.authorizeService(serviceName);
|
||||
const headers = await this.authService.buildAuthHeaders(serviceName);
|
||||
return this.fetchMainGroupsByPath(mainGroupsPath, headers);
|
||||
// Account-aware: tr for most P5 brands (no proxy), de + DE proxy for Fiat.
|
||||
// resolveAccount(undefined, …) → "de" only for fiatp_parts/fiatt_parts.
|
||||
const account = await this.resolveAccount(undefined, serviceName);
|
||||
await this.authService.authorizeServiceForAccount(serviceName, account);
|
||||
const headers = await this.authService.buildAuthHeadersForAccount(account, serviceName);
|
||||
const localizedPath = mainGroupsPath.replace(/lang=\w+/, `lang=${this.language}`);
|
||||
const response = await this.fetchWithRetry(
|
||||
`${this.baseUrl}${localizedPath}`,
|
||||
headers,
|
||||
serviceName,
|
||||
account,
|
||||
);
|
||||
const data = (await response.json()) as Record<string, any>;
|
||||
return this.parseMainGroupsResponse(data);
|
||||
} catch (error) {
|
||||
const err = error as Error;
|
||||
this.logger.error(`Fetch main groups error: ${err.message}`);
|
||||
@@ -1982,6 +1992,13 @@ export class PL24Service {
|
||||
return this.fordLegacyService.fetchVehicleListForVolvo(serviceName);
|
||||
}
|
||||
|
||||
// Fiat (p5fiat) has a two-level model hierarchy (families → model codes) that the
|
||||
// generic single-endpoint P5 flow below cannot express, and is licensed only on the
|
||||
// de-708171 account. Handle it via a dedicated expander.
|
||||
if (serviceName === "fiatp_parts" || serviceName === "fiatt_parts") {
|
||||
return this.fetchFiatVehicleList(serviceName);
|
||||
}
|
||||
|
||||
try {
|
||||
// Try de account; fall back to main token (demo mode) if that also fails
|
||||
let headers: Record<string, string>;
|
||||
@@ -2058,6 +2075,41 @@ export class PL24Service {
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: the primary modelPath returned nothing (e.g. p5fiat is not mapped
|
||||
// in BACKEND_MODEL_PATH yet). Try the other known P5 listing endpoints so an
|
||||
// unmapped backend self-discovers its model list instead of silently seeding 0.
|
||||
// Only runs when the primary yields nothing → no impact on mapped backends.
|
||||
const candidatePaths = [
|
||||
"/extern/vehicle/modelfamilies",
|
||||
"/extern/vehicle/models",
|
||||
"/extern/vehicle/modelFamilies",
|
||||
"/extern/vehicle/catalogs",
|
||||
"/extern/vehicle/scope",
|
||||
"/extern/vehicles/vehiclesOverview",
|
||||
"/extern/model/categories",
|
||||
].filter((p) => p !== modelPath);
|
||||
for (const candidate of candidatePaths) {
|
||||
try {
|
||||
const cu = `${this.baseUrl}${catalogBase}${candidate}?lang=${this.language}&serviceName=${serviceName}`;
|
||||
const cr = await fetch(cu, {
|
||||
method: "GET",
|
||||
headers,
|
||||
signal: AbortSignal.timeout(15000),
|
||||
});
|
||||
if (!cr.ok) continue;
|
||||
const cd = (await cr.json()) as Record<string, any>;
|
||||
const cv = this.parseVehicleListResponse(cd, serviceName);
|
||||
if (cv.length > 0) {
|
||||
this.logger.log(
|
||||
`fetchVehicleList: ${serviceName} discovered via ${candidate} (${cv.length} models) — pin BACKEND_MODEL_PATH["${backendKey}"]="${candidate}"`,
|
||||
);
|
||||
return cv;
|
||||
}
|
||||
} catch {
|
||||
// try next candidate
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.log(`No vehicle list available for ${serviceName} (HTTP ${response.status})`);
|
||||
return [];
|
||||
} catch (err) {
|
||||
@@ -2066,6 +2118,122 @@ export class PL24Service {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fiat (p5fiat) browse model list.
|
||||
*
|
||||
* Unlike other P5 backends, Fiat exposes a two-level hierarchy:
|
||||
* modelOverview → model FAMILIES (124 SPIDER, 500L, LINEA, TIPO …)
|
||||
* models?modelFamily=N → the actual model codes (+ year range)
|
||||
* Each model's link.path is the maingroups endpoint consumed by the generic P5
|
||||
* drill (getCategoryTree → fetchMainGroups → subgroups → bomDetails parts).
|
||||
*
|
||||
* Requires the de-708171 account (Fiat parts are licensed only there) and the
|
||||
* DataImpulse DE proxy — both applied via fetchWithRetry(account="de").
|
||||
*/
|
||||
private async fetchFiatVehicleList(serviceName: string): Promise<
|
||||
Array<{
|
||||
vehicleId: string;
|
||||
model: string;
|
||||
year?: string;
|
||||
engine?: string;
|
||||
bodyType?: string;
|
||||
transmission?: string;
|
||||
market?: string;
|
||||
catalogPath?: string;
|
||||
metadata?: Record<string, unknown>;
|
||||
}>
|
||||
> {
|
||||
const account = "de" as const;
|
||||
try {
|
||||
await this.authService.authorizeServiceForAccount(serviceName, account);
|
||||
const headers = await this.authService.buildAuthHeadersForAccount(account, serviceName);
|
||||
const base = getServiceApiPath(serviceName); // "/p5fiat"
|
||||
|
||||
// 1) Model families
|
||||
const ovUrl = `${this.baseUrl}${base}/extern/vehicle/modelOverview?lang=${this.language}&serviceName=${serviceName}`;
|
||||
const ovResp = await this.fetchWithRetry(ovUrl, headers, serviceName, account);
|
||||
const ovData = (await ovResp.json()) as Record<string, any>;
|
||||
const families: any[] = (ovData.data?.records || ovData.records || []).filter(
|
||||
(f: any) => !f.unavailable,
|
||||
);
|
||||
if (families.length === 0) {
|
||||
this.logger.warn(`Fiat ${serviceName}: modelOverview returned 0 families`);
|
||||
return [];
|
||||
}
|
||||
|
||||
// 2) Expand each family → model codes (bounded concurrency to keep the one-time seed fast)
|
||||
type FiatVehicle = {
|
||||
vehicleId: string;
|
||||
model: string;
|
||||
year?: string;
|
||||
catalogPath?: string;
|
||||
metadata?: Record<string, unknown>;
|
||||
};
|
||||
const out: FiatVehicle[] = [];
|
||||
const CONCURRENCY = 6;
|
||||
for (let i = 0; i < families.length; i += CONCURRENCY) {
|
||||
const batch = families.slice(i, i + CONCURRENCY);
|
||||
const batchResults = await Promise.all(
|
||||
batch.map(async (fam: any): Promise<FiatVehicle[]> => {
|
||||
const famName = String(fam.values?.description || fam.description || fam.id).trim();
|
||||
const modelsPath = fam.link?.path as string | undefined;
|
||||
const modelsUrl = modelsPath
|
||||
? modelsPath.startsWith("http")
|
||||
? modelsPath
|
||||
: `${this.baseUrl}${modelsPath}`
|
||||
: `${this.baseUrl}${base}/extern/vehicle/models?lang=${this.language}&serviceName=${serviceName}&modelFamily=${fam.id}`;
|
||||
try {
|
||||
const mResp = await this.fetchWithRetry(modelsUrl, headers, serviceName, account);
|
||||
const mData = (await mResp.json()) as Record<string, any>;
|
||||
const models: any[] = (mData.data?.records || mData.records || []).filter(
|
||||
(m: any) => !m.unavailable && m.link?.path,
|
||||
);
|
||||
return models.map((m: any): FiatVehicle => {
|
||||
const modelCode = String(m.values?.modelCode || m.id || "");
|
||||
const desc = String(m.values?.description || m.description || famName)
|
||||
.replace(/\\-/g, "-") // PL24 escapes hyphens as "\-"
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
const year = this.formatFiatYear(m.values?.year);
|
||||
return {
|
||||
vehicleId: `${fam.id}:${modelCode}`,
|
||||
model: year ? `${desc} (${year})` : desc,
|
||||
year: year || undefined,
|
||||
catalogPath: m.link.path as string,
|
||||
metadata: { modelFamily: String(fam.id), modelCode, family: famName },
|
||||
};
|
||||
});
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
`Fiat ${serviceName}: models fetch failed for family ${fam.id}: ${(err as Error).message}`,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
}),
|
||||
);
|
||||
for (const r of batchResults) out.push(...r);
|
||||
}
|
||||
|
||||
this.logger.log(`Fiat ${serviceName}: ${families.length} families → ${out.length} models`);
|
||||
return out;
|
||||
} catch (err) {
|
||||
this.logger.warn(`fetchFiatVehicleList failed for ${serviceName}: ${(err as Error).message}`);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/** Fiat year values arrive as "(2016,2020)" → "2016-2020"; single/open ranges handled. */
|
||||
private formatFiatYear(raw: unknown): string | undefined {
|
||||
if (typeof raw !== "string") return undefined;
|
||||
const trimmed = raw.replace(/[()]/g, "").trim();
|
||||
if (!trimmed) return undefined;
|
||||
const parts = trimmed
|
||||
.split(",")
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean);
|
||||
return parts.length >= 2 ? `${parts[0]}-${parts[parts.length - 1]}` : parts[0] || undefined;
|
||||
}
|
||||
|
||||
private parseVehicleListResponse(
|
||||
data: Record<string, any>,
|
||||
serviceName: string,
|
||||
|
||||
@@ -528,7 +528,9 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy {
|
||||
|
||||
private isLeafLinkPath(linkPath: string | null, source: string): boolean {
|
||||
if (!linkPath) return false;
|
||||
if (source === "emex") return true; // EMEX leaves always have a URL
|
||||
// EMEX: Vehicle.aspx group nodes are parents to drill; Unit.aspx (hierarchical
|
||||
// tree) and QuickDetails.aspx (legacy flat) leaves carry parts.
|
||||
if (source === "emex") return !linkPath.includes("Vehicle.aspx");
|
||||
if (source === "parts-catalogs") return linkPath.startsWith("pcat:"); // pcat: prefix = leaf
|
||||
// PL24 leaf indicators
|
||||
return (
|
||||
|
||||
@@ -116,7 +116,9 @@ export function ModelListColumns({
|
||||
const needsP5Restrictions =
|
||||
arch === "P5_MODERN" &&
|
||||
!!vehicle.catalogPath &&
|
||||
!vehicle.catalogPath.includes("/mainGroup");
|
||||
// Case-insensitive: Fiat's catalogPath is already a maingroups endpoint
|
||||
// (/extern/groups/mdl/maingroups) → no restriction step, go straight to categories.
|
||||
!/\/maingroup/i.test(vehicle.catalogPath);
|
||||
|
||||
setLoadingCol(colIdx);
|
||||
try {
|
||||
|
||||
@@ -72,7 +72,9 @@ function CatalogVehiclePage() {
|
||||
const isP5WithRestrictions =
|
||||
vehicle?.architecture === "P5_MODERN" &&
|
||||
!!vehicle?.catalogPath &&
|
||||
!vehicle.catalogPath.includes("/mainGroup");
|
||||
// Case-insensitive: Fiat's catalogPath is already a maingroups endpoint
|
||||
// (/extern/groups/mdl/maingroups) → no restriction step, categories load directly.
|
||||
!/\/maingroup/i.test(vehicle.catalogPath);
|
||||
const showPsaVariantSelector = isPsa && !hasVariant;
|
||||
const showFordVariantSelector = isP4Legacy && !hasVariant;
|
||||
const showP5RestrictionSelector = isP5WithRestrictions && !hasVariant;
|
||||
|
||||
Reference in New Issue
Block a user