root/sase.tr
1
0
Fork 0
Code Issues 5 Pull Requests 3 Actions 7 Packages Projects Releases Wiki Activity

fix(api): emex source-db lookup uses catalogCode+gid, not SSD

Browse Source 
Initial design routed emex lookups through vehicles.rawData.ssd → dump vehicles
→ vehicle_parts. Smoke test against prod ssd values: 0 / 10 matched. EMEX
regenerates the SSD on every decode session, so sase's stored SSD never
matches the SSD the dump scraper recorded for the same physical vehicle.

Pivot to a catalog-wide bridge that actually works:
  catalogs.code    ↔ vehicles.rawData.catalogCode  (e.g. "RENAULT201910")
  part_groups.group_id ↔ categories.externalId    (e.g. "11754")
  → parts via vehicle_parts.group_id (dump's parts.group_id is 100% NULL)

Verified coverage on prod's 8287 unique (catalogCode, gid) pairs: 25/26
catalog codes resolve, 7178 pairs hit a part_group (87%), 5919 of those
return actual parts via vehicle_parts (~71% net). Tradeoff: returns all
parts in the (catalog, group) across every variant in the catalog, so the
result is slightly noisier than the live per-vehicle scrape. Acceptable —
parts overlap heavily and the upstream-call savings outweigh the noise.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Semih Yesilyurt
2026-06-01 15:06:48 +03:00
parent e5ed6b9f36
commit 41acff2ab2
2 changed files with 49 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
apps/api/src/categories/categories.service.ts
View File

@@ -1092,16 +1092,19 @@ export class CategoriesService {
} else if (vehicle && category.source === "emex") {
// EMEX: fetch parts + schema image via Puppeteer from QuickDetails URL
try {
// Local dump first (uses vehicles.rawData.ssd persisted by the
// emex mapper). Falls back to live upstream on miss.
const emexSsd = (vehicle.rawData as { ssd?: string } | null)?.ssd;
// Local dump first (catalog-wide via vehicles.rawData.catalogCode +
// category.externalId). The dump's per-vehicle bridge is the EMEX
// SSD, which is session-regenerated and never matches what sase
// stored — so we accept over-return at the catalog level (~71% hit).
// Falls back to live upstream on miss.
const catalogCode = (vehicle.rawData as { catalogCode?: string } | null)?.catalogCode;
let emexResult = await this.emexSourceDb.fetchCategoryParts(
emexSsd,
category.linkPath ?? "",
catalogCode,
category.externalId,
);
if (emexResult) {
this.logger.debug(
`[source-db hit emex] ssd=${emexSsd?.slice(0, 12)}... gid=${category.externalId}`,
`[source-db hit emex] catalog=${catalogCode} gid=${category.externalId}`,
);
} else {
emexResult = await this.emexService.fetchCategoryParts(category.linkPath);

88
apps/api/src/integrations/catalog-source-db/emex-source-db.service.ts
View File

@@ -4,16 +4,22 @@ import mysql, { type Pool, type RowDataPacket } from "mysql2/promise";
import type { EmexHotspot, EmexHotspotArea, EmexPart, EmexPartsResult } from "../emex/emex.types";
/**
* Look up a vehicle's parts + schema for a category (PNC group) in the local
* EMEX dump (sase-catalog-src-emex MariaDB). Returns null on any miss so the
* caller falls through to the live emex scrape.
* Look up parts + schema for an EMEX category in the local dump
* (sase-catalog-src-emex MariaDB). Returns null on any miss so the caller
* falls through to the live emex scrape.
*
* Inputs:
* - vehicleSsd: emex's per-vehicle session-state-descriptor — captured into
* `vehicles.rawData.emexSsd` during live VIN decode. Required (no VIN
* column in the dump; SSD is the only stable vehicle identifier).
* - categoryUrl: the QuickDetails.aspx URL stored in `categories.linkPath`.
* We parse `gid` (the group id) out of it.
* Bridge: catalogs.code ↔ sase's vehicles.rawData.catalogCode,
* part_groups.group_id (varchar) ↔ sase's categories.externalId (gid).
*
* Why catalog-wide (no per-vehicle filter): the dump's only stable per-vehicle
* key is the EMEX session-state-descriptor (`ssd`), but SSD is regenerated
* every decode session, so the SSD sase stored for its decoded vehicle never
* matches the SSD the dump scraper recorded (0 / 10 sampled — verified). The
* vehicle_id we'd need to join `vehicle_parts` is unreachable. So we return
* all parts in (catalog, group) across every vehicle in the dump for that
* catalog — over-returns variants that may not apply to the user's specific
* vehicle, but parts overlap heavily and the cost is a slightly noisier list.
* Hits: ~71% of sase's 8287 (catalogCode, gid) pairs (verified 2026-06-01).
*/
@Injectable()
export class EmexSourceDbService implements OnModuleInit, OnModuleDestroy {
@@ -50,45 +56,41 @@ export class EmexSourceDbService implements OnModuleInit, OnModuleDestroy {
}
/**
* Mirror the live `EmexCatalogService.fetchCategoryParts` shape.
* Mirror the live `EmexService.fetchCategoryParts` shape.
* `catalogCode` = vehicles.rawData.catalogCode (e.g. "RENAULT201910"),
* `gid` = categories.externalId (e.g. "11754").
* Returns null on miss; never throws.
*/
async fetchCategoryParts(
vehicleSsd: string | null | undefined,
categoryUrl: string,
catalogCode: string | null | undefined,
gid: string | null | undefined,
): Promise<EmexPartsResult | null> {
if (!this.enabled || !this.pool) return null;
if (!vehicleSsd) return null; // no SSD captured → can't locate vehicle in dump
const gid = extractGid(categoryUrl);
if (!gid) return null;
if (!catalogCode || !gid) return null;
try {
// 1) Resolve vehicle.id from SSD. We try `ssd = ?` first; if the dump
// canonicalized into `unique_key` (hash), the second arm catches it.
const [vehicleRows] = await this.pool.execute<RowDataPacket[]>(
"SELECT id, catalog_id FROM vehicles WHERE ssd = ? OR unique_key = ? LIMIT 1",
[vehicleSsd, vehicleSsd],
);
if (vehicleRows.length === 0) return null;
const vehicleId = vehicleRows[0].id as number;
const catalogId = vehicleRows[0].catalog_id as number;
// 2) Resolve part_group.id from external gid scoped to this catalog.
// Resolve (catalog.id, part_group.id) in one round trip.
const [groupRows] = await this.pool.execute<RowDataPacket[]>(
"SELECT id FROM part_groups WHERE catalog_id = ? AND group_id = ? LIMIT 1",
[catalogId, gid],
`SELECT pg.id AS pg_id, pg.catalog_id
FROM catalogs c
JOIN part_groups pg ON pg.catalog_id = c.id
WHERE c.code = ? AND pg.group_id = ?
LIMIT 1`,
[catalogCode, gid],
);
if (groupRows.length === 0) return null;
const groupPk = groupRows[0].id as number;
const groupPk = groupRows[0].pg_id as number;
// 3) Parts for this vehicle in this group.
// Parts: catalog-wide via vehicle_parts (no vehicle filter — see class
// comment). DISTINCT collapses the same part being linked from many
// vehicles in the catalog.
const [partRows] = await this.pool.execute<RowDataPacket[]>(
`SELECT p.id AS part_id, p.part_number, p.name, p.position_number, p.pnc
`SELECT DISTINCT p.id AS part_id, p.part_number, p.name, p.position_number, p.pnc
FROM vehicle_parts vp
JOIN parts p ON p.id = vp.part_id
WHERE vp.vehicle_id = ? AND vp.group_id = ?
WHERE vp.group_id = ?
ORDER BY p.position_number, p.id`,
[vehicleId, groupPk],
[groupPk],
);
const parts: EmexPart[] = partRows.map((r) => ({
@@ -97,11 +99,11 @@ export class EmexSourceDbService implements OnModuleInit, OnModuleDestroy {
positionCode: r.position_number ?? r.pnc ?? undefined,
}));
// 4) Schema image + hotspots for this group.
// Schema image + hotspots for this group.
const [imgRows] = await this.pool.execute<RowDataPacket[]>(
`SELECT original_url, width, height, hotspots
FROM part_images
WHERE group_id = ? AND image_type IN ('DIAGRAM','SCHEMATIC')
WHERE group_id = ? AND image_type IN ('DIAGRAM', 'SCHEMATIC')
ORDER BY is_primary DESC, sort_order
LIMIT 1`,
[groupPk],
@@ -135,24 +137,14 @@ export class EmexSourceDbService implements OnModuleInit, OnModuleDestroy {
return { parts, schemaImageUrl, hotspots, schemaWidth, schemaHeight };
} catch (err) {
this.logger.warn(`[emex-src] lookup failed (gid=${gid}): ${(err as Error).message}`);
this.logger.warn(
`[emex-src] lookup failed (${catalogCode}/${gid}): ${(err as Error).message}`,
);
return null;
}
}
}
/** Extract `gid=...` from a QuickDetails.aspx / similar URL. */
function extractGid(url: string): string | null {
if (!url) return null;
const m = url.match(/[?&]gid=([^&#]+)/i);
if (!m) return null;
try {
return decodeURIComponent(m[1]);
} catch {
return m[1];
}
}
/**
* Convert the dump's hotspots JSON ([{x,y,w,h,part_id}]) into the EmexHotspot
* shape used by the live scraper (grouped by position code).