root/sase.tr
1
0
Fork 0
Code Issues 5 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

refactor(emex): rewrite EMEX integration, add category fallback

Browse Source 
Replace browser-based Puppeteer scraper with standalone scraper wrapper.
Add EMEX as fallback source for categories when PL24 is unavailable.
Update vehicle decoding to use new synchronous EMEX API.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Sase Dev
2026-02-12 19:48:03 +00:00
parent 8b106cbb1f
commit a8eb8f4bdc
20 changed files with 1494 additions and 1209 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/src/integrations/corgi/corgi.service.ts
View File

@@ -57,6 +57,8 @@ const WMI_DATABASE: Record<string, string> = {
WMW: "Mini",
// Dacia
UU1: "Dacia",
// Subaru
JF1: "Subaru", JF2: "Subaru",
};
const YEAR_MAP: Record<string, number> = {

22
apps/api/src/integrations/emex/browser.d.ts vendored
View File

@@ -1,22 +0,0 @@
/**
* Ambient type declarations for browser-context code used in Puppeteer evaluate().
* These functions are serialized and executed inside Chromium, not in Node.js.
* We declare minimal DOM types here to avoid adding "dom" to the global tsconfig lib.
*/
interface Element {
querySelector(selector: string): Element | null;
querySelectorAll(selector: string): NodeListOf<Element>;
getAttribute(name: string): string | null;
textContent: string | null;
}
interface NodeListOf<T> {
forEach(callback: (value: T, index: number) => void): void;
length: number;
}
declare const document: {
querySelector(selector: string): Element | null;
querySelectorAll(selector: string): NodeListOf<Element>;
};

178
apps/api/src/integrations/emex/emex-parser.service.ts
View File

@@ -1,178 +0,0 @@
import { Injectable, Logger } from "@nestjs/common";
import type {
EmexVehicleData,
EmexCategoryData,
EmexPartData,
} from "./emex.types";
@Injectable()
export class EmexParserService {
private readonly logger = new Logger(EmexParserService.name);
parseVehicleData(data: Record<string, unknown>): EmexVehicleData | null {
try {
const vehicleId = this.extractString(data, "vehicleId", "id", "vehicle_id");
if (!vehicleId) {
this.logger.warn("No vehicleId found in EMEX vehicle data");
return null;
}
const brandName = this.extractString(data, "brandName", "brand", "make") || "";
const name = this.extractString(data, "name", "title", "vehicleName") || "";
const modelCode = this.extractString(data, "modelCode", "model", "model_code");
const engine = this.extractString(data, "engine", "engineCode", "engine_code");
const yearFrom = this.extractNumber(data, "yearFrom", "year_from", "startYear");
const yearTo = this.extractNumber(data, "yearTo", "year_to", "endYear");
const catalogId = this.extractString(data, "catalogId", "catalog_id", "catalogueId") || "";
return {
vehicleId,
catalogId,
brandName,
name,
modelCode,
engine,
yearFrom,
yearTo,
rawData: data,
};
} catch (error) {
this.logger.error("Failed to parse EMEX vehicle data", error);
return null;
}
}
parseCategoryTree(data: unknown[]): EmexCategoryData[] {
try {
if (!Array.isArray(data)) {
this.logger.warn("Invalid category data: expected array");
return [];
}
const categories: EmexCategoryData[] = [];
for (const item of data) {
if (typeof item !== "object" || item === null) continue;
const record = item as Record<string, unknown>;
const groupId = this.extractString(record, "groupId", "id", "group_id");
const name = this.extractString(record, "name", "title", "groupName");
if (!groupId || !name) continue;
const category: EmexCategoryData = {
groupId,
name,
nameOriginal: this.extractString(record, "nameOriginal", "name_original", "originalName"),
parentGroupId: this.extractString(record, "parentGroupId", "parent_group_id", "parentId"),
sortOrder: this.extractNumber(record, "sortOrder", "sort_order", "order"),
};
categories.push(category);
// Recursively parse children if present
const children = record.children || record.subGroups || record.sub_groups;
if (Array.isArray(children) && children.length > 0) {
const childCategories = this.parseCategoryTree(
children.map((child: unknown) => ({
...(child as Record<string, unknown>),
parentGroupId: groupId,
})),
);
categories.push(...childCategories);
}
}
return categories;
} catch (error) {
this.logger.error("Failed to parse EMEX category tree", error);
return [];
}
}
parsePartsTable(data: unknown[]): EmexPartData[] {
try {
if (!Array.isArray(data)) {
this.logger.warn("Invalid parts data: expected array");
return [];
}
const parts: EmexPartData[] = [];
for (const item of data) {
if (typeof item !== "object" || item === null) continue;
const record = item as Record<string, unknown>;
const name = this.extractString(record, "name", "title", "partName");
if (!name) continue;
// Extract OEM codes
const oemCodes: string[] = [];
const rawOem = record.oemCodes || record.oem_codes || record.partNumbers || record.codes;
if (Array.isArray(rawOem)) {
for (const code of rawOem) {
if (typeof code === "string" && code.trim()) {
oemCodes.push(code.trim());
} else if (typeof code === "object" && code !== null) {
const codeStr = (code as Record<string, unknown>).code || (code as Record<string, unknown>).value;
if (typeof codeStr === "string" && codeStr.trim()) {
oemCodes.push(codeStr.trim());
}
}
}
} else if (typeof rawOem === "string" && rawOem.trim()) {
oemCodes.push(rawOem.trim());
}
// Try to extract single OEM code field
const singleOem = this.extractString(record, "oemCode", "oem_code", "partNumber");
if (singleOem && !oemCodes.includes(singleOem)) {
oemCodes.unshift(singleOem);
}
parts.push({
partId: this.extractString(record, "partId", "id", "part_id"),
name,
nameOriginal: this.extractString(record, "nameOriginal", "name_original", "originalName"),
description: this.extractString(record, "description", "desc", "note"),
quantity: this.extractNumber(record, "quantity", "qty", "count"),
position: this.extractString(record, "position", "pos", "location"),
hotspotIndex: this.extractNumber(record, "hotspotIndex", "hotspot_index", "hotspot"),
oemCodes,
});
}
return parts;
} catch (error) {
this.logger.error("Failed to parse EMEX parts table", error);
return [];
}
}
private extractString(data: Record<string, unknown>, ...keys: string[]): string | null {
for (const key of keys) {
const value = data[key];
if (typeof value === "string" && value.trim()) {
return value.trim();
}
}
return null;
}
private extractNumber(data: Record<string, unknown>, ...keys: string[]): number | null {
for (const key of keys) {
const value = data[key];
if (typeof value === "number" && !isNaN(value)) {
return value;
}
if (typeof value === "string") {
const parsed = parseInt(value, 10);
if (!isNaN(parsed)) return parsed;
}
}
return null;
}
}

136
apps/api/src/integrations/emex/emex-queue.service.ts
View File

@@ -1,136 +0,0 @@
import { Injectable, Logger, OnModuleDestroy } from "@nestjs/common";
import { ConfigService } from "@nestjs/config";
import { Queue, type JobsOptions } from "bullmq";
import { RedisService } from "../../redis/redis.service";
import type { EmexScrapeJobData, EmexJobStatus } from "./emex.types";
const QUEUE_NAME = "emex-scrape";
const DEFAULT_JOB_OPTIONS: JobsOptions = {
attempts: 3,
backoff: {
type: "exponential",
delay: 5000,
},
removeOnComplete: {
age: 86400, // 24h
count: 1000,
},
removeOnFail: {
age: 604800, // 7 days
count: 5000,
},
};
@Injectable()
export class EmexQueueService implements OnModuleDestroy {
private readonly logger = new Logger(EmexQueueService.name);
private readonly queue: Queue<EmexScrapeJobData>;
constructor(
private configService: ConfigService,
private redis: RedisService,
) {
const redisHost = this.configService.get<string>("redis.host", "127.0.0.1");
const redisPort = this.configService.get<number>("redis.port", 6379);
const redisPassword = this.configService.get<string>("redis.password");
this.queue = new Queue<EmexScrapeJobData>(QUEUE_NAME, {
connection: {
host: redisHost,
port: redisPort,
password: redisPassword,
maxRetriesPerRequest: null,
},
defaultJobOptions: DEFAULT_JOB_OPTIONS,
});
this.logger.log(`EMEX scrape queue initialized: ${QUEUE_NAME}`);
}
async onModuleDestroy() {
await this.queue.close();
this.logger.log("EMEX scrape queue closed");
}
async addScrapeJob(vin: string, userId: string): Promise<string> {
const jobData: EmexScrapeJobData = {
vin,
userId,
type: "full-decode",
};
const job = await this.queue.add("decode-vin", jobData, {
jobId: `emex-decode:${vin}:${Date.now()}`,
priority: 1,
});
this.logger.log(`Added EMEX scrape job for VIN: ${vin}, jobId: ${job.id}`);
return job.id!;
}
async addCategoriesJob(
emexVehicleId: string,
vin: string,
userId: string,
): Promise<string> {
const jobData: EmexScrapeJobData = {
vin,
userId,
type: "categories",
emexVehicleId,
};
const job = await this.queue.add("scrape-categories", jobData, {
jobId: `emex-categories:${emexVehicleId}:${Date.now()}`,
priority: 2,
});
this.logger.log(`Added EMEX categories job for vehicleId: ${emexVehicleId}, jobId: ${job.id}`);
return job.id!;
}
async addPartsJob(
emexVehicleId: string,
groupId: string,
vin: string,
userId: string,
): Promise<string> {
const jobData: EmexScrapeJobData = {
vin,
userId,
type: "parts",
emexVehicleId,
groupId,
};
const job = await this.queue.add("scrape-parts", jobData, {
jobId: `emex-parts:${emexVehicleId}:${groupId}:${Date.now()}`,
priority: 3,
});
this.logger.log(
`Added EMEX parts job for vehicleId: ${emexVehicleId}, groupId: ${groupId}, jobId: ${job.id}`,
);
return job.id!;
}
async getJobStatus(jobId: string): Promise<EmexJobStatus | null> {
const job = await this.queue.getJob(jobId);
if (!job) return null;
const state = await job.getState();
return {
jobId: job.id!,
status: state as EmexJobStatus["status"],
progress: typeof job.progress === "number" ? job.progress : 0,
result: state === "completed" ? (job.returnvalue as EmexJobStatus["result"]) : null,
failedReason: job.failedReason || null,
};
}
getQueue(): Queue<EmexScrapeJobData> {
return this.queue;
}
}

419
apps/api/src/integrations/emex/emex-scraper.service.ts
View File

@@ -1,419 +0,0 @@
import { Injectable, Logger, OnModuleDestroy } from "@nestjs/common";
import { ConfigService } from "@nestjs/config";
import type { Browser, Page } from "puppeteer";
import puppeteer from "puppeteer";
import { EmexParserService } from "./emex-parser.service";
import {
EmexCaptchaError,
EmexScraperError,
type EmexVehicleData,
type EmexCategoryData,
type EmexPartData,
type EmexCredentials,
} from "./emex.types";
const MAX_CONCURRENT_PAGES = 3;
const MAX_RETRIES = 3;
const PAGE_TIMEOUT = 30_000;
const EMEX_BASE_URL = "https://emex.ru";
@Injectable()
export class EmexScraperService implements OnModuleDestroy {
private readonly logger = new Logger(EmexScraperService.name);
private browser: Browser | null = null;
private activePagesCount = 0;
private readonly pageQueue: Array<{
resolve: (page: Page) => void;
reject: (error: Error) => void;
}> = [];
private readonly credentials: EmexCredentials;
constructor(
private configService: ConfigService,
private parser: EmexParserService,
) {
this.credentials = {
username: this.configService.get<string>("emex.username") || "",
password: this.configService.get<string>("emex.password") || "",
};
}
async onModuleDestroy() {
await this.closeBrowser();
}
async scrapeVehicle(vin: string): Promise<EmexVehicleData | null> {
return this.withRetry(`scrapeVehicle(${vin})`, async () => {
const page = await this.acquirePage();
try {
await this.ensureLoggedIn(page);
await page.goto(`${EMEX_BASE_URL}/catalogs/decode?vin=${encodeURIComponent(vin)}`, {
waitUntil: "networkidle2",
timeout: PAGE_TIMEOUT,
});
this.detectCaptcha(page);
const vehicleData = await page.evaluate(this.extractVehicleFromPage);
if (!vehicleData) {
this.logger.warn(`No vehicle data found for VIN: ${vin}`);
return null;
}
return this.parser.parseVehicleData(vehicleData);
} finally {
await this.releasePage(page);
}
});
}
async scrapeCategories(emexVehicleId: string): Promise<EmexCategoryData[]> {
return this.withRetry(`scrapeCategories(${emexVehicleId})`, async () => {
const page = await this.acquirePage();
try {
await this.ensureLoggedIn(page);
await page.goto(
`${EMEX_BASE_URL}/catalogs/vehicle/${encodeURIComponent(emexVehicleId)}/groups`,
{
waitUntil: "networkidle2",
timeout: PAGE_TIMEOUT,
},
);
this.detectCaptcha(page);
const categoriesData = await page.evaluate(this.extractCategoriesFromPage);
return this.parser.parseCategoryTree(categoriesData);
} finally {
await this.releasePage(page);
}
});
}
async scrapeParts(emexVehicleId: string, groupId: string): Promise<EmexPartData[]> {
return this.withRetry(`scrapeParts(${emexVehicleId}, ${groupId})`, async () => {
const page = await this.acquirePage();
try {
await this.ensureLoggedIn(page);
await page.goto(
`${EMEX_BASE_URL}/catalogs/vehicle/${encodeURIComponent(emexVehicleId)}/groups/${encodeURIComponent(groupId)}/parts`,
{
waitUntil: "networkidle2",
timeout: PAGE_TIMEOUT,
},
);
this.detectCaptcha(page);
const partsData = await page.evaluate(this.extractPartsFromPage);
return this.parser.parsePartsTable(partsData);
} finally {
await this.releasePage(page);
}
});
}
/**
* Browser-context function: extracts vehicle data from the EMEX decode page.
* Serialized and sent to Puppeteer's evaluate — runs inside Chromium, not Node.
*/
private extractVehicleFromPage(): Record<string, unknown> | null {
const vehicleInfo = document.querySelector(
"[data-vehicle-info], .vehicle-info, .decode-result",
);
if (!vehicleInfo) return null;
const data: Record<string, unknown> = {};
data.vehicleId = vehicleInfo.getAttribute("data-vehicle-id") || "";
data.catalogId = vehicleInfo.getAttribute("data-catalog-id") || "";
const fields = vehicleInfo.querySelectorAll("[data-field], .info-row, tr");
fields.forEach((field: Element) => {
const label =
field.querySelector(".label, th, [data-label]")?.textContent?.trim().toLowerCase() || "";
const value =
field.querySelector(".value, td, [data-value]")?.textContent?.trim() || "";
if (label.includes("brand") || label.includes("marka")) data.brandName = value;
if (label.includes("model")) data.modelCode = value;
if (label.includes("name") || label.includes("ad")) data.name = value;
if (label.includes("engine") || label.includes("motor")) data.engine = value;
if (label.includes("year") || label.includes("yil") || label.includes("yıl")) {
const years = value.match(/(\d{4})/g);
if (years) {
data.yearFrom = parseInt(years[0], 10);
if (years.length > 1) data.yearTo = parseInt(years[1], 10);
}
}
});
return data;
}
/**
* Browser-context function: extracts category groups from the EMEX groups page.
*/
private extractCategoriesFromPage(): Record<string, unknown>[] {
const groups: Record<string, unknown>[] = [];
const groupElements = document.querySelectorAll(
"[data-group], .group-item, .category-item, .tree-node",
);
groupElements.forEach((el: Element, index: number) => {
const group: Record<string, unknown> = {};
group.groupId =
el.getAttribute("data-group-id") || el.getAttribute("data-id") || `group-${index}`;
group.name =
el.querySelector(".group-name, .name, .title")?.textContent?.trim() || "";
group.nameOriginal = el.getAttribute("data-original-name") || null;
group.parentGroupId = el.getAttribute("data-parent-id") || null;
group.sortOrder = index;
if (group.name) {
groups.push(group);
}
});
return groups;
}
/**
* Browser-context function: extracts parts from the EMEX parts page.
*/
private extractPartsFromPage(): Record<string, unknown>[] {
const parts: Record<string, unknown>[] = [];
const partRows = document.querySelectorAll(
"[data-part], .part-row, .parts-table tbody tr, .part-item",
);
partRows.forEach((row: Element) => {
const part: Record<string, unknown> = {};
part.partId =
row.getAttribute("data-part-id") || row.getAttribute("data-id") || null;
part.name =
row.querySelector(".part-name, .name, td:nth-child(2)")?.textContent?.trim() || "";
part.nameOriginal = row.getAttribute("data-original-name") || null;
part.description =
row.querySelector(".part-desc, .description, td:nth-child(3)")?.textContent?.trim() ||
null;
const qtyText = row
.querySelector(".part-qty, .quantity, td:nth-child(4)")
?.textContent?.trim();
part.quantity = qtyText ? parseInt(qtyText, 10) : null;
part.position =
row.querySelector(".part-position, .position")?.textContent?.trim() || null;
const hotspotAttr = row.getAttribute("data-hotspot");
part.hotspotIndex = hotspotAttr ? parseInt(hotspotAttr, 10) : null;
// Extract OEM codes
const oemElements = row.querySelectorAll(".oem-code, .part-number, [data-oem]");
const codes: string[] = [];
oemElements.forEach((el: Element) => {
const code = el.textContent?.trim();
if (code) codes.push(code);
});
part.oemCodes = codes;
// Fallback: single OEM code field
if (codes.length === 0) {
const singleOem = row.querySelector("td:first-child")?.textContent?.trim();
if (singleOem) part.oemCode = singleOem;
}
if (part.name) {
parts.push(part);
}
});
return parts;
}
private async ensureLoggedIn(page: Page): Promise<void> {
if (!this.credentials.username || !this.credentials.password) {
throw new EmexScraperError("EMEX credentials not configured", false);
}
// Check if already logged in by looking for session cookie
const cookies = await page.cookies(EMEX_BASE_URL);
const sessionCookie = cookies.find(
(c) => c.name === "session" || c.name === "PHPSESSID" || c.name === "auth_token",
);
if (sessionCookie) return;
await page.goto(`${EMEX_BASE_URL}/login`, {
waitUntil: "networkidle2",
timeout: PAGE_TIMEOUT,
});
this.detectCaptcha(page);
await page.type(
'input[name="username"], input[name="email"], input[name="login"], #username, #email',
this.credentials.username,
);
await page.type(
'input[name="password"], input[type="password"], #password',
this.credentials.password,
);
await Promise.all([
page.waitForNavigation({ waitUntil: "networkidle2", timeout: PAGE_TIMEOUT }),
page.click('button[type="submit"], input[type="submit"], .login-btn, #login-btn'),
]);
this.detectCaptcha(page);
this.logger.log("Successfully logged in to EMEX");
}
private detectCaptcha(page: Page): void {
// Synchronous check of page URL for captcha indicators
const url = page.url();
if (url.includes("captcha") || url.includes("challenge")) {
throw new EmexCaptchaError();
}
}
private async withRetry<T>(operation: string, fn: () => Promise<T>): Promise<T> {
let lastError: Error | null = null;
for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
try {
return await fn();
} catch (error) {
lastError = error as Error;
if (error instanceof EmexCaptchaError) {
this.logger.error(`CAPTCHA detected during ${operation}, cannot retry`);
throw error;
}
if (error instanceof EmexScraperError && !error.retryable) {
throw error;
}
this.logger.warn(
`Attempt ${attempt}/${MAX_RETRIES} failed for ${operation}: ${lastError.message}`,
);
if (attempt < MAX_RETRIES) {
const delay = Math.min(1000 * Math.pow(2, attempt - 1), 10_000);
await new Promise((resolve) => setTimeout(resolve, delay));
}
}
}
throw new EmexScraperError(
`${operation} failed after ${MAX_RETRIES} attempts: ${lastError?.message}`,
false,
);
}
private async getBrowser(): Promise<Browser> {
if (!this.browser || !this.browser.connected) {
this.browser = await puppeteer.launch({
headless: true,
args: [
"--no-sandbox",
"--disable-setuid-sandbox",
"--disable-dev-shm-usage",
"--disable-gpu",
"--disable-extensions",
"--single-process",
],
});
this.browser.on("disconnected", () => {
this.logger.warn("Browser disconnected");
this.browser = null;
this.activePagesCount = 0;
});
this.logger.log("Puppeteer browser launched");
}
return this.browser;
}
private async acquirePage(): Promise<Page> {
if (this.activePagesCount >= MAX_CONCURRENT_PAGES) {
return new Promise<Page>((resolve, reject) => {
this.pageQueue.push({ resolve, reject });
});
}
this.activePagesCount++;
try {
const browser = await this.getBrowser();
const page = await browser.newPage();
await page.setDefaultTimeout(PAGE_TIMEOUT);
await page.setDefaultNavigationTimeout(PAGE_TIMEOUT);
await page.setViewport({ width: 1280, height: 800 });
await page.setUserAgent(
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
);
return page;
} catch (error) {
this.activePagesCount--;
this.processPageQueue();
throw error;
}
}
private async releasePage(page: Page): Promise<void> {
try {
if (!page.isClosed()) {
await page.close();
}
} catch {
// Page may already be closed
}
this.activePagesCount--;
this.processPageQueue();
}
private processPageQueue(): void {
if (this.pageQueue.length > 0 && this.activePagesCount < MAX_CONCURRENT_PAGES) {
const next = this.pageQueue.shift();
if (next) {
this.acquirePage().then(next.resolve).catch(next.reject);
}
}
}
private async closeBrowser(): Promise<void> {
// Reject queued page requests
for (const queued of this.pageQueue) {
queued.reject(new Error("Browser closing"));
}
this.pageQueue.length = 0;
if (this.browser) {
try {
await this.browser.close();
} catch {
// Browser may already be closed
}
this.browser = null;
this.activePagesCount = 0;
this.logger.log("Puppeteer browser closed");
}
}
}

538
apps/api/src/integrations/emex/emex.mapper.ts Normal file
View File

@@ -0,0 +1,538 @@
/**
* EMEX Response Mapper
*
* Transforms raw EmexVinScraper responses into standardized DecodedVehicle format.
* Includes Turkish translation support for common automotive terms.
*/
import {
EmexScraperResponse,
EmexCategory,
EmexCategoryTreeNode,
DecodedVehicle,
DecodedCategory,
CATALOG_MAP,
} from './emex.types';
// ==================== TURKISH TRANSLATIONS ====================
/**
* Turkish translations for common automotive terms
*/
const TR_TRANSLATIONS = {
// Body types
bodyTypes: {
sedan: 'Sedan',
coupe: 'Coupe',
hatchback: 'Hatchback',
wagon: 'Station Wagon',
'station wagon': 'Station Wagon',
estate: 'Station Wagon',
convertible: 'Ustu Acik',
cabriolet: 'Kabriyole',
suv: 'SUV',
crossover: 'Crossover',
pickup: 'Pikap',
van: 'Minivan',
minivan: 'Minivan',
mpv: 'Cok Amacli Arac',
roadster: 'Roadster',
} as Record<string, string>,
// Engine types
engineTypes: {
gasoline: 'Benzin',
petrol: 'Benzin',
benzin: 'Benzin',
diesel: 'Dizel',
electric: 'Elektrik',
hybrid: 'Hibrit',
'plug-in hybrid': 'Sarjli Hibrit',
phev: 'Sarjli Hibrit',
lpg: 'LPG',
cng: 'CNG',
hydrogen: 'Hidrojen',
} as Record<string, string>,
// Transmission types
transmissions: {
automatic: 'Otomatik',
manual: 'Manuel',
'semi-automatic': 'Yari Otomatik',
dct: 'Cift Kavramali',
cvt: 'CVT',
'dual clutch': 'Cift Kavramali',
dsg: 'DSG',
tiptronic: 'Tiptronic',
steptronic: 'Steptronic',
at: 'Otomatik',
mt: 'Manuel',
} as Record<string, string>,
// Drive types
driveTypes: {
fwd: 'Ondan Cekis',
rwd: 'Arkadan Itis',
awd: 'Dort Ceker',
'4wd': 'Dort Ceker',
'4x4': 'Dort Ceker',
'front-wheel drive': 'Ondan Cekis',
'rear-wheel drive': 'Arkadan Itis',
'all-wheel drive': 'Dort Ceker',
quattro: 'Quattro (Dort Ceker)',
xdrive: 'xDrive (Dort Ceker)',
'4matic': '4MATIC (Dort Ceker)',
} as Record<string, string>,
// Common part categories
categories: {
engine: 'Motor',
brake: 'Fren Sistemi',
brakes: 'Fren Sistemi',
suspension: 'Suspansiyon',
steering: 'Direksiyon',
transmission: 'Sanziman',
exhaust: 'Egzoz Sistemi',
cooling: 'Sogutma Sistemi',
electrical: 'Elektrik Sistemi',
interior: 'Ic Aksam',
exterior: 'Dis Aksam',
body: 'Kaporta',
lighting: 'Aydinlatma',
lights: 'Aydinlatma',
wheels: 'Jantlar',
tires: 'Lastikler',
fuel: 'Yakit Sistemi',
'fuel system': 'Yakit Sistemi',
air: 'Hava Sistemi',
'air conditioning': 'Klima',
climate: 'Klima',
filters: 'Filtreler',
oil: 'Yag',
battery: 'Akku',
alternator: 'Alternator',
starter: 'Mars Motoru',
clutch: 'Debriyaj',
gearbox: 'Vites Kutusu',
axle: 'Aks',
differential: 'Diferansiyel',
driveshaft: 'Saft',
'cv joint': 'Aks Kafasi',
'tie rod': 'Rot Kolu',
'ball joint': 'Rotil',
'control arm': 'Salincak',
shock: 'Amortisor',
'shock absorber': 'Amortisor',
spring: 'Yay',
strut: 'Makfersan',
'brake pad': 'Fren Balatasi',
'brake disc': 'Fren Diski',
'brake rotor': 'Fren Diski',
caliper: 'Fren Kaliperi',
'master cylinder': 'Ana Merkez',
'wheel bearing': 'Bilyali Rulman',
hub: 'Porya',
mirror: 'Ayna',
bumper: 'Tampon',
fender: 'Camurluk',
hood: 'Kaput',
bonnet: 'Kaput',
trunk: 'Bagaj',
boot: 'Bagaj',
door: 'Kapi',
window: 'Cam',
windshield: 'On Cam',
windscreen: 'On Cam',
wiper: 'Silecek',
headlight: 'Far',
taillight: 'Stop Lambasi',
'turn signal': 'Sinyal Lambasi',
indicator: 'Sinyal Lambasi',
seat: 'Koltuk',
dashboard: 'Gosterge Paneli',
'steering wheel': 'Direksiyon Simidi',
pedal: 'Pedal',
carpet: 'Hali',
mat: 'Paspas',
'water pump': 'Su Pompasi',
thermostat: 'Termostat',
radiator: 'Radyator',
fan: 'Fan',
hose: 'Hortum',
belt: 'Kayis',
'timing belt': 'Eksantrik Kayisi',
'timing chain': 'Eksantrik Zinciri',
'serpentine belt': 'V Kayis',
gasket: 'Conta',
seal: 'Simir',
'o-ring': 'O-Ring',
sensor: 'Sensor',
oxygen: 'Oksijen Sensoru',
abs: 'ABS Sensoru',
airbag: 'Hava Yastigi',
horn: 'Korna',
relay: 'Role',
fuse: 'Sigorta',
switch: 'Dugme',
motor: 'Motor',
pump: 'Pompa',
compressor: 'Kompresor',
condenser: 'Kondenser',
evaporator: 'Evaporator',
heater: 'Isitici',
blower: 'Ufleyici',
'spark plug': 'Buji',
'ignition coil': 'Atesleme Bobini',
injector: 'Enjktor',
'fuel pump': 'Yakit Pompasi',
'fuel filter': 'Yakit Filtresi',
'air filter': 'Hava Filtresi',
'oil filter': 'Yag Filtresi',
'cabin filter': 'Polen Filtresi',
'pollen filter': 'Polen Filtresi',
} as Record<string, string>,
// Common part names
parts: {
'oil filter': 'Yag Filtresi',
'air filter': 'Hava Filtresi',
'fuel filter': 'Yakit Filtresi',
'cabin filter': 'Polen Filtresi',
'spark plug': 'Buji',
'brake pad': 'Fren Balatasi',
'brake disc': 'Fren Diski',
'brake rotor': 'Fren Diski',
'timing belt': 'Eksantrik Kayisi',
'water pump': 'Su Pompasi',
thermostat: 'Termostat',
alternator: 'Alternator',
starter: 'Mars Motoru',
battery: 'Akku',
radiator: 'Radyator',
'shock absorber': 'Amortisor',
'control arm': 'Salincak',
'tie rod': 'Rot Kolu',
'ball joint': 'Rotil',
'cv joint': 'Aks Kafasi',
clutch: 'Debriyaj',
'clutch kit': 'Debriyaj Seti',
flywheel: 'Volan',
'wheel bearing': 'Bilyali Rulman',
'hub bearing': 'Porya Rulmani',
caliper: 'Fren Kaliperi',
'master cylinder': 'Ana Merkez',
'slave cylinder': 'Yardimci Merkez',
'brake hose': 'Fren Hortumu',
'brake line': 'Fren Borusu',
'abs sensor': 'ABS Sensoru',
'oxygen sensor': 'Oksijen Sensoru',
'crankshaft sensor': 'Krank Sensoru',
'camshaft sensor': 'Eksantrik Sensoru',
'coolant sensor': 'Su Isisi Sensoru',
'oil pressure sensor': 'Yag Basinci Sensoru',
'ignition coil': 'Atesleme Bobini',
injector: 'Enjktor',
'fuel pump': 'Yakit Pompasi',
'fuel injector': 'Yakit Enjektoru',
gasket: 'Conta',
'head gasket': 'Silindir Kapagi Contasi',
'valve cover gasket': 'Kulbtor Kapagi Contasi',
'oil pan gasket': 'Karter Contasi',
'intake manifold gasket': 'Emme Manifoldu Contasi',
'exhaust manifold gasket': 'Egzoz Manifoldu Contasi',
'serpentine belt': 'V Kayis',
'drive belt': 'Tahrik Kayisi',
tensioner: 'Gerdirici',
'belt tensioner': 'Kayis Gerdirici',
idler: 'Avare',
'idler pulley': 'Avare Kasnak',
pulley: 'Kasnak',
'crankshaft pulley': 'Krank Kasnagi',
'power steering pump': 'Hidrolik Direksiyon Pompasi',
'power steering hose': 'Hidrolik Direksiyon Hortumu',
'steering rack': 'Kremayer',
'steering gear': 'Direksiyon Kutusu',
'cv boot': 'Aks Korfezi',
'drive shaft': 'Saft',
'axle shaft': 'Aks Mili',
'wheel hub': 'Porya',
'wheel stud': 'Bijon',
'lug nut': 'Bijon Somunu',
headlight: 'Far',
'headlight bulb': 'Far Ampulu',
taillight: 'Stop Lambasi',
'turn signal': 'Sinyal Lambasi',
'fog light': 'Sis Lambasi',
mirror: 'Ayna',
'side mirror': 'Yan Ayna',
'rear view mirror': 'Ic Ayna',
wiper: 'Silecek',
'wiper blade': 'Silecek Lastigi',
'wiper motor': 'Silecek Motoru',
'window regulator': 'Cam Mekanizmasi',
'window motor': 'Cam Motoru',
'door handle': 'Kapi Kolu',
'door lock': 'Kapi Kilidi',
'door hinge': 'Kapi Mentesesi',
bumper: 'Tampon',
grille: 'Izgara',
hood: 'Kaput',
fender: 'Camurluk',
'splash guard': 'Paclama',
mudguard: 'Camurluk',
} as Record<string, string>,
};
// ==================== TRANSLATION HELPERS ====================
/**
* Translates a term to Turkish if available
*/
function translateToTurkish(
term: string | null | undefined,
dictionary: Record<string, string>,
): string | null {
if (!term) return null;
const normalized = term.toLowerCase().trim();
return dictionary[normalized] || null;
}
/**
* Translates body type to Turkish
*/
export function translateBodyType(bodyType: string | null): string | null {
return translateToTurkish(bodyType, TR_TRANSLATIONS.bodyTypes);
}
/**
* Translates engine type to Turkish
*/
export function translateEngineType(engineType: string | null): string | null {
return translateToTurkish(engineType, TR_TRANSLATIONS.engineTypes);
}
/**
* Translates transmission type to Turkish
*/
export function translateTransmission(
transmission: string | null,
): string | null {
return translateToTurkish(transmission, TR_TRANSLATIONS.transmissions);
}
/**
* Translates drive type to Turkish
*/
export function translateDriveType(driveType: string | null): string | null {
return translateToTurkish(driveType, TR_TRANSLATIONS.driveTypes);
}
/**
* Translates category name to Turkish
*/
export function translateCategoryName(name: string): string {
const normalized = name.toLowerCase().trim();
return TR_TRANSLATIONS.categories[normalized] || name;
}
/**
* Translates part name to Turkish
*/
export function translatePartName(name: string): string {
const normalized = name.toLowerCase().trim();
return TR_TRANSLATIONS.parts[normalized] || name;
}
// ==================== MAPPER FUNCTIONS ====================
/**
* Maps raw EMEX scraper response to standardized DecodedVehicle
*/
export function mapEmexResponse(response: EmexScraperResponse): DecodedVehicle {
const vehicle = response.vehicle;
// Get brand from catalog map or use the one from response
const wmi = response.vin.substring(0, 3);
const catalogEntry = CATALOG_MAP[wmi];
const brand = catalogEntry?.brand || vehicle.brand || 'Unknown';
return {
brand: brand.toUpperCase(),
model: vehicle.model || 'Unknown',
year: vehicle.year || extractYearFromVin(response.vin),
series: vehicle.series || null,
bodyType: vehicle.bodyType || null,
engineCode: vehicle.engineCode || null,
engineType: vehicle.engineType || null,
engineVolume: vehicle.engineVolume || null,
transmission: vehicle.transmission || null,
driveType: vehicle.driveType || null,
colorCode: null, // EMEX doesn't provide color info
raw: buildRawResponse(response),
categories: mapCategories(response.categories),
};
}
/**
* Extracts year from VIN (10th character)
*/
function extractYearFromVin(vin: string): number {
const yearChar = vin.charAt(9).toUpperCase();
const yearMap: Record<string, number> = {
'1': 2001,
'2': 2002,
'3': 2003,
'4': 2004,
'5': 2005,
'6': 2006,
'7': 2007,
'8': 2008,
'9': 2009,
A: 2010,
B: 2011,
C: 2012,
D: 2013,
E: 2014,
F: 2015,
G: 2016,
H: 2017,
J: 2018,
K: 2019,
L: 2020,
M: 2021,
N: 2022,
P: 2023,
R: 2024,
S: 2025,
T: 2026,
V: 2027,
W: 2028,
X: 2029,
Y: 2030,
};
return yearMap[yearChar] || new Date().getFullYear();
}
/**
* Builds the raw response object for storage
* Includes category URLs for on-demand parts fetching
*/
function buildRawResponse(
response: EmexScraperResponse,
): Record<string, unknown> {
return {
source: 'emex', // Explicit source identifier for on-demand loading
method: response.method,
vin: response.vin,
catalogCode: response.catalogCode,
ssd: response.ssd,
quickGroupsUrl: response.quickGroupsUrl,
timestamp: response.timestamp,
success: response.success,
message: response.message,
parsedOptions: response.parsedOptions,
rawResponse: response.rawResponse,
// Store category tree for hierarchical insertion (QuickGroups.aspx)
emexCategoryTree: response.categoryTree || [],
// Store flat category URLs for on-demand parts fetching (fallback)
emexCategories: response.categories?.map((cat) => ({
gid: cat.gid,
name: cat.name,
url: cat.url,
})) || [],
};
}
/**
* Maps EMEX categories to standardized DecodedCategory format
* NOTE: Parts are NOT included here - they will be fetched on-demand when user clicks a category
*/
function mapCategories(
categories?: EmexCategory[],
): DecodedCategory[] {
if (!categories || categories.length === 0) {
return [];
}
return categories.map((cat, index) => {
return {
code: cat.gid || `CAT_${index}`,
nameEn: cat.name,
nameTr: translateCategoryName(cat.name),
description: null,
iconName: deriveIconName(cat.name),
schemaImageUrl: null,
parts: [], // Parts will be fetched on-demand
};
});
}
/**
* Derives icon name from category name
*/
function deriveIconName(categoryName: string): string | null {
const normalized = categoryName.toLowerCase();
const iconMap: Record<string, string> = {
engine: 'engine',
motor: 'engine',
brake: 'brake',
brakes: 'brake',
suspension: 'suspension',
steering: 'steering',
transmission: 'transmission',
gearbox: 'transmission',
exhaust: 'exhaust',
cooling: 'cooling',
electrical: 'electrical',
interior: 'interior',
exterior: 'exterior',
body: 'body',
lighting: 'lighting',
lights: 'lighting',
wheels: 'wheels',
fuel: 'fuel',
air: 'air',
climate: 'climate',
filters: 'filters',
};
for (const [key, icon] of Object.entries(iconMap)) {
if (normalized.includes(key)) {
return icon;
}
}
return null;
}
/**
* Creates an empty/default DecodedVehicle for error cases
*/
export function createEmptyDecodedVehicle(
vin: string,
errorMessage?: string,
): DecodedVehicle {
const wmi = vin.substring(0, 3);
const catalogEntry = CATALOG_MAP[wmi];
return {
brand: catalogEntry?.brand?.toUpperCase() || 'UNKNOWN',
model: 'Unknown',
year: extractYearFromVin(vin),
series: null,
bodyType: null,
engineCode: null,
engineType: null,
engineVolume: null,
transmission: null,
driveType: null,
colorCode: null,
raw: {
vin,
error: errorMessage || 'Vehicle data not found',
source: 'emexdwc.ae',
},
categories: [],
};
}

5
apps/api/src/integrations/emex/emex.module.ts
View File

@@ -1,11 +1,8 @@
import { Module } from "@nestjs/common";
import { EmexService } from "./emex.service";
import { EmexScraperService } from "./emex-scraper.service";
import { EmexParserService } from "./emex-parser.service";
import { EmexQueueService } from "./emex-queue.service";
@Module({
providers: [EmexService, EmexScraperService, EmexParserService, EmexQueueService],
providers: [EmexService],
exports: [EmexService],
})
export class EmexModule {}

764
apps/api/src/integrations/emex/emex.service.ts
View File

@@ -1,371 +1,445 @@
import { Inject, Injectable, Logger } from "@nestjs/common";
import { eq, and } from "drizzle-orm";
import { DATABASE, type Database } from "../../database/database.provider";
import { RedisService } from "../../redis/redis.service";
import { EmexScraperService } from "./emex-scraper.service";
import { EmexQueueService } from "./emex-queue.service";
import {
emexVehicles,
emexVehicleVins,
emexPartGroups,
emexParts,
emexPartNumbers,
emexCatalogs,
emexScrapeSessions,
} from "../../database/schema/emex";
import type {
EmexVehicleData,
EmexCategoryData,
EmexPartData,
EmexJobStatus,
} from "./emex.types";
/**
* EMEX VIN Service
*
* NestJS service for emexdwc.ae VIN integration.
* Wraps the EmexVinScraper from scripts/emex-vin-scraper.js
* and provides standardized DecodedVehicle responses.
*/
const CACHE_PREFIX = "emex:";
const VEHICLE_CACHE_TTL = 86400; // 24h
const CATEGORY_CACHE_TTL = 3600; // 1h
const PARTS_CACHE_TTL = 3600; // 1h
import {
Injectable,
Logger,
BadRequestException,
ServiceUnavailableException,
InternalServerErrorException,
OnModuleDestroy,
} from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import * as path from 'path';
import {
EmexScraperResponse,
EmexCategoryTreeNode,
EmexPartsResult,
DecodedVehicle,
CATALOG_MAP,
} from './emex.types';
import { mapEmexResponse, createEmptyDecodedVehicle } from './emex.mapper';
// Type definition for the imported scraper module
interface EmexScraperModule {
EmexVinScraper: new () => EmexVinScraperInstance;
getCatalogCode: (vin: string) => string | null;
getYearFromVIN: (vin: string) => number | null;
CONFIG: Record<string, unknown>;
}
interface EmexCategoryResult {
gid: string;
name: string;
url: string | null;
}
interface EmexVinScraperInstance {
init(): Promise<void>;
close(): Promise<void>;
searchByVIN(vin: string): Promise<EmexScraperResponse>;
getCategories(quickGroupsUrl: string): Promise<EmexCategoryResult[]>;
getCategoryTree(quickGroupsUrl: string): Promise<EmexCategoryTreeNode[]>;
getParts(detailsUrl: string): Promise<EmexPartsResult>;
}
@Injectable()
export class EmexService {
export class EmexService implements OnModuleDestroy {
private readonly logger = new Logger(EmexService.name);
private scraperModule: EmexScraperModule | null = null;
private isInitialized = false;
private initializationPromise: Promise<void> | null = null;
constructor(
@Inject(DATABASE) private db: Database,
private redis: RedisService,
private scraper: EmexScraperService,
private queue: EmexQueueService,
) {}
private readonly scraperPath: string;
private readonly timeout: number;
private readonly debug: boolean;
async decodeVin(vin: string, userId: string): Promise<{ jobId: string }> {
// Check if we already have a recent scrape session
const existingSession = await this.db
.select()
.from(emexScrapeSessions)
.where(and(eq(emexScrapeSessions.vin, vin), eq(emexScrapeSessions.status, "pending")))
.limit(1);
if (existingSession.length > 0 && existingSession[0].jobId) {
this.logger.log(`Reusing existing scrape session for VIN: ${vin}`);
return { jobId: existingSession[0].jobId };
}
const jobId = await this.queue.addScrapeJob(vin, userId);
// Create scrape session record
await this.db.insert(emexScrapeSessions).values({
vin,
status: "pending",
jobId,
startedAt: new Date(),
});
return { jobId };
}
async getJobStatus(jobId: string): Promise<EmexJobStatus | null> {
return this.queue.getJobStatus(jobId);
}
async getScrapedVehicle(vin: string): Promise<EmexVehicleData | null> {
// Redis cache check
const cacheKey = `${CACHE_PREFIX}vehicle:${vin}`;
const cached = await this.redis.getJson<EmexVehicleData>(cacheKey);
if (cached) return cached;
// Database check via VIN link
const vinRecord = await this.db
.select()
.from(emexVehicleVins)
.where(eq(emexVehicleVins.vin, vin))
.limit(1);
if (vinRecord.length === 0 || !vinRecord[0].emexVehicleId) return null;
const vehicleRecord = await this.db
.select()
.from(emexVehicles)
.where(eq(emexVehicles.id, vinRecord[0].emexVehicleId))
.limit(1);
if (vehicleRecord.length === 0) return null;
const vehicle = vehicleRecord[0];
const result: EmexVehicleData = {
vehicleId: vehicle.vehicleId,
catalogId: vehicle.catalogId || "",
brandName: "",
name: vehicle.name || "",
modelCode: vehicle.modelCode || null,
engine: vehicle.engine || null,
yearFrom: vehicle.yearFrom || null,
yearTo: vehicle.yearTo || null,
rawData: (vehicle.rawData as Record<string, unknown>) || null,
};
// Resolve brand name from catalog
if (vehicle.catalogId) {
const catalog = await this.db
.select()
.from(emexCatalogs)
.where(eq(emexCatalogs.id, vehicle.catalogId))
.limit(1);
if (catalog.length > 0) {
result.brandName = catalog[0].brandName;
}
}
await this.redis.setJson(cacheKey, result, VEHICLE_CACHE_TTL);
return result;
}
async getScrapedCategories(vehicleId: string): Promise<EmexCategoryData[]> {
const cacheKey = `${CACHE_PREFIX}categories:${vehicleId}`;
const cached = await this.redis.getJson<EmexCategoryData[]>(cacheKey);
if (cached) return cached;
// Look up the internal UUID from the emex vehicleId string
const vehicleRecord = await this.db
.select()
.from(emexVehicles)
.where(eq(emexVehicles.vehicleId, vehicleId))
.limit(1);
if (vehicleRecord.length === 0) return [];
const emexVehicleUuid = vehicleRecord[0].id;
const groups = await this.db
.select()
.from(emexPartGroups)
.where(eq(emexPartGroups.emexVehicleId, emexVehicleUuid));
const categories: EmexCategoryData[] = groups.map((g) => ({
groupId: g.groupId,
name: g.name,
nameOriginal: g.nameOriginal || null,
parentGroupId: g.parentGroupId || null,
sortOrder: g.sortOrder || null,
}));
if (categories.length > 0) {
await this.redis.setJson(cacheKey, categories, CATEGORY_CACHE_TTL);
}
return categories;
}
async getScrapedParts(vehicleId: string, groupId: string): Promise<EmexPartData[]> {
const cacheKey = `${CACHE_PREFIX}parts:${vehicleId}:${groupId}`;
const cached = await this.redis.getJson<EmexPartData[]>(cacheKey);
if (cached) return cached;
// Look up internal UUIDs
const vehicleRecord = await this.db
.select()
.from(emexVehicles)
.where(eq(emexVehicles.vehicleId, vehicleId))
.limit(1);
if (vehicleRecord.length === 0) return [];
const emexVehicleUuid = vehicleRecord[0].id;
const groupRecord = await this.db
.select()
.from(emexPartGroups)
.where(
and(
eq(emexPartGroups.emexVehicleId, emexVehicleUuid),
eq(emexPartGroups.groupId, groupId),
),
)
.limit(1);
if (groupRecord.length === 0) return [];
const groupUuid = groupRecord[0].id;
// Fetch parts for this group
const partsRecords = await this.db
.select()
.from(emexParts)
.where(
and(
eq(emexParts.emexVehicleId, emexVehicleUuid),
eq(emexParts.groupId, groupUuid),
),
);
// Fetch OEM codes for each part
const parts: EmexPartData[] = await Promise.all(
partsRecords.map(async (part) => {
const partNumbers = await this.db
.select()
.from(emexPartNumbers)
.where(eq(emexPartNumbers.emexPartId, part.id));
return {
partId: part.partId || null,
name: part.name,
nameOriginal: part.nameOriginal || null,
description: part.description || null,
quantity: part.quantity || null,
position: part.position || null,
hotspotIndex: part.hotspotIndex || null,
oemCodes: partNumbers.map((pn) => pn.oemCode),
};
}),
constructor(private configService: ConfigService) {
// __dirname is apps/api/src/integrations/emex/ or dist/integrations/emex/
// Scraper lives at <monorepo-root>/scripts/emex-vin-scraper.js
const monorepoRoot = path.resolve(__dirname, '..', '..', '..', '..', '..');
const defaultPath = path.resolve(monorepoRoot, 'scripts/emex-vin-scraper.js');
this.scraperPath = this.configService.get<string>(
'EMEX_SCRAPER_PATH',
defaultPath,
);
if (parts.length > 0) {
await this.redis.setJson(cacheKey, parts, PARTS_CACHE_TTL);
}
this.timeout = this.configService.get<number>('EMEX_TIMEOUT', 60000);
this.debug = this.configService.get<boolean>('EMEX_DEBUG', false);
return parts;
this.logger.log(`EMEX Service initialized with scraper path: ${this.scraperPath}`);
}
async saveScrapedVehicle(vin: string, data: EmexVehicleData): Promise<string> {
// Upsert catalog
let catalogUuid: string | null = null;
if (data.catalogId) {
const existingCatalog = await this.db
.select()
.from(emexCatalogs)
.where(eq(emexCatalogs.catalogId, data.catalogId))
.limit(1);
async onModuleDestroy(): Promise<void> {
// Nothing to clean up — each scraper instance is created and closed per-call
}
if (existingCatalog.length > 0) {
catalogUuid = existingCatalog[0].id;
} else {
const [inserted] = await this.db
.insert(emexCatalogs)
.values({
catalogId: data.catalogId,
brandName: data.brandName,
})
.returning();
catalogUuid = inserted.id;
/**
* Lazily initialize the scraper module
*/
private async initializeScraper(): Promise<void> {
if (this.isInitialized) {
return;
}
if (this.initializationPromise) {
return this.initializationPromise;
}
this.initializationPromise = this.doInitialize();
return this.initializationPromise;
}
private async doInitialize(): Promise<void> {
try {
this.logger.log(`Loading EMEX scraper module from: ${this.scraperPath}`);
const fs = require('fs');
if (!fs.existsSync(this.scraperPath)) {
this.logger.error(`Scraper file not found at: ${this.scraperPath}`);
this.logger.error(`Current working directory: ${process.cwd()}`);
throw new Error(`Scraper file not found: ${this.scraperPath}`);
}
// Clear require cache to always load the latest scraper version
delete require.cache[require.resolve(this.scraperPath)];
// eslint-disable-next-line @typescript-eslint/no-var-requires
this.scraperModule = require(this.scraperPath) as EmexScraperModule;
this.logger.log('EMEX scraper module loaded successfully');
this.isInitialized = true;
} catch (error) {
const err = error as Error;
this.logger.error(
`Failed to load EMEX scraper module: ${err.message}`,
err.stack,
);
throw new InternalServerErrorException(
'EMEX servis modulu yuklenemedi',
);
}
// Upsert vehicle
const existingVehicle = await this.db
.select()
.from(emexVehicles)
.where(eq(emexVehicles.vehicleId, data.vehicleId))
.limit(1);
let vehicleUuid: string;
if (existingVehicle.length > 0) {
vehicleUuid = existingVehicle[0].id;
await this.db
.update(emexVehicles)
.set({
catalogId: catalogUuid,
name: data.name,
modelCode: data.modelCode,
engine: data.engine,
yearFrom: data.yearFrom,
yearTo: data.yearTo,
rawData: data.rawData,
})
.where(eq(emexVehicles.id, vehicleUuid));
} else {
const [inserted] = await this.db
.insert(emexVehicles)
.values({
vehicleId: data.vehicleId,
catalogId: catalogUuid,
name: data.name,
modelCode: data.modelCode,
engine: data.engine,
yearFrom: data.yearFrom,
yearTo: data.yearTo,
rawData: data.rawData,
})
.returning();
vehicleUuid = inserted.id;
}
// Link VIN to vehicle
const existingVinLink = await this.db
.select()
.from(emexVehicleVins)
.where(eq(emexVehicleVins.vin, vin))
.limit(1);
if (existingVinLink.length === 0) {
await this.db.insert(emexVehicleVins).values({
emexVehicleId: vehicleUuid,
vin,
});
}
// Invalidate cache
await this.redis.del(`${CACHE_PREFIX}vehicle:${vin}`);
return vehicleUuid;
}
async saveScrapedCategories(
emexVehicleUuid: string,
categories: EmexCategoryData[],
): Promise<void> {
for (const category of categories) {
const existing = await this.db
.select()
.from(emexPartGroups)
.where(
and(
eq(emexPartGroups.emexVehicleId, emexVehicleUuid),
eq(emexPartGroups.groupId, category.groupId),
),
)
.limit(1);
/**
* Creates a new scraper instance and initializes browser
*/
private async createScraperInstance(): Promise<EmexVinScraperInstance> {
await this.initializeScraper();
if (existing.length === 0) {
await this.db.insert(emexPartGroups).values({
emexVehicleId: emexVehicleUuid,
groupId: category.groupId,
name: category.name,
nameOriginal: category.nameOriginal,
parentGroupId: category.parentGroupId,
sortOrder: category.sortOrder,
});
if (!this.scraperModule) {
throw new InternalServerErrorException('EMEX scraper modulu yuklenemedi');
}
const instance = new this.scraperModule.EmexVinScraper();
await instance.init();
return instance;
}
/**
* Validates VIN format
*/
private validateVin(vin: string): void {
if (!vin) {
throw new BadRequestException('VIN numarasi gereklidir');
}
const cleanVin = vin.toUpperCase().replace(/[^A-HJ-NPR-Z0-9]/g, '');
if (cleanVin.length !== 17) {
throw new BadRequestException(
'VIN numarasi 17 karakter olmalidir',
);
}
if (/[IOQ]/i.test(cleanVin)) {
throw new BadRequestException(
'VIN numarasi gecersiz karakterler iceriyor (I, O, Q kullanilamaz)',
);
}
}
/**
* Decodes a VIN number using EMEX scraper
*/
async decodeVin(vin: string): Promise<DecodedVehicle> {
const cleanVin = vin.toUpperCase().replace(/[^A-HJ-NPR-Z0-9]/g, '');
this.validateVin(cleanVin);
this.logger.log(`Decoding VIN: ${cleanVin}`);
let scraper: EmexVinScraperInstance | null = null;
try {
scraper = await this.createScraperInstance();
const response = await this.executeWithTimeout(
scraper.searchByVIN(cleanVin),
this.timeout,
);
if (this.debug) {
this.logger.debug(
`EMEX raw response: ${JSON.stringify(response, null, 2)}`,
);
}
if (!response.success) {
this.logger.warn(
`EMEX search unsuccessful: ${response.message || response.error}`,
);
if (response.vehicle && response.vehicle.brand) {
return mapEmexResponse(response);
}
return createEmptyDecodedVehicle(
cleanVin,
response.message || response.error,
);
}
// Try category tree from QuickGroups.aspx (hierarchical)
if (response.quickGroupsUrl) {
this.logger.log(`Fetching category tree from QuickGroups.aspx: ${response.quickGroupsUrl}`);
try {
const tree = await this.executeWithTimeout(
scraper.getCategoryTree(response.quickGroupsUrl),
this.timeout,
);
if (tree && tree.length > 0) {
this.logger.log(`Found ${tree.length} top-level category groups`);
response.categoryTree = tree;
}
} catch (treeError) {
const err = treeError as Error;
this.logger.warn(`Failed to fetch category tree: ${err.message}`);
}
}
// Fall back to flat categories if tree failed
if (!response.categoryTree?.length && response.quickGroupsUrl) {
this.logger.log(`Falling back to flat categories from: ${response.quickGroupsUrl}`);
try {
const categories = await this.executeWithTimeout(
scraper.getCategories(response.quickGroupsUrl),
this.timeout,
);
if (categories && categories.length > 0) {
this.logger.log(`Found ${categories.length} flat categories (on-demand parts loading enabled)`);
response.categories = categories;
}
} catch (catError) {
const err = catError as Error;
this.logger.warn(`Failed to fetch categories: ${err.message}`);
}
}
const decodedVehicle = mapEmexResponse(response);
this.logger.log(
`VIN decoded successfully: ${decodedVehicle.brand} ${decodedVehicle.model} (${decodedVehicle.year})`,
);
return decodedVehicle;
} catch (error) {
const err = error as Error;
if (
err instanceof BadRequestException ||
err instanceof ServiceUnavailableException ||
err instanceof InternalServerErrorException
) {
throw err;
}
if (err.message?.includes('timeout') || err.name === 'TimeoutError') {
this.logger.error(`VIN decode timeout for: ${cleanVin}`);
throw new ServiceUnavailableException(
'EMEX servisi zaman asimina ugradi. Lutfen tekrar deneyin.',
);
}
if (
err.message?.includes('browser') ||
err.message?.includes('puppeteer') ||
err.message?.includes('navigation')
) {
this.logger.error(`Browser error: ${err.message}`, err.stack);
throw new ServiceUnavailableException(
'EMEX servisine baglanamadi. Lutfen daha sonra tekrar deneyin.',
);
}
this.logger.error(`VIN decode error: ${err.message}`, err.stack);
throw new ServiceUnavailableException(
'VIN sorgulama sirasinda bir hata olustu',
);
} finally {
if (scraper) {
try {
await scraper.close();
} catch (closeError) {
const err = closeError as Error;
this.logger.warn(`Error closing scraper: ${err.message}`);
}
}
}
}
async saveScrapedParts(
emexVehicleUuid: string,
groupUuid: string,
parts: EmexPartData[],
): Promise<void> {
for (const part of parts) {
const [insertedPart] = await this.db
.insert(emexParts)
.values({
emexVehicleId: emexVehicleUuid,
groupId: groupUuid,
partId: part.partId,
name: part.name,
nameOriginal: part.nameOriginal,
description: part.description,
quantity: part.quantity,
position: part.position,
hotspotIndex: part.hotspotIndex,
})
.returning();
/**
* Executes a promise with timeout
*/
private async executeWithTimeout<T>(
promise: Promise<T>,
timeoutMs: number,
): Promise<T> {
let timeoutId: NodeJS.Timeout;
// Insert OEM codes
for (let i = 0; i < part.oemCodes.length; i++) {
await this.db.insert(emexPartNumbers).values({
emexPartId: insertedPart.id,
oemCode: part.oemCodes[i],
isMain: i === 0,
});
const timeoutPromise = new Promise<never>((_, reject) => {
timeoutId = setTimeout(() => {
const error = new Error(`Operation timed out after ${timeoutMs}ms`);
error.name = 'TimeoutError';
reject(error);
}, timeoutMs);
});
try {
const result = await Promise.race([promise, timeoutPromise]);
clearTimeout(timeoutId!);
return result;
} catch (error) {
clearTimeout(timeoutId!);
throw error;
}
}
/**
* Gets the catalog code for a VIN
*/
getCatalogCode(vin: string): string | null {
const wmi = vin.substring(0, 3).toUpperCase();
return CATALOG_MAP[wmi]?.code || null;
}
/**
* Checks if a VIN's manufacturer is supported
*/
isSupported(vin: string): boolean {
if (!vin || vin.length < 3) {
return false;
}
const wmi = vin.substring(0, 3).toUpperCase();
return wmi in CATALOG_MAP;
}
/**
* Gets list of supported manufacturers
*/
getSupportedBrands(): string[] {
const brands = new Set<string>();
for (const entry of Object.values(CATALOG_MAP)) {
brands.add(entry.brand);
}
return Array.from(brands).sort();
}
/**
* Fetches parts + schema image for a specific category (on-demand)
*/
async fetchCategoryParts(categoryUrl: string): Promise<EmexPartsResult> {
if (!categoryUrl) {
this.logger.warn('fetchCategoryParts called with empty URL');
return { parts: [], schemaImageUrl: null, hotspots: [], schemaWidth: 0, schemaHeight: 0 };
}
this.logger.log(`Fetching parts from category URL: ${categoryUrl}`);
let scraper: EmexVinScraperInstance | null = null;
try {
scraper = await this.createScraperInstance();
const result = await this.executeWithTimeout(
scraper.getParts(categoryUrl),
this.timeout,
);
if (result && result.parts.length > 0) {
this.logger.log(`Fetched ${result.parts.length} parts from category`);
if (result.schemaImageUrl) {
this.logger.log(`Schema image found: ${result.schemaImageUrl}`);
}
return result;
}
return { parts: [], schemaImageUrl: null, hotspots: [], schemaWidth: 0, schemaHeight: 0 };
} catch (error) {
const err = error as Error;
this.logger.error(`Failed to fetch category parts: ${err.message}`);
return { parts: [], schemaImageUrl: null, hotspots: [], schemaWidth: 0, schemaHeight: 0 };
} finally {
if (scraper) {
try {
await scraper.close();
} catch (closeError) {
const err = closeError as Error;
this.logger.warn(`Error closing scraper: ${err.message}`);
}
}
}
}
/**
* Extracts year from VIN (10th character)
*/
getYearFromVin(vin: string): number | null {
if (!vin || vin.length < 10) {
return null;
}
const yearChar = vin.charAt(9).toUpperCase();
const yearMap: Record<string, number> = {
'1': 2001,
'2': 2002,
'3': 2003,
'4': 2004,
'5': 2005,
'6': 2006,
'7': 2007,
'8': 2008,
'9': 2009,
A: 2010,
B: 2011,
C: 2012,
D: 2013,
E: 2014,
F: 2015,
G: 2016,
H: 2017,
J: 2018,
K: 2019,
L: 2020,
M: 2021,
N: 2022,
P: 2023,
R: 2024,
S: 2025,
T: 2026,
V: 2027,
W: 2028,
X: 2029,
Y: 2030,
};
return yearMap[yearChar] || null;
}
}

293
apps/api/src/integrations/emex/emex.types.ts
View File

@@ -1,74 +1,259 @@
/**
* EMEX VIN API Types
*
* Type definitions for emexdwc.ae VIN scraping integration.
* These types represent the raw response from the EmexVinScraper.
*/
// ==================== RAW EMEX RESPONSE TYPES ====================
/**
* Raw vehicle data from EMEX API/scraper response
*/
export interface EmexVehicleData {
vehicleId: string;
catalogId: string;
brandName: string;
name: string;
modelCode: string | null;
engine: string | null;
yearFrom: number | null;
yearTo: number | null;
rawData: Record<string, unknown> | null;
brand: string | null;
model: string | null;
year: number | null;
series?: string | null;
bodyType?: string | null;
engineCode?: string | null;
engineType?: string | null;
engineVolume?: string | null;
transmission?: string | null;
driveType?: string | null;
}
export interface EmexCategoryData {
groupId: string;
name: string;
nameOriginal: string | null;
parentGroupId: string | null;
sortOrder: number | null;
/**
* Parsed options from vehicle HTML/API response
*/
export interface EmexParsedOptions {
vehicle_type?: string;
engine_type?: string;
gearbox_type?: string;
[key: string]: string | undefined;
}
export interface EmexPartData {
partId: string | null;
/**
* Vehicle entry from HTML parsing
*/
export interface EmexHtmlVehicle {
name: string;
nameOriginal: string | null;
description: string | null;
quantity: number | null;
position: string | null;
hotspotIndex: number | null;
oemCodes: string[];
engine: string;
options: string;
quickGroupsUrl: string;
}
export interface EmexScrapeJobData {
/**
* Part category from EMEX
*/
export interface EmexCategory {
gid: string;
name: string;
url: string | null;
}
/**
* Recursive tree node from QuickGroups.aspx nested ul/li
* Parents have children[], leaves have gid + url to QuickDetails.aspx
*/
export interface EmexCategoryTreeNode {
name: string;
gid: string | null;
url: string | null;
children: EmexCategoryTreeNode[];
}
/**
* Part data from EMEX
*/
export interface EmexPart {
oemCode: string;
nameEn: string;
positionCode?: string;
}
/**
* Single hotspot area rectangle (pixel coordinates relative to schema image)
*/
export interface EmexHotspotArea {
left: number;
top: number;
width: number;
height: number;
}
/**
* Hotspot group keyed by position code (PNC)
* One position code can map to multiple highlight areas on the diagram
*/
export interface EmexHotspot {
key: string; // position code (PNC), e.g. "15010", "A7065"
areas: EmexHotspotArea[];
}
/**
* Result from getParts() — parts + optional schema image URL + hotspots
*/
export interface EmexPartsResult {
parts: EmexPart[];
schemaImageUrl: string | null;
hotspots: EmexHotspot[];
schemaWidth: number;
schemaHeight: number;
}
/**
* Wizard step from EMEX API
*/
export interface EmexWizardStep {
name: string;
determined: boolean;
allowlistvehicles?: boolean;
options?: EmexWizardOption[];
}
/**
* Wizard option from EMEX API
*/
export interface EmexWizardOption {
key: string;
value: string;
}
/**
* Main response from EmexVinScraper.searchByVIN()
*/
export interface EmexScraperResponse {
success: boolean;
source: string;
method: 'api' | 'vin_url' | 'wizard' | 'html_parse' | 'fallback';
vin: string;
userId: string;
type: "full-decode" | "categories" | "parts";
emexVehicleId?: string;
groupId?: string;
catalogCode: string;
ssd?: string;
vehicle: EmexVehicleData;
message?: string;
error?: string;
rawResponse?: Record<string, unknown>;
wizardSteps?: EmexWizardStep[];
allVehicles?: EmexHtmlVehicle[];
quickGroupsUrl?: string | null;
parsedOptions?: EmexParsedOptions;
categories?: EmexCategory[];
categoryTree?: EmexCategoryTreeNode[];
sampleParts?: EmexPart[];
timestamp: string;
}
export interface EmexScrapeResult {
vehicle: EmexVehicleData | null;
categories: EmexCategoryData[];
parts: EmexPartData[];
// ==================== STANDARDIZED OUTPUT TYPES ====================
/**
* Standardized decoded vehicle response
* Matches the sase.tr schema structure
*/
export interface DecodedVehicle {
brand: string;
model: string;
year: number;
series: string | null;
bodyType: string | null;
engineCode: string | null;
engineType: string | null;
engineVolume: string | null;
transmission: string | null;
driveType: string | null;
colorCode: string | null;
raw: Record<string, unknown>;
categories: DecodedCategory[];
}
export interface EmexJobStatus {
jobId: string;
status: "waiting" | "active" | "completed" | "failed" | "delayed";
progress: number;
result: EmexScrapeResult | null;
failedReason: string | null;
/**
* Standardized category structure
*/
export interface DecodedCategory {
code: string;
nameEn: string;
nameTr?: string;
description: string | null;
iconName: string | null;
schemaImageUrl: string | null;
parts: DecodedPart[];
}
export interface EmexCredentials {
username: string;
password: string;
/**
* Standardized part structure
*/
export interface DecodedPart {
oemCode: string;
alternativeOems?: string[];
nameEn: string;
nameTr?: string;
description: string | null;
positionCode?: string;
positionX?: number;
positionY?: number;
imageUrl?: string;
prices: DecodedPrice[];
}
export class EmexCaptchaError extends Error {
constructor(message = "CAPTCHA detected on EMEX page") {
super(message);
this.name = "EmexCaptchaError";
}
/**
* Standardized price structure
*/
export interface DecodedPrice {
brand: string;
price: number;
currency: string;
inStock: boolean;
}
export class EmexScraperError extends Error {
constructor(
message: string,
public readonly retryable = true,
) {
super(message);
this.name = "EmexScraperError";
}
// ==================== SERVICE CONFIG TYPES ====================
/**
* EMEX service configuration
*/
export interface EmexConfig {
/** Timeout for scraper operations in milliseconds */
timeout: number;
/** Whether to enable debug logging */
debug: boolean;
/** Path to the scraper script */
scraperPath: string;
}
/**
* Catalog mapping entry
*/
export interface CatalogEntry {
code: string;
brand: string;
}
/**
* WMI (World Manufacturer Identifier) to catalog mapping
*/
export const CATALOG_MAP: Record<string, CatalogEntry> = {
WBA: { code: 'BMW202501', brand: 'BMW' },
WBS: { code: 'BMW202501', brand: 'BMW' },
WBY: { code: 'BMW202501', brand: 'BMW' },
WDB: { code: 'MB201810', brand: 'Mercedes-Benz' },
WDD: { code: 'MB201810', brand: 'Mercedes-Benz' },
WDC: { code: 'MB201810', brand: 'Mercedes-Benz' },
WDF: { code: 'MB201810', brand: 'Mercedes-Benz' },
WAU: { code: 'AU1587', brand: 'Audi' },
WVW: { code: 'VW1587', brand: 'Volkswagen' },
WVG: { code: 'VW1587', brand: 'Volkswagen' },
VF1: { code: 'RENAULT201910', brand: 'Renault' },
VF7: { code: 'CPSA01', brand: 'Peugeot' },
VF3: { code: 'CPSA01', brand: 'Peugeot' },
ZFA: { code: 'CFIAT84', brand: 'Fiat' },
ZAR: { code: 'RFIAT84', brand: 'Alfa Romeo' },
WF0: { code: 'FORD202201', brand: 'Ford' },
JTD: { code: 'TOYOTA00', brand: 'Toyota' },
JTE: { code: 'TOYOTA00', brand: 'Toyota' },
SHH: { code: 'HONDA00', brand: 'Honda' },
KNM: { code: 'HYUNDAI00', brand: 'Hyundai' },
KNA: { code: 'KIA00', brand: 'Kia' },
WP0: { code: 'PO799', brand: 'Porsche' },
WP1: { code: 'PO799', brand: 'Porsche' },
JF1: { code: 'SUBARU201802', brand: 'Subaru' },
JF2: { code: 'SUBARU201802', brand: 'Subaru' },
};

6
apps/api/src/integrations/pl24/pl24.types.ts
View File

@@ -213,15 +213,15 @@ export const PL24_SERVICE_CATALOGS: Record<string, PL24CatalogConfig> = {
architecture: "P5_MODERN",
},
// Porsche
// Porsche (uses VWAG backend like other VAG brands)
porsche_parts: {
basePath: "/pl24-app/porsche_parts",
apiPath: "/p5porsche",
apiPath: "/p5vwag",
architecture: "P5_MODERN",
},
porscheclassic_parts: {
basePath: "/pl24-app/porscheclassic_parts",
apiPath: "/p5porsche",
apiPath: "/p5vwag",
architecture: "P5_MODERN",
},