4a06ba5fdb
- Add PL24 Ford legacy service for fordt_parts architecture - Refactor EMEX to use persistent browser pool instead of per-call instances - Make vehicle decode resilient: fallback to PL24 when Corgi doesn't recognize VIN - Add collapsible sidebar with persistent user preference - Improve brand access guard and categories service - Add debug/test scripts for VIN e2e testing Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1130 lines
43 KiB
JavaScript
1130 lines
43 KiB
JavaScript
/**
|
|
* EMEX VIN Scraper for sase.tr
|
|
*
|
|
* This script queries emexdwc.ae with a VIN number and returns vehicle data.
|
|
* Designed for integration with sase.tr VIN decode service.
|
|
*
|
|
* Supports two modes:
|
|
* - Standalone (CLI): launches its own browser
|
|
* - Managed (NestJS): receives a pre-created page via { page } option
|
|
*
|
|
* Usage: node emex-vin-scraper.js <VIN>
|
|
* Example: node emex-vin-scraper.js WBAEY710X5FT11264
|
|
*/
|
|
|
|
const { chromium } = require('playwright');
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
|
|
// Configuration
|
|
const CONFIG = {
|
|
baseUrl: 'https://emexdwc.ae',
|
|
searchUrl: 'https://emexdwc.ae/Search.aspx',
|
|
timeout: 60000,
|
|
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
// DataImpulse Proxy Configuration
|
|
proxy: {
|
|
enabled: process.env.EMEX_USE_PROXY === 'true' || false,
|
|
host: '74.81.81.81',
|
|
portStart: 10000,
|
|
portEnd: 10099,
|
|
username: '1726bbe361918676d44e',
|
|
password: 'f11c7b6128cc86c6',
|
|
},
|
|
// Catalog codes for major brands (WMI prefix -> catalog code)
|
|
catalogMap: {
|
|
'WBA': 'BMW202501', // BMW
|
|
'WBS': 'BMW202501', // BMW M
|
|
'WBY': 'BMW202501', // BMW i
|
|
'WDB': 'MB201810', // Mercedes-Benz
|
|
'WDD': 'MB201810', // Mercedes-Benz
|
|
'WDC': 'MB201810', // Mercedes-Benz
|
|
'WDF': 'MB201810', // Mercedes-Benz
|
|
'WAU': 'AU1587', // Audi
|
|
'WVW': 'VW1587', // Volkswagen
|
|
'WVG': 'VW1587', // Volkswagen
|
|
'VF1': 'RENAULT201910', // Renault
|
|
'VF7': 'CPSA01', // Peugeot
|
|
'VF3': 'CPSA01', // Peugeot
|
|
'ZFA': 'CFIAT84', // Fiat
|
|
'ZAR': 'RFIAT84', // Alfa Romeo
|
|
'WF0': 'FORD202201', // Ford
|
|
'NM0': 'FORD202201', // Ford Turkey
|
|
'JTD': 'TOYOTA00', // Toyota
|
|
'JTE': 'TOYOTA00', // Toyota
|
|
'SHH': 'HONDA00', // Honda
|
|
'KNM': 'HYUNDAI00', // Hyundai
|
|
'KNA': 'KIA00', // Kia
|
|
'JF1': 'SUBARU201802', // Subaru
|
|
'JF2': 'SUBARU201802', // Subaru
|
|
'WP0': 'PO799', // Porsche
|
|
'WP1': 'PO799', // Porsche (Cayenne/Macan)
|
|
'JMZ': 'MAZDA2020', // Mazda (Japan/EU)
|
|
'JM1': 'MAZDA2020', // Mazda (Japan/US)
|
|
'JM3': 'MAZDA2020', // Mazda (Japan/US SUVs)
|
|
}
|
|
};
|
|
|
|
// Get catalog code from VIN
|
|
function getCatalogCode(vin) {
|
|
const wmi = vin.substring(0, 3);
|
|
return CONFIG.catalogMap[wmi] || null;
|
|
}
|
|
|
|
// Parse year from VIN (10th character)
|
|
function getYearFromVIN(vin) {
|
|
const yearChar = vin.charAt(9);
|
|
const yearMap = {
|
|
'1': 2001, '2': 2002, '3': 2003, '4': 2004, '5': 2005,
|
|
'6': 2006, '7': 2007, '8': 2008, '9': 2009, 'A': 2010,
|
|
'B': 2011, 'C': 2012, 'D': 2013, 'E': 2014, 'F': 2015,
|
|
'G': 2016, 'H': 2017, 'J': 2018, 'K': 2019, 'L': 2020,
|
|
'M': 2021, 'N': 2022, 'P': 2023, 'R': 2024, 'S': 2025,
|
|
'T': 2026, 'V': 2027, 'W': 2028, 'X': 2029, 'Y': 2030
|
|
};
|
|
return yearMap[yearChar.toUpperCase()] || null;
|
|
}
|
|
|
|
// Get random proxy port
|
|
function getRandomProxyPort() {
|
|
const { portStart, portEnd } = CONFIG.proxy;
|
|
return Math.floor(Math.random() * (portEnd - portStart + 1)) + portStart;
|
|
}
|
|
|
|
// Main scraper class
|
|
class EmexVinScraper {
|
|
/**
|
|
* @param {object} [options]
|
|
* @param {import('playwright').Page} [options.page] - Pre-created page (managed mode)
|
|
* @param {boolean} [options.useProxy]
|
|
* @param {number} [options.proxyPort]
|
|
*/
|
|
constructor(options = {}) {
|
|
this.browser = null;
|
|
this.context = null;
|
|
this.page = options.page || null;
|
|
this.managed = !!options.page; // true when NestJS provides the page
|
|
this.sessionCookie = null;
|
|
this.useProxy = options.useProxy !== undefined ? options.useProxy : CONFIG.proxy.enabled;
|
|
this.proxyPort = options.proxyPort || getRandomProxyPort();
|
|
}
|
|
|
|
async init() {
|
|
// In managed mode the page is already provided — skip browser launch
|
|
if (this.managed) {
|
|
this._setupResponseCapture();
|
|
return;
|
|
}
|
|
|
|
console.log('Initializing browser...');
|
|
|
|
const launchOptions = {
|
|
headless: true,
|
|
args: [
|
|
'--no-sandbox',
|
|
'--disable-setuid-sandbox',
|
|
'--disable-dev-shm-usage',
|
|
'--disable-accelerated-2d-canvas',
|
|
'--disable-gpu',
|
|
],
|
|
};
|
|
|
|
// Add proxy if enabled
|
|
if (this.useProxy) {
|
|
const proxyUrl = `http://${CONFIG.proxy.host}:${this.proxyPort}`;
|
|
launchOptions.proxy = {
|
|
server: proxyUrl,
|
|
username: CONFIG.proxy.username,
|
|
password: CONFIG.proxy.password,
|
|
};
|
|
console.log(`Using proxy: ${proxyUrl}`);
|
|
}
|
|
|
|
this.browser = await chromium.launch(launchOptions);
|
|
this.context = await this.browser.newContext({
|
|
viewport: { width: 1920, height: 1080 },
|
|
userAgent: CONFIG.userAgent,
|
|
});
|
|
this.page = await this.context.newPage();
|
|
|
|
this._setupResponseCapture();
|
|
|
|
console.log('Browser initialized.');
|
|
}
|
|
|
|
_setupResponseCapture() {
|
|
// Capture network responses
|
|
this.apiResponses = [];
|
|
this.page.on('response', async (response) => {
|
|
const url = response.url();
|
|
if (url.includes('/api/') && url.includes('.svc/')) {
|
|
try {
|
|
const json = await response.json();
|
|
this.apiResponses.push({
|
|
url: url,
|
|
data: json
|
|
});
|
|
} catch (e) {
|
|
// Not JSON
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
async close() {
|
|
// In managed mode, don't close anything — the caller owns the page
|
|
if (this.managed) return;
|
|
if (this.browser) {
|
|
await this.browser.close();
|
|
}
|
|
}
|
|
|
|
// Search by VIN using multiple strategies
|
|
async searchByVIN(vin) {
|
|
console.log(`\nSearching for VIN: ${vin}`);
|
|
|
|
const catalogCode = getCatalogCode(vin);
|
|
console.log(`Detected catalog: ${catalogCode || 'NONE (will use VIN URL fallback)'}`);
|
|
|
|
// First, establish a session
|
|
console.log('Establishing session...');
|
|
await this.page.goto(CONFIG.baseUrl, {
|
|
waitUntil: 'networkidle',
|
|
timeout: CONFIG.timeout
|
|
});
|
|
|
|
// Get session cookie
|
|
const cookies = await this.page.context().cookies();
|
|
this.sessionCookie = cookies.find(c => c.name === 'ASP.NET_SessionId')?.value || '';
|
|
console.log(`Session: ${this.sessionCookie ? 'established' : 'none'}`);
|
|
|
|
// Strategy 1: Try API-based VIN search (requires catalog code)
|
|
if (catalogCode) {
|
|
console.log('\nTrying API-based VIN search...');
|
|
const apiResult = await this.tryAPIVinSearch(vin, catalogCode);
|
|
if (apiResult && apiResult.success) {
|
|
return apiResult;
|
|
}
|
|
}
|
|
|
|
// Strategy 2: Try direct VIN URL search (works without catalog code)
|
|
console.log('\nTrying direct VIN URL search...');
|
|
const vinUrlResult = await this.searchViaVinUrl(vin);
|
|
if (vinUrlResult && vinUrlResult.success) {
|
|
return vinUrlResult;
|
|
}
|
|
|
|
// Strategy 3: Fall back to wizard-based approach (requires catalog code)
|
|
if (catalogCode) {
|
|
console.log('\nVIN URL search failed, trying wizard approach...');
|
|
return await this.searchViaWizard(vin, catalogCode);
|
|
}
|
|
|
|
// No catalog code and VIN URL failed
|
|
return this.createBasicResponse(vin, catalogCode || 'UNKNOWN', 'Brand not in catalog, VIN URL search returned no results');
|
|
}
|
|
|
|
// Try API-based VIN search
|
|
async tryAPIVinSearch(vin, catalogCode) {
|
|
// First get catalog info to check VIN support
|
|
const catalogInfo = await this.page.evaluate(async (code) => {
|
|
try {
|
|
const res = await fetch(`/api/Catalog.svc/GetCatalogInfo?catalogCode=${code}&_tstamp=${Date.now()}`, {
|
|
headers: {
|
|
'Accept': 'application/json, text/javascript, */*; q=0.01',
|
|
'X-Requested-With': 'XMLHttpRequest'
|
|
}
|
|
});
|
|
if (res.ok) {
|
|
return await res.json();
|
|
}
|
|
} catch (e) {
|
|
console.log('Catalog info error:', e.message);
|
|
}
|
|
return null;
|
|
}, catalogCode);
|
|
|
|
console.log('Catalog info:', JSON.stringify(catalogInfo, null, 2));
|
|
|
|
if (!catalogInfo) {
|
|
return { success: false, error: 'Could not get catalog info' };
|
|
}
|
|
|
|
// Try various VIN search API endpoints
|
|
const vinSearchResults = await this.page.evaluate(async ({ code, vinNumber }) => {
|
|
const endpoints = [
|
|
`/api/Catalog.svc/FindVehicleByVin?catalogCode=${code}&vin=${vinNumber}`,
|
|
`/api/Catalog.svc/FindByVin?catalogCode=${code}&vin=${vinNumber}`,
|
|
`/api/Catalog.svc/SearchByVin?catalogCode=${code}&vin=${vinNumber}`,
|
|
`/api/Catalog.svc/GetVehicleByVin?catalogCode=${code}&vin=${vinNumber}`,
|
|
`/api/Catalog.svc/FindVehiclesByVIN?catalogCode=${code}&vin=${vinNumber}`,
|
|
`/api/Catalog.svc/IdentifyByVin?catalogCode=${code}&vin=${vinNumber}`
|
|
];
|
|
|
|
for (const endpoint of endpoints) {
|
|
try {
|
|
const res = await fetch(endpoint + `&_tstamp=${Date.now()}`, {
|
|
headers: {
|
|
'Accept': 'application/json, text/javascript, */*; q=0.01',
|
|
'X-Requested-With': 'XMLHttpRequest'
|
|
}
|
|
});
|
|
if (res.ok) {
|
|
const data = await res.json();
|
|
if (data && (data.length > 0 || Object.keys(data).length > 0)) {
|
|
return { endpoint, data };
|
|
}
|
|
}
|
|
} catch (e) {
|
|
// Continue to next endpoint
|
|
}
|
|
}
|
|
return null;
|
|
}, { code: catalogCode, vinNumber: vin });
|
|
|
|
if (vinSearchResults) {
|
|
console.log(`VIN search successful via: ${vinSearchResults.endpoint}`);
|
|
return this.formatApiResponse(vinSearchResults.data, vin, catalogCode);
|
|
}
|
|
|
|
return { success: false, error: 'No VIN search API found' };
|
|
}
|
|
|
|
// Search via direct VIN URL (mimics Search.aspx VIN tab behavior)
|
|
async searchViaVinUrl(vin) {
|
|
const vinUrl = `${CONFIG.baseUrl}/Vehicles.aspx?ft=findByVIN&c=&ssd=&vin=${vin}`;
|
|
console.log(`Navigating to: ${vinUrl}`);
|
|
|
|
await this.page.goto(vinUrl, {
|
|
waitUntil: 'networkidle',
|
|
timeout: CONFIG.timeout
|
|
});
|
|
|
|
await new Promise(r => setTimeout(r, 2000));
|
|
|
|
// Extract vehicle links from the results page
|
|
const vehicleLinks = await this.page.evaluate(() => {
|
|
const results = [];
|
|
|
|
// Look for links to Vehicle.aspx (vehicle detail pages)
|
|
const links = document.querySelectorAll('a[href*="Vehicle.aspx"], a[href*="QuickGroups.aspx"]');
|
|
links.forEach(link => {
|
|
const href = link.href;
|
|
const name = link.textContent.trim();
|
|
if (name && !results.some(r => r.name === name)) {
|
|
// Extract catalog code from URL
|
|
const cMatch = href.match(/[?&]c=([^&]+)/);
|
|
const ssdMatch = href.match(/[?&]ssd=([^&]+)/);
|
|
results.push({
|
|
name,
|
|
href,
|
|
catalogCode: cMatch ? cMatch[1] : '',
|
|
ssd: ssdMatch ? decodeURIComponent(ssdMatch[1]) : '',
|
|
isQuickGroups: href.includes('QuickGroups.aspx'),
|
|
isVehicle: href.includes('Vehicle.aspx'),
|
|
});
|
|
}
|
|
});
|
|
|
|
return results;
|
|
});
|
|
|
|
console.log(`Found ${vehicleLinks.length} vehicle links via VIN URL`);
|
|
|
|
if (vehicleLinks.length === 0) {
|
|
return { success: false, error: 'No results from VIN URL search' };
|
|
}
|
|
|
|
// Use the first vehicle result
|
|
const firstVehicle = vehicleLinks[0];
|
|
const catalogCode = firstVehicle.catalogCode || getCatalogCode(vin);
|
|
console.log(`Vehicle: ${firstVehicle.name}, Catalog: ${catalogCode}`);
|
|
|
|
// Build QuickGroups URL from the vehicle link params
|
|
let quickGroupsUrl = null;
|
|
const qgLink = vehicleLinks.find(l => l.isQuickGroups);
|
|
|
|
if (qgLink) {
|
|
quickGroupsUrl = qgLink.href;
|
|
} else if (firstVehicle.isVehicle && firstVehicle.ssd) {
|
|
// Construct QuickGroups URL from Vehicle.aspx params
|
|
const vidMatch = firstVehicle.href.match(/[?&]vid=([^&]+)/);
|
|
const vid = vidMatch ? vidMatch[1] : '0';
|
|
quickGroupsUrl = `${CONFIG.baseUrl}/QuickGroups.aspx?c=${catalogCode}&vid=${vid}&ssd=${encodeURIComponent(firstVehicle.ssd)}`;
|
|
}
|
|
|
|
console.log(`QuickGroups URL: ${quickGroupsUrl ? 'found' : 'none'}`);
|
|
|
|
return {
|
|
success: true,
|
|
source: 'emexdwc.ae',
|
|
method: 'vin_url',
|
|
vin: vin,
|
|
catalogCode: catalogCode,
|
|
ssd: firstVehicle.ssd,
|
|
vehicle: {
|
|
brand: this.extractBrandFromCatalog(catalogCode),
|
|
model: firstVehicle.name || 'Unknown',
|
|
year: getYearFromVIN(vin),
|
|
series: null,
|
|
bodyType: null,
|
|
engineCode: null,
|
|
engineType: null,
|
|
engineVolume: null,
|
|
transmission: null,
|
|
driveType: null,
|
|
},
|
|
allVehicles: vehicleLinks.map(v => ({
|
|
name: v.name,
|
|
engine: '',
|
|
options: '',
|
|
quickGroupsUrl: v.isQuickGroups ? v.href : ''
|
|
})),
|
|
quickGroupsUrl: quickGroupsUrl,
|
|
timestamp: new Date().toISOString()
|
|
};
|
|
}
|
|
|
|
// Search via wizard approach - navigate step by step
|
|
async searchViaWizard(vin, catalogCode) {
|
|
console.log('\nStarting wizard-based search...');
|
|
|
|
// Get initial wizard data
|
|
const wizardData = await this.navigateWizard(catalogCode, vin);
|
|
|
|
if (wizardData.vehicles && wizardData.vehicles.length > 0) {
|
|
// Found vehicles, get details
|
|
return this.formatWizardResponse(wizardData, vin, catalogCode);
|
|
}
|
|
|
|
// If still no results, try navigating to Vehicles.aspx page
|
|
console.log('\nTrying HTML-based vehicle search...');
|
|
return await this.searchViaHTML(vin, catalogCode, wizardData.ssd);
|
|
}
|
|
|
|
// Navigate wizard to get vehicle configurations
|
|
async navigateWizard(catalogCode, vin) {
|
|
let ssd = '';
|
|
let wizardSteps = [];
|
|
let finalSSD = '';
|
|
|
|
// Get initial wizard steps
|
|
const initialSteps = await this.page.evaluate(async (code) => {
|
|
try {
|
|
const res = await fetch(`/api/Catalog.svc/GetWizard2?catalogCode=${code}&ssd=&_tstamp=${Date.now()}`, {
|
|
headers: {
|
|
'Accept': 'application/json, text/javascript, */*; q=0.01',
|
|
'X-Requested-With': 'XMLHttpRequest'
|
|
}
|
|
});
|
|
if (res.ok) {
|
|
return await res.json();
|
|
}
|
|
} catch (e) {
|
|
console.log('Wizard error:', e.message);
|
|
}
|
|
return null;
|
|
}, catalogCode);
|
|
|
|
console.log('Initial wizard steps:', initialSteps?.length || 0);
|
|
|
|
if (!initialSteps || initialSteps.length === 0) {
|
|
return { ssd: '', vehicles: [] };
|
|
}
|
|
|
|
wizardSteps = initialSteps;
|
|
|
|
// Try to find a matching model based on VIN (if pattern is recognizable)
|
|
// For BMW, the 4th-8th characters indicate model
|
|
const modelCode = vin.substring(3, 7);
|
|
console.log(`Model code from VIN: ${modelCode}`);
|
|
|
|
// Navigate wizard by selecting options
|
|
for (let i = 0; i < 5; i++) { // Max 5 iterations
|
|
const currentStep = wizardSteps.find(s => !s.determined && s.options?.length > 0);
|
|
|
|
if (!currentStep) {
|
|
console.log('All steps determined or no more options');
|
|
break;
|
|
}
|
|
|
|
console.log(`\nWizard Step: ${currentStep.name}`);
|
|
console.log(`Options: ${currentStep.options.length}`);
|
|
|
|
// Find best matching option
|
|
let selectedOption = currentStep.options[0]; // Default to first
|
|
|
|
// Try to match based on VIN patterns
|
|
// For BMW VIN WBAEY710X5FT11264:
|
|
// WBA = BMW AG, E = 3 Series, Y7 = body type (sedan E90), 10 = engine
|
|
const stepNameLower = currentStep.name.toLowerCase();
|
|
|
|
if (stepNameLower.includes('series') || stepNameLower.includes('model')) {
|
|
// BMW VIN 4th character: E = 3 series
|
|
const vinSeriesChar = vin.charAt(3);
|
|
const seriesMap = {
|
|
'A': '1', 'B': '2', 'C': '3', 'D': '4', 'E': '3',
|
|
'F': '5', 'G': '6', 'H': '7', 'J': '5', 'K': '6',
|
|
'L': '7', 'M': '8', 'N': '2', 'P': 'X', 'R': 'Z',
|
|
'S': 'X', 'T': 'X', 'U': 'X', 'V': 'X', 'W': 'i',
|
|
'X': 'X', 'Y': 'X', 'Z': 'Z'
|
|
};
|
|
const targetSeries = seriesMap[vinSeriesChar] || '3';
|
|
console.log(`VIN char: ${vinSeriesChar}, Target series: ${targetSeries}`);
|
|
|
|
for (const opt of currentStep.options) {
|
|
const optValue = opt.value.toLowerCase();
|
|
// Look for matching series (3', 3 series, E90, E46, etc.)
|
|
if (optValue.includes(`${targetSeries}'`) ||
|
|
optValue.includes(`${targetSeries} series`) ||
|
|
optValue.includes(`${targetSeries}-series`) ||
|
|
(targetSeries === '3' && (optValue.includes('e90') || optValue.includes('e46') || optValue.includes('f30'))) ||
|
|
optValue.includes('318') ||
|
|
optValue.includes('320') ||
|
|
optValue.includes('325') ||
|
|
optValue.includes('330')) {
|
|
selectedOption = opt;
|
|
console.log(`Matched option: ${opt.value}`);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
ssd = selectedOption.key;
|
|
finalSSD = ssd;
|
|
console.log(`Selected: ${selectedOption.value}`);
|
|
|
|
// Get next wizard steps
|
|
wizardSteps = await this.page.evaluate(async ({ code, ssdValue }) => {
|
|
try {
|
|
const res = await fetch(`/api/Catalog.svc/GetWizard2?catalogCode=${code}&ssd=${encodeURIComponent(ssdValue)}&_tstamp=${Date.now()}`, {
|
|
headers: {
|
|
'Accept': 'application/json, text/javascript, */*; q=0.01',
|
|
'X-Requested-With': 'XMLHttpRequest'
|
|
}
|
|
});
|
|
if (res.ok) {
|
|
return await res.json();
|
|
}
|
|
} catch (e) {
|
|
console.log('Wizard step error:', e.message);
|
|
}
|
|
return [];
|
|
}, { code: catalogCode, ssdValue: ssd });
|
|
|
|
// Check if we can list vehicles now
|
|
const canListVehicles = wizardSteps.some(s => s.allowlistvehicles);
|
|
if (canListVehicles) {
|
|
console.log('Can list vehicles now!');
|
|
break;
|
|
}
|
|
}
|
|
|
|
return {
|
|
ssd: finalSSD,
|
|
steps: wizardSteps,
|
|
vehicles: []
|
|
};
|
|
}
|
|
|
|
// Search via HTML page parsing
|
|
async searchViaHTML(vin, catalogCode, ssd) {
|
|
if (!ssd) {
|
|
return this.createBasicResponse(vin, catalogCode, 'No SSD obtained from wizard');
|
|
}
|
|
|
|
// Navigate to Vehicles.aspx
|
|
const vehiclesUrl = `${CONFIG.baseUrl}/Vehicles.aspx?ft=findByWizard2&c=${catalogCode}&ssd=${encodeURIComponent(ssd)}`;
|
|
console.log(`Navigating to: ${vehiclesUrl}`);
|
|
|
|
await this.page.goto(vehiclesUrl, {
|
|
waitUntil: 'networkidle',
|
|
timeout: CONFIG.timeout
|
|
});
|
|
|
|
await new Promise(r => setTimeout(r, 3000));
|
|
|
|
// Take a screenshot for debugging
|
|
await this.page.screenshot({ path: path.join(__dirname, 'vehicles-page.png'), fullPage: true });
|
|
console.log('Screenshot saved to vehicles-page.png');
|
|
|
|
// Extract vehicle data from HTML
|
|
const vehicles = await this.page.evaluate(() => {
|
|
const results = [];
|
|
|
|
// Try multiple table selectors
|
|
const tables = document.querySelectorAll('table');
|
|
tables.forEach(table => {
|
|
const rows = table.querySelectorAll('tbody tr');
|
|
rows.forEach(row => {
|
|
const cells = row.querySelectorAll('td');
|
|
if (cells.length >= 2) {
|
|
const links = row.querySelectorAll('a[href*="QuickGroups"]');
|
|
let quickGroupsUrl = '';
|
|
if (links.length > 0) {
|
|
quickGroupsUrl = links[0].href;
|
|
}
|
|
|
|
const name = cells[0]?.textContent?.trim() || '';
|
|
const engine = cells[1]?.textContent?.trim() || '';
|
|
const options = cells[2]?.textContent?.trim() || '';
|
|
|
|
if (name || engine) {
|
|
results.push({
|
|
name,
|
|
engine,
|
|
options,
|
|
quickGroupsUrl
|
|
});
|
|
}
|
|
}
|
|
});
|
|
});
|
|
|
|
// If no table found, try other selectors
|
|
if (results.length === 0) {
|
|
const vehicleElements = document.querySelectorAll('[class*="vehicle"], [class*="car"], [data-vehicle]');
|
|
vehicleElements.forEach(el => {
|
|
results.push({
|
|
name: el.textContent?.trim()?.substring(0, 100) || '',
|
|
engine: '',
|
|
options: '',
|
|
quickGroupsUrl: ''
|
|
});
|
|
});
|
|
}
|
|
|
|
return results;
|
|
});
|
|
|
|
console.log(`Found ${vehicles.length} vehicles on page`);
|
|
|
|
if (vehicles.length > 0) {
|
|
return this.formatHtmlResponse(vehicles, vin, catalogCode, ssd);
|
|
}
|
|
|
|
// Get page HTML for debugging
|
|
const pageHtml = await this.page.content();
|
|
fs.writeFileSync(path.join(__dirname, 'vehicles-page.html'), pageHtml);
|
|
console.log('HTML saved to vehicles-page.html');
|
|
|
|
return this.createBasicResponse(vin, catalogCode, 'No vehicles found in HTML');
|
|
}
|
|
|
|
// Format API response
|
|
formatApiResponse(apiData, vin, catalogCode) {
|
|
const data = Array.isArray(apiData) ? apiData[0] : apiData;
|
|
|
|
return {
|
|
success: true,
|
|
source: 'emexdwc.ae',
|
|
method: 'api',
|
|
vin: vin,
|
|
catalogCode: catalogCode,
|
|
vehicle: {
|
|
brand: this.extractBrandFromCatalog(catalogCode),
|
|
model: data.name || data.model || data.description || 'Unknown',
|
|
year: data.year || getYearFromVIN(vin),
|
|
series: data.series || null,
|
|
bodyType: data.bodyType || data.vehicleType || null,
|
|
engineCode: data.engine || data.engineCode || null,
|
|
engineType: data.engineType || data.fuelType || null,
|
|
engineVolume: data.engineVolume || data.displacement || null,
|
|
transmission: data.transmission || data.gearbox || null,
|
|
driveType: data.driveType || null,
|
|
},
|
|
rawResponse: apiData,
|
|
timestamp: new Date().toISOString()
|
|
};
|
|
}
|
|
|
|
// Format wizard response
|
|
formatWizardResponse(wizardData, vin, catalogCode) {
|
|
return {
|
|
success: true,
|
|
source: 'emexdwc.ae',
|
|
method: 'wizard',
|
|
vin: vin,
|
|
catalogCode: catalogCode,
|
|
ssd: wizardData.ssd,
|
|
vehicle: {
|
|
brand: this.extractBrandFromCatalog(catalogCode),
|
|
model: 'Multiple configurations found',
|
|
year: getYearFromVIN(vin),
|
|
},
|
|
wizardSteps: wizardData.steps,
|
|
timestamp: new Date().toISOString()
|
|
};
|
|
}
|
|
|
|
// Format HTML-parsed response
|
|
formatHtmlResponse(vehicles, vin, catalogCode, ssd) {
|
|
const vehicle = vehicles[0] || {};
|
|
|
|
// Parse engine code and options
|
|
const engineMatch = vehicle.engine?.match(/([A-Z0-9]+)/i);
|
|
const optionsText = vehicle.options || '';
|
|
|
|
// Extract options
|
|
const options = {};
|
|
optionsText.split(';').forEach(opt => {
|
|
const parts = opt.split(':').map(s => s?.trim());
|
|
if (parts.length === 2) {
|
|
options[parts[0].toLowerCase().replace(/\s+/g, '_')] = parts[1];
|
|
}
|
|
});
|
|
|
|
return {
|
|
success: true,
|
|
source: 'emexdwc.ae',
|
|
method: 'html_parse',
|
|
vin: vin,
|
|
catalogCode: catalogCode,
|
|
ssd: ssd,
|
|
vehicle: {
|
|
brand: this.extractBrandFromCatalog(catalogCode),
|
|
model: vehicle.name || 'Unknown',
|
|
year: getYearFromVIN(vin),
|
|
series: null,
|
|
bodyType: options.vehicle_type || null,
|
|
engineCode: vehicle.engine || engineMatch?.[1] || null,
|
|
engineType: options.engine_type || null,
|
|
engineVolume: null,
|
|
transmission: options.gearbox_type || null,
|
|
driveType: null,
|
|
},
|
|
allVehicles: vehicles,
|
|
quickGroupsUrl: vehicle.quickGroupsUrl || null,
|
|
parsedOptions: options,
|
|
timestamp: new Date().toISOString()
|
|
};
|
|
}
|
|
|
|
// Create basic response
|
|
createBasicResponse(vin, catalogCode, message) {
|
|
return {
|
|
success: false,
|
|
source: 'emexdwc.ae',
|
|
method: 'fallback',
|
|
vin: vin,
|
|
catalogCode: catalogCode,
|
|
vehicle: {
|
|
brand: this.extractBrandFromCatalog(catalogCode),
|
|
model: null,
|
|
year: getYearFromVIN(vin),
|
|
},
|
|
message: message,
|
|
timestamp: new Date().toISOString()
|
|
};
|
|
}
|
|
|
|
// Extract brand name from catalog code
|
|
extractBrandFromCatalog(catalogCode) {
|
|
const brandMap = {
|
|
'BMW202501': 'BMW',
|
|
'MB201810': 'Mercedes-Benz',
|
|
'AU1587': 'Audi',
|
|
'VW1587': 'Volkswagen',
|
|
'RENAULT201910': 'Renault',
|
|
'CPSA01': 'Peugeot',
|
|
'CFIAT84': 'Fiat',
|
|
'RFIAT84': 'Alfa Romeo',
|
|
'FORD202201': 'Ford',
|
|
'TOYOTA00': 'Toyota',
|
|
'HONDA00': 'Honda',
|
|
'HYUNDAI00': 'Hyundai',
|
|
'KIA00': 'Kia',
|
|
'PO799': 'Porsche',
|
|
'SUBARU201802': 'Subaru',
|
|
'MAZDA2020': 'Mazda'
|
|
};
|
|
return brandMap[catalogCode] || catalogCode;
|
|
}
|
|
|
|
// Get part categories for a vehicle
|
|
async getCategories(quickGroupsUrl) {
|
|
if (!quickGroupsUrl) return [];
|
|
|
|
console.log(`\nFetching categories from: ${quickGroupsUrl}`);
|
|
await this.page.goto(quickGroupsUrl, { waitUntil: 'networkidle', timeout: CONFIG.timeout });
|
|
await new Promise(r => setTimeout(r, 2000));
|
|
|
|
// Screenshot for debugging
|
|
await this.page.screenshot({ path: path.join(__dirname, 'categories-page.png'), fullPage: true });
|
|
|
|
const categories = await this.page.evaluate(() => {
|
|
const cats = [];
|
|
|
|
// Method 1: Links to QuickDetails
|
|
const links = document.querySelectorAll('a[href*="QuickDetails"], a[href*="gid="]');
|
|
links.forEach(link => {
|
|
const href = link.href;
|
|
const gidMatch = href.match(/gid=(\d+)/);
|
|
if (gidMatch) {
|
|
cats.push({
|
|
gid: gidMatch[1],
|
|
name: link.textContent.trim(),
|
|
url: href
|
|
});
|
|
}
|
|
});
|
|
|
|
// Method 2: List items with category data
|
|
if (cats.length === 0) {
|
|
const items = document.querySelectorAll('li[data-gid], .category-item, .group-item, [class*="category"]');
|
|
items.forEach(item => {
|
|
const gid = item.dataset?.gid || item.getAttribute('data-gid');
|
|
const name = item.textContent?.trim() || '';
|
|
if (gid || name) {
|
|
cats.push({
|
|
gid: gid || '',
|
|
name: name.substring(0, 100),
|
|
url: null
|
|
});
|
|
}
|
|
});
|
|
}
|
|
|
|
// Method 3: Tables
|
|
if (cats.length === 0) {
|
|
const rows = document.querySelectorAll('table tr');
|
|
rows.forEach(row => {
|
|
const cells = row.querySelectorAll('td');
|
|
if (cells.length >= 1) {
|
|
const link = row.querySelector('a');
|
|
const gidMatch = link?.href?.match(/gid=(\d+)/);
|
|
if (gidMatch || cells[0]?.textContent) {
|
|
cats.push({
|
|
gid: gidMatch?.[1] || '',
|
|
name: cells[0]?.textContent?.trim() || '',
|
|
url: link?.href || null
|
|
});
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
return cats;
|
|
});
|
|
|
|
console.log(`Found ${categories.length} categories`);
|
|
return categories;
|
|
}
|
|
|
|
// Get hierarchical category tree from QuickGroups.aspx
|
|
// Parses the nested <ul>/<li> tree structure recursively
|
|
// Returns: [{name, gid, url, children: [...]}]
|
|
async getCategoryTree(quickGroupsUrl) {
|
|
if (!quickGroupsUrl) return [];
|
|
|
|
console.log(`\nFetching category tree from QuickGroups.aspx: ${quickGroupsUrl}`);
|
|
await this.page.goto(quickGroupsUrl, { waitUntil: 'networkidle', timeout: CONFIG.timeout });
|
|
await new Promise(r => setTimeout(r, 2000));
|
|
|
|
// Screenshot for debugging
|
|
await this.page.screenshot({ path: path.join(__dirname, 'quickgroups-tree.png'), fullPage: true });
|
|
|
|
const tree = await this.page.evaluate(() => {
|
|
function parseNode(li) {
|
|
const contentDiv = li.querySelector(':scope > .qgContent');
|
|
const childUl = li.querySelector(':scope > ul.qgContainer');
|
|
const link = contentDiv?.querySelector('a');
|
|
|
|
let name = '';
|
|
let gid = null;
|
|
let url = null;
|
|
|
|
if (link) {
|
|
// Leaf node with QuickDetails link
|
|
name = link.textContent?.trim() || '';
|
|
const gidMatch = link.href?.match(/gid=(\d+)/);
|
|
gid = gidMatch ? gidMatch[1] : null;
|
|
url = link.href || null;
|
|
} else if (contentDiv) {
|
|
// Parent node — plain text
|
|
name = contentDiv.textContent?.trim() || '';
|
|
}
|
|
|
|
const node = { name, gid, url, children: [] };
|
|
|
|
if (childUl) {
|
|
const childLis = childUl.querySelectorAll(':scope > li.qgNode');
|
|
childLis.forEach(childLi => {
|
|
const child = parseNode(childLi);
|
|
if (child.name) {
|
|
node.children.push(child);
|
|
}
|
|
});
|
|
}
|
|
|
|
return node;
|
|
}
|
|
|
|
// Find the tree root
|
|
const treeRoot = document.querySelector('#qgTree > ul.qgContainer');
|
|
if (!treeRoot) {
|
|
// Fallback: any ul.qgContainer
|
|
const fallback = document.querySelector('ul.qgContainer');
|
|
if (!fallback) return [];
|
|
const topLis = fallback.querySelectorAll(':scope > li.qgNode');
|
|
const tree = [];
|
|
topLis.forEach(li => {
|
|
const node = parseNode(li);
|
|
if (node.name) tree.push(node);
|
|
});
|
|
return tree;
|
|
}
|
|
|
|
const topLevelLis = treeRoot.querySelectorAll(':scope > li.qgNode');
|
|
const tree = [];
|
|
topLevelLis.forEach(li => {
|
|
const node = parseNode(li);
|
|
if (node.name) tree.push(node);
|
|
});
|
|
return tree;
|
|
});
|
|
|
|
// Count nodes
|
|
function countNodes(nodes) {
|
|
let count = 0;
|
|
for (const n of nodes) {
|
|
count++;
|
|
if (n.children) count += countNodes(n.children);
|
|
}
|
|
return count;
|
|
}
|
|
const total = countNodes(tree);
|
|
const leaves = tree.reduce(function countLeaves(sum, n) {
|
|
if (!n.children || n.children.length === 0) return sum + 1;
|
|
return n.children.reduce(countLeaves, sum);
|
|
}, 0);
|
|
|
|
console.log(`Category tree: ${tree.length} top-level groups, ${total} total nodes, ${leaves} leaves`);
|
|
for (const g of tree) {
|
|
console.log(` ${g.name}: ${g.children?.length || 0} children`);
|
|
}
|
|
|
|
return tree;
|
|
}
|
|
|
|
// Get parts for a category
|
|
// Navigates to QuickDetails.aspx, then follows the Unit.aspx link for full
|
|
// interactive schema with hotspot overlays.
|
|
// Returns { parts, schemaImageUrl, hotspots, schemaWidth, schemaHeight }
|
|
async getParts(detailsUrl) {
|
|
if (!detailsUrl) return { parts: [], schemaImageUrl: null, hotspots: [], schemaWidth: 0, schemaHeight: 0 };
|
|
|
|
console.log(`\nFetching parts from: ${detailsUrl}`);
|
|
await this.page.goto(detailsUrl, { waitUntil: 'networkidle', timeout: CONFIG.timeout });
|
|
await new Promise(r => setTimeout(r, 2000));
|
|
|
|
// QuickDetails.aspx has a link to Unit.aspx which contains the full
|
|
// interactive diagram with hotspot overlay divs. Navigate there.
|
|
const unitUrl = await this.page.evaluate(() => {
|
|
const link = document.querySelector('a[href*="Unit.aspx"]');
|
|
return link ? link.href : null;
|
|
});
|
|
|
|
if (unitUrl) {
|
|
console.log(` Following Unit.aspx link: ${unitUrl}`);
|
|
await this.page.goto(unitUrl, { waitUntil: 'networkidle', timeout: CONFIG.timeout });
|
|
await new Promise(r => setTimeout(r, 2000));
|
|
}
|
|
|
|
const result = await this.page.evaluate(() => {
|
|
// --- 1. Get schema image rect and compute coordinate mapping ---
|
|
let schemaImageUrl = null;
|
|
let schemaWidth = 0;
|
|
let schemaHeight = 0;
|
|
let imgRect = null;
|
|
let scaleX = 1;
|
|
let scaleY = 1;
|
|
|
|
const draggerImg = document.querySelector('img.dragger[src*="laximo"]');
|
|
if (draggerImg) {
|
|
schemaImageUrl = draggerImg.src;
|
|
schemaWidth = draggerImg.naturalWidth || 0;
|
|
schemaHeight = draggerImg.naturalHeight || 0;
|
|
imgRect = draggerImg.getBoundingClientRect();
|
|
if (imgRect.width > 0 && schemaWidth > 0) scaleX = schemaWidth / imgRect.width;
|
|
if (imgRect.height > 0 && schemaHeight > 0) scaleY = schemaHeight / imgRect.height;
|
|
}
|
|
|
|
// --- 2. Extract hotspot overlay divs, mapped to natural image coords ---
|
|
const hotspotMap = {};
|
|
const hotspotDivs = document.querySelectorAll('div.dragger.g_highlight[name]');
|
|
for (const div of hotspotDivs) {
|
|
const key = div.getAttribute('name');
|
|
if (!key) continue;
|
|
const divRect = div.getBoundingClientRect();
|
|
const relLeft = imgRect ? (divRect.left - imgRect.left) : divRect.left;
|
|
const relTop = imgRect ? (divRect.top - imgRect.top) : divRect.top;
|
|
const area = {
|
|
left: Math.round(relLeft * scaleX),
|
|
top: Math.round(relTop * scaleY),
|
|
width: Math.round(divRect.width * scaleX),
|
|
height: Math.round(divRect.height * scaleY),
|
|
};
|
|
if (!hotspotMap[key]) {
|
|
hotspotMap[key] = { key, areas: [] };
|
|
}
|
|
hotspotMap[key].areas.push(area);
|
|
}
|
|
const hotspots = Object.values(hotspotMap);
|
|
|
|
// Fallback: any laximo image or full attribute on zoom element
|
|
if (!schemaImageUrl) {
|
|
const zoomEl = document.querySelector('.guayaquil_zoom[full]');
|
|
if (zoomEl) {
|
|
schemaImageUrl = zoomEl.getAttribute('full');
|
|
}
|
|
}
|
|
if (!schemaImageUrl) {
|
|
const imgs = document.querySelectorAll('img[src*="img.laximo.net"]');
|
|
for (const img of imgs) {
|
|
const src = img.src || '';
|
|
if (src.includes('img.laximo.net')) {
|
|
schemaImageUrl = src.replace(/\/\d+\//, '/source/');
|
|
schemaWidth = schemaWidth || img.naturalWidth || 0;
|
|
schemaHeight = schemaHeight || img.naturalHeight || 0;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
// --- 3. Extract parts from named table rows ---
|
|
const partsData = [];
|
|
const namedRows = document.querySelectorAll('tr[name]');
|
|
|
|
if (namedRows.length > 0) {
|
|
for (const row of namedRows) {
|
|
const pncCell = row.querySelector('td[name="c_pnc"]');
|
|
const oemCell = row.querySelector('td[name="c_oem"]');
|
|
const nameCell = row.querySelector('td[name="c_name"]');
|
|
|
|
const oemCode = (oemCell?.textContent || '').trim();
|
|
if (!oemCode) continue;
|
|
|
|
partsData.push({
|
|
oemCode,
|
|
nameEn: (nameCell?.textContent || '').trim(),
|
|
positionCode: (pncCell?.textContent || '').trim(),
|
|
});
|
|
}
|
|
} else {
|
|
// Fallback: generic table row extraction
|
|
const rows = document.querySelectorAll('table tr, .part-row, [class*="part-item"]');
|
|
rows.forEach(row => {
|
|
const cells = row.querySelectorAll('td');
|
|
if (cells.length >= 2) {
|
|
let partNumber = '';
|
|
let description = '';
|
|
let position = '';
|
|
|
|
cells.forEach((cell, idx) => {
|
|
const text = cell.textContent.trim();
|
|
if (/^[A-Z0-9]{6,14}$/.test(text) && !partNumber) {
|
|
partNumber = text;
|
|
} else if (text.length > 10 && !description) {
|
|
description = text;
|
|
} else if (/^\d{1,3}$/.test(text) && idx === 0) {
|
|
position = text;
|
|
}
|
|
});
|
|
|
|
if (partNumber) {
|
|
partsData.push({
|
|
oemCode: partNumber,
|
|
nameEn: description,
|
|
positionCode: position
|
|
});
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
return { parts: partsData, schemaImageUrl, hotspots, schemaWidth, schemaHeight };
|
|
});
|
|
|
|
console.log(`Found ${result.parts.length} parts, ${result.hotspots.length} hotspot groups, schema: ${result.schemaImageUrl ? 'found' : 'none'} (${result.schemaWidth}x${result.schemaHeight})`);
|
|
return result;
|
|
}
|
|
}
|
|
|
|
// CLI execution
|
|
async function main() {
|
|
const vin = process.argv[2] || 'WBAEY710X5FT11264';
|
|
|
|
if (!vin || vin.length !== 17) {
|
|
console.error('Usage: node emex-vin-scraper.js <VIN>');
|
|
console.error('VIN must be exactly 17 characters');
|
|
process.exit(1);
|
|
}
|
|
|
|
console.log('='.repeat(80));
|
|
console.log('EMEX VIN SCRAPER');
|
|
console.log('='.repeat(80));
|
|
console.log(`VIN: ${vin}`);
|
|
console.log(`Year (from VIN): ${getYearFromVIN(vin)}`);
|
|
console.log(`Catalog: ${getCatalogCode(vin) || 'Unknown'}`);
|
|
console.log('='.repeat(80));
|
|
|
|
const scraper = new EmexVinScraper();
|
|
|
|
try {
|
|
await scraper.init();
|
|
|
|
// Search for vehicle
|
|
const vehicleData = await scraper.searchByVIN(vin);
|
|
|
|
console.log('\n' + '='.repeat(80));
|
|
console.log('RESULT');
|
|
console.log('='.repeat(80));
|
|
console.log(JSON.stringify(vehicleData, null, 2));
|
|
|
|
// Save result to file
|
|
const outputPath = path.join(__dirname, `vin-result-${vin}.json`);
|
|
fs.writeFileSync(outputPath, JSON.stringify(vehicleData, null, 2));
|
|
console.log(`\nResult saved to: ${outputPath}`);
|
|
|
|
// If we have quickGroupsUrl, get categories
|
|
if (vehicleData.quickGroupsUrl) {
|
|
console.log('\n' + '-'.repeat(80));
|
|
console.log('FETCHING CATEGORIES...');
|
|
const categories = await scraper.getCategories(vehicleData.quickGroupsUrl);
|
|
vehicleData.categories = categories;
|
|
|
|
// Save updated result
|
|
fs.writeFileSync(outputPath, JSON.stringify(vehicleData, null, 2));
|
|
console.log(`Updated result with ${categories.length} categories`);
|
|
|
|
// Get parts for first category
|
|
if (categories.length > 0 && categories[0].url) {
|
|
console.log('\n' + '-'.repeat(80));
|
|
console.log('FETCHING PARTS FOR FIRST CATEGORY...');
|
|
const parts = await scraper.getParts(categories[0].url);
|
|
vehicleData.sampleParts = parts;
|
|
fs.writeFileSync(outputPath, JSON.stringify(vehicleData, null, 2));
|
|
console.log(`Added ${parts.parts.length} sample parts`);
|
|
}
|
|
}
|
|
|
|
return vehicleData;
|
|
|
|
} catch (error) {
|
|
console.error('\nError:', error.message);
|
|
console.error(error.stack);
|
|
process.exit(1);
|
|
} finally {
|
|
await scraper.close();
|
|
}
|
|
}
|
|
|
|
// Export for module use
|
|
module.exports = { EmexVinScraper, getCatalogCode, getYearFromVIN, CONFIG };
|
|
|
|
// Run if called directly
|
|
if (require.main === module) {
|
|
main().catch(console.error);
|
|
}
|