Semih Yesilyurt
3514272936
EMEX leaf parts fetch was averaging 12.5s cold per category (Honda Civic
prod traces). Almost all of that lived in the scraper's two waterfall page
loads, each of which (a) waited for `networkidle` — which only fires after
analytics/ads stop chattering — then (b) slept an unconditional 2 seconds.
The actual extraction work runs in <100ms once the DOM is parsed.
Three focused changes in scripts/emex-vin-scraper.js:
1) getParts() resolves the Unit.aspx URL via a plain `fetch()` of
QuickDetails.aspx instead of opening it in the browser. Probe showed
the page is fully server-rendered and the anchor is present in the
initial HTML, so the browser stage is dead weight (~3s saved). Falls
back to a browser load if the fetch fails or the anchor isn't there,
so catalogs that gate the link behind JS still work.
2) Both remaining `page.goto()` calls in getParts/getCategories/
getCategoryTree drop `waitUntil:'networkidle'` for `'domcontentloaded'`
plus a targeted `waitForSelector('img.dragger' | 'a[href*=…]', {timeout:5000-8000})`.
`.catch(()=>null)` makes the wait advisory — the evaluate() below has
its own null-safe fallbacks — but in practice the selector is present
well before networkidle would have fired.
3) Removes the two unconditional `setTimeout(r,2000)` sleeps in getParts.
They predate the selector-wait pattern and were belt-and-braces.
Expected: ~12.5s → ~3-5s cold leaf fetch. Warm path (DB-cached) is
unchanged at ~45ms. Dev-only push for verification before main merge.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1173 lines
46 KiB
JavaScript
1173 lines
46 KiB
JavaScript
/**
|
|
* EMEX VIN Scraper for sase.tr
|
|
*
|
|
* This script queries emexdwc.ae with a VIN number and returns vehicle data.
|
|
* Designed for integration with sase.tr VIN decode service.
|
|
*
|
|
* Supports two modes:
|
|
* - Standalone (CLI): launches its own browser
|
|
* - Managed (NestJS): receives a pre-created page via { page } option
|
|
*
|
|
* Usage: node emex-vin-scraper.js <VIN>
|
|
* Example: node emex-vin-scraper.js WBAEY710X5FT11264
|
|
*/
|
|
|
|
const { chromium } = require('playwright');
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
|
|
// Configuration
|
|
const CONFIG = {
|
|
baseUrl: 'https://emexdwc.ae',
|
|
searchUrl: 'https://emexdwc.ae/Search.aspx',
|
|
timeout: 60000,
|
|
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
// DataImpulse Proxy Configuration
|
|
proxy: {
|
|
enabled: process.env.EMEX_USE_PROXY === 'true' || false,
|
|
host: '74.81.81.81',
|
|
portStart: 10000,
|
|
portEnd: 10099,
|
|
username: '1726bbe361918676d44e',
|
|
password: 'f11c7b6128cc86c6',
|
|
},
|
|
// Catalog codes for major brands (WMI prefix -> catalog code)
|
|
catalogMap: {
|
|
'WBA': 'BMW202501', // BMW
|
|
'WBS': 'BMW202501', // BMW M
|
|
'WBY': 'BMW202501', // BMW i
|
|
'WDB': 'MB201810', // Mercedes-Benz
|
|
'WDD': 'MB201810', // Mercedes-Benz
|
|
'WDC': 'MB201810', // Mercedes-Benz
|
|
'WDF': 'MB201810', // Mercedes-Benz
|
|
'WAU': 'AU1587', // Audi
|
|
'WVW': 'VW1587', // Volkswagen
|
|
'WVG': 'VW1587', // Volkswagen
|
|
'VF1': 'RENAULT201910', // Renault
|
|
'VF7': 'CPSA01', // Peugeot
|
|
'VF3': 'CPSA01', // Peugeot
|
|
'ZFA': 'CFIAT84', // Fiat
|
|
'ZAR': 'RFIAT84', // Alfa Romeo
|
|
'WF0': 'FORD202201', // Ford
|
|
'NM0': 'FORD202201', // Ford Turkey
|
|
'JTD': 'TOYOTA00', // Toyota
|
|
'JTE': 'TOYOTA00', // Toyota
|
|
'SHH': 'HONDA00', // Honda
|
|
'KNM': 'HYUNDAI00', // Hyundai
|
|
'KNA': 'KIA00', // Kia
|
|
'JF1': 'SUBARU201802', // Subaru
|
|
'JF2': 'SUBARU201802', // Subaru
|
|
'WP0': 'PO799', // Porsche
|
|
'WP1': 'PO799', // Porsche (Cayenne/Macan)
|
|
'JMZ': 'MAZDA2020', // Mazda (Japan/EU)
|
|
'JM1': 'MAZDA2020', // Mazda (Japan/US)
|
|
'JM3': 'MAZDA2020', // Mazda (Japan/US SUVs)
|
|
}
|
|
};
|
|
|
|
// Get catalog code from VIN
|
|
function getCatalogCode(vin) {
|
|
const wmi = vin.substring(0, 3);
|
|
return CONFIG.catalogMap[wmi] || null;
|
|
}
|
|
|
|
// Parse year from VIN (10th character)
|
|
function getYearFromVIN(vin) {
|
|
const yearChar = vin.charAt(9);
|
|
const yearMap = {
|
|
'1': 2001, '2': 2002, '3': 2003, '4': 2004, '5': 2005,
|
|
'6': 2006, '7': 2007, '8': 2008, '9': 2009, 'A': 2010,
|
|
'B': 2011, 'C': 2012, 'D': 2013, 'E': 2014, 'F': 2015,
|
|
'G': 2016, 'H': 2017, 'J': 2018, 'K': 2019, 'L': 2020,
|
|
'M': 2021, 'N': 2022, 'P': 2023, 'R': 2024, 'S': 2025,
|
|
'T': 2026, 'V': 2027, 'W': 2028, 'X': 2029, 'Y': 2030
|
|
};
|
|
return yearMap[yearChar.toUpperCase()] || null;
|
|
}
|
|
|
|
// Get random proxy port
|
|
function getRandomProxyPort() {
|
|
const { portStart, portEnd } = CONFIG.proxy;
|
|
return Math.floor(Math.random() * (portEnd - portStart + 1)) + portStart;
|
|
}
|
|
|
|
// Main scraper class
|
|
class EmexVinScraper {
|
|
/**
|
|
* @param {object} [options]
|
|
* @param {import('playwright').Page} [options.page] - Pre-created page (managed mode)
|
|
* @param {boolean} [options.useProxy]
|
|
* @param {number} [options.proxyPort]
|
|
*/
|
|
constructor(options = {}) {
|
|
this.browser = null;
|
|
this.context = null;
|
|
this.page = options.page || null;
|
|
this.managed = !!options.page; // true when NestJS provides the page
|
|
this.sessionCookie = null;
|
|
this.useProxy = options.useProxy !== undefined ? options.useProxy : CONFIG.proxy.enabled;
|
|
this.proxyPort = options.proxyPort || getRandomProxyPort();
|
|
}
|
|
|
|
async init() {
|
|
// In managed mode the page is already provided — skip browser launch
|
|
if (this.managed) {
|
|
this._setupResponseCapture();
|
|
return;
|
|
}
|
|
|
|
console.log('Initializing browser...');
|
|
|
|
const launchOptions = {
|
|
headless: true,
|
|
args: [
|
|
'--no-sandbox',
|
|
'--disable-setuid-sandbox',
|
|
'--disable-dev-shm-usage',
|
|
'--disable-accelerated-2d-canvas',
|
|
'--disable-gpu',
|
|
],
|
|
};
|
|
|
|
// Add proxy if enabled
|
|
if (this.useProxy) {
|
|
const proxyUrl = `http://${CONFIG.proxy.host}:${this.proxyPort}`;
|
|
launchOptions.proxy = {
|
|
server: proxyUrl,
|
|
username: CONFIG.proxy.username,
|
|
password: CONFIG.proxy.password,
|
|
};
|
|
console.log(`Using proxy: ${proxyUrl}`);
|
|
}
|
|
|
|
this.browser = await chromium.launch(launchOptions);
|
|
this.context = await this.browser.newContext({
|
|
viewport: { width: 1920, height: 1080 },
|
|
userAgent: CONFIG.userAgent,
|
|
});
|
|
this.page = await this.context.newPage();
|
|
|
|
this._setupResponseCapture();
|
|
|
|
console.log('Browser initialized.');
|
|
}
|
|
|
|
_setupResponseCapture() {
|
|
// Capture network responses
|
|
this.apiResponses = [];
|
|
this.page.on('response', async (response) => {
|
|
const url = response.url();
|
|
if (url.includes('/api/') && url.includes('.svc/')) {
|
|
try {
|
|
const json = await response.json();
|
|
this.apiResponses.push({
|
|
url: url,
|
|
data: json
|
|
});
|
|
} catch (e) {
|
|
// Not JSON
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
async close() {
|
|
// In managed mode, don't close anything — the caller owns the page
|
|
if (this.managed) return;
|
|
if (this.browser) {
|
|
await this.browser.close();
|
|
}
|
|
}
|
|
|
|
// Search by VIN using multiple strategies
|
|
async searchByVIN(vin) {
|
|
console.log(`\nSearching for VIN: ${vin}`);
|
|
|
|
const catalogCode = getCatalogCode(vin);
|
|
console.log(`Detected catalog: ${catalogCode || 'NONE (will use VIN URL fallback)'}`);
|
|
|
|
// First, establish a session
|
|
console.log('Establishing session...');
|
|
await this.page.goto(CONFIG.baseUrl, {
|
|
waitUntil: 'networkidle',
|
|
timeout: CONFIG.timeout
|
|
});
|
|
|
|
// Get session cookie
|
|
const cookies = await this.page.context().cookies();
|
|
this.sessionCookie = cookies.find(c => c.name === 'ASP.NET_SessionId')?.value || '';
|
|
console.log(`Session: ${this.sessionCookie ? 'established' : 'none'}`);
|
|
|
|
// Strategy 1: Try API-based VIN search (requires catalog code)
|
|
if (catalogCode) {
|
|
console.log('\nTrying API-based VIN search...');
|
|
const apiResult = await this.tryAPIVinSearch(vin, catalogCode);
|
|
if (apiResult && apiResult.success) {
|
|
return apiResult;
|
|
}
|
|
}
|
|
|
|
// Strategy 2: Try direct VIN URL search (works without catalog code)
|
|
console.log('\nTrying direct VIN URL search...');
|
|
const vinUrlResult = await this.searchViaVinUrl(vin);
|
|
if (vinUrlResult && vinUrlResult.success) {
|
|
return vinUrlResult;
|
|
}
|
|
|
|
// Strategy 3: Fall back to wizard-based approach (requires catalog code)
|
|
if (catalogCode) {
|
|
console.log('\nVIN URL search failed, trying wizard approach...');
|
|
return await this.searchViaWizard(vin, catalogCode);
|
|
}
|
|
|
|
// No catalog code and VIN URL failed
|
|
return this.createBasicResponse(vin, catalogCode || 'UNKNOWN', 'Brand not in catalog, VIN URL search returned no results');
|
|
}
|
|
|
|
// Try API-based VIN search
|
|
async tryAPIVinSearch(vin, catalogCode) {
|
|
// First get catalog info to check VIN support
|
|
const catalogInfo = await this.page.evaluate(async (code) => {
|
|
try {
|
|
const res = await fetch(`/api/Catalog.svc/GetCatalogInfo?catalogCode=${code}&_tstamp=${Date.now()}`, {
|
|
headers: {
|
|
'Accept': 'application/json, text/javascript, */*; q=0.01',
|
|
'X-Requested-With': 'XMLHttpRequest'
|
|
}
|
|
});
|
|
if (res.ok) {
|
|
return await res.json();
|
|
}
|
|
} catch (e) {
|
|
console.log('Catalog info error:', e.message);
|
|
}
|
|
return null;
|
|
}, catalogCode);
|
|
|
|
console.log('Catalog info:', JSON.stringify(catalogInfo, null, 2));
|
|
|
|
if (!catalogInfo) {
|
|
return { success: false, error: 'Could not get catalog info' };
|
|
}
|
|
|
|
// Try various VIN search API endpoints
|
|
const vinSearchResults = await this.page.evaluate(async ({ code, vinNumber }) => {
|
|
const endpoints = [
|
|
`/api/Catalog.svc/FindVehicleByVin?catalogCode=${code}&vin=${vinNumber}`,
|
|
`/api/Catalog.svc/FindByVin?catalogCode=${code}&vin=${vinNumber}`,
|
|
`/api/Catalog.svc/SearchByVin?catalogCode=${code}&vin=${vinNumber}`,
|
|
`/api/Catalog.svc/GetVehicleByVin?catalogCode=${code}&vin=${vinNumber}`,
|
|
`/api/Catalog.svc/FindVehiclesByVIN?catalogCode=${code}&vin=${vinNumber}`,
|
|
`/api/Catalog.svc/IdentifyByVin?catalogCode=${code}&vin=${vinNumber}`
|
|
];
|
|
|
|
for (const endpoint of endpoints) {
|
|
try {
|
|
const res = await fetch(endpoint + `&_tstamp=${Date.now()}`, {
|
|
headers: {
|
|
'Accept': 'application/json, text/javascript, */*; q=0.01',
|
|
'X-Requested-With': 'XMLHttpRequest'
|
|
}
|
|
});
|
|
if (res.ok) {
|
|
const data = await res.json();
|
|
if (data && (data.length > 0 || Object.keys(data).length > 0)) {
|
|
return { endpoint, data };
|
|
}
|
|
}
|
|
} catch (e) {
|
|
// Continue to next endpoint
|
|
}
|
|
}
|
|
return null;
|
|
}, { code: catalogCode, vinNumber: vin });
|
|
|
|
if (vinSearchResults) {
|
|
console.log(`VIN search successful via: ${vinSearchResults.endpoint}`);
|
|
return this.formatApiResponse(vinSearchResults.data, vin, catalogCode);
|
|
}
|
|
|
|
return { success: false, error: 'No VIN search API found' };
|
|
}
|
|
|
|
// Search via direct VIN URL (mimics Search.aspx VIN tab behavior)
|
|
async searchViaVinUrl(vin) {
|
|
const vinUrl = `${CONFIG.baseUrl}/Vehicles.aspx?ft=findByVIN&c=&ssd=&vin=${vin}`;
|
|
console.log(`Navigating to: ${vinUrl}`);
|
|
|
|
await this.page.goto(vinUrl, {
|
|
waitUntil: 'networkidle',
|
|
timeout: CONFIG.timeout
|
|
});
|
|
|
|
await new Promise(r => setTimeout(r, 2000));
|
|
|
|
// Extract vehicle links from the results page
|
|
const vehicleLinks = await this.page.evaluate(() => {
|
|
const results = [];
|
|
|
|
// Look for links to Vehicle.aspx (vehicle detail pages)
|
|
const links = document.querySelectorAll('a[href*="Vehicle.aspx"], a[href*="QuickGroups.aspx"]');
|
|
links.forEach(link => {
|
|
const href = link.href;
|
|
const name = link.textContent.trim();
|
|
if (name && !results.some(r => r.name === name)) {
|
|
// Extract catalog code from URL
|
|
const cMatch = href.match(/[?&]c=([^&]+)/);
|
|
const ssdMatch = href.match(/[?&]ssd=([^&]+)/);
|
|
results.push({
|
|
name,
|
|
href,
|
|
catalogCode: cMatch ? cMatch[1] : '',
|
|
ssd: ssdMatch ? decodeURIComponent(ssdMatch[1]) : '',
|
|
isQuickGroups: href.includes('QuickGroups.aspx'),
|
|
isVehicle: href.includes('Vehicle.aspx'),
|
|
});
|
|
}
|
|
});
|
|
|
|
return results;
|
|
});
|
|
|
|
console.log(`Found ${vehicleLinks.length} vehicle links via VIN URL`);
|
|
|
|
if (vehicleLinks.length === 0) {
|
|
return { success: false, error: 'No results from VIN URL search' };
|
|
}
|
|
|
|
// Use the first vehicle result
|
|
const firstVehicle = vehicleLinks[0];
|
|
const catalogCode = firstVehicle.catalogCode || getCatalogCode(vin);
|
|
console.log(`Vehicle: ${firstVehicle.name}, Catalog: ${catalogCode}`);
|
|
|
|
// Build QuickGroups URL from the vehicle link params
|
|
let quickGroupsUrl = null;
|
|
const qgLink = vehicleLinks.find(l => l.isQuickGroups);
|
|
|
|
if (qgLink) {
|
|
quickGroupsUrl = qgLink.href;
|
|
} else if (firstVehicle.isVehicle && firstVehicle.ssd) {
|
|
// Construct QuickGroups URL from Vehicle.aspx params
|
|
const vidMatch = firstVehicle.href.match(/[?&]vid=([^&]+)/);
|
|
const vid = vidMatch ? vidMatch[1] : '0';
|
|
quickGroupsUrl = `${CONFIG.baseUrl}/QuickGroups.aspx?c=${catalogCode}&vid=${vid}&ssd=${encodeURIComponent(firstVehicle.ssd)}`;
|
|
}
|
|
|
|
console.log(`QuickGroups URL: ${quickGroupsUrl ? 'found' : 'none'}`);
|
|
|
|
return {
|
|
success: true,
|
|
source: 'emexdwc.ae',
|
|
method: 'vin_url',
|
|
vin: vin,
|
|
catalogCode: catalogCode,
|
|
ssd: firstVehicle.ssd,
|
|
vehicle: {
|
|
brand: this.extractBrandFromCatalog(catalogCode),
|
|
model: firstVehicle.name || 'Unknown',
|
|
year: getYearFromVIN(vin),
|
|
series: null,
|
|
bodyType: null,
|
|
engineCode: null,
|
|
engineType: null,
|
|
engineVolume: null,
|
|
transmission: null,
|
|
driveType: null,
|
|
},
|
|
allVehicles: vehicleLinks.map(v => ({
|
|
name: v.name,
|
|
engine: '',
|
|
options: '',
|
|
quickGroupsUrl: v.isQuickGroups ? v.href : ''
|
|
})),
|
|
quickGroupsUrl: quickGroupsUrl,
|
|
timestamp: new Date().toISOString()
|
|
};
|
|
}
|
|
|
|
// Search via wizard approach - navigate step by step
|
|
async searchViaWizard(vin, catalogCode) {
|
|
console.log('\nStarting wizard-based search...');
|
|
|
|
// Get initial wizard data
|
|
const wizardData = await this.navigateWizard(catalogCode, vin);
|
|
|
|
if (wizardData.vehicles && wizardData.vehicles.length > 0) {
|
|
// Found vehicles, get details
|
|
return this.formatWizardResponse(wizardData, vin, catalogCode);
|
|
}
|
|
|
|
// If still no results, try navigating to Vehicles.aspx page
|
|
console.log('\nTrying HTML-based vehicle search...');
|
|
return await this.searchViaHTML(vin, catalogCode, wizardData.ssd);
|
|
}
|
|
|
|
// Navigate wizard to get vehicle configurations
|
|
async navigateWizard(catalogCode, vin) {
|
|
let ssd = '';
|
|
let wizardSteps = [];
|
|
let finalSSD = '';
|
|
|
|
// Get initial wizard steps
|
|
const initialSteps = await this.page.evaluate(async (code) => {
|
|
try {
|
|
const res = await fetch(`/api/Catalog.svc/GetWizard2?catalogCode=${code}&ssd=&_tstamp=${Date.now()}`, {
|
|
headers: {
|
|
'Accept': 'application/json, text/javascript, */*; q=0.01',
|
|
'X-Requested-With': 'XMLHttpRequest'
|
|
}
|
|
});
|
|
if (res.ok) {
|
|
return await res.json();
|
|
}
|
|
} catch (e) {
|
|
console.log('Wizard error:', e.message);
|
|
}
|
|
return null;
|
|
}, catalogCode);
|
|
|
|
console.log('Initial wizard steps:', initialSteps?.length || 0);
|
|
|
|
if (!initialSteps || initialSteps.length === 0) {
|
|
return { ssd: '', vehicles: [] };
|
|
}
|
|
|
|
wizardSteps = initialSteps;
|
|
|
|
// Try to find a matching model based on VIN (if pattern is recognizable)
|
|
// For BMW, the 4th-8th characters indicate model
|
|
const modelCode = vin.substring(3, 7);
|
|
console.log(`Model code from VIN: ${modelCode}`);
|
|
|
|
// Navigate wizard by selecting options
|
|
for (let i = 0; i < 5; i++) { // Max 5 iterations
|
|
const currentStep = wizardSteps.find(s => !s.determined && s.options?.length > 0);
|
|
|
|
if (!currentStep) {
|
|
console.log('All steps determined or no more options');
|
|
break;
|
|
}
|
|
|
|
console.log(`\nWizard Step: ${currentStep.name}`);
|
|
console.log(`Options: ${currentStep.options.length}`);
|
|
|
|
// Find best matching option
|
|
let selectedOption = currentStep.options[0]; // Default to first
|
|
|
|
// Try to match based on VIN patterns
|
|
// For BMW VIN WBAEY710X5FT11264:
|
|
// WBA = BMW AG, E = 3 Series, Y7 = body type (sedan E90), 10 = engine
|
|
const stepNameLower = currentStep.name.toLowerCase();
|
|
|
|
if (stepNameLower.includes('series') || stepNameLower.includes('model')) {
|
|
// BMW VIN 4th character: E = 3 series
|
|
const vinSeriesChar = vin.charAt(3);
|
|
const seriesMap = {
|
|
'A': '1', 'B': '2', 'C': '3', 'D': '4', 'E': '3',
|
|
'F': '5', 'G': '6', 'H': '7', 'J': '5', 'K': '6',
|
|
'L': '7', 'M': '8', 'N': '2', 'P': 'X', 'R': 'Z',
|
|
'S': 'X', 'T': 'X', 'U': 'X', 'V': 'X', 'W': 'i',
|
|
'X': 'X', 'Y': 'X', 'Z': 'Z'
|
|
};
|
|
const targetSeries = seriesMap[vinSeriesChar] || '3';
|
|
console.log(`VIN char: ${vinSeriesChar}, Target series: ${targetSeries}`);
|
|
|
|
for (const opt of currentStep.options) {
|
|
const optValue = opt.value.toLowerCase();
|
|
// Look for matching series (3', 3 series, E90, E46, etc.)
|
|
if (optValue.includes(`${targetSeries}'`) ||
|
|
optValue.includes(`${targetSeries} series`) ||
|
|
optValue.includes(`${targetSeries}-series`) ||
|
|
(targetSeries === '3' && (optValue.includes('e90') || optValue.includes('e46') || optValue.includes('f30'))) ||
|
|
optValue.includes('318') ||
|
|
optValue.includes('320') ||
|
|
optValue.includes('325') ||
|
|
optValue.includes('330')) {
|
|
selectedOption = opt;
|
|
console.log(`Matched option: ${opt.value}`);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
ssd = selectedOption.key;
|
|
finalSSD = ssd;
|
|
console.log(`Selected: ${selectedOption.value}`);
|
|
|
|
// Get next wizard steps
|
|
wizardSteps = await this.page.evaluate(async ({ code, ssdValue }) => {
|
|
try {
|
|
const res = await fetch(`/api/Catalog.svc/GetWizard2?catalogCode=${code}&ssd=${encodeURIComponent(ssdValue)}&_tstamp=${Date.now()}`, {
|
|
headers: {
|
|
'Accept': 'application/json, text/javascript, */*; q=0.01',
|
|
'X-Requested-With': 'XMLHttpRequest'
|
|
}
|
|
});
|
|
if (res.ok) {
|
|
return await res.json();
|
|
}
|
|
} catch (e) {
|
|
console.log('Wizard step error:', e.message);
|
|
}
|
|
return [];
|
|
}, { code: catalogCode, ssdValue: ssd });
|
|
|
|
// Check if we can list vehicles now
|
|
const canListVehicles = wizardSteps.some(s => s.allowlistvehicles);
|
|
if (canListVehicles) {
|
|
console.log('Can list vehicles now!');
|
|
break;
|
|
}
|
|
}
|
|
|
|
return {
|
|
ssd: finalSSD,
|
|
steps: wizardSteps,
|
|
vehicles: []
|
|
};
|
|
}
|
|
|
|
// Search via HTML page parsing
|
|
async searchViaHTML(vin, catalogCode, ssd) {
|
|
if (!ssd) {
|
|
return this.createBasicResponse(vin, catalogCode, 'No SSD obtained from wizard');
|
|
}
|
|
|
|
// Navigate to Vehicles.aspx
|
|
const vehiclesUrl = `${CONFIG.baseUrl}/Vehicles.aspx?ft=findByWizard2&c=${catalogCode}&ssd=${encodeURIComponent(ssd)}`;
|
|
console.log(`Navigating to: ${vehiclesUrl}`);
|
|
|
|
await this.page.goto(vehiclesUrl, {
|
|
waitUntil: 'networkidle',
|
|
timeout: CONFIG.timeout
|
|
});
|
|
|
|
await new Promise(r => setTimeout(r, 3000));
|
|
|
|
// Take a screenshot for debugging
|
|
await this.page.screenshot({ path: path.join(__dirname, 'vehicles-page.png'), fullPage: true });
|
|
console.log('Screenshot saved to vehicles-page.png');
|
|
|
|
// Extract vehicle data from HTML
|
|
const vehicles = await this.page.evaluate(() => {
|
|
const results = [];
|
|
|
|
// Try multiple table selectors
|
|
const tables = document.querySelectorAll('table');
|
|
tables.forEach(table => {
|
|
const rows = table.querySelectorAll('tbody tr');
|
|
rows.forEach(row => {
|
|
const cells = row.querySelectorAll('td');
|
|
if (cells.length >= 2) {
|
|
const links = row.querySelectorAll('a[href*="QuickGroups"]');
|
|
let quickGroupsUrl = '';
|
|
if (links.length > 0) {
|
|
quickGroupsUrl = links[0].href;
|
|
}
|
|
|
|
const name = cells[0]?.textContent?.trim() || '';
|
|
const engine = cells[1]?.textContent?.trim() || '';
|
|
const options = cells[2]?.textContent?.trim() || '';
|
|
|
|
if (name || engine) {
|
|
results.push({
|
|
name,
|
|
engine,
|
|
options,
|
|
quickGroupsUrl
|
|
});
|
|
}
|
|
}
|
|
});
|
|
});
|
|
|
|
// If no table found, try other selectors
|
|
if (results.length === 0) {
|
|
const vehicleElements = document.querySelectorAll('[class*="vehicle"], [class*="car"], [data-vehicle]');
|
|
vehicleElements.forEach(el => {
|
|
results.push({
|
|
name: el.textContent?.trim()?.substring(0, 100) || '',
|
|
engine: '',
|
|
options: '',
|
|
quickGroupsUrl: ''
|
|
});
|
|
});
|
|
}
|
|
|
|
return results;
|
|
});
|
|
|
|
console.log(`Found ${vehicles.length} vehicles on page`);
|
|
|
|
if (vehicles.length > 0) {
|
|
return this.formatHtmlResponse(vehicles, vin, catalogCode, ssd);
|
|
}
|
|
|
|
// Get page HTML for debugging
|
|
const pageHtml = await this.page.content();
|
|
fs.writeFileSync(path.join(__dirname, 'vehicles-page.html'), pageHtml);
|
|
console.log('HTML saved to vehicles-page.html');
|
|
|
|
return this.createBasicResponse(vin, catalogCode, 'No vehicles found in HTML');
|
|
}
|
|
|
|
// Format API response
|
|
formatApiResponse(apiData, vin, catalogCode) {
|
|
const data = Array.isArray(apiData) ? apiData[0] : apiData;
|
|
|
|
return {
|
|
success: true,
|
|
source: 'emexdwc.ae',
|
|
method: 'api',
|
|
vin: vin,
|
|
catalogCode: catalogCode,
|
|
vehicle: {
|
|
brand: this.extractBrandFromCatalog(catalogCode),
|
|
model: data.name || data.model || data.description || 'Unknown',
|
|
year: data.year || getYearFromVIN(vin),
|
|
series: data.series || null,
|
|
bodyType: data.bodyType || data.vehicleType || null,
|
|
engineCode: data.engine || data.engineCode || null,
|
|
engineType: data.engineType || data.fuelType || null,
|
|
engineVolume: data.engineVolume || data.displacement || null,
|
|
transmission: data.transmission || data.gearbox || null,
|
|
driveType: data.driveType || null,
|
|
},
|
|
rawResponse: apiData,
|
|
timestamp: new Date().toISOString()
|
|
};
|
|
}
|
|
|
|
// Format wizard response
|
|
formatWizardResponse(wizardData, vin, catalogCode) {
|
|
return {
|
|
success: true,
|
|
source: 'emexdwc.ae',
|
|
method: 'wizard',
|
|
vin: vin,
|
|
catalogCode: catalogCode,
|
|
ssd: wizardData.ssd,
|
|
vehicle: {
|
|
brand: this.extractBrandFromCatalog(catalogCode),
|
|
model: 'Multiple configurations found',
|
|
year: getYearFromVIN(vin),
|
|
},
|
|
wizardSteps: wizardData.steps,
|
|
timestamp: new Date().toISOString()
|
|
};
|
|
}
|
|
|
|
// Format HTML-parsed response
|
|
formatHtmlResponse(vehicles, vin, catalogCode, ssd) {
|
|
const vehicle = vehicles[0] || {};
|
|
|
|
// Parse engine code and options
|
|
const engineMatch = vehicle.engine?.match(/([A-Z0-9]+)/i);
|
|
const optionsText = vehicle.options || '';
|
|
|
|
// Extract options
|
|
const options = {};
|
|
optionsText.split(';').forEach(opt => {
|
|
const parts = opt.split(':').map(s => s?.trim());
|
|
if (parts.length === 2) {
|
|
options[parts[0].toLowerCase().replace(/\s+/g, '_')] = parts[1];
|
|
}
|
|
});
|
|
|
|
return {
|
|
success: true,
|
|
source: 'emexdwc.ae',
|
|
method: 'html_parse',
|
|
vin: vin,
|
|
catalogCode: catalogCode,
|
|
ssd: ssd,
|
|
vehicle: {
|
|
brand: this.extractBrandFromCatalog(catalogCode),
|
|
model: vehicle.name || 'Unknown',
|
|
year: getYearFromVIN(vin),
|
|
series: null,
|
|
bodyType: options.vehicle_type || null,
|
|
engineCode: vehicle.engine || engineMatch?.[1] || null,
|
|
engineType: options.engine_type || null,
|
|
engineVolume: null,
|
|
transmission: options.gearbox_type || null,
|
|
driveType: null,
|
|
},
|
|
allVehicles: vehicles,
|
|
quickGroupsUrl: vehicle.quickGroupsUrl || null,
|
|
parsedOptions: options,
|
|
timestamp: new Date().toISOString()
|
|
};
|
|
}
|
|
|
|
// Create basic response
|
|
createBasicResponse(vin, catalogCode, message) {
|
|
return {
|
|
success: false,
|
|
source: 'emexdwc.ae',
|
|
method: 'fallback',
|
|
vin: vin,
|
|
catalogCode: catalogCode,
|
|
vehicle: {
|
|
brand: this.extractBrandFromCatalog(catalogCode),
|
|
model: null,
|
|
year: getYearFromVIN(vin),
|
|
},
|
|
message: message,
|
|
timestamp: new Date().toISOString()
|
|
};
|
|
}
|
|
|
|
// Extract brand name from catalog code
|
|
extractBrandFromCatalog(catalogCode) {
|
|
const brandMap = {
|
|
'BMW202501': 'BMW',
|
|
'MB201810': 'Mercedes-Benz',
|
|
'AU1587': 'Audi',
|
|
'VW1587': 'Volkswagen',
|
|
'RENAULT201910': 'Renault',
|
|
'CPSA01': 'Peugeot',
|
|
'CFIAT84': 'Fiat',
|
|
'RFIAT84': 'Alfa Romeo',
|
|
'FORD202201': 'Ford',
|
|
'TOYOTA00': 'Toyota',
|
|
'HONDA00': 'Honda',
|
|
'HYUNDAI00': 'Hyundai',
|
|
'KIA00': 'Kia',
|
|
'PO799': 'Porsche',
|
|
'SUBARU201802': 'Subaru',
|
|
'MAZDA2020': 'Mazda'
|
|
};
|
|
return brandMap[catalogCode] || catalogCode;
|
|
}
|
|
|
|
// Get part categories for a vehicle
|
|
async getCategories(quickGroupsUrl) {
|
|
if (!quickGroupsUrl) return [];
|
|
|
|
console.log(`\nFetching categories from: ${quickGroupsUrl}`);
|
|
// domcontentloaded + selector wait shaves ~3-5s vs networkidle (which
|
|
// waits for analytics/ads to settle and adds nothing for our extraction).
|
|
await this.page.goto(quickGroupsUrl, { waitUntil: 'domcontentloaded', timeout: CONFIG.timeout });
|
|
await this.page.waitForSelector('a[href*="QuickDetails.aspx"], a[href*="QuickGroups.aspx"]', { timeout: 8000 }).catch(() => null);
|
|
|
|
// Screenshot for debugging
|
|
await this.page.screenshot({ path: path.join(__dirname, 'categories-page.png'), fullPage: true });
|
|
|
|
const categories = await this.page.evaluate(() => {
|
|
const cats = [];
|
|
|
|
// Method 1: Links to QuickDetails
|
|
const links = document.querySelectorAll('a[href*="QuickDetails"], a[href*="gid="]');
|
|
links.forEach(link => {
|
|
const href = link.href;
|
|
const gidMatch = href.match(/gid=(\d+)/);
|
|
if (gidMatch) {
|
|
cats.push({
|
|
gid: gidMatch[1],
|
|
name: link.textContent.trim(),
|
|
url: href
|
|
});
|
|
}
|
|
});
|
|
|
|
// Method 2: List items with category data
|
|
if (cats.length === 0) {
|
|
const items = document.querySelectorAll('li[data-gid], .category-item, .group-item, [class*="category"]');
|
|
items.forEach(item => {
|
|
const gid = item.dataset?.gid || item.getAttribute('data-gid');
|
|
const name = item.textContent?.trim() || '';
|
|
if (gid || name) {
|
|
cats.push({
|
|
gid: gid || '',
|
|
name: name.substring(0, 100),
|
|
url: null
|
|
});
|
|
}
|
|
});
|
|
}
|
|
|
|
// Method 3: Tables
|
|
if (cats.length === 0) {
|
|
const rows = document.querySelectorAll('table tr');
|
|
rows.forEach(row => {
|
|
const cells = row.querySelectorAll('td');
|
|
if (cells.length >= 1) {
|
|
const link = row.querySelector('a');
|
|
const gidMatch = link?.href?.match(/gid=(\d+)/);
|
|
if (gidMatch || cells[0]?.textContent) {
|
|
cats.push({
|
|
gid: gidMatch?.[1] || '',
|
|
name: cells[0]?.textContent?.trim() || '',
|
|
url: link?.href || null
|
|
});
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
return cats;
|
|
});
|
|
|
|
console.log(`Found ${categories.length} categories`);
|
|
return categories;
|
|
}
|
|
|
|
// Get hierarchical category tree from QuickGroups.aspx
|
|
// Parses the nested <ul>/<li> tree structure recursively
|
|
// Returns: [{name, gid, url, children: [...]}]
|
|
async getCategoryTree(quickGroupsUrl) {
|
|
if (!quickGroupsUrl) return [];
|
|
|
|
console.log(`\nFetching category tree from QuickGroups.aspx: ${quickGroupsUrl}`);
|
|
// See getParts/getCategories: networkidle replaced with selector wait.
|
|
await this.page.goto(quickGroupsUrl, { waitUntil: 'domcontentloaded', timeout: CONFIG.timeout });
|
|
await this.page.waitForSelector('a[href*="QuickDetails.aspx"], a[href*="QuickGroups.aspx"]', { timeout: 8000 }).catch(() => null);
|
|
|
|
// Screenshot for debugging
|
|
await this.page.screenshot({ path: path.join(__dirname, 'quickgroups-tree.png'), fullPage: true });
|
|
|
|
const tree = await this.page.evaluate(() => {
|
|
function parseNode(li) {
|
|
const contentDiv = li.querySelector(':scope > .qgContent');
|
|
const childUl = li.querySelector(':scope > ul.qgContainer');
|
|
const link = contentDiv?.querySelector('a');
|
|
|
|
let name = '';
|
|
let gid = null;
|
|
let url = null;
|
|
|
|
if (link) {
|
|
// Leaf node with QuickDetails link
|
|
name = link.textContent?.trim() || '';
|
|
const gidMatch = link.href?.match(/gid=(\d+)/);
|
|
gid = gidMatch ? gidMatch[1] : null;
|
|
url = link.href || null;
|
|
} else if (contentDiv) {
|
|
// Parent node — plain text
|
|
name = contentDiv.textContent?.trim() || '';
|
|
}
|
|
|
|
const node = { name, gid, url, children: [] };
|
|
|
|
if (childUl) {
|
|
const childLis = childUl.querySelectorAll(':scope > li.qgNode');
|
|
childLis.forEach(childLi => {
|
|
const child = parseNode(childLi);
|
|
if (child.name) {
|
|
node.children.push(child);
|
|
}
|
|
});
|
|
}
|
|
|
|
return node;
|
|
}
|
|
|
|
// Find the tree root
|
|
const treeRoot = document.querySelector('#qgTree > ul.qgContainer');
|
|
if (!treeRoot) {
|
|
// Fallback: any ul.qgContainer
|
|
const fallback = document.querySelector('ul.qgContainer');
|
|
if (!fallback) return [];
|
|
const topLis = fallback.querySelectorAll(':scope > li.qgNode');
|
|
const tree = [];
|
|
topLis.forEach(li => {
|
|
const node = parseNode(li);
|
|
if (node.name) tree.push(node);
|
|
});
|
|
return tree;
|
|
}
|
|
|
|
const topLevelLis = treeRoot.querySelectorAll(':scope > li.qgNode');
|
|
const tree = [];
|
|
topLevelLis.forEach(li => {
|
|
const node = parseNode(li);
|
|
if (node.name) tree.push(node);
|
|
});
|
|
return tree;
|
|
});
|
|
|
|
// Count nodes
|
|
function countNodes(nodes) {
|
|
let count = 0;
|
|
for (const n of nodes) {
|
|
count++;
|
|
if (n.children) count += countNodes(n.children);
|
|
}
|
|
return count;
|
|
}
|
|
const total = countNodes(tree);
|
|
const leaves = tree.reduce(function countLeaves(sum, n) {
|
|
if (!n.children || n.children.length === 0) return sum + 1;
|
|
return n.children.reduce(countLeaves, sum);
|
|
}, 0);
|
|
|
|
console.log(`Category tree: ${tree.length} top-level groups, ${total} total nodes, ${leaves} leaves`);
|
|
for (const g of tree) {
|
|
console.log(` ${g.name}: ${g.children?.length || 0} children`);
|
|
}
|
|
|
|
return tree;
|
|
}
|
|
|
|
// Get parts for a category
|
|
// Two-stage navigation: QuickDetails.aspx → Unit.aspx (where the interactive
|
|
// hotspot diagram lives). Both pages are fully server-rendered for the data
|
|
// we need; networkidle previously waited 3-5s extra per page just for
|
|
// analytics/ads to settle. Switched to domcontentloaded + targeted selector
|
|
// waits, and skipped the QuickDetails browser load entirely by resolving the
|
|
// Unit.aspx href via a single plain HTTP GET (~3s saved). Falls back to the
|
|
// old browser-first path if the fetch can't find the Unit.aspx link, so
|
|
// catalogs that gate the link behind JS still work.
|
|
// Returns { parts, schemaImageUrl, hotspots, schemaWidth, schemaHeight }
|
|
async getParts(detailsUrl) {
|
|
if (!detailsUrl) return { parts: [], schemaImageUrl: null, hotspots: [], schemaWidth: 0, schemaHeight: 0 };
|
|
|
|
console.log(`\nFetching parts from: ${detailsUrl}`);
|
|
|
|
// Fast path: pull the QuickDetails.aspx HTML via plain HTTP (no JS,
|
|
// no analytics-driven networkidle wait), grab the Unit.aspx link.
|
|
let unitUrl = null;
|
|
try {
|
|
const resp = await fetch(detailsUrl, {
|
|
headers: { 'User-Agent': CONFIG.userAgent },
|
|
redirect: 'follow',
|
|
});
|
|
if (resp.ok) {
|
|
const html = await resp.text();
|
|
const m = html.match(/href="([^"]*Unit\.aspx[^"]*)"/i);
|
|
if (m) {
|
|
const raw = m[1].replace(/&/g, '&');
|
|
unitUrl = raw.startsWith('http') ? raw : new URL(raw, detailsUrl).toString();
|
|
console.log(` Resolved Unit.aspx via plain HTTP: ${unitUrl.substring(0, 120)}…`);
|
|
}
|
|
} else {
|
|
console.log(` Plain HTTP QuickDetails failed (${resp.status}), falling back to browser`);
|
|
}
|
|
} catch (err) {
|
|
console.log(` Plain HTTP QuickDetails error: ${err.message}, falling back to browser`);
|
|
}
|
|
|
|
// Slow path: open QuickDetails in the browser only if plain HTTP
|
|
// didn't yield a Unit.aspx URL. Keeps catalogs that gate the link
|
|
// behind JS rendering working.
|
|
if (!unitUrl) {
|
|
await this.page.goto(detailsUrl, { waitUntil: 'domcontentloaded', timeout: CONFIG.timeout });
|
|
await this.page.waitForSelector('a[href*="Unit.aspx"]', { timeout: 5000 }).catch(() => null);
|
|
unitUrl = await this.page.evaluate(() => {
|
|
const link = document.querySelector('a[href*="Unit.aspx"]');
|
|
return link ? link.href : null;
|
|
});
|
|
}
|
|
|
|
if (unitUrl) {
|
|
console.log(` Loading Unit.aspx for hotspots…`);
|
|
await this.page.goto(unitUrl, { waitUntil: 'domcontentloaded', timeout: CONFIG.timeout });
|
|
// The img.dragger element is what the extractor reads to compute
|
|
// hotspot scale; waiting on it is both necessary and sufficient.
|
|
// Falls through after 5s if the catalog uses a different layout —
|
|
// the evaluate() below has its own null-safe fallbacks.
|
|
await this.page.waitForSelector('img.dragger', { timeout: 5000 }).catch(() => null);
|
|
} else {
|
|
// No Unit.aspx link found anywhere — operate on the already-loaded
|
|
// QuickDetails.aspx page (which was loaded as a browser fallback).
|
|
// Keep the existing extraction behaviour.
|
|
}
|
|
|
|
const result = await this.page.evaluate(() => {
|
|
// --- 1. Get schema image rect and compute coordinate mapping ---
|
|
let schemaImageUrl = null;
|
|
let schemaWidth = 0;
|
|
let schemaHeight = 0;
|
|
let imgRect = null;
|
|
let scaleX = 1;
|
|
let scaleY = 1;
|
|
|
|
const draggerImg = document.querySelector('img.dragger[src*="laximo"]');
|
|
if (draggerImg) {
|
|
schemaImageUrl = draggerImg.src;
|
|
schemaWidth = draggerImg.naturalWidth || 0;
|
|
schemaHeight = draggerImg.naturalHeight || 0;
|
|
imgRect = draggerImg.getBoundingClientRect();
|
|
if (imgRect.width > 0 && schemaWidth > 0) scaleX = schemaWidth / imgRect.width;
|
|
if (imgRect.height > 0 && schemaHeight > 0) scaleY = schemaHeight / imgRect.height;
|
|
}
|
|
|
|
// --- 2. Extract hotspot overlay divs, mapped to natural image coords ---
|
|
const hotspotMap = {};
|
|
const hotspotDivs = document.querySelectorAll('div.dragger.g_highlight[name]');
|
|
for (const div of hotspotDivs) {
|
|
const key = div.getAttribute('name');
|
|
if (!key) continue;
|
|
const divRect = div.getBoundingClientRect();
|
|
const relLeft = imgRect ? (divRect.left - imgRect.left) : divRect.left;
|
|
const relTop = imgRect ? (divRect.top - imgRect.top) : divRect.top;
|
|
const area = {
|
|
left: Math.round(relLeft * scaleX),
|
|
top: Math.round(relTop * scaleY),
|
|
width: Math.round(divRect.width * scaleX),
|
|
height: Math.round(divRect.height * scaleY),
|
|
};
|
|
if (!hotspotMap[key]) {
|
|
hotspotMap[key] = { key, areas: [] };
|
|
}
|
|
hotspotMap[key].areas.push(area);
|
|
}
|
|
const hotspots = Object.values(hotspotMap);
|
|
|
|
// Fallback: any laximo image or full attribute on zoom element
|
|
if (!schemaImageUrl) {
|
|
const zoomEl = document.querySelector('.guayaquil_zoom[full]');
|
|
if (zoomEl) {
|
|
schemaImageUrl = zoomEl.getAttribute('full');
|
|
}
|
|
}
|
|
if (!schemaImageUrl) {
|
|
const imgs = document.querySelectorAll('img[src*="img.laximo.net"]');
|
|
for (const img of imgs) {
|
|
const src = img.src || '';
|
|
if (src.includes('img.laximo.net')) {
|
|
schemaImageUrl = src.replace(/\/\d+\//, '/source/');
|
|
schemaWidth = schemaWidth || img.naturalWidth || 0;
|
|
schemaHeight = schemaHeight || img.naturalHeight || 0;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
// --- 3. Extract parts from named table rows ---
|
|
const partsData = [];
|
|
const namedRows = document.querySelectorAll('tr[name]');
|
|
|
|
if (namedRows.length > 0) {
|
|
for (const row of namedRows) {
|
|
const pncCell = row.querySelector('td[name="c_pnc"]');
|
|
const oemCell = row.querySelector('td[name="c_oem"]');
|
|
const nameCell = row.querySelector('td[name="c_name"]');
|
|
|
|
const oemCode = (oemCell?.textContent || '').trim();
|
|
if (!oemCode) continue;
|
|
|
|
partsData.push({
|
|
oemCode,
|
|
nameEn: (nameCell?.textContent || '').trim(),
|
|
positionCode: (pncCell?.textContent || '').trim(),
|
|
});
|
|
}
|
|
} else {
|
|
// Fallback: generic table row extraction
|
|
const rows = document.querySelectorAll('table tr, .part-row, [class*="part-item"]');
|
|
rows.forEach(row => {
|
|
const cells = row.querySelectorAll('td');
|
|
if (cells.length >= 2) {
|
|
let partNumber = '';
|
|
let description = '';
|
|
let position = '';
|
|
|
|
cells.forEach((cell, idx) => {
|
|
const text = cell.textContent.trim();
|
|
if (/^[A-Z0-9]{6,14}$/.test(text) && !partNumber) {
|
|
partNumber = text;
|
|
} else if (text.length > 10 && !description) {
|
|
description = text;
|
|
} else if (/^\d{1,3}$/.test(text) && idx === 0) {
|
|
position = text;
|
|
}
|
|
});
|
|
|
|
if (partNumber) {
|
|
partsData.push({
|
|
oemCode: partNumber,
|
|
nameEn: description,
|
|
positionCode: position
|
|
});
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
return { parts: partsData, schemaImageUrl, hotspots, schemaWidth, schemaHeight };
|
|
});
|
|
|
|
console.log(`Found ${result.parts.length} parts, ${result.hotspots.length} hotspot groups, schema: ${result.schemaImageUrl ? 'found' : 'none'} (${result.schemaWidth}x${result.schemaHeight})`);
|
|
return result;
|
|
}
|
|
}
|
|
|
|
// CLI execution
|
|
async function main() {
|
|
const vin = process.argv[2] || 'WBAEY710X5FT11264';
|
|
|
|
if (!vin || vin.length !== 17) {
|
|
console.error('Usage: node emex-vin-scraper.js <VIN>');
|
|
console.error('VIN must be exactly 17 characters');
|
|
process.exit(1);
|
|
}
|
|
|
|
console.log('='.repeat(80));
|
|
console.log('EMEX VIN SCRAPER');
|
|
console.log('='.repeat(80));
|
|
console.log(`VIN: ${vin}`);
|
|
console.log(`Year (from VIN): ${getYearFromVIN(vin)}`);
|
|
console.log(`Catalog: ${getCatalogCode(vin) || 'Unknown'}`);
|
|
console.log('='.repeat(80));
|
|
|
|
const scraper = new EmexVinScraper();
|
|
|
|
try {
|
|
await scraper.init();
|
|
|
|
// Search for vehicle
|
|
const vehicleData = await scraper.searchByVIN(vin);
|
|
|
|
console.log('\n' + '='.repeat(80));
|
|
console.log('RESULT');
|
|
console.log('='.repeat(80));
|
|
console.log(JSON.stringify(vehicleData, null, 2));
|
|
|
|
// Save result to file
|
|
const outputPath = path.join(__dirname, `vin-result-${vin}.json`);
|
|
fs.writeFileSync(outputPath, JSON.stringify(vehicleData, null, 2));
|
|
console.log(`\nResult saved to: ${outputPath}`);
|
|
|
|
// If we have quickGroupsUrl, get categories
|
|
if (vehicleData.quickGroupsUrl) {
|
|
console.log('\n' + '-'.repeat(80));
|
|
console.log('FETCHING CATEGORIES...');
|
|
const categories = await scraper.getCategories(vehicleData.quickGroupsUrl);
|
|
vehicleData.categories = categories;
|
|
|
|
// Save updated result
|
|
fs.writeFileSync(outputPath, JSON.stringify(vehicleData, null, 2));
|
|
console.log(`Updated result with ${categories.length} categories`);
|
|
|
|
// Get parts for first category
|
|
if (categories.length > 0 && categories[0].url) {
|
|
console.log('\n' + '-'.repeat(80));
|
|
console.log('FETCHING PARTS FOR FIRST CATEGORY...');
|
|
const parts = await scraper.getParts(categories[0].url);
|
|
vehicleData.sampleParts = parts;
|
|
fs.writeFileSync(outputPath, JSON.stringify(vehicleData, null, 2));
|
|
console.log(`Added ${parts.parts.length} sample parts`);
|
|
}
|
|
}
|
|
|
|
return vehicleData;
|
|
|
|
} catch (error) {
|
|
console.error('\nError:', error.message);
|
|
console.error(error.stack);
|
|
process.exit(1);
|
|
} finally {
|
|
await scraper.close();
|
|
}
|
|
}
|
|
|
|
// Export for module use
|
|
module.exports = { EmexVinScraper, getCatalogCode, getYearFromVIN, CONFIG };
|
|
|
|
// Run if called directly
|
|
if (require.main === module) {
|
|
main().catch(console.error);
|
|
}
|