fix: Volvo parser - VIN decode & category flow improvements
Volvo VIN Decode: - Changed URL pattern from vehicle.action to vin-group.action - Added parseVolvoInfoTable() for Volvo's specific table format (multiple label/value pairs per row) - Fixed brand detection using WMI codes (YV1/YV4 = Volvo) - Updated selectors for vinInfoTable and nav-group tables Category & SubGroup Flow: - Added LEGACY_VOLVO to architectures using vin-group.action - Implemented 4-level Volvo hierarchy: 1. vin-group.action → Main groups (group1) 2. vin-group.action?group1=... → Sub-groups (group2) 3. vin-group.action?group1=...&group2=... → Illustrations 4. vin-image-board.action → Parts (dynamic JS loading) - Added fetchVolvoPartsFromIllustrations() method - Added parseVolvoIllustrations() for nav-group3-table TODO: Volvo parts are loaded dynamically via JavaScript. The nav-bom-table in vin-image-board.action is empty in static HTML. Need to investigate json-vin-bom.action or similar for BOM data. Test Results: - VIN decode: Volvo V40 (13-), 2015, 7 categories ✓ - SubGroups fetch: Working ✓ - Parts fetch: Needs dynamic JS handling Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Semih
@@ -67,29 +67,32 @@ export class VolvoParser extends BaseLegacyParser {
|
||||
link: 'a[href]',
|
||||
},
|
||||
subGroups: {
|
||||
container: '#subGroupList, .sub-groups, #sgList',
|
||||
item: '.sub-group, .sg-item, tr.sg',
|
||||
code: '.sg-code, td.code',
|
||||
name: '.sg-name, td.name, a',
|
||||
description: '.sg-desc, td.description',
|
||||
// Volvo uses nav-group2-table for sub-groups (group2) and nav-group3-table for illustrations
|
||||
container: '#nav-group2-table tbody, #nav-group3-table tbody',
|
||||
item: 'tr.tc-data-row',
|
||||
code: '.group2, td.group2, .name, td.name',
|
||||
name: '.group2 a, td.group2 a, .name a, td.name a',
|
||||
description: '.restrictionHtml, td.restrictionHtml',
|
||||
link: 'a[href]',
|
||||
illustrationNumber: '.illus-no, td.illus',
|
||||
illustrationNumber: '.illNo, td.illNo',
|
||||
},
|
||||
parts: {
|
||||
container: '#partsContainer, .parts-list, #bomContainer',
|
||||
table: 'table.parts, table#partsList, .bom-table',
|
||||
row: 'tr.part, tr[data-partno], tbody tr',
|
||||
oemCode: 'td.partno, .part-number',
|
||||
name: 'td.description, .part-name',
|
||||
quantity: 'td.qty, .quantity',
|
||||
positionCode: 'td.pos, .position',
|
||||
remark: 'td.remark, .note',
|
||||
// Volvo uses vin-image-board.action for parts
|
||||
container: '#partsContainer, .parts-list, #bomContainer, #bom-container',
|
||||
table: 'table.parts, table#partsList, .bom-table, #bom-table',
|
||||
row: 'tr.part, tr[data-partno], tbody tr.bom-row, #bom-table tbody tr',
|
||||
oemCode: 'td.partno, .part-number, td.bom-partno',
|
||||
name: 'td.description, .part-name, td.bom-descr',
|
||||
quantity: 'td.qty, .quantity, td.bom-qty',
|
||||
positionCode: 'td.pos, .position, td.bom-pos',
|
||||
remark: 'td.remark, .note, td.bom-remark',
|
||||
modelCodes: 'td.model, .app-info',
|
||||
},
|
||||
schema: {
|
||||
container: '#illustrationContainer, .illustration-container',
|
||||
image: 'img#illustrationImage, img.schema',
|
||||
hotspotArea: 'area[data-pos], map area',
|
||||
// Volvo schema images use ImageViewer
|
||||
container: '#illustrationContainer, .illustration-container, #imageViewerContainer',
|
||||
image: 'img#illustrationImage, img.schema, #schemaImage img',
|
||||
hotspotArea: 'area[data-pos], map area, area[onclick]',
|
||||
},
|
||||
navigation: {
|
||||
breadcrumbs: '#breadcrumbs, .breadcrumb',
|
||||
@@ -289,7 +292,16 @@ export class VolvoParser extends BaseLegacyParser {
|
||||
}
|
||||
|
||||
private findSubGroupLinks(doc: Document): import('domhandler').Element[] {
|
||||
return this.htmlParser.querySelectorAll(doc, 'a[href*="subgroup"], a[href*="sg="], a[href*="illus"]');
|
||||
// Volvo uses nav-group2-table for sub-groups (group2)
|
||||
const rows = this.htmlParser.querySelectorAll(doc, '#nav-group2-table tbody tr.tc-data-row');
|
||||
if (rows.length > 0) return rows;
|
||||
|
||||
// Also check for nav-group3-table (illustrations/sections)
|
||||
const illusRows = this.htmlParser.querySelectorAll(doc, '#nav-group3-table tbody tr.tc-data-row');
|
||||
if (illusRows.length > 0) return illusRows;
|
||||
|
||||
// Fallback to generic selectors
|
||||
return this.htmlParser.querySelectorAll(doc, 'a[href*="subgroup"], a[href*="sg="], a[href*="group2="], a[href*="sectionCode="]');
|
||||
}
|
||||
|
||||
private parseCategoryItem(item: import('domhandler').Element): LegacyCategoryData | null {
|
||||
|
||||
@@ -315,7 +315,7 @@ export class PL24LegacyScraperService {
|
||||
// Otherwise use HTML scraping
|
||||
const parser = this.parserFactory.getParser(config.architecture);
|
||||
|
||||
// Build sub-groups URL - Hyundai/Opel use vin-group.action, others use sub-groups.action
|
||||
// Build sub-groups URL - Hyundai/Opel/Volvo use vin-group.action, others use sub-groups.action
|
||||
let fullUrl: string;
|
||||
if (config.architecture === 'LEGACY_HYUNDAI_KIA') {
|
||||
// Hyundai uses vin-group.action with mainGroup parameter
|
||||
@@ -328,6 +328,24 @@ export class PL24LegacyScraperService {
|
||||
mode: 'A0LW0TRTR',
|
||||
});
|
||||
fullUrl = `${this.baseUrl}${config.basePath}/vin-group.action?${subGroupsParams}`;
|
||||
} else if (config.architecture === 'LEGACY_VOLVO') {
|
||||
// Volvo uses vin-group.action with group1 parameter
|
||||
// If linkPath is provided, use it directly (it has the correct format)
|
||||
if (linkPath && linkPath.includes('group1=')) {
|
||||
fullUrl = linkPath.startsWith('http')
|
||||
? linkPath
|
||||
: `${this.baseUrl}${config.basePath}/${linkPath}`;
|
||||
} else {
|
||||
const subGroupsParams = new URLSearchParams({
|
||||
lang: this.language,
|
||||
group1: categoryCode,
|
||||
openVinDialog: 'true',
|
||||
startup: 'false',
|
||||
vin: cleanVin,
|
||||
mode: 'A0LW0TRTR',
|
||||
});
|
||||
fullUrl = `${this.baseUrl}${config.basePath}/vin-group.action?${subGroupsParams}`;
|
||||
}
|
||||
} else if (config.architecture === 'LEGACY_OPEL') {
|
||||
// Opel uses json-vin-sub-group.action for subgroups (JSON endpoint)
|
||||
const subGroupsParams = new URLSearchParams({
|
||||
@@ -551,6 +569,14 @@ export class PL24LegacyScraperService {
|
||||
return this.fetchHyundaiPartsFromSubgroups(fullUrl, serviceName, cookies, parser);
|
||||
}
|
||||
|
||||
// VOLVO SPECIAL HANDLING:
|
||||
// Volvo has 4 levels: group1 -> group2 -> illustrations -> parts
|
||||
// vin-group.action?group1=...&group2=... returns illustrations list, not parts
|
||||
// We need to fetch illustrations first, then get parts from vin-image-board.action
|
||||
if (config.architecture === 'LEGACY_VOLVO' && linkPath.includes('vin-group.action') && linkPath.includes('group2=')) {
|
||||
return this.fetchVolvoPartsFromIllustrations(fullUrl, serviceName, cookies, parser);
|
||||
}
|
||||
|
||||
this.logger.log(`Fetching parts page: ${fullUrl}`);
|
||||
|
||||
let partsData: import('./parsers/base-parser').LegacyPartsPageData;
|
||||
@@ -1686,6 +1712,263 @@ export class PL24LegacyScraperService {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch Volvo parts from illustrations page
|
||||
* Volvo has 4 levels: group1 -> group2 -> illustrations (nav-group3) -> parts (vin-image-board)
|
||||
*/
|
||||
private async fetchVolvoPartsFromIllustrations(
|
||||
illustrationsUrl: string,
|
||||
serviceName: string,
|
||||
cookies: string,
|
||||
parser: import('./parsers/base-parser').BaseLegacyParser,
|
||||
): Promise<PL24PartsResponse> {
|
||||
this.logger.log(`=== VOLVO ILLUSTRATIONS FLOW ===`);
|
||||
this.logger.log(`Fetching Volvo illustrations page: ${illustrationsUrl}`);
|
||||
|
||||
// Parse URL to extract parameters
|
||||
const url = new URL(illustrationsUrl);
|
||||
const vin = url.searchParams.get('vin') || '';
|
||||
const group1 = url.searchParams.get('group1') || '';
|
||||
const group2 = url.searchParams.get('group2') || '';
|
||||
const mode = url.searchParams.get('mode') || 'A0LW0TRTR';
|
||||
|
||||
// Fetch illustrations page (nav-group3-table)
|
||||
const illustrationsHtml = await this.fetchService.fetchHtml(illustrationsUrl, cookies);
|
||||
|
||||
// Parse illustrations from nav-group3-table
|
||||
const illustrations = this.parseVolvoIllustrations(illustrationsHtml);
|
||||
this.logger.log(`Found ${illustrations.length} Volvo illustrations for group1=${group1}, group2=${group2}`);
|
||||
|
||||
if (illustrations.length === 0) {
|
||||
this.logger.warn(`No illustrations found for Volvo group2=${group2}`);
|
||||
return {
|
||||
success: true,
|
||||
groupId: `${group1}-${group2}`,
|
||||
groupName: group2,
|
||||
parts: [],
|
||||
};
|
||||
}
|
||||
|
||||
// Fetch parts from each illustration (limit to first 5 to avoid timeout)
|
||||
const allParts: PL24Part[] = [];
|
||||
let firstSchemaUrl: string | undefined;
|
||||
let firstHotspots: Array<{ key: string; areas: Array<{ left: number; top: number; width: number; height: number }> }> | undefined;
|
||||
let schemaWidth: number | undefined;
|
||||
let schemaHeight: number | undefined;
|
||||
|
||||
const config = getServiceConfig(serviceName);
|
||||
const maxIllustrations = 5; // Limit to prevent timeout
|
||||
|
||||
for (let i = 0; i < Math.min(illustrations.length, maxIllustrations); i++) {
|
||||
const illus = illustrations[i];
|
||||
this.logger.log(`Fetching Volvo illustration ${i + 1}/${Math.min(illustrations.length, maxIllustrations)}: ${illus.code} - ${illus.name}`);
|
||||
|
||||
// Build vin-image-board.action URL for this illustration
|
||||
// If illustration has a full linkPath, use it; otherwise build the URL
|
||||
let partsUrl: string;
|
||||
if (illus.linkPath && illus.linkPath.includes('vin-image-board.action')) {
|
||||
partsUrl = illus.linkPath.startsWith('http')
|
||||
? illus.linkPath
|
||||
: `${this.baseUrl}${config!.basePath}/${illus.linkPath}`;
|
||||
} else {
|
||||
// Build URL with sectionCode
|
||||
const partsParams = new URLSearchParams({
|
||||
group1,
|
||||
group2,
|
||||
lang: this.language,
|
||||
openVinDialog: 'true',
|
||||
startup: 'false',
|
||||
vin,
|
||||
mode,
|
||||
});
|
||||
if (illus.sectionCode) {
|
||||
partsParams.set('sectionCode', illus.sectionCode);
|
||||
}
|
||||
partsUrl = `${this.baseUrl}${config!.basePath}/vin-image-board.action?${partsParams}`;
|
||||
}
|
||||
|
||||
try {
|
||||
const partsHtml = await this.fetchService.fetchHtml(partsUrl, cookies);
|
||||
|
||||
const partsData = parser.parsePartsPage(partsHtml, partsUrl);
|
||||
|
||||
// TODO: Volvo parts are loaded dynamically via JavaScript after page load
|
||||
// The nav-bom-table in vin-image-board.action is empty in the static HTML
|
||||
// Need to investigate json-vin-bom.action or similar endpoint for BOM data
|
||||
// For now, this will return 0 parts until the dynamic loading is implemented
|
||||
this.logger.log(`Volvo illustration ${illus.code}: found ${partsData.parts.length} parts`);
|
||||
|
||||
// Store first schema URL and dimensions
|
||||
if (!firstSchemaUrl && partsData.schema?.imageUrl) {
|
||||
// Download schema image for Volvo
|
||||
const imageResult = await this.downloadVolvoSchemaImage(partsUrl, cookies, config!);
|
||||
if (imageResult) {
|
||||
firstSchemaUrl = imageResult.localPath;
|
||||
firstHotspots = imageResult.hotspots;
|
||||
schemaWidth = imageResult.width;
|
||||
schemaHeight = imageResult.height;
|
||||
this.logger.log(`Volvo schema image saved: ${imageResult.localPath}`);
|
||||
} else {
|
||||
firstSchemaUrl = partsData.schema.imageUrl;
|
||||
schemaWidth = partsData.schema.width;
|
||||
schemaHeight = partsData.schema.height;
|
||||
}
|
||||
if (partsData.schema.hotspots && !firstHotspots) {
|
||||
firstHotspots = partsData.schema.hotspots.map(h => ({
|
||||
key: h.key,
|
||||
areas: h.areas,
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
// Convert and add parts
|
||||
const convertedParts = this.convertLegacyParts(partsData.parts);
|
||||
allParts.push(...convertedParts);
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to fetch Volvo illustration ${illus.code}: ${(error as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.log(`Total Volvo parts fetched: ${allParts.length} from ${Math.min(illustrations.length, maxIllustrations)} illustrations`);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
groupId: `${group1}-${group2}`,
|
||||
groupName: group2,
|
||||
schemaImageUrl: firstSchemaUrl,
|
||||
schemaWidth,
|
||||
schemaHeight,
|
||||
parts: allParts,
|
||||
hotspots: firstHotspots,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse Volvo illustrations from nav-group3-table
|
||||
*/
|
||||
private parseVolvoIllustrations(html: string): Array<{ code: string; name: string; linkPath?: string; sectionCode?: string }> {
|
||||
const illustrations: Array<{ code: string; name: string; linkPath?: string; sectionCode?: string }> = [];
|
||||
|
||||
// Use regex to find nav-group3-table rows
|
||||
// Each row has: <tr ... url="vin-image-board.action?...§ionCode=...">
|
||||
const rowRegex = /<tr[^>]*class="[^"]*tc-data-row[^"]*"[^>]*url="([^"]+)"[^>]*>/gi;
|
||||
const nameRegex = /<td[^>]*class="[^"]*name[^"]*"[^>]*>.*?<a[^>]*>([^<]+)<\/a>/gi;
|
||||
const sectionRegex = /sectionCode=([^&"]+)/i;
|
||||
|
||||
let match;
|
||||
let rowIndex = 0;
|
||||
const rows: { url: string; startIndex: number }[] = [];
|
||||
|
||||
// Find all rows with URLs
|
||||
while ((match = rowRegex.exec(html)) !== null) {
|
||||
rows.push({ url: match[1], startIndex: match.index });
|
||||
}
|
||||
|
||||
// For each row, find the name
|
||||
for (const row of rows) {
|
||||
// Extract sectionCode from URL
|
||||
const sectionMatch = row.url.match(sectionRegex);
|
||||
const sectionCode = sectionMatch ? sectionMatch[1] : undefined;
|
||||
|
||||
// Find name in the HTML after this row
|
||||
const rowHtml = html.substring(row.startIndex, row.startIndex + 500);
|
||||
const nameMatch = rowHtml.match(/<td[^>]*class="[^"]*name[^"]*"[^>]*>.*?<a[^>]*>([^<]+)<\/a>/i);
|
||||
const name = nameMatch ? nameMatch[1].trim() : `Illustration ${rowIndex + 1}`;
|
||||
|
||||
// Decode URL entities
|
||||
const linkPath = row.url.replace(/&/g, '&');
|
||||
|
||||
illustrations.push({
|
||||
code: sectionCode || `illus_${rowIndex}`,
|
||||
name,
|
||||
linkPath,
|
||||
sectionCode,
|
||||
});
|
||||
|
||||
rowIndex++;
|
||||
}
|
||||
|
||||
this.logger.log(`Parsed ${illustrations.length} Volvo illustrations from nav-group3-table`);
|
||||
|
||||
return illustrations;
|
||||
}
|
||||
|
||||
/**
|
||||
* Download Volvo schema image with ticket-based authentication
|
||||
*/
|
||||
private async downloadVolvoSchemaImage(
|
||||
pageUrl: string,
|
||||
cookies: string,
|
||||
config: { basePath: string },
|
||||
): Promise<{ localPath: string; width?: number; height?: number; hotspots?: Array<{ key: string; areas: Array<{ left: number; top: number; width: number; height: number }> }> } | null> {
|
||||
try {
|
||||
// Fetch the page to get the image ticket
|
||||
const ticketUrl = pageUrl.replace('vin-image-board.action', 'json-vin-image-ticket.action');
|
||||
const ticketResponse = await this.fetchService.fetch(ticketUrl, {
|
||||
headers: {
|
||||
Cookie: cookies,
|
||||
Accept: 'application/json, text/javascript, */*',
|
||||
'X-Requested-With': 'XMLHttpRequest',
|
||||
},
|
||||
});
|
||||
|
||||
const ticketData = JSON.parse(ticketResponse.body);
|
||||
if (!ticketData.ticket || !ticketData.path) {
|
||||
this.logger.warn(`Volvo image ticket not found in response`);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Build image URL with ticket
|
||||
const imageUrl = `${this.baseUrl}/pl24images/ImageViewer?path=${encodeURIComponent(ticketData.path)}&request=GetImage&ticket=${ticketData.ticket}`;
|
||||
|
||||
// Fetch the image
|
||||
const imageResponse = await this.fetchService.fetch(imageUrl, {
|
||||
headers: {
|
||||
Cookie: cookies,
|
||||
Accept: 'image/*',
|
||||
},
|
||||
});
|
||||
|
||||
if (!imageResponse.body || imageResponse.status !== 200) {
|
||||
this.logger.warn(`Failed to fetch Volvo image: ${imageResponse.status}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Generate image ID from path
|
||||
const pathMatch = ticketData.path.match(/\/([^/]+)\.[^.]+$/);
|
||||
const imageId = pathMatch ? pathMatch[1] : `volvo_${Date.now()}`;
|
||||
|
||||
// Save image
|
||||
const localPath = `/images/schemas/${imageId}.png`;
|
||||
const fullPath = path.join(process.cwd(), 'public', localPath);
|
||||
await fs.mkdir(path.dirname(fullPath), { recursive: true });
|
||||
await fs.writeFile(fullPath, imageResponse.body, 'binary');
|
||||
|
||||
// Get image info for dimensions
|
||||
let width: number | undefined;
|
||||
let height: number | undefined;
|
||||
try {
|
||||
const infoUrl = `${this.baseUrl}/pl24images/ImageViewer?path=${encodeURIComponent(ticketData.path)}&request=GetImageInfo&ticket=${ticketData.ticket}`;
|
||||
const infoResponse = await this.fetchService.fetch(infoUrl, {
|
||||
headers: {
|
||||
Cookie: cookies,
|
||||
Accept: '*/*',
|
||||
},
|
||||
});
|
||||
const infoData = JSON.parse(infoResponse.body);
|
||||
width = infoData.width;
|
||||
height = infoData.height;
|
||||
} catch (e) {
|
||||
// Ignore info fetch errors
|
||||
}
|
||||
|
||||
return { localPath, width, height };
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to download Volvo schema image: ${(error as Error).message}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch FINIS numbers for Ford Trucks parts
|
||||
* Ford Trucks catalog doesn't include part numbers in the BOM table HTML,
|
||||
|
||||
Reference in New Issue
Block a user