feat: add parts catalogs integration, catalog prefetch worker, and vehicle select modal
Integrate external parts catalogs API with auth service, add BullMQ-based catalog prefetch worker for background data caching, expand vehicles service with shared vehicle support, and add vehicle select modal to frontend. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Sase Dev
@@ -28,6 +28,7 @@ import {
|
||||
} from './emex.types';
|
||||
import { mapEmexResponse, createEmptyDecodedVehicle } from './emex.mapper';
|
||||
import { EmexBrowserService } from './emex.browser';
|
||||
import { RedisService } from '../../redis/redis.service';
|
||||
|
||||
// Type definition for the imported scraper module
|
||||
interface EmexScraperModule {
|
||||
@@ -66,6 +67,7 @@ export class EmexService {
|
||||
constructor(
|
||||
private configService: ConfigService,
|
||||
private browserService: EmexBrowserService,
|
||||
private redis: RedisService,
|
||||
) {
|
||||
// __dirname is apps/api/src/integrations/emex/ or dist/integrations/emex/
|
||||
// Scraper lives at <monorepo-root>/scripts/emex-vin-scraper.js
|
||||
@@ -82,6 +84,15 @@ export class EmexService {
|
||||
this.logger.log(`EMEX Service initialized with scraper path: ${this.scraperPath}`);
|
||||
}
|
||||
|
||||
/** Mark EMEX as actively used (5min TTL) to defer prefetch worker */
|
||||
private async touchActivity(): Promise<void> {
|
||||
try {
|
||||
await this.redis.set("prefetch:activity:emex", String(Date.now()), 300);
|
||||
} catch {
|
||||
// Non-critical — don't break the request
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Lazily initialize the scraper module
|
||||
*/
|
||||
@@ -184,6 +195,7 @@ export class EmexService {
|
||||
|
||||
const supported = this.isSupported(cleanVin);
|
||||
this.logger.log(`Decoding VIN: ${cleanVin} (catalog supported: ${supported})`);
|
||||
await this.touchActivity();
|
||||
|
||||
let release: (() => Promise<void>) | null = null;
|
||||
|
||||
@@ -375,6 +387,7 @@ export class EmexService {
|
||||
}
|
||||
|
||||
this.logger.log(`Fetching parts from category URL: ${categoryUrl}`);
|
||||
await this.touchActivity();
|
||||
|
||||
let release: (() => Promise<void>) | null = null;
|
||||
|
||||
|
||||
@@ -0,0 +1,685 @@
|
||||
/**
|
||||
* Parts-Catalogs Auth Service — JWT warm pool via Playwright + DataImpulse proxy
|
||||
*
|
||||
* JWT is captured by navigating to partner sites and intercepting
|
||||
* the Authorization header from requests to parts-catalogs.com.
|
||||
* JWT is IP-bound (~10 min TTL), so the same proxy port must be used for both
|
||||
* browser capture and subsequent API calls.
|
||||
*
|
||||
* Warm pool behavior:
|
||||
* 09:00-19:00 Istanbul → proactive: maintain >= 1 slot, auto-refresh before expiry
|
||||
* 19:00-09:00 → on-demand only: capture only when needed
|
||||
*
|
||||
* Each slot manages its own refresh timer (no polling loop).
|
||||
* Dynamic scaling: 1 JWT per 6 req/min, capped at 5 slots.
|
||||
*/
|
||||
|
||||
import {
|
||||
Injectable,
|
||||
Logger,
|
||||
OnModuleInit,
|
||||
OnModuleDestroy,
|
||||
} from "@nestjs/common";
|
||||
import { ConfigService } from "@nestjs/config";
|
||||
import type { Browser, BrowserContext } from "playwright";
|
||||
import type { PcatJwtToken, JwtSlot, PcatSession } from "./parts-catalogs.types";
|
||||
|
||||
const REFRESH_BUFFER = 90; // Refresh 90s before expiry
|
||||
const CAPTURE_POLL_INTERVAL = 500; // ms
|
||||
const CAPTURE_POLL_MAX = 40; // 40 × 500ms = 20s max wait
|
||||
const PAGE_TIMEOUT = 30_000; // 30s navigation timeout
|
||||
const CONTEXT_CLOSE_TIMEOUT = 5_000;
|
||||
const SITE_COOLDOWN = 10 * 60 * 1000; // 10 min per site
|
||||
const MAX_POOL_SIZE = 5;
|
||||
const RPM_WINDOW = 60_000; // 1-minute rolling window
|
||||
const RPM_PER_SLOT = 6; // 1 JWT per 6 req/min
|
||||
|
||||
/**
|
||||
* Sites that embed the parts-catalogs.com widget.
|
||||
* Widget loads JS → calls /api/start → then calls /v1/catalogs/ with JWT.
|
||||
* Each site uses a different proxy port (IP) to avoid rate limiting.
|
||||
*/
|
||||
const JWT_SITES = [
|
||||
"https://www.e-acca.com/cats/#/catalogs",
|
||||
"https://www.alkatalog.com/cats/#/catalogs",
|
||||
"https://auto-komplekt.ru/goodvin#/catalogs",
|
||||
"https://www.autotrade.md/cats/#/catalogs",
|
||||
"https://www.e-trak.ru/cats/#/catalogs",
|
||||
"https://www.autopolyus.ru/cats/#/catalogs",
|
||||
"https://knkauto.ru/goodvin#/catalogs",
|
||||
"https://www.autodo.kz/#/catalogs",
|
||||
"https://avtoman124.ru/goodvin#/catalogs",
|
||||
"https://flynestauto.com/auto-parts-oem-catalog",
|
||||
"http://en.demo.tradesoft.hk.com/cats/#/catalogs",
|
||||
];
|
||||
|
||||
// DataImpulse proxy defaults (port-based IP rotation)
|
||||
const DI_HOST = "gw.dataimpulse.com";
|
||||
const DI_PORT_MIN = 10000;
|
||||
const DI_PORT_MAX = 10999;
|
||||
const DI_DEFAULT_USER = "1726bbe361918676d44e";
|
||||
const DI_DEFAULT_PASS = "f11c7b6128cc86c6";
|
||||
|
||||
/** Simple counting semaphore (same pattern as EmexBrowserService) */
|
||||
class Semaphore {
|
||||
private current = 0;
|
||||
private queue: Array<() => void> = [];
|
||||
|
||||
constructor(private readonly max: number) {}
|
||||
|
||||
acquire(): Promise<void> {
|
||||
if (this.current < this.max) {
|
||||
this.current++;
|
||||
return Promise.resolve();
|
||||
}
|
||||
return new Promise<void>((resolve) => {
|
||||
this.queue.push(resolve);
|
||||
});
|
||||
}
|
||||
|
||||
release(): void {
|
||||
const next = this.queue.shift();
|
||||
if (next) {
|
||||
next();
|
||||
} else {
|
||||
this.current--;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class PartsCatalogsAuthService implements OnModuleInit, OnModuleDestroy {
|
||||
private readonly logger = new Logger(PartsCatalogsAuthService.name);
|
||||
|
||||
private browser: Browser | null = null;
|
||||
private launching: Promise<void> | null = null;
|
||||
private readonly semaphore = new Semaphore(1); // Max 1 concurrent JWT capture
|
||||
|
||||
// Pool state
|
||||
private pool: JwtSlot[] = [];
|
||||
private siteLastUsedAt = new Map<string, number>();
|
||||
private siteIndex = 0; // round-robin across JWT_SITES
|
||||
private requestRoundRobin = 0; // round-robin across pool slots
|
||||
|
||||
// Business hours scheduling
|
||||
private businessHoursTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
// RPM tracking
|
||||
private requestTimestamps: number[] = [];
|
||||
|
||||
// Config
|
||||
private readonly useProxy: boolean;
|
||||
private readonly proxyHost: string;
|
||||
private readonly proxyUser: string;
|
||||
private readonly proxyPass: string;
|
||||
|
||||
constructor(private configService: ConfigService) {
|
||||
this.useProxy =
|
||||
this.configService.get<string>("PCAT_USE_PROXY", "true") === "true";
|
||||
this.proxyHost = this.configService.get<string>("PCAT_PROXY_HOST", DI_HOST);
|
||||
this.proxyUser = this.configService.get<string>(
|
||||
"PCAT_PROXY_USER",
|
||||
DI_DEFAULT_USER,
|
||||
);
|
||||
this.proxyPass = this.configService.get<string>(
|
||||
"PCAT_PROXY_PASS",
|
||||
DI_DEFAULT_PASS,
|
||||
);
|
||||
}
|
||||
|
||||
async onModuleInit(): Promise<void> {
|
||||
try {
|
||||
await this.launchBrowser();
|
||||
this.logger.log("Browser launched for JWT capture");
|
||||
} catch (err) {
|
||||
this.logger.error(
|
||||
`Failed to launch browser on init: ${(err as Error).message}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Start business hours scheduling
|
||||
if (this.isBusinessHours()) {
|
||||
this.captureToPool().catch((err) => {
|
||||
this.logger.error(`Initial pool capture failed: ${(err as Error).message}`);
|
||||
});
|
||||
}
|
||||
this.scheduleBusinessHours();
|
||||
}
|
||||
|
||||
async onModuleDestroy(): Promise<void> {
|
||||
this.clearAllRefreshTimers();
|
||||
if (this.businessHoursTimer) {
|
||||
clearTimeout(this.businessHoursTimer);
|
||||
this.businessHoursTimer = null;
|
||||
}
|
||||
await this.closeBrowser();
|
||||
this.logger.log("Browser closed on module destroy");
|
||||
}
|
||||
|
||||
// ─── Public API ───────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Acquire a matched JWT + proxy session from the pool.
|
||||
* Returns a valid slot (round-robin) or captures on-demand if pool is empty.
|
||||
*/
|
||||
async acquireSession(): Promise<PcatSession> {
|
||||
this.trackRequest();
|
||||
|
||||
// Try to find a valid slot in the pool
|
||||
const slot = this.pickValidSlot();
|
||||
if (slot) {
|
||||
// Check if we should scale up in the background
|
||||
this.maybeScaleUp();
|
||||
return this.slotToSession(slot);
|
||||
}
|
||||
|
||||
// No valid slot — capture on-demand
|
||||
this.logger.log("JWT pool empty — capturing on-demand...");
|
||||
const newSlot = await this.captureToPool();
|
||||
return this.slotToSession(newSlot);
|
||||
}
|
||||
|
||||
/**
|
||||
* Invalidate a session after 401/403, remove the slot, and capture a replacement.
|
||||
*/
|
||||
async invalidateSession(session: PcatSession): Promise<void> {
|
||||
const idx = this.pool.indexOf(session._slot);
|
||||
if (idx !== -1) {
|
||||
this.clearSlotTimer(this.pool[idx]);
|
||||
this.pool.splice(idx, 1);
|
||||
this.logger.log(
|
||||
`JWT pool: slot invalidated (port ${session._slot.proxyPort}), ${this.pool.length} remaining`,
|
||||
);
|
||||
}
|
||||
|
||||
// Capture replacement in background (don't block the caller's retry)
|
||||
if (this.isBusinessHours() || this.pool.length === 0) {
|
||||
this.captureToPool().catch((err) => {
|
||||
this.logger.error(`Replacement capture failed: ${(err as Error).message}`);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Pool management ──────────────────────────────────────
|
||||
|
||||
private pickValidSlot(): JwtSlot | null {
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
// Remove expired slots
|
||||
this.pool = this.pool.filter((s) => {
|
||||
if (s.jwt.exp - now < 30) {
|
||||
this.clearSlotTimer(s);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
|
||||
if (this.pool.length === 0) return null;
|
||||
|
||||
// Round-robin across valid slots
|
||||
this.requestRoundRobin = this.requestRoundRobin % this.pool.length;
|
||||
const slot = this.pool[this.requestRoundRobin];
|
||||
this.requestRoundRobin++;
|
||||
return slot;
|
||||
}
|
||||
|
||||
private slotToSession(slot: JwtSlot): PcatSession {
|
||||
const proxyUrl = this.useProxy
|
||||
? `http://${this.proxyUser}:${this.proxyPass}@${this.proxyHost}:${slot.proxyPort}`
|
||||
: null;
|
||||
const proxyConfig = this.useProxy
|
||||
? {
|
||||
server: `http://${this.proxyHost}:${slot.proxyPort}`,
|
||||
username: this.proxyUser,
|
||||
password: this.proxyPass,
|
||||
}
|
||||
: null;
|
||||
return {
|
||||
authorization: slot.jwt.raw,
|
||||
proxyUrl,
|
||||
proxyConfig,
|
||||
_slot: slot,
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Core capture + pool add ──────────────────────────────
|
||||
|
||||
private async captureToPool(): Promise<JwtSlot> {
|
||||
await this.semaphore.acquire();
|
||||
try {
|
||||
await this.ensureBrowser();
|
||||
|
||||
const maxRetries = 4;
|
||||
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
||||
const port = this.allocatePort();
|
||||
const siteUrl = this.getAvailableSite();
|
||||
|
||||
const jwt = await this.attemptCapture(siteUrl, port);
|
||||
if (jwt) {
|
||||
const slot: JwtSlot = {
|
||||
jwt,
|
||||
proxyPort: port,
|
||||
siteUsed: siteUrl,
|
||||
capturedAt: Date.now(),
|
||||
refreshTimer: null,
|
||||
};
|
||||
|
||||
// Mark site as used
|
||||
this.siteLastUsedAt.set(siteUrl, Date.now());
|
||||
|
||||
// Add to pool
|
||||
this.pool.push(slot);
|
||||
|
||||
// Schedule refresh if in business hours
|
||||
if (this.isBusinessHours()) {
|
||||
this.scheduleSlotRefresh(slot);
|
||||
}
|
||||
|
||||
const ttl = jwt.exp - Math.floor(Date.now() / 1000);
|
||||
const refreshIn = this.isBusinessHours()
|
||||
? Math.max(ttl - REFRESH_BUFFER, 30)
|
||||
: null;
|
||||
this.logger.log(
|
||||
`JWT pool: slot captured, TTL: ${ttl}s${refreshIn ? `, refresh in ${refreshIn}s` : ""}, pool size: ${this.pool.length}`,
|
||||
);
|
||||
return slot;
|
||||
}
|
||||
this.logger.warn(
|
||||
`JWT capture attempt ${attempt + 1}/${maxRetries} failed (${new URL(siteUrl).hostname}), trying next site...`,
|
||||
);
|
||||
}
|
||||
|
||||
throw new Error("Failed to capture JWT after all retries");
|
||||
} finally {
|
||||
this.semaphore.release();
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Timer-based refresh per slot ─────────────────────────
|
||||
|
||||
private scheduleSlotRefresh(slot: JwtSlot): void {
|
||||
this.clearSlotTimer(slot);
|
||||
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
const delayMs = Math.max((slot.jwt.exp - now - REFRESH_BUFFER) * 1000, 30_000);
|
||||
|
||||
slot.refreshTimer = setTimeout(async () => {
|
||||
// Don't refresh outside business hours
|
||||
if (!this.isBusinessHours()) {
|
||||
this.logger.debug("JWT pool: refresh timer fired outside business hours, skipping");
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const newSlot = await this.captureToPool();
|
||||
// Remove old slot
|
||||
const idx = this.pool.indexOf(slot);
|
||||
if (idx !== -1) {
|
||||
this.pool.splice(idx, 1);
|
||||
}
|
||||
this.logger.log(
|
||||
`JWT pool: slot refreshed, TTL: ${newSlot.jwt.exp - Math.floor(Date.now() / 1000)}s`,
|
||||
);
|
||||
} catch (err) {
|
||||
this.logger.error(`JWT pool: refresh failed: ${(err as Error).message}`);
|
||||
// Schedule a retry in 30s if still in business hours
|
||||
if (this.isBusinessHours()) {
|
||||
slot.refreshTimer = setTimeout(() => {
|
||||
this.scheduleSlotRefresh(slot);
|
||||
}, 30_000);
|
||||
}
|
||||
}
|
||||
}, delayMs);
|
||||
}
|
||||
|
||||
private clearSlotTimer(slot: JwtSlot): void {
|
||||
if (slot.refreshTimer) {
|
||||
clearTimeout(slot.refreshTimer);
|
||||
slot.refreshTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
private clearAllRefreshTimers(): void {
|
||||
for (const slot of this.pool) {
|
||||
this.clearSlotTimer(slot);
|
||||
}
|
||||
this.logger.log("JWT pool: all refresh timers cleared");
|
||||
}
|
||||
|
||||
// ─── Business hours scheduling ────────────────────────────
|
||||
|
||||
/**
|
||||
* Get current Istanbul hour and minute using Intl.DateTimeFormat.
|
||||
* This works correctly regardless of the server's local timezone.
|
||||
*/
|
||||
private getIstanbulTime(): { hour: number; minute: number } {
|
||||
const parts = new Intl.DateTimeFormat("en-US", {
|
||||
timeZone: "Europe/Istanbul",
|
||||
hour: "numeric",
|
||||
minute: "numeric",
|
||||
hour12: false,
|
||||
}).formatToParts(new Date());
|
||||
|
||||
const hour = parseInt(parts.find((p) => p.type === "hour")!.value, 10);
|
||||
const minute = parseInt(parts.find((p) => p.type === "minute")!.value, 10);
|
||||
return { hour, minute };
|
||||
}
|
||||
|
||||
private isBusinessHours(): boolean {
|
||||
const { hour } = this.getIstanbulTime();
|
||||
return hour >= 9 && hour < 19;
|
||||
}
|
||||
|
||||
private scheduleBusinessHours(): void {
|
||||
if (this.businessHoursTimer) {
|
||||
clearTimeout(this.businessHoursTimer);
|
||||
this.businessHoursTimer = null;
|
||||
}
|
||||
|
||||
const { hour, minute } = this.getIstanbulTime();
|
||||
|
||||
let delayMs: number;
|
||||
let nextEvent: string;
|
||||
|
||||
if (this.isBusinessHours()) {
|
||||
// Schedule 19:00 stop
|
||||
const minsUntil19 = (19 - hour - 1) * 60 + (60 - minute);
|
||||
delayMs = minsUntil19 * 60 * 1000;
|
||||
nextEvent = "stop (19:00)";
|
||||
|
||||
this.businessHoursTimer = setTimeout(() => {
|
||||
this.logger.log("JWT pool: business hours ended, timers cleared");
|
||||
this.clearAllRefreshTimers();
|
||||
this.scheduleBusinessHours(); // schedule next 09:00 start
|
||||
}, delayMs);
|
||||
} else {
|
||||
// Schedule 09:00 start
|
||||
let minsUntil9: number;
|
||||
if (hour >= 19) {
|
||||
// Same day evening → next day 09:00
|
||||
minsUntil9 = (24 - hour + 9 - 1) * 60 + (60 - minute);
|
||||
} else {
|
||||
// Before 09:00
|
||||
minsUntil9 = (9 - hour - 1) * 60 + (60 - minute);
|
||||
}
|
||||
delayMs = minsUntil9 * 60 * 1000;
|
||||
nextEvent = "start (09:00)";
|
||||
|
||||
this.businessHoursTimer = setTimeout(() => {
|
||||
this.logger.log("JWT pool: business hours started, capturing initial slot");
|
||||
this.captureToPool().catch((err) => {
|
||||
this.logger.error(`Business hours initial capture failed: ${(err as Error).message}`);
|
||||
});
|
||||
this.scheduleBusinessHours(); // schedule 19:00 stop
|
||||
}, delayMs);
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`JWT pool: next ${nextEvent} in ${Math.round(delayMs / 60_000)}min (Istanbul: ${hour}:${String(minute).padStart(2, "0")})`,
|
||||
);
|
||||
}
|
||||
|
||||
// ─── RPM tracking & dynamic scaling ───────────────────────
|
||||
|
||||
private trackRequest(): void {
|
||||
const now = Date.now();
|
||||
this.requestTimestamps.push(now);
|
||||
// Prune old timestamps outside the window
|
||||
const cutoff = now - RPM_WINDOW;
|
||||
while (this.requestTimestamps.length > 0 && this.requestTimestamps[0] < cutoff) {
|
||||
this.requestTimestamps.shift();
|
||||
}
|
||||
}
|
||||
|
||||
private getRPM(): number {
|
||||
const now = Date.now();
|
||||
const cutoff = now - RPM_WINDOW;
|
||||
return this.requestTimestamps.filter((t) => t >= cutoff).length;
|
||||
}
|
||||
|
||||
private getDesiredPoolSize(): number {
|
||||
const rpm = this.getRPM();
|
||||
return Math.min(Math.max(1, Math.ceil(rpm / RPM_PER_SLOT)), MAX_POOL_SIZE);
|
||||
}
|
||||
|
||||
private maybeScaleUp(): void {
|
||||
if (!this.isBusinessHours()) return;
|
||||
|
||||
const desired = this.getDesiredPoolSize();
|
||||
if (this.pool.length < desired) {
|
||||
this.logger.log(
|
||||
`JWT pool: scaling up, ${this.pool.length}/${desired} slots (RPM: ${this.getRPM()})`,
|
||||
);
|
||||
this.captureToPool().catch((err) => {
|
||||
this.logger.warn(`JWT pool: scale-up capture failed: ${(err as Error).message}`);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Site cooldown & port allocation ──────────────────────
|
||||
|
||||
private getAvailableSite(): string {
|
||||
const now = Date.now();
|
||||
|
||||
// Try round-robin, preferring sites not on cooldown
|
||||
for (let i = 0; i < JWT_SITES.length; i++) {
|
||||
const idx = (this.siteIndex + i) % JWT_SITES.length;
|
||||
const site = JWT_SITES[idx];
|
||||
const lastUsed = this.siteLastUsedAt.get(site) || 0;
|
||||
|
||||
if (now - lastUsed >= SITE_COOLDOWN) {
|
||||
this.siteIndex = (idx + 1) % JWT_SITES.length;
|
||||
return site;
|
||||
}
|
||||
}
|
||||
|
||||
// All on cooldown — pick the one with oldest usage
|
||||
let oldestIdx = 0;
|
||||
let oldestTime = Infinity;
|
||||
for (let i = 0; i < JWT_SITES.length; i++) {
|
||||
const lastUsed = this.siteLastUsedAt.get(JWT_SITES[i]) || 0;
|
||||
if (lastUsed < oldestTime) {
|
||||
oldestTime = lastUsed;
|
||||
oldestIdx = i;
|
||||
}
|
||||
}
|
||||
this.siteIndex = (oldestIdx + 1) % JWT_SITES.length;
|
||||
return JWT_SITES[oldestIdx];
|
||||
}
|
||||
|
||||
private allocatePort(): number {
|
||||
return DI_PORT_MIN + Math.floor(Math.random() * (DI_PORT_MAX - DI_PORT_MIN + 1));
|
||||
}
|
||||
|
||||
// ─── JWT capture via Playwright ───────────────────────────
|
||||
|
||||
private async attemptCapture(
|
||||
siteUrl: string,
|
||||
port: number,
|
||||
): Promise<PcatJwtToken | null> {
|
||||
let context: BrowserContext | null = null;
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
// Build context options with proxy
|
||||
const contextOptions: Record<string, unknown> = {};
|
||||
if (this.useProxy) {
|
||||
contextOptions.proxy = {
|
||||
server: `http://${this.proxyHost}:${port}`,
|
||||
username: this.proxyUser,
|
||||
password: this.proxyPass,
|
||||
};
|
||||
}
|
||||
|
||||
context = await this.browser!.newContext(contextOptions);
|
||||
const page = await context.newPage();
|
||||
|
||||
// Intercept requests to parts-catalogs.com
|
||||
let capturedJwt: string | null = null;
|
||||
|
||||
page.on("request", (request) => {
|
||||
if (capturedJwt) return;
|
||||
const url = request.url();
|
||||
if (
|
||||
url.includes("parts-catalogs.com") ||
|
||||
url.includes("api.parts-catalogs.com")
|
||||
) {
|
||||
const auth = request.headers()["authorization"];
|
||||
if (auth) {
|
||||
capturedJwt = auth;
|
||||
this.logger.debug("JWT intercepted from request");
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Block heavy resources to save proxy bandwidth
|
||||
await page.route("**/*", (route) => {
|
||||
const url = route.request().url();
|
||||
const type = route.request().resourceType();
|
||||
|
||||
// Block images, fonts, media
|
||||
if (["image", "font", "media"].includes(type)) {
|
||||
return route.abort();
|
||||
}
|
||||
|
||||
// Block known trackers/analytics
|
||||
if (
|
||||
url.includes("google-analytics.com") ||
|
||||
url.includes("googletagmanager.com") ||
|
||||
url.includes("mc.yandex.ru") ||
|
||||
url.includes("facebook.net") ||
|
||||
url.includes("doubleclick.net") ||
|
||||
url.includes("hotjar.com")
|
||||
) {
|
||||
return route.abort();
|
||||
}
|
||||
|
||||
return route.continue();
|
||||
});
|
||||
|
||||
// Navigate — networkidle waits for widget JS to load + make API calls
|
||||
try {
|
||||
await page.goto(siteUrl, {
|
||||
timeout: PAGE_TIMEOUT,
|
||||
waitUntil: "networkidle",
|
||||
});
|
||||
} catch (navErr) {
|
||||
// Navigation may timeout but JWT could still be captured
|
||||
this.logger.debug(
|
||||
`Navigation ended: ${(navErr as Error).message?.slice(0, 80)}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Poll for JWT
|
||||
for (let i = 0; i < CAPTURE_POLL_MAX; i++) {
|
||||
if (capturedJwt) break;
|
||||
await new Promise((r) => setTimeout(r, CAPTURE_POLL_INTERVAL));
|
||||
}
|
||||
|
||||
const elapsed = Date.now() - startTime;
|
||||
|
||||
if (capturedJwt) {
|
||||
const token = this.parseJwt(capturedJwt);
|
||||
this.logger.log(
|
||||
`JWT captured in ${elapsed}ms from ${new URL(siteUrl).hostname}`,
|
||||
);
|
||||
return token;
|
||||
}
|
||||
|
||||
this.logger.debug(`No JWT after ${elapsed}ms from ${siteUrl}`);
|
||||
return null;
|
||||
} catch (err) {
|
||||
this.logger.warn(`JWT capture error: ${(err as Error).message}`);
|
||||
return null;
|
||||
} finally {
|
||||
if (context) {
|
||||
try {
|
||||
await Promise.race([
|
||||
context.close(),
|
||||
new Promise((r) => setTimeout(r, CONTEXT_CLOSE_TIMEOUT)),
|
||||
]);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private parseJwt(rawToken: string): PcatJwtToken {
|
||||
const parts = rawToken.split(".");
|
||||
if (parts.length !== 3) {
|
||||
throw new Error("Invalid JWT format");
|
||||
}
|
||||
|
||||
// Decode payload with proper base64url padding
|
||||
let payloadB64 = parts[1];
|
||||
const padding = 4 - (payloadB64.length % 4);
|
||||
if (padding !== 4) {
|
||||
payloadB64 += "=".repeat(padding);
|
||||
}
|
||||
|
||||
const payload = JSON.parse(
|
||||
Buffer.from(payloadB64, "base64url").toString("utf-8"),
|
||||
);
|
||||
|
||||
return {
|
||||
raw: rawToken,
|
||||
exp: payload.exp || 0,
|
||||
host: payload.host || "",
|
||||
apiKey: payload.apiKey || "",
|
||||
apiPath: payload.apiPath || "",
|
||||
ip: payload.ip || "",
|
||||
hash: payload.h || "",
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Browser lifecycle ───────────────────────────────────
|
||||
|
||||
private async launchBrowser(): Promise<void> {
|
||||
if (this.launching) {
|
||||
return this.launching;
|
||||
}
|
||||
this.launching = this._doLaunch();
|
||||
try {
|
||||
await this.launching;
|
||||
} finally {
|
||||
this.launching = null;
|
||||
}
|
||||
}
|
||||
|
||||
private async _doLaunch(): Promise<void> {
|
||||
const { chromium } = await import("playwright");
|
||||
|
||||
this.browser = await chromium.launch({
|
||||
headless: true,
|
||||
args: [
|
||||
"--no-sandbox",
|
||||
"--disable-setuid-sandbox",
|
||||
"--disable-dev-shm-usage",
|
||||
"--disable-accelerated-2d-canvas",
|
||||
"--disable-gpu",
|
||||
],
|
||||
});
|
||||
|
||||
this.browser.on("disconnected", () => {
|
||||
this.logger.warn("Browser disconnected — will relaunch on next request");
|
||||
this.browser = null;
|
||||
});
|
||||
}
|
||||
|
||||
private async closeBrowser(): Promise<void> {
|
||||
if (this.browser) {
|
||||
try {
|
||||
await this.browser.close();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
this.browser = null;
|
||||
}
|
||||
}
|
||||
|
||||
private async ensureBrowser(): Promise<void> {
|
||||
if (this.browser?.isConnected()) return;
|
||||
this.logger.log("Browser not connected — relaunching");
|
||||
await this.launchBrowser();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
import { Module } from "@nestjs/common";
|
||||
import { PartsCatalogsAuthService } from "./parts-catalogs-auth.service";
|
||||
import { PartsCatalogsService } from "./parts-catalogs.service";
|
||||
|
||||
@Module({
|
||||
providers: [PartsCatalogsAuthService, PartsCatalogsService],
|
||||
exports: [PartsCatalogsService],
|
||||
})
|
||||
export class PartsCatalogsModule {}
|
||||
@@ -0,0 +1,240 @@
|
||||
/**
|
||||
* Parts-Catalogs API Service — HTTP client for parts-catalogs.com
|
||||
*
|
||||
* All requests go through the same DataImpulse proxy as the JWT capture
|
||||
* to ensure the JWT's IP-bound constraint is satisfied.
|
||||
*/
|
||||
|
||||
import { Injectable, Logger } from "@nestjs/common";
|
||||
import { PartsCatalogsAuthService } from "./parts-catalogs-auth.service";
|
||||
import { RedisService } from "../../redis/redis.service";
|
||||
import type {
|
||||
PcatVinResult,
|
||||
PcatCar,
|
||||
PcatGroup,
|
||||
PcatPartsResult,
|
||||
PcatSession,
|
||||
} from "./parts-catalogs.types";
|
||||
|
||||
const API_BASE = "https://api.parts-catalogs.com/v1";
|
||||
const REQUEST_TIMEOUT = 30_000;
|
||||
|
||||
@Injectable()
|
||||
export class PartsCatalogsService {
|
||||
private readonly logger = new Logger(PartsCatalogsService.name);
|
||||
|
||||
constructor(
|
||||
private authService: PartsCatalogsAuthService,
|
||||
private redis: RedisService,
|
||||
) {}
|
||||
|
||||
/** Mark parts-catalogs as actively used (5min TTL) to defer prefetch worker */
|
||||
private async touchActivity(): Promise<void> {
|
||||
try {
|
||||
await this.redis.set("prefetch:activity:parts-catalogs", String(Date.now()), 300);
|
||||
} catch {
|
||||
// Non-critical
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* VIN decode — returns one or more car matches.
|
||||
*/
|
||||
async decodeVin(vin: string): Promise<PcatVinResult | null> {
|
||||
try {
|
||||
const data = await this.fetchWithAuth("/car/info", { q: vin });
|
||||
|
||||
if (!data || typeof data !== "object") {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Response can be a single car or array of cars depending on VIN
|
||||
const rawCars = Array.isArray(data) ? data : [data];
|
||||
|
||||
const cars: PcatCar[] = [];
|
||||
for (const item of rawCars) {
|
||||
if (!item) continue;
|
||||
|
||||
// Each car result may have nested catalog info
|
||||
const catalogId = item.catalogId || item.catalog?.id || "";
|
||||
const carId = item.id || item.carId || "";
|
||||
|
||||
if (!carId) continue;
|
||||
|
||||
cars.push({
|
||||
id: String(carId),
|
||||
name: item.name || item.title || "",
|
||||
description: item.description || item.modelName || undefined,
|
||||
parameters: Array.isArray(item.parameters) ? item.parameters : undefined,
|
||||
catalogId: String(catalogId),
|
||||
});
|
||||
}
|
||||
|
||||
if (cars.length === 0) return null;
|
||||
|
||||
return { cars };
|
||||
} catch (err) {
|
||||
this.logger.warn(`VIN decode failed for ${vin}: ${(err as Error).message}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get category groups for a car.
|
||||
* Pass groupId for subgroups.
|
||||
*/
|
||||
async fetchGroups(
|
||||
catalogId: string,
|
||||
carId: string,
|
||||
groupId?: string,
|
||||
carParams?: Record<string, string>,
|
||||
): Promise<PcatGroup[]> {
|
||||
await this.touchActivity();
|
||||
const params: Record<string, string> = { carId };
|
||||
if (groupId) params.groupId = groupId;
|
||||
if (carParams) Object.assign(params, carParams);
|
||||
|
||||
const data = await this.fetchWithAuth(
|
||||
`/catalogs/${catalogId}/groups2/`,
|
||||
params,
|
||||
);
|
||||
|
||||
if (!Array.isArray(data)) return [];
|
||||
|
||||
return data.map((g: any) => ({
|
||||
id: String(g.id),
|
||||
parentId: g.parentId ? String(g.parentId) : undefined,
|
||||
name: g.name || "",
|
||||
img: g.img || undefined,
|
||||
hasSubgroups: !!g.hasSubgroups,
|
||||
hasParts: !!g.hasParts,
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get parts + schema image + hotspot positions for a group.
|
||||
*/
|
||||
async fetchParts(
|
||||
catalogId: string,
|
||||
carId: string,
|
||||
groupId: string,
|
||||
carParams?: Record<string, string>,
|
||||
): Promise<PcatPartsResult | null> {
|
||||
await this.touchActivity();
|
||||
const params: Record<string, string> = { carId, groupId };
|
||||
if (carParams) Object.assign(params, carParams);
|
||||
|
||||
const data = await this.fetchWithAuth(
|
||||
`/catalogs/${catalogId}/parts2`,
|
||||
params,
|
||||
);
|
||||
|
||||
if (!data || typeof data !== "object") return null;
|
||||
|
||||
return {
|
||||
img: data.img || "",
|
||||
imgDescription: data.imgDescription || undefined,
|
||||
partGroups: Array.isArray(data.partGroups)
|
||||
? data.partGroups.map((pg: any) => ({
|
||||
name: pg.name || undefined,
|
||||
number: pg.number || undefined,
|
||||
positionNumber: pg.positionNumber || undefined,
|
||||
parts: Array.isArray(pg.parts)
|
||||
? pg.parts.map((p: any) => ({
|
||||
id: p.id ? String(p.id) : undefined,
|
||||
number: p.number || "",
|
||||
name: p.name || "",
|
||||
nameId: p.nameId || undefined,
|
||||
notice: p.notice || undefined,
|
||||
positionNumber: p.positionNumber || undefined,
|
||||
}))
|
||||
: [],
|
||||
}))
|
||||
: [],
|
||||
positions: Array.isArray(data.positions)
|
||||
? data.positions.map((pos: any) => ({
|
||||
number: String(pos.number),
|
||||
coordinates: pos.coordinates || [0, 0, 0, 0],
|
||||
}))
|
||||
: [],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a VIN is potentially supported.
|
||||
* Parts-catalogs.com covers most brands, so this is broadly true.
|
||||
*/
|
||||
isSupported(_vin: string): boolean {
|
||||
return true;
|
||||
}
|
||||
|
||||
// ─── Private ─────────────────────────────────────────────
|
||||
|
||||
private async fetchWithAuth(
|
||||
endpoint: string,
|
||||
params?: Record<string, string>,
|
||||
): Promise<any> {
|
||||
const maxRetries = 2;
|
||||
|
||||
let session: PcatSession | null = null;
|
||||
|
||||
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
||||
session = await this.authService.acquireSession();
|
||||
|
||||
const url = new URL(`${API_BASE}${endpoint}`);
|
||||
if (params) {
|
||||
for (const [key, value] of Object.entries(params)) {
|
||||
url.searchParams.set(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const fetchOptions: RequestInit & { dispatcher?: any } = {
|
||||
method: "GET",
|
||||
headers: {
|
||||
Authorization: session.authorization,
|
||||
Accept: "application/json",
|
||||
"User-Agent":
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
},
|
||||
signal: AbortSignal.timeout(REQUEST_TIMEOUT),
|
||||
};
|
||||
|
||||
// Use undici ProxyAgent if proxy is configured
|
||||
if (session.proxyUrl) {
|
||||
const { ProxyAgent } = await import("undici");
|
||||
fetchOptions.dispatcher = new ProxyAgent(session.proxyUrl);
|
||||
}
|
||||
|
||||
const response = await fetch(url.toString(), fetchOptions);
|
||||
|
||||
if (response.ok) {
|
||||
return await response.json();
|
||||
}
|
||||
|
||||
if (response.status === 401 || response.status === 403) {
|
||||
this.logger.warn(
|
||||
`Auth error (${response.status}) on ${endpoint}, attempt ${attempt + 1}/${maxRetries + 1}`,
|
||||
);
|
||||
if (attempt < maxRetries) {
|
||||
await this.authService.invalidateSession(session);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
const text = await response.text().catch(() => "");
|
||||
throw new Error(
|
||||
`HTTP ${response.status} from ${endpoint}: ${text.slice(0, 200)}`,
|
||||
);
|
||||
} catch (err) {
|
||||
if ((err as Error).name === "TimeoutError") {
|
||||
this.logger.warn(`Timeout on ${endpoint}, attempt ${attempt + 1}`);
|
||||
if (attempt < maxRetries) continue;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(`Max retries exceeded for ${endpoint}`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
export interface PcatJwtToken {
|
||||
raw: string;
|
||||
exp: number;
|
||||
host: string;
|
||||
apiKey: string;
|
||||
apiPath: string;
|
||||
ip: string;
|
||||
hash: string;
|
||||
}
|
||||
|
||||
export interface JwtSlot {
|
||||
jwt: PcatJwtToken;
|
||||
proxyPort: number;
|
||||
siteUsed: string;
|
||||
capturedAt: number;
|
||||
refreshTimer: ReturnType<typeof setTimeout> | null;
|
||||
}
|
||||
|
||||
export interface PcatSession {
|
||||
authorization: string;
|
||||
proxyUrl: string | null;
|
||||
proxyConfig: { server: string; username: string; password: string } | null;
|
||||
_slot: JwtSlot;
|
||||
}
|
||||
|
||||
export interface PcatCarParameter {
|
||||
key: string;
|
||||
idx: string;
|
||||
value: string;
|
||||
}
|
||||
|
||||
export interface PcatCar {
|
||||
id: string;
|
||||
name: string;
|
||||
description?: string;
|
||||
parameters?: PcatCarParameter[];
|
||||
catalogId: string;
|
||||
}
|
||||
|
||||
export interface PcatVinResult {
|
||||
cars: PcatCar[];
|
||||
}
|
||||
|
||||
export interface PcatGroup {
|
||||
id: string;
|
||||
parentId?: string;
|
||||
name: string;
|
||||
img?: string;
|
||||
hasSubgroups: boolean;
|
||||
hasParts: boolean;
|
||||
}
|
||||
|
||||
export interface PcatPart {
|
||||
id?: string;
|
||||
number: string; // OEM code
|
||||
name: string;
|
||||
nameId?: number;
|
||||
notice?: string;
|
||||
positionNumber?: string;
|
||||
}
|
||||
|
||||
export interface PcatPartGroup {
|
||||
name?: string;
|
||||
number?: string;
|
||||
positionNumber?: string;
|
||||
parts: PcatPart[];
|
||||
}
|
||||
|
||||
export interface PcatPosition {
|
||||
number: string;
|
||||
coordinates: [number, number, number, number]; // x, y, w, h
|
||||
}
|
||||
|
||||
export interface PcatPartsResult {
|
||||
img: string;
|
||||
imgDescription?: string;
|
||||
partGroups: PcatPartGroup[];
|
||||
positions: PcatPosition[];
|
||||
}
|
||||
@@ -261,6 +261,7 @@ export class PL24Service {
|
||||
return this.fordLegacyService.fetchPartsByPath(linkPath, serviceName);
|
||||
}
|
||||
|
||||
await this.touchActivity();
|
||||
const pathHash = createHash("sha256").update(linkPath).digest("hex").substring(0, 16);
|
||||
const cacheKey = `${PL24_DEFAULTS.CACHE_PREFIX}parts:path:${pathHash}`;
|
||||
const cached = await this.redis.getJson<PL24PartsResponse>(cacheKey);
|
||||
@@ -384,6 +385,7 @@ export class PL24Service {
|
||||
}
|
||||
|
||||
this.logger.log(`Fetching sub-groups by path: ${linkPath}`);
|
||||
await this.touchActivity();
|
||||
|
||||
try {
|
||||
await this.authService.authorizeService(serviceName);
|
||||
@@ -411,6 +413,7 @@ export class PL24Service {
|
||||
serviceName: string,
|
||||
mainGroupsPath: string,
|
||||
): Promise<PL24DecodedCategory[]> {
|
||||
await this.touchActivity();
|
||||
try {
|
||||
await this.authService.authorizeService(serviceName);
|
||||
const headers = await this.authService.buildAuthHeaders(serviceName);
|
||||
@@ -435,6 +438,7 @@ export class PL24Service {
|
||||
hotspots: PL24Hotspot[];
|
||||
} | null> {
|
||||
if (!imageUrl) return null;
|
||||
await this.touchActivity();
|
||||
|
||||
// Extract image ID for dedup
|
||||
const imageId = this.extractImageIdFromUrl(imageUrl);
|
||||
@@ -572,6 +576,15 @@ export class PL24Service {
|
||||
return SERVICE_TO_BRAND[serviceName] || null;
|
||||
}
|
||||
|
||||
/** Mark PL24 as actively used (5min TTL) to defer prefetch worker */
|
||||
private async touchActivity(): Promise<void> {
|
||||
try {
|
||||
await this.redis.set("prefetch:activity:pl24", String(Date.now()), 300);
|
||||
} catch {
|
||||
// Non-critical — don't break the request
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== PRIVATE: Request helpers ====================
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user