Some checks are pending
QA Gate (P0/P1) / Test affected app (pull_request) Waiting to run
Dev'de ölçüldü: upstream 429 gövdesi "Customer Hourly Request Limit Exceeded", ~110 çağrı sonra; 47 dk sonraki denemede hâlâ 429 (kayan saatlik kota, sabit kilit değil). Kilide çarpıp her araç için baştan başlamak yerine: - Throttle: `CARCATONLINE_HOURLY_CALL_CAP` (varsayılan 90) — Istanbul saat dilimi başına Redis sayaç; dolunca iş kendini bir sonraki saat başına erteler. - Tarama cache'i: groups2 cevapları Redis'te 24 s (`carcatonline:groups:*`); tekrar koşan iş cache'ten okur, çağrı harcamaz. - Varyant çözümü metadata'ya `status:"matched"` olarak yazılır; tarama saat ortasında kesilirse sonraki koşu kademeli seçimi atlar. Scan 'matched' satırları 7 günlük bekleme olmadan yeniden kuyruklar. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
192 lines
5.9 KiB
TypeScript
192 lines
5.9 KiB
TypeScript
/**
|
|
* Resolve a carcatonline model to one representative car (cascading parameter
|
|
* selection) and crawl its full group tree. Pure orchestration over an injected
|
|
* client; the caller supplies `beforeCall` (throttle) and `checkAbort` (window).
|
|
*/
|
|
import type { CarcatCar, CarcatGroup, CarcatParameter } from "./carcatonline.client";
|
|
|
|
export interface TreeClient {
|
|
carsParameters(
|
|
catalogId: string,
|
|
modelId: string,
|
|
selected: string[],
|
|
): Promise<CarcatParameter[]>;
|
|
cars(catalogId: string, modelId: string, selected: string[]): Promise<CarcatCar[]>;
|
|
groups(catalogId: string, carId: string, groupId?: string): Promise<CarcatGroup[]>;
|
|
}
|
|
|
|
export interface CrawlHooks {
|
|
beforeCall(): Promise<void>;
|
|
/** Throw to abort (e.g. bulk window closed). */
|
|
checkAbort?(): void;
|
|
/** Optional response cache so a re-run after a quota stop resumes without re-spending calls. */
|
|
cache?: {
|
|
get(key: string): Promise<CarcatGroup[] | null>;
|
|
set(key: string, groups: CarcatGroup[]): Promise<void>;
|
|
};
|
|
}
|
|
|
|
export interface SelectedParameter {
|
|
key: string;
|
|
name: string;
|
|
idx: string;
|
|
value: string;
|
|
}
|
|
|
|
export interface ResolvedCar {
|
|
carId: string;
|
|
parameters: SelectedParameter[];
|
|
car: CarcatCar;
|
|
}
|
|
|
|
/** Preferred values when a parameter offers several (first match wins, else first value). */
|
|
const PREFERRED_TOKENS = [
|
|
"1.5",
|
|
"1.6",
|
|
"DIESEL",
|
|
"DIZEL",
|
|
"MANUAL",
|
|
"MEKANIK",
|
|
"LHD",
|
|
"EUROPE",
|
|
"EU",
|
|
];
|
|
|
|
function pickValue(param: CarcatParameter): CarcatParameter["values"][number] {
|
|
for (const tok of PREFERRED_TOKENS) {
|
|
const hit = param.values.find((v) => v.value.toUpperCase().includes(tok));
|
|
if (hit) return hit;
|
|
}
|
|
return param.values[0];
|
|
}
|
|
|
|
/**
|
|
* Walk `cars-parameters` picking one value per required parameter until
|
|
* `cars2` returns a car. At most `maxSteps` API calls per step pair.
|
|
*/
|
|
export async function resolveRepresentativeCar(
|
|
client: TreeClient,
|
|
hooks: CrawlHooks,
|
|
catalogId: string,
|
|
modelId: string,
|
|
maxSteps = 8,
|
|
): Promise<ResolvedCar | null> {
|
|
const selected: SelectedParameter[] = [];
|
|
for (let step = 0; step < maxSteps; step += 1) {
|
|
hooks.checkAbort?.();
|
|
await hooks.beforeCall();
|
|
const params = await client.carsParameters(
|
|
catalogId,
|
|
modelId,
|
|
selected.map((s) => s.idx),
|
|
);
|
|
const chosen = new Set(selected.map((s) => s.idx));
|
|
const pending = params.filter(
|
|
(p) => p.values?.length > 0 && !p.values.some((v) => chosen.has(v.idx)),
|
|
);
|
|
if (pending.length > 0) {
|
|
const p = pending[0];
|
|
const v = pickValue(p);
|
|
selected.push({ key: p.key, name: p.name, idx: v.idx, value: v.value });
|
|
}
|
|
hooks.checkAbort?.();
|
|
await hooks.beforeCall();
|
|
try {
|
|
const cars = await client.cars(
|
|
catalogId,
|
|
modelId,
|
|
selected.map((s) => s.idx),
|
|
);
|
|
if (Array.isArray(cars) && cars.length > 0) {
|
|
return { carId: cars[0].id, parameters: selected, car: cars[0] };
|
|
}
|
|
} catch (err) {
|
|
// 404 "Not all required parameters are selected" → keep cascading;
|
|
// anything else (429/401/5xx) propagates.
|
|
const status = (err as { status?: number }).status;
|
|
if (status !== 404) throw err;
|
|
}
|
|
if (pending.length === 0) return null; // nothing left to select and still no car
|
|
}
|
|
return null;
|
|
}
|
|
|
|
export interface CrawledNode {
|
|
id: string; // carcatonline group id
|
|
parentId: string | null; // carcatonline parent group id
|
|
name: string;
|
|
hasSubgroups: boolean;
|
|
hasParts: boolean;
|
|
img: string | null;
|
|
illustration: string | null;
|
|
depth: number;
|
|
}
|
|
|
|
/** Breadth-first crawl of the whole group tree (one call per node that has subgroups). */
|
|
export async function crawlGroupTree(
|
|
client: TreeClient,
|
|
hooks: CrawlHooks,
|
|
catalogId: string,
|
|
carId: string,
|
|
maxNodes = 5000,
|
|
): Promise<{ nodes: CrawledNode[]; calls: number }> {
|
|
const nodes: CrawledNode[] = [];
|
|
let calls = 0;
|
|
const queue: { groupId: string | null; depth: number }[] = [{ groupId: null, depth: 1 }];
|
|
const seen = new Set<string>();
|
|
while (queue.length > 0 && nodes.length < maxNodes) {
|
|
const { groupId, depth } = queue.shift() as { groupId: string | null; depth: number };
|
|
hooks.checkAbort?.();
|
|
const cacheKey = `${catalogId}:${carId}:${groupId ?? "root"}`;
|
|
let groups = (await hooks.cache?.get(cacheKey)) ?? null;
|
|
if (!groups) {
|
|
await hooks.beforeCall();
|
|
groups = await client.groups(catalogId, carId, groupId ?? undefined);
|
|
calls += 1;
|
|
await hooks.cache?.set(cacheKey, groups);
|
|
}
|
|
for (const g of groups) {
|
|
if (!g?.id || seen.has(g.id)) continue;
|
|
seen.add(g.id);
|
|
nodes.push({
|
|
id: g.id,
|
|
parentId: groupId,
|
|
name: (g.name || "").trim() || g.id,
|
|
hasSubgroups: !!g.hasSubgroups,
|
|
hasParts: !!g.hasParts,
|
|
img: g.img || null,
|
|
illustration: g.description?.illustration || null,
|
|
depth,
|
|
});
|
|
if (g.hasSubgroups) queue.push({ groupId: g.id, depth: depth + 1 });
|
|
}
|
|
}
|
|
return { nodes, calls };
|
|
}
|
|
|
|
/**
|
|
* The `categories` unique index is (vehicle, catalog_vehicle, name, source), so
|
|
* two nodes with the same display name under one catalog vehicle would collide
|
|
* and orphan a branch. Disambiguate duplicates with their parent's name, then
|
|
* with an ordinal.
|
|
*/
|
|
export function disambiguateNames(nodes: CrawledNode[]): Map<string, string> {
|
|
const byId = new Map(nodes.map((n) => [n.id, n]));
|
|
const counts = new Map<string, number>();
|
|
for (const n of nodes) counts.set(n.name, (counts.get(n.name) ?? 0) + 1);
|
|
const used = new Set<string>();
|
|
const out = new Map<string, string>();
|
|
for (const n of nodes) {
|
|
let name = n.name;
|
|
if ((counts.get(n.name) ?? 0) > 1 && n.parentId) {
|
|
const parent = byId.get(n.parentId);
|
|
if (parent) name = `${n.name} (${parent.name})`;
|
|
}
|
|
let candidate = name;
|
|
for (let i = 2; used.has(candidate); i += 1) candidate = `${name} #${i}`;
|
|
used.add(candidate);
|
|
out.set(n.id, candidate);
|
|
}
|
|
return out;
|
|
}
|