Files
sase.tr/apps/api/src/integrations/carcatonline/carcatonline.tree.ts
Semih Yesilyurt 337192685c
Some checks are pending
QA Gate (P0/P1) / Test affected app (pull_request) Waiting to run
feat(carcatonline): saatlik istek kotası + kaldığı yerden devam eden tarama
Dev'de ölçüldü: upstream 429 gövdesi "Customer Hourly Request Limit Exceeded",
~110 çağrı sonra; 47 dk sonraki denemede hâlâ 429 (kayan saatlik kota, sabit
kilit değil). Kilide çarpıp her araç için baştan başlamak yerine:
- Throttle: `CARCATONLINE_HOURLY_CALL_CAP` (varsayılan 90) — Istanbul saat
  dilimi başına Redis sayaç; dolunca iş kendini bir sonraki saat başına erteler.
- Tarama cache'i: groups2 cevapları Redis'te 24 s (`carcatonline:groups:*`);
  tekrar koşan iş cache'ten okur, çağrı harcamaz.
- Varyant çözümü metadata'ya `status:"matched"` olarak yazılır; tarama saat
  ortasında kesilirse sonraki koşu kademeli seçimi atlar. Scan 'matched'
  satırları 7 günlük bekleme olmadan yeniden kuyruklar.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-26 16:25:00 +03:00

192 lines
5.9 KiB
TypeScript

/**
* Resolve a carcatonline model to one representative car (cascading parameter
* selection) and crawl its full group tree. Pure orchestration over an injected
* client; the caller supplies `beforeCall` (throttle) and `checkAbort` (window).
*/
import type { CarcatCar, CarcatGroup, CarcatParameter } from "./carcatonline.client";
export interface TreeClient {
carsParameters(
catalogId: string,
modelId: string,
selected: string[],
): Promise<CarcatParameter[]>;
cars(catalogId: string, modelId: string, selected: string[]): Promise<CarcatCar[]>;
groups(catalogId: string, carId: string, groupId?: string): Promise<CarcatGroup[]>;
}
export interface CrawlHooks {
beforeCall(): Promise<void>;
/** Throw to abort (e.g. bulk window closed). */
checkAbort?(): void;
/** Optional response cache so a re-run after a quota stop resumes without re-spending calls. */
cache?: {
get(key: string): Promise<CarcatGroup[] | null>;
set(key: string, groups: CarcatGroup[]): Promise<void>;
};
}
export interface SelectedParameter {
key: string;
name: string;
idx: string;
value: string;
}
export interface ResolvedCar {
carId: string;
parameters: SelectedParameter[];
car: CarcatCar;
}
/** Preferred values when a parameter offers several (first match wins, else first value). */
const PREFERRED_TOKENS = [
"1.5",
"1.6",
"DIESEL",
"DIZEL",
"MANUAL",
"MEKANIK",
"LHD",
"EUROPE",
"EU",
];
function pickValue(param: CarcatParameter): CarcatParameter["values"][number] {
for (const tok of PREFERRED_TOKENS) {
const hit = param.values.find((v) => v.value.toUpperCase().includes(tok));
if (hit) return hit;
}
return param.values[0];
}
/**
* Walk `cars-parameters` picking one value per required parameter until
* `cars2` returns a car. At most `maxSteps` API calls per step pair.
*/
export async function resolveRepresentativeCar(
client: TreeClient,
hooks: CrawlHooks,
catalogId: string,
modelId: string,
maxSteps = 8,
): Promise<ResolvedCar | null> {
const selected: SelectedParameter[] = [];
for (let step = 0; step < maxSteps; step += 1) {
hooks.checkAbort?.();
await hooks.beforeCall();
const params = await client.carsParameters(
catalogId,
modelId,
selected.map((s) => s.idx),
);
const chosen = new Set(selected.map((s) => s.idx));
const pending = params.filter(
(p) => p.values?.length > 0 && !p.values.some((v) => chosen.has(v.idx)),
);
if (pending.length > 0) {
const p = pending[0];
const v = pickValue(p);
selected.push({ key: p.key, name: p.name, idx: v.idx, value: v.value });
}
hooks.checkAbort?.();
await hooks.beforeCall();
try {
const cars = await client.cars(
catalogId,
modelId,
selected.map((s) => s.idx),
);
if (Array.isArray(cars) && cars.length > 0) {
return { carId: cars[0].id, parameters: selected, car: cars[0] };
}
} catch (err) {
// 404 "Not all required parameters are selected" → keep cascading;
// anything else (429/401/5xx) propagates.
const status = (err as { status?: number }).status;
if (status !== 404) throw err;
}
if (pending.length === 0) return null; // nothing left to select and still no car
}
return null;
}
export interface CrawledNode {
id: string; // carcatonline group id
parentId: string | null; // carcatonline parent group id
name: string;
hasSubgroups: boolean;
hasParts: boolean;
img: string | null;
illustration: string | null;
depth: number;
}
/** Breadth-first crawl of the whole group tree (one call per node that has subgroups). */
export async function crawlGroupTree(
client: TreeClient,
hooks: CrawlHooks,
catalogId: string,
carId: string,
maxNodes = 5000,
): Promise<{ nodes: CrawledNode[]; calls: number }> {
const nodes: CrawledNode[] = [];
let calls = 0;
const queue: { groupId: string | null; depth: number }[] = [{ groupId: null, depth: 1 }];
const seen = new Set<string>();
while (queue.length > 0 && nodes.length < maxNodes) {
const { groupId, depth } = queue.shift() as { groupId: string | null; depth: number };
hooks.checkAbort?.();
const cacheKey = `${catalogId}:${carId}:${groupId ?? "root"}`;
let groups = (await hooks.cache?.get(cacheKey)) ?? null;
if (!groups) {
await hooks.beforeCall();
groups = await client.groups(catalogId, carId, groupId ?? undefined);
calls += 1;
await hooks.cache?.set(cacheKey, groups);
}
for (const g of groups) {
if (!g?.id || seen.has(g.id)) continue;
seen.add(g.id);
nodes.push({
id: g.id,
parentId: groupId,
name: (g.name || "").trim() || g.id,
hasSubgroups: !!g.hasSubgroups,
hasParts: !!g.hasParts,
img: g.img || null,
illustration: g.description?.illustration || null,
depth,
});
if (g.hasSubgroups) queue.push({ groupId: g.id, depth: depth + 1 });
}
}
return { nodes, calls };
}
/**
* The `categories` unique index is (vehicle, catalog_vehicle, name, source), so
* two nodes with the same display name under one catalog vehicle would collide
* and orphan a branch. Disambiguate duplicates with their parent's name, then
* with an ordinal.
*/
export function disambiguateNames(nodes: CrawledNode[]): Map<string, string> {
const byId = new Map(nodes.map((n) => [n.id, n]));
const counts = new Map<string, number>();
for (const n of nodes) counts.set(n.name, (counts.get(n.name) ?? 0) + 1);
const used = new Set<string>();
const out = new Map<string, string>();
for (const n of nodes) {
let name = n.name;
if ((counts.get(n.name) ?? 0) > 1 && n.parentId) {
const parent = byId.get(n.parentId);
if (parent) name = `${n.name} (${parent.name})`;
}
let candidate = name;
for (let i = 2; used.has(candidate); i += 1) candidate = `${name} #${i}`;
used.add(candidate);
out.set(n.id, candidate);
}
return out;
}