Merge pull request 'dev' (#60) from dev into main
Reviewed-on: #60
This commit was merged in pull request #60.
This commit is contained in:
@@ -1,4 +1,4 @@
|
|||||||
import { Controller, Get, Param } from "@nestjs/common";
|
import { Controller, Get, Param, Query } from "@nestjs/common";
|
||||||
import { CategoriesService } from "./categories.service";
|
import { CategoriesService } from "./categories.service";
|
||||||
|
|
||||||
@Controller("categories")
|
@Controller("categories")
|
||||||
@@ -10,6 +10,11 @@ export class CategoriesController {
|
|||||||
return this.categoriesService.getCategoryTree(vehicleId);
|
return this.categoriesService.getCategoryTree(vehicleId);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Get("search/:vehicleId")
|
||||||
|
async searchCatalog(@Param("vehicleId") vehicleId: string, @Query("q") q: string) {
|
||||||
|
return this.categoriesService.searchCatalog(vehicleId, q ?? "");
|
||||||
|
}
|
||||||
|
|
||||||
@Get(":id/children")
|
@Get(":id/children")
|
||||||
async getChildren(@Param("id") id: string) {
|
async getChildren(@Param("id") id: string) {
|
||||||
return this.categoriesService.getChildren(id);
|
return this.categoriesService.getChildren(id);
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { Inject, Injectable, Logger, NotFoundException } from "@nestjs/common";
|
import { Inject, Injectable, Logger, NotFoundException } from "@nestjs/common";
|
||||||
import { eq, inArray, isNull, sql } from "drizzle-orm";
|
import { and, eq, ilike, inArray, isNull, or, sql } from "drizzle-orm";
|
||||||
import { DATABASE, type Database } from "../database/database.provider";
|
import { DATABASE, type Database } from "../database/database.provider";
|
||||||
import { categories, parts, schemaPics, vehicles } from "../database/schema/core";
|
import { categories, parts, schemaPics, vehicles } from "../database/schema/core";
|
||||||
import { EmexService } from "../integrations/emex/emex.service";
|
import { EmexService } from "../integrations/emex/emex.service";
|
||||||
@@ -615,6 +615,172 @@ export class CategoriesService {
|
|||||||
return rows.map((r) => ({ id: r.id, name: r.name }));
|
return rows.map((r) => ({ id: r.id, name: r.name }));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build root→parent breadcrumb trails for many categories in one round-trip.
|
||||||
|
* The per-node trail excludes the node itself, ordered root-first. Used by the
|
||||||
|
* catalog search so each hit can show where it sits in the tree.
|
||||||
|
*/
|
||||||
|
private async buildBreadcrumbs(
|
||||||
|
ids: string[],
|
||||||
|
): Promise<Map<string, Array<{ id: string; name: string }>>> {
|
||||||
|
const map = new Map<string, Array<{ id: string; name: string }>>();
|
||||||
|
if (ids.length === 0) return map;
|
||||||
|
|
||||||
|
const rows = await this.db.execute<{
|
||||||
|
start_id: string;
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
depth: number;
|
||||||
|
}>(sql`
|
||||||
|
WITH RECURSIVE anc AS (
|
||||||
|
SELECT id AS start_id, id, name, parent_id, 0 AS depth
|
||||||
|
FROM categories
|
||||||
|
WHERE id IN (${sql.join(
|
||||||
|
ids.map((i) => sql`${i}`),
|
||||||
|
sql`, `,
|
||||||
|
)})
|
||||||
|
UNION ALL
|
||||||
|
SELECT a.start_id, c.id, c.name, c.parent_id, a.depth + 1
|
||||||
|
FROM anc a
|
||||||
|
JOIN categories c ON c.id = a.parent_id
|
||||||
|
)
|
||||||
|
SELECT start_id, id, name, depth FROM anc WHERE depth > 0 ORDER BY start_id, depth DESC
|
||||||
|
`);
|
||||||
|
|
||||||
|
for (const r of rows) {
|
||||||
|
const trail = map.get(r.start_id) ?? [];
|
||||||
|
trail.push({ id: r.id, name: r.name });
|
||||||
|
map.set(r.start_id, trail);
|
||||||
|
}
|
||||||
|
return map;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Full-catalog search for a single vehicle, over what has already been drilled
|
||||||
|
* into the DB. Returns two sections:
|
||||||
|
* - categories: leaf (and parent) categories whose name matches every token
|
||||||
|
* - parts: parts whose name matches every token, or whose OEM code contains
|
||||||
|
* the raw query
|
||||||
|
* Multi-word queries are AND-ed across tokens; matching is case-insensitive
|
||||||
|
* (ILIKE) and checks both the Turkish `name` and the original `nameOriginal`.
|
||||||
|
* Pure DB read — does not trigger any upstream drill (see getChildren for that).
|
||||||
|
*/
|
||||||
|
async searchCatalog(vehicleId: string, rawQuery: string) {
|
||||||
|
const query = (rawQuery ?? "").trim();
|
||||||
|
if (query.length < 2) return { query, categories: [], parts: [], treeIncomplete: false };
|
||||||
|
|
||||||
|
const tokens = query.toLocaleLowerCase("tr").split(/\s+/).filter(Boolean).slice(0, 6);
|
||||||
|
if (tokens.length === 0) return { query, categories: [], parts: [], treeIncomplete: false };
|
||||||
|
|
||||||
|
// ── Section 1: categories (match every token on name OR nameOriginal) ──
|
||||||
|
const catTokenConds = tokens.map((tok) =>
|
||||||
|
or(ilike(categories.name, `%${tok}%`), ilike(categories.nameOriginal, `%${tok}%`)),
|
||||||
|
);
|
||||||
|
const matchedCats = await this.db
|
||||||
|
.select({
|
||||||
|
id: categories.id,
|
||||||
|
name: categories.name,
|
||||||
|
source: categories.source,
|
||||||
|
unavailable: categories.unavailable,
|
||||||
|
})
|
||||||
|
.from(categories)
|
||||||
|
.where(and(eq(categories.vehicleId, vehicleId), ...catTokenConds))
|
||||||
|
.limit(80);
|
||||||
|
|
||||||
|
// ── Section 2: parts (every token on name, OR raw query on OEM code) ──
|
||||||
|
const partTokenConds = tokens.map((tok) => ilike(parts.name, `%${tok}%`));
|
||||||
|
const matchedParts = await this.db
|
||||||
|
.select({
|
||||||
|
oemCode: parts.oemCode,
|
||||||
|
name: parts.name,
|
||||||
|
categoryId: parts.categoryId,
|
||||||
|
categoryName: categories.name,
|
||||||
|
unavailable: parts.unavailable,
|
||||||
|
})
|
||||||
|
.from(parts)
|
||||||
|
.innerJoin(categories, eq(parts.categoryId, categories.id))
|
||||||
|
.where(
|
||||||
|
and(
|
||||||
|
eq(parts.vehicleId, vehicleId),
|
||||||
|
or(and(...partTokenConds), ilike(parts.oemCode, `%${query}%`)),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.limit(80);
|
||||||
|
|
||||||
|
// Section 1 = categories whose NAME matches ∪ the leaf categories that CONTAIN
|
||||||
|
// a matching part. "fren balatası" matches no leaf literally named that, but
|
||||||
|
// the leaves holding those parts (e.g. "Disk freni") are exactly what the user
|
||||||
|
// is after — so fold the matched parts' categories in, with a hit count.
|
||||||
|
const partCatInfo = new Map<string, { name: string; count: number }>();
|
||||||
|
for (const p of matchedParts) {
|
||||||
|
const cur = partCatInfo.get(p.categoryId);
|
||||||
|
if (cur) cur.count += 1;
|
||||||
|
else partCatInfo.set(p.categoryId, { name: p.categoryName, count: 1 });
|
||||||
|
}
|
||||||
|
const namedCats = new Map(matchedCats.map((c) => [c.id, c]));
|
||||||
|
const allCatIds = [...new Set([...namedCats.keys(), ...partCatInfo.keys()])];
|
||||||
|
|
||||||
|
// A category is a "leaf" (a real parts page) when nothing points to it as a
|
||||||
|
// parent. Un-drilled mid-groups can be mis-flagged as leaves, but clicking one
|
||||||
|
// just drills it like any browse, so this stays safe.
|
||||||
|
const parentRows = allCatIds.length
|
||||||
|
? await this.db
|
||||||
|
.selectDistinct({ parentId: categories.parentId })
|
||||||
|
.from(categories)
|
||||||
|
.where(inArray(categories.parentId, allCatIds))
|
||||||
|
: [];
|
||||||
|
const parentSet = new Set(parentRows.map((r) => r.parentId).filter(Boolean) as string[]);
|
||||||
|
|
||||||
|
// Breadcrumbs for every category referenced by either section, in one query.
|
||||||
|
const crumbs = await this.buildBreadcrumbs(allCatIds);
|
||||||
|
|
||||||
|
const categoryResults = allCatIds
|
||||||
|
.map((id) => {
|
||||||
|
const named = namedCats.get(id);
|
||||||
|
const viaPart = partCatInfo.get(id);
|
||||||
|
return {
|
||||||
|
id,
|
||||||
|
name: named?.name ?? viaPart?.name ?? "",
|
||||||
|
source: named?.source,
|
||||||
|
unavailable: named?.unavailable ?? false,
|
||||||
|
isLeaf: !parentSet.has(id),
|
||||||
|
matchingPartCount: viaPart?.count ?? 0,
|
||||||
|
breadcrumb: crumbs.get(id) ?? [],
|
||||||
|
};
|
||||||
|
})
|
||||||
|
// Categories that actually contain matching parts lead, then other leaves.
|
||||||
|
.sort(
|
||||||
|
(a, b) =>
|
||||||
|
b.matchingPartCount - a.matchingPartCount ||
|
||||||
|
Number(b.isLeaf) - Number(a.isLeaf) ||
|
||||||
|
a.name.localeCompare(b.name, "tr"),
|
||||||
|
)
|
||||||
|
.slice(0, 50);
|
||||||
|
|
||||||
|
const partResults = matchedParts.slice(0, 50).map((p) => ({
|
||||||
|
oemCode: p.oemCode,
|
||||||
|
name: p.name,
|
||||||
|
categoryId: p.categoryId,
|
||||||
|
categoryName: p.categoryName,
|
||||||
|
unavailable: p.unavailable,
|
||||||
|
breadcrumb: crumbs.get(p.categoryId) ?? [],
|
||||||
|
}));
|
||||||
|
|
||||||
|
// Hint the UI when this vehicle's tree looks barely drilled (roots only, no
|
||||||
|
// parts): search only sees what's in the DB, so results may be sparse.
|
||||||
|
const [catAgg] = await this.db
|
||||||
|
.select({ n: sql<number>`count(*)`.mapWith(Number) })
|
||||||
|
.from(categories)
|
||||||
|
.where(eq(categories.vehicleId, vehicleId));
|
||||||
|
const [partAgg] = await this.db
|
||||||
|
.select({ n: sql<number>`count(*)`.mapWith(Number) })
|
||||||
|
.from(parts)
|
||||||
|
.where(eq(parts.vehicleId, vehicleId));
|
||||||
|
const treeIncomplete = partAgg.n === 0 && catAgg.n <= 20;
|
||||||
|
|
||||||
|
return { query, categories: categoryResults, parts: partResults, treeIncomplete };
|
||||||
|
}
|
||||||
|
|
||||||
async getCategoryWithParts(categoryId: string) {
|
async getCategoryWithParts(categoryId: string) {
|
||||||
const result = await this.getCategoryWithPartsInner(categoryId);
|
const result = await this.getCategoryWithPartsInner(categoryId);
|
||||||
// Attach the full ancestor trail so the client can render a complete,
|
// Attach the full ancestor trail so the client can render a complete,
|
||||||
|
|||||||
@@ -88,6 +88,30 @@ export class JobsModule implements OnModuleInit, OnModuleDestroy {
|
|||||||
},
|
},
|
||||||
);
|
);
|
||||||
console.log("[jobs] Registered lifecycle-email cron: 0 9 * * *");
|
console.log("[jobs] Registered lifecycle-email cron: 0 9 * * *");
|
||||||
|
|
||||||
|
// Catalog backfill: every hour at :00. Scans for decoded vehicles whose catalog
|
||||||
|
// isn't fully prefetched and queues them. Self-throttled (queue-depth guard),
|
||||||
|
// cooldown- and business-hours-aware — handled by PrefetchWorkerService.
|
||||||
|
// PRODUCTION ONLY: dev runs against a separate DB and must not sweep/scrape.
|
||||||
|
if (process.env.NODE_ENV === "production") {
|
||||||
|
await this.catalogPrefetchQueue.upsertJobScheduler(
|
||||||
|
"catalog-backfill-hourly",
|
||||||
|
{ pattern: "0 * * * *" },
|
||||||
|
{
|
||||||
|
name: "backfill-scan",
|
||||||
|
data: {},
|
||||||
|
opts: {
|
||||||
|
removeOnComplete: { count: 48 },
|
||||||
|
removeOnFail: { count: 100 },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
console.log("[jobs] Registered catalog-backfill cron: 0 * * * *");
|
||||||
|
} else {
|
||||||
|
// Clean up any stale scheduler (e.g. if NODE_ENV changed) and stay idle.
|
||||||
|
await this.catalogPrefetchQueue.removeJobScheduler("catalog-backfill-hourly").catch(() => {});
|
||||||
|
console.log("[jobs] Skipped catalog-backfill cron (NODE_ENV != production)");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async onModuleDestroy() {
|
async onModuleDestroy() {
|
||||||
|
|||||||
@@ -24,22 +24,34 @@ export async function checkCooldown(redis: RedisService, source: string): Promis
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** Current hour (0–23) in Europe/Istanbul. */
|
||||||
* Check PL24 business hours (09:00–18:00 Europe/Istanbul).
|
function currentIstanbulHour(): number {
|
||||||
* Throws RateLimitError with delay until next 09:00 if outside window.
|
|
||||||
*/
|
|
||||||
export function checkTimeWindow(source: string): void {
|
|
||||||
if (source !== "pl24" && source !== "parts-catalogs") return;
|
|
||||||
|
|
||||||
const hourStr = new Intl.DateTimeFormat("en-US", {
|
const hourStr = new Intl.DateTimeFormat("en-US", {
|
||||||
timeZone: "Europe/Istanbul",
|
timeZone: "Europe/Istanbul",
|
||||||
hour: "numeric",
|
hour: "numeric",
|
||||||
hour12: false,
|
hour12: false,
|
||||||
}).format(new Date());
|
}).format(new Date());
|
||||||
const h = Number.parseInt(hourStr, 10);
|
return Number.parseInt(hourStr, 10);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Whether `source` may be scraped right now.
|
||||||
|
* PL24 → 09:00–18:00, parts-catalogs → 09:00–19:00 (Europe/Istanbul).
|
||||||
|
* EMEX and everything else have no window (always true).
|
||||||
|
*/
|
||||||
|
export function isWithinTimeWindow(source: string): boolean {
|
||||||
|
if (source !== "pl24" && source !== "parts-catalogs") return true;
|
||||||
|
const h = currentIstanbulHour();
|
||||||
const endHour = source === "parts-catalogs" ? 19 : 18;
|
const endHour = source === "parts-catalogs" ? 19 : 18;
|
||||||
if (h < 9 || h >= endHour) {
|
return h >= 9 && h < endHour;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check PL24 business hours (09:00–18:00 Europe/Istanbul).
|
||||||
|
* Throws RateLimitError with delay until next 09:00 if outside window.
|
||||||
|
*/
|
||||||
|
export function checkTimeWindow(source: string): void {
|
||||||
|
if (!isWithinTimeWindow(source)) {
|
||||||
throw new RateLimitError(msUntilNext9AM());
|
throw new RateLimitError(msUntilNext9AM());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import {
|
|||||||
type OnModuleInit,
|
type OnModuleInit,
|
||||||
} from "@nestjs/common";
|
} from "@nestjs/common";
|
||||||
import { type Job, type Queue, Worker } from "bullmq";
|
import { type Job, type Queue, Worker } from "bullmq";
|
||||||
import { and, eq, isNull } from "drizzle-orm";
|
import { and, asc, eq, gt, inArray, isNull, notExists, sql } from "drizzle-orm";
|
||||||
import { CategoriesService } from "../categories/categories.service";
|
import { CategoriesService } from "../categories/categories.service";
|
||||||
import { DATABASE, type Database } from "../database/database.provider";
|
import { DATABASE, type Database } from "../database/database.provider";
|
||||||
import { categories, parts, vehicles } from "../database/schema/core";
|
import { categories, parts, vehicles } from "../database/schema/core";
|
||||||
@@ -17,6 +17,7 @@ import {
|
|||||||
checkCooldown,
|
checkCooldown,
|
||||||
checkTimeWindow,
|
checkTimeWindow,
|
||||||
initProgress,
|
initProgress,
|
||||||
|
isWithinTimeWindow,
|
||||||
updateProgress,
|
updateProgress,
|
||||||
} from "./prefetch-utils";
|
} from "./prefetch-utils";
|
||||||
import { PrefetchCategoryJobData, PrefetchInitJobData } from "./prefetch.types";
|
import { PrefetchCategoryJobData, PrefetchInitJobData } from "./prefetch.types";
|
||||||
@@ -24,6 +25,18 @@ import { CATALOG_PREFETCH_QUEUE } from "./queues/catalog-prefetch.queue";
|
|||||||
|
|
||||||
const MAX_DEPTH = 5;
|
const MAX_DEPTH = 5;
|
||||||
|
|
||||||
|
// ── Backfill scan (hourly cron) tuning ──
|
||||||
|
/** Vehicles queued per scan wave. */
|
||||||
|
const BACKFILL_BATCH_SIZE = 20;
|
||||||
|
/** Skip the wave entirely if the queue already has more than this many jobs pending. */
|
||||||
|
const BACKFILL_MAX_BACKLOG = 1000;
|
||||||
|
/** In-flight guard TTL (seconds) — safety net if a run dies without clearing. */
|
||||||
|
const BACKFILL_SCHEDULED_TTL = 6 * 60 * 60;
|
||||||
|
/** Only these decode sources have catalogs worth prefetching. */
|
||||||
|
const BACKFILL_SOURCES = ["pl24", "emex", "parts-catalogs"];
|
||||||
|
/** Redis key holding the rolling rescan cursor (last createdAt seen). */
|
||||||
|
const BACKFILL_CURSOR_KEY = "prefetch:backfill:cursor";
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy {
|
export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy {
|
||||||
private readonly logger = new Logger(PrefetchWorkerService.name);
|
private readonly logger = new Logger(PrefetchWorkerService.name);
|
||||||
@@ -74,6 +87,9 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy {
|
|||||||
await new Promise((r) => setTimeout(r, 15_000));
|
await new Promise((r) => setTimeout(r, 15_000));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (job.name === "backfill-scan") {
|
||||||
|
return this.processBackfillScan();
|
||||||
|
}
|
||||||
if (job.name === "prefetch-init") {
|
if (job.name === "prefetch-init") {
|
||||||
return this.processInit(job as Job<PrefetchInitJobData>);
|
return this.processInit(job as Job<PrefetchInitJobData>);
|
||||||
}
|
}
|
||||||
@@ -113,11 +129,29 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy {
|
|||||||
await initProgress(this.redis, vehicleId);
|
await initProgress(this.redis, vehicleId);
|
||||||
|
|
||||||
// Get all top-level categories for this vehicle
|
// Get all top-level categories for this vehicle
|
||||||
const topCategories = await this.db
|
let topCategories = await this.db
|
||||||
.select()
|
.select()
|
||||||
.from(categories)
|
.from(categories)
|
||||||
.where(and(eq(categories.vehicleId, vehicleId), isNull(categories.parentId)));
|
.where(and(eq(categories.vehicleId, vehicleId), isNull(categories.parentId)));
|
||||||
|
|
||||||
|
// Self-seed: a vehicle that was decoded but never viewed has no categories in
|
||||||
|
// DB yet. getCategoryTree fetches & inserts the top-level groups from upstream
|
||||||
|
// (PL24/PSA/EMEX) so backfill can proceed without a user opening the page.
|
||||||
|
// Cooldown/time-window were already enforced above, so this only runs in-window.
|
||||||
|
if (topCategories.length === 0) {
|
||||||
|
try {
|
||||||
|
await this.categoriesService.getCategoryTree(vehicleId);
|
||||||
|
topCategories = await this.db
|
||||||
|
.select()
|
||||||
|
.from(categories)
|
||||||
|
.where(and(eq(categories.vehicleId, vehicleId), isNull(categories.parentId)));
|
||||||
|
} catch (err) {
|
||||||
|
this.logger.warn(
|
||||||
|
`[prefetch] Top-category seed failed for ${vehicleId}: ${(err as Error).message}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (topCategories.length === 0) {
|
if (topCategories.length === 0) {
|
||||||
this.logger.log(`[prefetch] No categories for vehicle=${vehicleId}`);
|
this.logger.log(`[prefetch] No categories for vehicle=${vehicleId}`);
|
||||||
await updateProgress(this.redis, vehicleId, {
|
await updateProgress(this.redis, vehicleId, {
|
||||||
@@ -251,6 +285,124 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Backfill scan (hourly cron): find decoded vehicles whose catalog isn't fully
|
||||||
|
* prefetched and queue prefetch-init for them. prefetch-init is idempotent — it
|
||||||
|
* only fetches missing categories/parts, so re-running over a complete vehicle is
|
||||||
|
* cheap. Self-throttling via a queue-depth guard, per-source cooldown, and the
|
||||||
|
* business-hours window. Goal: no decoded vehicle is left without catalog data.
|
||||||
|
*/
|
||||||
|
private async processBackfillScan(): Promise<void> {
|
||||||
|
// Production only — dev uses a separate DB and must never sweep/scrape.
|
||||||
|
// Defense-in-depth in case a scan job lands here via a shared Redis/queue.
|
||||||
|
if (process.env.NODE_ENV !== "production") {
|
||||||
|
this.logger.log("[backfill] Skip — NODE_ENV != production");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Self-throttle: don't pile on if the queue is already deep — let it drain.
|
||||||
|
const counts = await this.queue.getJobCounts("waiting", "delayed", "active");
|
||||||
|
const backlog = (counts.waiting ?? 0) + (counts.delayed ?? 0) + (counts.active ?? 0);
|
||||||
|
if (backlog > BACKFILL_MAX_BACKLOG) {
|
||||||
|
this.logger.log(`[backfill] Skip — queue backlog ${backlog} > ${BACKFILL_MAX_BACKLOG}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Only target sources eligible right now: not in cooldown (user active) and
|
||||||
|
// inside their scrape window (PL24/parts-catalogs office hours; EMEX always).
|
||||||
|
const eligible: string[] = [];
|
||||||
|
for (const s of BACKFILL_SOURCES) {
|
||||||
|
if (await this.redis.exists(`prefetch:activity:${s}`)) continue;
|
||||||
|
if (!isWithinTimeWindow(s)) continue;
|
||||||
|
eligible.push(s);
|
||||||
|
}
|
||||||
|
if (eligible.length === 0) {
|
||||||
|
this.logger.log("[backfill] Skip — no eligible sources (cooldown / off-hours)");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const picked: Array<{ id: string; source: string }> = [];
|
||||||
|
const seen = new Set<string>();
|
||||||
|
const overfetch = BACKFILL_BATCH_SIZE * 4; // headroom for in-flight skips
|
||||||
|
|
||||||
|
const tryPick = async (v: { id: string; source: string | null }): Promise<void> => {
|
||||||
|
if (picked.length >= BACKFILL_BATCH_SIZE || seen.has(v.id) || !v.source) return;
|
||||||
|
if (await this.redis.exists(`prefetch:scheduled:${v.id}`)) return; // already in flight
|
||||||
|
seen.add(v.id);
|
||||||
|
picked.push({ id: v.id, source: v.source });
|
||||||
|
};
|
||||||
|
|
||||||
|
// Phase 1 — clear the obvious backlog first: decoded vehicles with zero parts.
|
||||||
|
const noParts = await this.db
|
||||||
|
.select({ id: vehicles.id, source: vehicles.source })
|
||||||
|
.from(vehicles)
|
||||||
|
.where(
|
||||||
|
and(
|
||||||
|
inArray(vehicles.source, eligible),
|
||||||
|
notExists(
|
||||||
|
this.db.select({ one: sql`1` }).from(parts).where(eq(parts.vehicleId, vehicles.id)),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.orderBy(asc(vehicles.createdAt))
|
||||||
|
.limit(overfetch);
|
||||||
|
|
||||||
|
for (const v of noParts) await tryPick(v);
|
||||||
|
|
||||||
|
// Phase 2 — rolling rescan of ALL decoded vehicles to gap-fill partially-fetched
|
||||||
|
// ones. A createdAt cursor walks forward and wraps around at the end.
|
||||||
|
if (picked.length < BACKFILL_BATCH_SIZE) {
|
||||||
|
const cursorObj = await this.redis.getJson<{ ts: string }>(BACKFILL_CURSOR_KEY);
|
||||||
|
const cursor = cursorObj?.ts ? new Date(cursorObj.ts) : new Date(0);
|
||||||
|
|
||||||
|
const rolling = await this.db
|
||||||
|
.select({ id: vehicles.id, source: vehicles.source, createdAt: vehicles.createdAt })
|
||||||
|
.from(vehicles)
|
||||||
|
.where(and(inArray(vehicles.source, eligible), gt(vehicles.createdAt, cursor)))
|
||||||
|
.orderBy(asc(vehicles.createdAt))
|
||||||
|
.limit(overfetch);
|
||||||
|
|
||||||
|
if (rolling.length === 0 && cursor.getTime() > 0) {
|
||||||
|
// Reached the end — restart the rescan from the beginning next wave.
|
||||||
|
await this.redis.setJson(
|
||||||
|
BACKFILL_CURSOR_KEY,
|
||||||
|
{ ts: new Date(0).toISOString() },
|
||||||
|
30 * 86400,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let lastTs: Date | null = null;
|
||||||
|
for (const v of rolling) {
|
||||||
|
lastTs = v.createdAt;
|
||||||
|
await tryPick(v);
|
||||||
|
}
|
||||||
|
if (lastTs) {
|
||||||
|
await this.redis.setJson(BACKFILL_CURSOR_KEY, { ts: lastTs.toISOString() }, 30 * 86400);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (picked.length === 0) {
|
||||||
|
this.logger.log("[backfill] No candidates this wave");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const v of picked) await this.enqueueInit(v.id, v.source);
|
||||||
|
this.logger.log(
|
||||||
|
`[backfill] Queued ${picked.length} vehicle(s) (sources=${eligible.join(",")}, backlog=${backlog})`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Queue a prefetch-init for a vehicle and set the in-flight guard. */
|
||||||
|
private async enqueueInit(vehicleId: string, source: string): Promise<void> {
|
||||||
|
await this.queue.add(
|
||||||
|
"prefetch-init",
|
||||||
|
{ vehicleId, source: source as PrefetchInitJobData["source"] },
|
||||||
|
{ removeOnComplete: { count: 1000 }, removeOnFail: { count: 5000 } },
|
||||||
|
);
|
||||||
|
// Guard cleared on completion (incrementCompleted) or by TTL if the run dies.
|
||||||
|
await this.redis.set(`prefetch:scheduled:${vehicleId}`, "1", BACKFILL_SCHEDULED_TTL);
|
||||||
|
}
|
||||||
|
|
||||||
// ==================== Helpers ====================
|
// ==================== Helpers ====================
|
||||||
|
|
||||||
private async queueCategoryJob(
|
private async queueCategoryJob(
|
||||||
|
|||||||
246
apps/web/src/components/categories/catalog-search.tsx
Normal file
246
apps/web/src/components/categories/catalog-search.tsx
Normal file
@@ -0,0 +1,246 @@
|
|||||||
|
import { api } from "@/lib/api-client";
|
||||||
|
import { getCategoryIcon } from "@/lib/category-icons";
|
||||||
|
import { Badge, Input } from "@sase/ui";
|
||||||
|
import { useQuery } from "@tanstack/react-query";
|
||||||
|
import { useNavigate } from "@tanstack/react-router";
|
||||||
|
import { ChevronRight, Info, Loader2, Package, Search, X } from "lucide-react";
|
||||||
|
import { useEffect, useState } from "react";
|
||||||
|
|
||||||
|
interface Crumb {
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface CategoryHit {
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
source?: string;
|
||||||
|
unavailable?: boolean;
|
||||||
|
isLeaf: boolean;
|
||||||
|
matchingPartCount: number;
|
||||||
|
breadcrumb: Crumb[];
|
||||||
|
}
|
||||||
|
|
||||||
|
interface PartHit {
|
||||||
|
oemCode: string;
|
||||||
|
name: string;
|
||||||
|
categoryId: string;
|
||||||
|
categoryName: string;
|
||||||
|
unavailable?: boolean;
|
||||||
|
breadcrumb: Crumb[];
|
||||||
|
}
|
||||||
|
|
||||||
|
interface SearchResponse {
|
||||||
|
query: string;
|
||||||
|
categories: CategoryHit[];
|
||||||
|
parts: PartHit[];
|
||||||
|
treeIncomplete: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Strip PL24 raw escapes (\r\n, literal \-) and collapse whitespace — part and
|
||||||
|
* category names often arrive as multi-line illustration captions. */
|
||||||
|
function clean(s: string): string {
|
||||||
|
return (s || "").replace(/\r?\n/g, " ").replace(/\\/g, "").replace(/\s+/g, " ").trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
function useDebounced<T>(value: T, ms: number): T {
|
||||||
|
const [debounced, setDebounced] = useState(value);
|
||||||
|
useEffect(() => {
|
||||||
|
const id = setTimeout(() => setDebounced(value), ms);
|
||||||
|
return () => clearTimeout(id);
|
||||||
|
}, [value, ms]);
|
||||||
|
return debounced;
|
||||||
|
}
|
||||||
|
|
||||||
|
function Breadcrumb({ trail, tail }: { trail: Crumb[]; tail?: string }) {
|
||||||
|
const parts = [...trail.map((c) => clean(c.name)), ...(tail ? [clean(tail)] : [])];
|
||||||
|
if (parts.length === 0) return null;
|
||||||
|
return <span className="block truncate text-xs text-muted-foreground">{parts.join(" › ")}</span>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Cross-tree catalog search for one vehicle. Queries the server over everything
|
||||||
|
* already drilled into the DB and renders two sections: matching leaf categories
|
||||||
|
* and matching parts (OEM). Calls `onActiveChange` so the parent can hide the
|
||||||
|
* normal category browse while a search is active.
|
||||||
|
*/
|
||||||
|
export function CatalogSearch({
|
||||||
|
vehicleId,
|
||||||
|
onActiveChange,
|
||||||
|
}: {
|
||||||
|
vehicleId: string;
|
||||||
|
onActiveChange?: (active: boolean) => void;
|
||||||
|
}) {
|
||||||
|
const navigate = useNavigate();
|
||||||
|
const [input, setInput] = useState("");
|
||||||
|
const debounced = useDebounced(input.trim(), 300);
|
||||||
|
const active = debounced.length >= 2;
|
||||||
|
|
||||||
|
// biome-ignore lint/correctness/useExhaustiveDependencies: setter from parent is stable
|
||||||
|
useEffect(() => {
|
||||||
|
onActiveChange?.(active);
|
||||||
|
}, [active]);
|
||||||
|
|
||||||
|
const { data, isLoading, isFetching } = useQuery({
|
||||||
|
queryKey: ["catalog-search", vehicleId, debounced],
|
||||||
|
queryFn: () =>
|
||||||
|
api.get<SearchResponse>(`/categories/search/${vehicleId}?q=${encodeURIComponent(debounced)}`),
|
||||||
|
enabled: active,
|
||||||
|
staleTime: 60_000,
|
||||||
|
});
|
||||||
|
|
||||||
|
const goCategory = (categoryId: string) =>
|
||||||
|
navigate({
|
||||||
|
to: "/dashboard/vehicles/$id/categories/$categoryId",
|
||||||
|
params: { id: vehicleId, categoryId },
|
||||||
|
});
|
||||||
|
|
||||||
|
const categories = data?.categories ?? [];
|
||||||
|
const parts = data?.parts ?? [];
|
||||||
|
const nothing = active && !isLoading && categories.length === 0 && parts.length === 0;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div>
|
||||||
|
<div className="relative mb-4">
|
||||||
|
<Search className="pointer-events-none absolute left-3 top-1/2 h-4 w-4 -translate-y-1/2 text-muted-foreground" />
|
||||||
|
<Input
|
||||||
|
type="search"
|
||||||
|
value={input}
|
||||||
|
onChange={(e) => setInput(e.target.value)}
|
||||||
|
placeholder="Tüm katalogda ara — ör. fren balatası veya OEM kodu"
|
||||||
|
aria-label="Tüm katalogda ara"
|
||||||
|
className="pl-9 pr-9"
|
||||||
|
/>
|
||||||
|
{active && isFetching ? (
|
||||||
|
<Loader2 className="absolute right-3 top-1/2 h-4 w-4 -translate-y-1/2 animate-spin text-muted-foreground" />
|
||||||
|
) : input ? (
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={() => setInput("")}
|
||||||
|
aria-label="Aramayı temizle"
|
||||||
|
className="absolute right-3 top-1/2 -translate-y-1/2 text-muted-foreground hover:text-foreground"
|
||||||
|
>
|
||||||
|
<X className="h-4 w-4" />
|
||||||
|
</button>
|
||||||
|
) : null}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{active && (
|
||||||
|
<div className="space-y-6">
|
||||||
|
{isLoading ? (
|
||||||
|
<div className="flex items-center justify-center gap-2 py-8 text-sm text-muted-foreground">
|
||||||
|
<Loader2 className="h-4 w-4 animate-spin" /> Aranıyor…
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
<>
|
||||||
|
{data?.treeIncomplete && (
|
||||||
|
<div className="flex items-start gap-2 rounded-lg border border-amber-500/30 bg-amber-500/5 p-3 text-xs text-muted-foreground">
|
||||||
|
<Info className="mt-0.5 h-4 w-4 flex-shrink-0 text-amber-500" />
|
||||||
|
<span>
|
||||||
|
Bu aracın kataloğu henüz tam indirilmemiş. Arama yalnızca şu ana kadar açılmış
|
||||||
|
kategorileri kapsar — kategorileri gezdikçe kapsam genişler.
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{nothing ? (
|
||||||
|
<p className="py-8 text-center text-sm text-muted-foreground">
|
||||||
|
"{debounced}" ile eşleşen kategori veya parça bulunamadı.
|
||||||
|
</p>
|
||||||
|
) : (
|
||||||
|
<>
|
||||||
|
{/* Section 1 — leaf categories */}
|
||||||
|
<section>
|
||||||
|
<h3 className="mb-2 text-xs font-semibold uppercase tracking-wide text-muted-foreground">
|
||||||
|
Kategoriler {categories.length > 0 && `(${categories.length})`}
|
||||||
|
</h3>
|
||||||
|
{categories.length === 0 ? (
|
||||||
|
<p className="text-sm text-muted-foreground">Eşleşen kategori yok.</p>
|
||||||
|
) : (
|
||||||
|
<ul className="space-y-1.5">
|
||||||
|
{categories.map((c) => {
|
||||||
|
const Icon = getCategoryIcon(c.name);
|
||||||
|
return (
|
||||||
|
<li key={c.id}>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={() => goCategory(c.id)}
|
||||||
|
disabled={c.unavailable}
|
||||||
|
className={`group flex w-full items-center gap-3 rounded-lg border border-border p-3 text-left transition-colors hover:border-foreground/20 hover:bg-muted/40 ${c.unavailable ? "opacity-40" : ""}`}
|
||||||
|
>
|
||||||
|
<Icon className="h-5 w-5 flex-shrink-0 text-primary" />
|
||||||
|
<span className="min-w-0 flex-1">
|
||||||
|
<span className="flex items-center gap-2">
|
||||||
|
<span className="truncate text-sm font-medium">
|
||||||
|
{clean(c.name)}
|
||||||
|
</span>
|
||||||
|
{c.matchingPartCount > 0 && (
|
||||||
|
<Badge
|
||||||
|
variant="secondary"
|
||||||
|
className="flex-shrink-0 text-[11px]"
|
||||||
|
>
|
||||||
|
{c.matchingPartCount} parça
|
||||||
|
</Badge>
|
||||||
|
)}
|
||||||
|
</span>
|
||||||
|
<Breadcrumb trail={c.breadcrumb} />
|
||||||
|
</span>
|
||||||
|
<ChevronRight className="h-4 w-4 flex-shrink-0 text-muted-foreground transition-transform group-hover:translate-x-0.5" />
|
||||||
|
</button>
|
||||||
|
</li>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</ul>
|
||||||
|
)}
|
||||||
|
</section>
|
||||||
|
|
||||||
|
{/* Section 2 — parts (OEM) */}
|
||||||
|
<section>
|
||||||
|
<h3 className="mb-2 text-xs font-semibold uppercase tracking-wide text-muted-foreground">
|
||||||
|
Parçalar (OEM) {parts.length > 0 && `(${parts.length})`}
|
||||||
|
</h3>
|
||||||
|
{parts.length === 0 ? (
|
||||||
|
<p className="text-sm text-muted-foreground">
|
||||||
|
Eşleşen parça yok. (Yalnızca açılmış kategorilerin parçaları aranır.)
|
||||||
|
</p>
|
||||||
|
) : (
|
||||||
|
<ul className="space-y-1.5">
|
||||||
|
{parts.map((p, i) => (
|
||||||
|
<li key={`${p.oemCode}-${p.categoryId}-${i}`}>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={() => goCategory(p.categoryId)}
|
||||||
|
disabled={p.unavailable}
|
||||||
|
className={`group flex w-full items-center gap-3 rounded-lg border border-border p-3 text-left transition-colors hover:border-foreground/20 hover:bg-muted/40 ${p.unavailable ? "opacity-40" : ""}`}
|
||||||
|
>
|
||||||
|
<Package className="h-5 w-5 flex-shrink-0 text-muted-foreground" />
|
||||||
|
<span className="min-w-0 flex-1">
|
||||||
|
<span className="flex items-center gap-2">
|
||||||
|
<Badge
|
||||||
|
variant="secondary"
|
||||||
|
className="flex-shrink-0 font-mono text-[11px]"
|
||||||
|
>
|
||||||
|
{p.oemCode}
|
||||||
|
</Badge>
|
||||||
|
<span className="truncate text-sm font-medium">
|
||||||
|
{clean(p.name)}
|
||||||
|
</span>
|
||||||
|
</span>
|
||||||
|
<Breadcrumb trail={p.breadcrumb} tail={p.categoryName} />
|
||||||
|
</span>
|
||||||
|
<ChevronRight className="h-4 w-4 flex-shrink-0 text-muted-foreground transition-transform group-hover:translate-x-0.5" />
|
||||||
|
</button>
|
||||||
|
</li>
|
||||||
|
))}
|
||||||
|
</ul>
|
||||||
|
)}
|
||||||
|
</section>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -25,6 +25,9 @@ interface CategoryGridProps {
|
|||||||
brandName?: string;
|
brandName?: string;
|
||||||
parentId?: string;
|
parentId?: string;
|
||||||
variantSearch?: { body?: string; engine?: string; gearbox?: string };
|
variantSearch?: { body?: string; engine?: string; gearbox?: string };
|
||||||
|
/** Hide the per-level "Kategori ara…" filter — used where a broader
|
||||||
|
* cross-tree catalog search already sits above the grid. */
|
||||||
|
hideFilter?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function CategoryGrid({
|
export function CategoryGrid({
|
||||||
@@ -34,6 +37,7 @@ export function CategoryGrid({
|
|||||||
brandName,
|
brandName,
|
||||||
parentId,
|
parentId,
|
||||||
variantSearch,
|
variantSearch,
|
||||||
|
hideFilter,
|
||||||
}: CategoryGridProps) {
|
}: CategoryGridProps) {
|
||||||
const { t } = useTranslation();
|
const { t } = useTranslation();
|
||||||
const queryClient = useQueryClient();
|
const queryClient = useQueryClient();
|
||||||
@@ -58,7 +62,7 @@ export function CategoryGrid({
|
|||||||
const visibleCategories = trimmedFilter
|
const visibleCategories = trimmedFilter
|
||||||
? categories.filter((c) => c.name.toLocaleLowerCase("tr").includes(trimmedFilter))
|
? categories.filter((c) => c.name.toLocaleLowerCase("tr").includes(trimmedFilter))
|
||||||
: categories;
|
: categories;
|
||||||
const showFilter = categories.length > 8;
|
const showFilter = !hideFilter && categories.length > 8;
|
||||||
|
|
||||||
// Prefetch schema images for leaf categories in batches of 2
|
// Prefetch schema images for leaf categories in batches of 2
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import { CatalogSearch } from "@/components/categories/catalog-search";
|
||||||
import { CategoryBreadcrumb } from "@/components/categories/category-breadcrumb";
|
import { CategoryBreadcrumb } from "@/components/categories/category-breadcrumb";
|
||||||
import { CategoryColumns } from "@/components/categories/category-columns";
|
import { CategoryColumns } from "@/components/categories/category-columns";
|
||||||
import { CategoryGrid } from "@/components/categories/category-grid";
|
import { CategoryGrid } from "@/components/categories/category-grid";
|
||||||
@@ -34,6 +35,9 @@ function VehicleDetailPage() {
|
|||||||
const [viewMode, setViewMode] = useState<"grid" | "tree" | "columns">(
|
const [viewMode, setViewMode] = useState<"grid" | "tree" | "columns">(
|
||||||
() => getUserSettings().categoryViewMode ?? "grid",
|
() => getUserSettings().categoryViewMode ?? "grid",
|
||||||
);
|
);
|
||||||
|
// When a cross-tree catalog search is active, the normal browse (and its view
|
||||||
|
// toggle) are hidden in favour of the search results.
|
||||||
|
const [searchActive, setSearchActive] = useState(false);
|
||||||
|
|
||||||
const changeViewMode = (mode: "grid" | "tree" | "columns") => {
|
const changeViewMode = (mode: "grid" | "tree" | "columns") => {
|
||||||
setViewMode(mode);
|
setViewMode(mode);
|
||||||
@@ -215,24 +219,32 @@ function VehicleDetailPage() {
|
|||||||
<Card>
|
<Card>
|
||||||
<CardHeader className="flex flex-row items-center justify-between">
|
<CardHeader className="flex flex-row items-center justify-between">
|
||||||
<CardTitle className="text-base">Yedek Parça Kategorileri</CardTitle>
|
<CardTitle className="text-base">Yedek Parça Kategorileri</CardTitle>
|
||||||
<CategoryViewToggle value={viewMode} onChange={changeViewMode} />
|
{!searchActive && <CategoryViewToggle value={viewMode} onChange={changeViewMode} />}
|
||||||
</CardHeader>
|
</CardHeader>
|
||||||
<CardContent
|
<CardContent
|
||||||
className={viewMode === "columns" ? "p-0 overflow-hidden rounded-b-lg" : undefined}
|
className={
|
||||||
|
viewMode === "columns" && !searchActive ? "p-0 overflow-hidden rounded-b-lg" : undefined
|
||||||
|
}
|
||||||
>
|
>
|
||||||
{categoriesLoading ? (
|
{/* Search box keeps its own padding when the columns view zeroes out
|
||||||
<div className="space-y-2">
|
CardContent padding. */}
|
||||||
{KEYS_8.map((__k) => (
|
<div className={viewMode === "columns" && !searchActive ? "p-6 pb-0" : undefined}>
|
||||||
<Skeleton key={__k} className="h-8 w-full" />
|
<CatalogSearch vehicleId={id} onActiveChange={setSearchActive} />
|
||||||
))}
|
</div>
|
||||||
</div>
|
{!searchActive &&
|
||||||
) : viewMode === "grid" ? (
|
(categoriesLoading ? (
|
||||||
<CategoryGrid categories={categoryTree || []} vehicleId={id} />
|
<div className="space-y-2">
|
||||||
) : viewMode === "tree" ? (
|
{KEYS_8.map((__k) => (
|
||||||
<CategoryTree categories={categoryTree || []} vehicleId={id} />
|
<Skeleton key={__k} className="h-8 w-full" />
|
||||||
) : (
|
))}
|
||||||
<CategoryColumns categories={categoryTree || []} vehicleId={id} />
|
</div>
|
||||||
)}
|
) : viewMode === "grid" ? (
|
||||||
|
<CategoryGrid categories={categoryTree || []} vehicleId={id} hideFilter />
|
||||||
|
) : viewMode === "tree" ? (
|
||||||
|
<CategoryTree categories={categoryTree || []} vehicleId={id} />
|
||||||
|
) : (
|
||||||
|
<CategoryColumns categories={categoryTree || []} vehicleId={id} />
|
||||||
|
))}
|
||||||
</CardContent>
|
</CardContent>
|
||||||
</Card>
|
</Card>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
Reference in New Issue
Block a user