From f25ef2d54fd88c47c6e3ccb7ca8d726209737fd9 Mon Sep 17 00:00:00 2001 From: Semih Yesilyurt Date: Mon, 6 Jul 2026 08:46:59 +0300 Subject: [PATCH] =?UTF-8?q?feat(canonical):=20marka/platform=20yap=C4=B1?= =?UTF-8?q?=20=C5=9Fablonu=20=E2=80=94=20decode=20an=C4=B1nda=20tam=20kano?= =?UTF-8?q?nik=20yap=C4=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit canonical_structure_template(source, brand, bucket, sub, vehicle_count): drill edilmiş araçlardan her (kaynak, marka) için görülen kova/alt-yaprak kümesi. Backfill sonunda yeniden üretilir (global run). Serving overlayTemplate ile aracın ağacına markanın bilinen ama bu araçta henüz drill edilmemiş kova/alt- yapraklarını BOŞ ekler → VIN decode olur olmaz FULL kanonik yapı görünür; parçalar backfill ile dolar. migration 0029. vehicle + catalog akışı. Co-Authored-By: Claude Opus 4.8 --- .../0029_canonical_structure_template.sql | 11 ++++ apps/api/drizzle/meta/_journal.json | 7 ++ apps/api/src/catalog/catalog.service.ts | 31 ++++++++- apps/api/src/categories/canonical-grouping.ts | 66 +++++++++++++++++++ apps/api/src/categories/categories.service.ts | 33 +++++++++- apps/api/src/database/schema/core.ts | 22 +++++++ .../canonical-backfill.processor.ts | 29 ++++++++ 7 files changed, 193 insertions(+), 6 deletions(-) create mode 100644 apps/api/drizzle/0029_canonical_structure_template.sql diff --git a/apps/api/drizzle/0029_canonical_structure_template.sql b/apps/api/drizzle/0029_canonical_structure_template.sql new file mode 100644 index 0000000..c2b834c --- /dev/null +++ b/apps/api/drizzle/0029_canonical_structure_template.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS "canonical_structure_template" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "source" varchar(20) NOT NULL, + "brand" varchar(120) NOT NULL, + "canonical_category_id" uuid NOT NULL, + "canonical_subcategory_id" uuid, + "vehicle_count" integer DEFAULT 0 NOT NULL, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "canonical_structure_template_source_brand_idx" ON "canonical_structure_template" ("source","brand"); diff --git a/apps/api/drizzle/meta/_journal.json b/apps/api/drizzle/meta/_journal.json index 5b0a7df..0bbe977 100644 --- a/apps/api/drizzle/meta/_journal.json +++ b/apps/api/drizzle/meta/_journal.json @@ -204,6 +204,13 @@ "when": 1783089194145, "tag": "0028_canonical_subcat_v3", "breakpoints": true + }, + { + "idx": 29, + "version": "7", + "when": 1783090194145, + "tag": "0029_canonical_structure_template", + "breakpoints": true } ] } \ No newline at end of file diff --git a/apps/api/src/catalog/catalog.service.ts b/apps/api/src/catalog/catalog.service.ts index a3cda92..a232803 100644 --- a/apps/api/src/catalog/catalog.service.ts +++ b/apps/api/src/catalog/catalog.service.ts @@ -1,10 +1,11 @@ import { ForbiddenException, Inject, Injectable, Logger, NotFoundException } from "@nestjs/common"; -import { and, eq, inArray, isNotNull, isNull, or, sql } from "drizzle-orm"; -import { groupCategoriesByCanonical } from "../categories/canonical-grouping"; +import { and, eq, gte, inArray, isNotNull, isNull, or, sql } from "drizzle-orm"; +import { groupCategoriesByCanonical, overlayTemplate } from "../categories/canonical-grouping"; import { DATABASE, type Database } from "../database/database.provider"; import { brands, canonicalCategories, + canonicalStructureTemplate, catalogVehicles, categories, parts, @@ -432,11 +433,35 @@ export class CatalogService { const drilledRoots = new Set(partLeaves.map((c) => rootOf(c.id))); const undrilledRoots = cats.filter((c) => c.parentId === null && !drilledRoots.has(c.id)); const input = [...partLeaves, ...undrilledRoots]; - return groupCategoriesByCanonical(input, partCounts, buckets, { + const groups = groupCategoriesByCanonical(input, partCounts, buckets, { requireParts: false, subLeaves, hasChildren, }); + + // Overlay the per-brand structure template (see CategoriesService.getCanonicalTree). + const cv = await this.db + .select({ brand: catalogVehicles.brandName, source: catalogVehicles.source }) + .from(catalogVehicles) + .where(eq(catalogVehicles.id, catalogVehicleId)) + .limit(1); + const brand = cv[0]?.brand; + const source = cats[0]?.source ?? cv[0]?.source; + if (!brand || !source) return groups; + const template = await this.db + .select({ + bucketId: canonicalStructureTemplate.canonicalCategoryId, + subId: canonicalStructureTemplate.canonicalSubcategoryId, + }) + .from(canonicalStructureTemplate) + .where( + and( + eq(canonicalStructureTemplate.source, source), + eq(canonicalStructureTemplate.brand, brand), + gte(canonicalStructureTemplate.vehicleCount, 2), + ), + ); + return overlayTemplate(groups, template, buckets, subLeaves); } async getCategoryTree( diff --git a/apps/api/src/categories/canonical-grouping.ts b/apps/api/src/categories/canonical-grouping.ts index 535c373..8136609 100644 --- a/apps/api/src/categories/canonical-grouping.ts +++ b/apps/api/src/categories/canonical-grouping.ts @@ -54,6 +54,72 @@ export interface CanonicalGroup { type LeafWithSub = CanonicalLeaf & { subId: string | null }; +export interface TemplateRow { + bucketId: string; + subId: string | null; +} + +// Overlay the per-brand structure template onto a vehicle's grouped tree so the +// FULL canonical structure shows immediately: any bucket / sub-leaf the brand is +// known to have but this vehicle hasn't deep-drilled yet is added as an empty +// (partCount 0) entry. As the prefetch backfill drills, these fill with real +// source categories + parts. Never removes the vehicle's own data. +export function overlayTemplate( + groups: CanonicalGroup[], + template: TemplateRow[], + buckets: CanonicalBucket[], + subLeaves: SubLeafRef[], +): CanonicalGroup[] { + if (template.length === 0) return groups; + const bucketById = new Map(buckets.map((b) => [b.id, b])); + const subById = new Map(subLeaves.map((s) => [s.id, s])); + const byBucket = new Map(groups.map((g) => [g.canonical.id, g])); + + const templBuckets = new Set(); + const templSubsByBucket = new Map>(); + for (const t of template) { + templBuckets.add(t.bucketId); + if (t.subId) { + const set = templSubsByBucket.get(t.bucketId) ?? new Set(); + set.add(t.subId); + templSubsByBucket.set(t.bucketId, set); + } + } + + const result = [...groups]; + for (const bid of templBuckets) { + let g = byBucket.get(bid); + if (!g) { + const b = bucketById.get(bid); + if (!b) continue; + g = { canonical: { id: b.id, slug: b.slug, name: b.name }, partCount: 0, categories: [] }; + result.push(g); + byBucket.set(bid, g); + } + const wantSubs = templSubsByBucket.get(bid); + if (!wantSubs || wantSubs.size === 0) continue; + if (!g.subgroups) g.subgroups = []; + const present = new Set(g.subgroups.map((sg) => sg.canonical.id).filter(Boolean)); + for (const sid of wantSubs) { + if (present.has(sid)) continue; + const s = subById.get(sid); + if (!s) continue; + g.subgroups.push({ + canonical: { id: s.id, slug: s.slug, name: s.name }, + partCount: 0, + categories: [], + }); + } + g.subgroups.sort((a, z) => z.partCount - a.partCount); + } + + const order = new Map(buckets.map((b, i) => [b.id, i])); + result.sort( + (a, z) => (order.get(a.canonical.id ?? "") ?? 999) - (order.get(z.canonical.id ?? "") ?? 999), + ); + return result; +} + export function groupCategoriesByCanonical( cats: RawCategory[], partCounts: Map, diff --git a/apps/api/src/categories/categories.service.ts b/apps/api/src/categories/categories.service.ts index f52f573..04102cd 100644 --- a/apps/api/src/categories/categories.service.ts +++ b/apps/api/src/categories/categories.service.ts @@ -1,5 +1,5 @@ import { Inject, Injectable, Logger, NotFoundException } from "@nestjs/common"; -import { and, eq, ilike, inArray, isNotNull, isNull, or, sql } from "drizzle-orm"; +import { and, eq, gte, ilike, inArray, isNotNull, isNull, or, sql } from "drizzle-orm"; import { type CatalogDegradationKind, reportCatalogDegradation, @@ -7,6 +7,7 @@ import { import { DATABASE, type Database } from "../database/database.provider"; import { canonicalCategories, + canonicalStructureTemplate, categories, parts, schemaPics, @@ -24,7 +25,7 @@ import { classifyNode, foldName, mapToCanonical } from "../jobs/canonical-lexico import { RedisService } from "../redis/redis.service"; import { StorageService } from "../storage/storage.service"; import { TranslationsService } from "../translations/translations.service"; -import { groupCategoriesByCanonical } from "./canonical-grouping"; +import { groupCategoriesByCanonical, overlayTemplate } from "./canonical-grouping"; @Injectable() export class CategoriesService { @@ -619,11 +620,37 @@ export class CategoriesService { const drilledRoots = new Set(partLeaves.map((c) => rootOf(c.id))); const undrilledRoots = cats.filter((c) => c.parentId === null && !drilledRoots.has(c.id)); const input = [...partLeaves, ...undrilledRoots]; - return groupCategoriesByCanonical(input, partCounts, buckets, { + const groups = groupCategoriesByCanonical(input, partCounts, buckets, { requireParts: false, subLeaves, hasChildren, }); + + // Overlay the per-brand structure template: buckets / sub-leaves this brand + // is known to have but this vehicle hasn't deep-drilled yet appear as empty + // entries, so the FULL canonical structure shows immediately after decode. + const veh = await this.db + .select({ brand: vehicles.brandName, source: vehicles.source }) + .from(vehicles) + .where(eq(vehicles.id, vehicleId)) + .limit(1); + const brand = veh[0]?.brand; + const source = cats[0]?.source ?? veh[0]?.source; + if (!brand || !source) return groups; + const template = await this.db + .select({ + bucketId: canonicalStructureTemplate.canonicalCategoryId, + subId: canonicalStructureTemplate.canonicalSubcategoryId, + }) + .from(canonicalStructureTemplate) + .where( + and( + eq(canonicalStructureTemplate.source, source), + eq(canonicalStructureTemplate.brand, brand), + gte(canonicalStructureTemplate.vehicleCount, 2), + ), + ); + return overlayTemplate(groups, template, buckets, subLeaves); } /** diff --git a/apps/api/src/database/schema/core.ts b/apps/api/src/database/schema/core.ts index a6e2aba..e99d192 100644 --- a/apps/api/src/database/schema/core.ts +++ b/apps/api/src/database/schema/core.ts @@ -1004,3 +1004,25 @@ export const categoryCanonicalMap = pgTable( index("category_canonical_map_canonical_idx").on(table.canonicalCategoryId), ], ); + +// ─── Canonical structure template (per source+brand) ────────────────────── +// The canonical STRUCTURE (which buckets/sub-leaves a platform has) is shared +// across vehicles of the same brand — only the parts differ. This table records, +// per (source, brand), each (bucket, sub-leaf) seen across already-drilled +// vehicles with a vehicle_count (support). On decode the serving overlays the +// brand template so the user sees the FULL canonical structure instantly, before +// this vehicle's own tree is deep-drilled by the prefetch backfill. Rebuilt at +// the end of each global canonical-backfill run (derived, so full-refresh). +export const canonicalStructureTemplate = pgTable( + "canonical_structure_template", + { + id: uuid("id").primaryKey().defaultRandom(), + source: varchar("source", { length: 20 }).notNull(), + brand: varchar("brand", { length: 120 }).notNull(), + canonicalCategoryId: uuid("canonical_category_id").notNull(), + canonicalSubcategoryId: uuid("canonical_subcategory_id"), + vehicleCount: integer("vehicle_count").default(0).notNull(), + updatedAt: timestamp("updated_at", { withTimezone: true }).defaultNow().notNull(), + }, + (table) => [index("canonical_structure_template_source_brand_idx").on(table.source, table.brand)], +); diff --git a/apps/api/src/jobs/processors/canonical-backfill.processor.ts b/apps/api/src/jobs/processors/canonical-backfill.processor.ts index c4fd999..a60dd99 100644 --- a/apps/api/src/jobs/processors/canonical-backfill.processor.ts +++ b/apps/api/src/jobs/processors/canonical-backfill.processor.ts @@ -263,6 +263,35 @@ export async function processCanonicalBackfill( `); stats.subcategoriesUpdated = subCount[0]?.n ?? 0; + // 6. rebuild the per (source, brand) structure template — the shared canonical + // skeleton the serving overlays so a freshly-decoded vehicle shows the FULL + // structure instantly (before its own deep-drill). Derived from all mapped + // categories, so only on a GLOBAL run (brand-scoped runs see a partial world). + if (!brands && !dryRun) { + await db.execute(sql`DELETE FROM canonical_structure_template`); + const tpl = await db.execute<{ n: number }>(sql` + WITH ins AS ( + INSERT INTO canonical_structure_template + (source, brand, canonical_category_id, canonical_subcategory_id, vehicle_count) + SELECT c.source, + coalesce(cv.brand_name, v.brand_name) AS brand, + c.canonical_category_id, + c.canonical_subcategory_id, + count(DISTINCT coalesce(c.catalog_vehicle_id, c.vehicle_id))::int + FROM categories c + LEFT JOIN catalog_vehicles cv ON c.catalog_vehicle_id = cv.id + LEFT JOIN vehicles v ON c.vehicle_id = v.id + WHERE c.canonical_category_id IS NOT NULL + AND coalesce(cv.brand_name, v.brand_name) IS NOT NULL + GROUP BY c.source, coalesce(cv.brand_name, v.brand_name), + c.canonical_category_id, c.canonical_subcategory_id + RETURNING 1 + ) + SELECT count(*)::int AS n FROM ins + `); + console.log(`[canonical-backfill] structure template rebuilt — ${tpl[0]?.n ?? 0} rows`); + } + console.log( `[canonical-backfill] done — ${rows.length} map rows (${stats.subMapped} with sub, ${stats.mapRowsPruned} pruned), ${stats.categoriesUpdated} materialized, ${stats.inherited} inherited, ${stats.subcategoriesUpdated} sub-categorized`, );