feat: EMEX catalog integration — schema, data migration, backend & frontend
Some checks failed
CI / Lint, Typecheck, Test & Build (push) Has been cancelled
Some checks failed
CI / Lint, Typecheck, Test & Build (push) Has been cancelled
Redesign emex_* tables from vehicle-centric to catalog-centric structure with junction tables (emex_vehicle_group_links, emex_vehicle_part_links). Migrate 92M+ rows from emex DB via postgres_fdw. Add EmexCatalogService for browse and VIN pre-scraped matching, with Redis caching. Frontend catalog page now has PL24/Emex tabs with full drill-down routes (brand → vehicle → group → parts). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
222
scripts/emex-migrate.sql
Normal file
222
scripts/emex-migrate.sql
Normal file
@@ -0,0 +1,222 @@
|
||||
-- EMEX Data Migration: emex DB → sase DB via postgres_fdw
|
||||
-- Both DBs on same PG server (100.66.11.79)
|
||||
-- Run as: psql -h 100.66.11.79 -U sase -d sase -f scripts/emex-migrate.sql
|
||||
|
||||
BEGIN;
|
||||
|
||||
-- ─── 1. Setup FDW ──────────────────────────────────
|
||||
CREATE EXTENSION IF NOT EXISTS postgres_fdw;
|
||||
|
||||
DO $$ BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_foreign_server WHERE srvname = 'emex_server') THEN
|
||||
CREATE SERVER emex_server FOREIGN DATA WRAPPER postgres_fdw OPTIONS (dbname 'emex');
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
DO $$ BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_user_mapping
|
||||
WHERE umuser = (SELECT usesysid FROM pg_user WHERE usename = 'sase')
|
||||
AND umserver = (SELECT oid FROM pg_foreign_server WHERE srvname = 'emex_server')
|
||||
) THEN
|
||||
CREATE USER MAPPING FOR sase SERVER emex_server OPTIONS (user 'sase', password 'f2bbcab1e2ad5c00cef3aeb4f8f48c8d774e1f2436cd86ab');
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
-- Create schema for foreign tables
|
||||
CREATE SCHEMA IF NOT EXISTS emex_foreign;
|
||||
|
||||
-- Import foreign tables
|
||||
IMPORT FOREIGN SCHEMA public
|
||||
LIMIT TO (catalogs, vehicles, part_groups, parts, part_numbers, vehicle_groups, vehicle_parts, part_images)
|
||||
FROM SERVER emex_server INTO emex_foreign;
|
||||
|
||||
-- ─── 2. Temp mapping tables (emex int id → sase uuid) ──────
|
||||
CREATE TEMP TABLE _map_catalogs (source_id int PRIMARY KEY, uuid uuid NOT NULL);
|
||||
CREATE TEMP TABLE _map_vehicles (source_id int PRIMARY KEY, uuid uuid NOT NULL);
|
||||
CREATE TEMP TABLE _map_groups (source_id int PRIMARY KEY, uuid uuid NOT NULL);
|
||||
CREATE TEMP TABLE _map_parts (source_id int PRIMARY KEY, uuid uuid NOT NULL);
|
||||
|
||||
-- ─── 3. Migrate catalogs (55 rows) ─────────────────
|
||||
INSERT INTO emex_catalogs (catalog_id, code, brand_name, description, source_id)
|
||||
SELECT
|
||||
c.code,
|
||||
c.code,
|
||||
COALESCE(c.brand, c.name),
|
||||
c.name,
|
||||
c.id
|
||||
FROM emex_foreign.catalogs c
|
||||
ON CONFLICT (catalog_id) DO NOTHING;
|
||||
|
||||
INSERT INTO _map_catalogs (source_id, uuid)
|
||||
SELECT ec.source_id, ec.id
|
||||
FROM emex_catalogs ec
|
||||
WHERE ec.source_id IS NOT NULL;
|
||||
|
||||
-- ─── 4. Migrate vehicles (12,561 rows) ─────────────
|
||||
INSERT INTO emex_vehicles (
|
||||
catalog_id, vehicle_id, name, engine, engine_code,
|
||||
body_type, transmission, drive_type, fuel_type,
|
||||
year_from, year_to, ssd, options_raw, raw_data, source_id
|
||||
)
|
||||
SELECT
|
||||
mc.uuid,
|
||||
'emex-' || v.id::text,
|
||||
v.name,
|
||||
v.engine,
|
||||
v.engine_code,
|
||||
v.body_type,
|
||||
v.transmission,
|
||||
v.drive_type,
|
||||
v.fuel_type,
|
||||
v.year_from,
|
||||
v.year_to,
|
||||
v.ssd,
|
||||
v.options->>'raw',
|
||||
v.options,
|
||||
v.id
|
||||
FROM emex_foreign.vehicles v
|
||||
JOIN _map_catalogs mc ON mc.source_id = v.catalog_id
|
||||
ON CONFLICT (vehicle_id) DO NOTHING;
|
||||
|
||||
INSERT INTO _map_vehicles (source_id, uuid)
|
||||
SELECT ev.source_id, ev.id
|
||||
FROM emex_vehicles ev
|
||||
WHERE ev.source_id IS NOT NULL;
|
||||
|
||||
-- ─── 5. Migrate part_groups (24,415 rows) ───────────
|
||||
INSERT INTO emex_part_groups (
|
||||
emex_catalog_id, group_id, name, name_original,
|
||||
parent_group_id, sort_order, has_parts, has_children, source_id
|
||||
)
|
||||
SELECT
|
||||
mc.uuid,
|
||||
pg.group_id,
|
||||
COALESCE(pg.name_local, pg.name),
|
||||
pg.name,
|
||||
pg.parent_id::text,
|
||||
pg.sort_order,
|
||||
COALESCE(pg.has_parts, false),
|
||||
COALESCE(pg.has_children, false),
|
||||
pg.id
|
||||
FROM emex_foreign.part_groups pg
|
||||
JOIN _map_catalogs mc ON mc.source_id = pg.catalog_id
|
||||
ON CONFLICT (emex_catalog_id, group_id) DO NOTHING;
|
||||
|
||||
INSERT INTO _map_groups (source_id, uuid)
|
||||
SELECT epg.source_id, epg.id
|
||||
FROM emex_part_groups epg
|
||||
WHERE epg.source_id IS NOT NULL;
|
||||
|
||||
COMMIT;
|
||||
|
||||
-- ─── 6. Migrate parts (1.59M rows) — batched ───────
|
||||
-- Run outside transaction for large data
|
||||
\echo 'Migrating parts (1.59M rows)...'
|
||||
|
||||
INSERT INTO emex_parts (
|
||||
emex_catalog_id, group_id, part_number, name, name_original,
|
||||
description, oem_number, source_id
|
||||
)
|
||||
SELECT
|
||||
mc.uuid,
|
||||
mg.uuid,
|
||||
p.part_number,
|
||||
COALESCE(p.name_local, p.name),
|
||||
p.name,
|
||||
p.description,
|
||||
p.oem_number,
|
||||
p.id
|
||||
FROM emex_foreign.parts p
|
||||
JOIN _map_catalogs mc ON mc.source_id = p.catalog_id
|
||||
LEFT JOIN _map_groups mg ON mg.source_id = p.group_id
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
INSERT INTO _map_parts (source_id, uuid)
|
||||
SELECT ep.source_id, ep.id
|
||||
FROM emex_parts ep
|
||||
WHERE ep.source_id IS NOT NULL;
|
||||
|
||||
-- ─── 7. Migrate schema_pics (deduplicated by local_path) ────
|
||||
\echo 'Migrating schema pics (deduplicated)...'
|
||||
|
||||
INSERT INTO emex_schema_pics (
|
||||
group_id, image_url, original_url, local_path, sort_order
|
||||
)
|
||||
SELECT DISTINCT ON (mg.uuid, pi.local_path)
|
||||
mg.uuid,
|
||||
CASE
|
||||
WHEN pi.local_path IS NOT NULL THEN 'https://storage.sase.tr/es/' || replace(pi.local_path, 'images/', '')
|
||||
ELSE pi.original_url
|
||||
END,
|
||||
pi.original_url,
|
||||
pi.local_path,
|
||||
pi.sort_order
|
||||
FROM emex_foreign.part_images pi
|
||||
JOIN _map_groups mg ON mg.source_id = pi.group_id
|
||||
WHERE pi.group_id IS NOT NULL
|
||||
AND pi.local_path IS NOT NULL
|
||||
ORDER BY mg.uuid, pi.local_path, pi.sort_order
|
||||
ON CONFLICT (group_id, local_path) DO NOTHING;
|
||||
|
||||
-- ─── 8. Migrate vehicle_groups → emex_vehicle_group_links (4.69M) ──
|
||||
\echo 'Migrating vehicle-group links (4.69M rows)...'
|
||||
|
||||
INSERT INTO emex_vehicle_group_links (emex_vehicle_id, emex_group_id, ssd)
|
||||
SELECT
|
||||
mv.uuid,
|
||||
mg.uuid,
|
||||
vg.ssd
|
||||
FROM emex_foreign.vehicle_groups vg
|
||||
JOIN _map_vehicles mv ON mv.source_id = vg.vehicle_id
|
||||
JOIN _map_groups mg ON mg.source_id = vg.group_id
|
||||
ON CONFLICT (emex_vehicle_id, emex_group_id) DO NOTHING;
|
||||
|
||||
-- ─── 9. Migrate vehicle_parts → emex_vehicle_part_links (83.5M) ──
|
||||
\echo 'Migrating vehicle-part links (83.5M rows) — this will take a while...'
|
||||
|
||||
INSERT INTO emex_vehicle_part_links (emex_vehicle_id, emex_part_id, emex_group_id, quantity, position)
|
||||
SELECT
|
||||
mv.uuid,
|
||||
mp.uuid,
|
||||
mg.uuid,
|
||||
vp.quantity::integer,
|
||||
vp.position
|
||||
FROM emex_foreign.vehicle_parts vp
|
||||
JOIN _map_vehicles mv ON mv.source_id = vp.vehicle_id
|
||||
JOIN _map_parts mp ON mp.source_id = vp.part_id
|
||||
LEFT JOIN _map_groups mg ON mg.source_id = vp.group_id
|
||||
ON CONFLICT (emex_vehicle_id, emex_part_id, emex_group_id) DO NOTHING;
|
||||
|
||||
-- ─── 10. Migrate part_numbers (285M) ───────────────
|
||||
\echo 'Migrating part numbers (285M rows) — this will take the longest...'
|
||||
|
||||
INSERT INTO emex_part_numbers (emex_part_id, oem_code, is_main)
|
||||
SELECT
|
||||
mp.uuid,
|
||||
pn.number,
|
||||
CASE WHEN pn.number_type = 'OEM' THEN true ELSE false END
|
||||
FROM emex_foreign.part_numbers pn
|
||||
JOIN _map_parts mp ON mp.source_id = pn.part_id
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
-- ─── 11. Verification ──────────────────────────────
|
||||
\echo 'Verifying migration...'
|
||||
|
||||
SELECT 'emex_catalogs' as tbl, count(*) FROM emex_catalogs
|
||||
UNION ALL SELECT 'emex_vehicles', count(*) FROM emex_vehicles
|
||||
UNION ALL SELECT 'emex_part_groups', count(*) FROM emex_part_groups
|
||||
UNION ALL SELECT 'emex_parts', count(*) FROM emex_parts
|
||||
UNION ALL SELECT 'emex_schema_pics', count(*) FROM emex_schema_pics
|
||||
UNION ALL SELECT 'emex_vehicle_group_links', count(*) FROM emex_vehicle_group_links
|
||||
UNION ALL SELECT 'emex_vehicle_part_links', count(*) FROM emex_vehicle_part_links
|
||||
UNION ALL SELECT 'emex_part_numbers', count(*) FROM emex_part_numbers
|
||||
ORDER BY 1;
|
||||
|
||||
-- ─── 12. Cleanup ───────────────────────────────────
|
||||
DROP TABLE IF EXISTS _map_catalogs;
|
||||
DROP TABLE IF EXISTS _map_vehicles;
|
||||
DROP TABLE IF EXISTS _map_groups;
|
||||
DROP TABLE IF EXISTS _map_parts;
|
||||
|
||||
\echo 'Migration complete!'
|
||||
Reference in New Issue
Block a user