feat(FN-094): add comment line for deployment verification
Some checks failed
Sync dev → Gitea / Mirror dev to Gitea (push) Has been cancelled
Some checks failed
Sync dev → Gitea / Mirror dev to Gitea (push) Has been cancelled
- Added a comment line to main.ts for deployment verification purposes
This commit is contained in:
@@ -5,6 +5,11 @@ set -euo pipefail
|
||||
# Zero-downtime deploy script for sase-v2
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
# Load NVM so node/pnpm are on PATH under non-interactive SSH sessions
|
||||
export NVM_DIR="$HOME/.nvm"
|
||||
# shellcheck disable=SC1091
|
||||
[ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"
|
||||
|
||||
DEPLOY_DIR="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
LOG_PREFIX="[deploy]"
|
||||
|
||||
@@ -22,9 +27,13 @@ cd "$DEPLOY_DIR" || error_exit "Cannot change to project directory: $DEPLOY_DIR"
|
||||
log "Starting zero-downtime deployment..."
|
||||
log "Working directory: $DEPLOY_DIR"
|
||||
|
||||
# ── Step 1: Pull latest changes ──
|
||||
log "Pulling latest changes from origin..."
|
||||
git pull origin main || error_exit "git pull failed"
|
||||
# ── Step 1: Sync to origin/main ──
|
||||
# Hard reset (not pull) so stale build artifacts on the deploy server
|
||||
# (e.g. apps/web/tsconfig.tsbuildinfo) don't block updates. The deploy
|
||||
# server is treated as a deployment target, not a development checkout.
|
||||
log "Fetching and resetting to origin/main..."
|
||||
git fetch origin main || error_exit "git fetch failed"
|
||||
git reset --hard origin/main || error_exit "git reset --hard failed"
|
||||
|
||||
# ── Step 2: Install dependencies ──
|
||||
log "Installing dependencies (frozen lockfile)..."
|
||||
@@ -42,11 +51,13 @@ log "Pre-rendering public pages..."
|
||||
pnpm prerender || log "WARNING: Pre-render failed (non-fatal, continuing deploy)"
|
||||
cd "$DEPLOY_DIR"
|
||||
|
||||
# ── Step 4: Database migrations (safe push) ──
|
||||
log "Running database migrations (drizzle-kit push)..."
|
||||
cd apps/api
|
||||
pnpm db:push || error_exit "Database migration failed"
|
||||
cd "$DEPLOY_DIR"
|
||||
# ── Step 4: Database migrations (manual) ──
|
||||
# Schema migrations are NOT run automatically.
|
||||
# `pnpm db:push` is interactive and may prompt for destructive changes
|
||||
# (column drops, etc.) which cannot be answered over a non-interactive
|
||||
# SSH session. Run schema changes manually from apps/api:
|
||||
# pnpm db:push # interactive, dev/staging
|
||||
# pnpm db:generate # produce a versioned migration (recommended for prod)
|
||||
|
||||
# ── Step 5: Zero-downtime PM2 reload ──
|
||||
log "Reloading PM2 processes (zero-downtime)..."
|
||||
|
||||
194
scripts/emex-backfill-tr-names.ts
Normal file
194
scripts/emex-backfill-tr-names.ts
Normal file
@@ -0,0 +1,194 @@
|
||||
/**
|
||||
* EMEX Türkçe Backfill
|
||||
*
|
||||
* `emex_category_translations` tablosu doldurulduktan SONRA çalıştır.
|
||||
* `categories` ve `parts` tablolarındaki source='emex' kayıtlarının
|
||||
* `name` kolonunu çevirilmiş hali ile günceller. `name_original` korunur
|
||||
* (rollback edilebilir).
|
||||
*
|
||||
* İlk çalıştırmada UPDATE'leri hızlandıran partial index'leri oluşturur.
|
||||
* Sonunda EMEX kategori tree'lerinin Redis cache'ini flush eder.
|
||||
*
|
||||
* Kullanım:
|
||||
* pnpm tsx scripts/emex-backfill-tr-names.ts --dry-run
|
||||
* pnpm tsx scripts/emex-backfill-tr-names.ts --target=categories
|
||||
* pnpm tsx scripts/emex-backfill-tr-names.ts --target=parts
|
||||
* pnpm tsx scripts/emex-backfill-tr-names.ts # all
|
||||
*/
|
||||
|
||||
import * as path from "node:path";
|
||||
import * as dotenv from "dotenv";
|
||||
import postgres from "postgres";
|
||||
import Redis from "ioredis";
|
||||
|
||||
dotenv.config({ path: path.join(__dirname, "../apps/api/.env") });
|
||||
|
||||
// ---------- CLI ----------
|
||||
const argv = process.argv.slice(2);
|
||||
const dryRun = argv.includes("--dry-run");
|
||||
const targetArg = argv.find((a) => a.startsWith("--target="));
|
||||
const target = targetArg ? targetArg.split("=")[1] : "all";
|
||||
const chunkArg = argv.find((a) => a.startsWith("--chunk-size="));
|
||||
const chunkSize = chunkArg ? parseInt(chunkArg.split("=")[1], 10) : 50_000;
|
||||
|
||||
if (!["all", "categories", "parts"].includes(target)) {
|
||||
console.error(`Invalid --target: ${target}. Use one of: all, categories, parts`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
if (!process.env.DATABASE_URL) {
|
||||
throw new Error("DATABASE_URL env var is required");
|
||||
}
|
||||
const sql = postgres(process.env.DATABASE_URL, { max: 1 });
|
||||
|
||||
// Step 1: ensure partial indexes exist (idempotent)
|
||||
if (!dryRun) {
|
||||
console.log("Ensuring partial indexes...");
|
||||
await sql`
|
||||
CREATE INDEX IF NOT EXISTS categories_name_original_emex_idx
|
||||
ON categories(name_original) WHERE source = 'emex'
|
||||
`;
|
||||
await sql`
|
||||
CREATE INDEX IF NOT EXISTS parts_name_original_emex_idx
|
||||
ON parts(name_original) WHERE source = 'emex'
|
||||
`;
|
||||
}
|
||||
|
||||
// Step 2: report scope
|
||||
console.log("\nScope:");
|
||||
|
||||
const [{ count: trCount }] = (await sql`
|
||||
SELECT COUNT(*)::int AS count FROM emex_category_translations
|
||||
`) as unknown as { count: number }[];
|
||||
console.log(` emex_category_translations rows: ${trCount}`);
|
||||
if (trCount === 0) {
|
||||
console.warn("\nWARNING: emex_category_translations is empty.");
|
||||
console.warn("Run scripts/emex-translate-bootstrap.ts first.");
|
||||
}
|
||||
|
||||
if (target === "all" || target === "categories") {
|
||||
const [{ count: catUpd }] = (await sql`
|
||||
SELECT COUNT(*)::int AS count
|
||||
FROM categories c
|
||||
JOIN emex_category_translations t ON c.name_original = t.original_name
|
||||
WHERE c.source IN ('emex', 'parts-catalogs')
|
||||
AND c.name_original IS NOT NULL
|
||||
AND c.name <> t.translated_name
|
||||
`) as unknown as { count: number }[];
|
||||
console.log(` categories to update: ${catUpd}`);
|
||||
}
|
||||
|
||||
if (target === "all" || target === "parts") {
|
||||
const [{ count: partUpd }] = (await sql`
|
||||
SELECT COUNT(*)::int AS count
|
||||
FROM parts p
|
||||
JOIN emex_category_translations t ON p.name_original = t.original_name
|
||||
WHERE p.source IN ('emex', 'parts-catalogs')
|
||||
AND p.name_original IS NOT NULL
|
||||
AND p.name <> t.translated_name
|
||||
`) as unknown as { count: number }[];
|
||||
console.log(` parts to update: ${partUpd}`);
|
||||
}
|
||||
|
||||
if (dryRun) {
|
||||
console.log("\nDRY RUN — no UPDATE / Redis flush executed.");
|
||||
await sql.end();
|
||||
return;
|
||||
}
|
||||
|
||||
// Step 3: chunked UPDATE
|
||||
if (target === "all" || target === "categories") {
|
||||
console.log("\nUpdating categories...");
|
||||
let totalUpdated = 0;
|
||||
while (true) {
|
||||
const result = await sql`
|
||||
WITH targets AS (
|
||||
SELECT c.id, t.translated_name
|
||||
FROM categories c
|
||||
JOIN emex_category_translations t ON c.name_original = t.original_name
|
||||
WHERE c.source IN ('emex', 'parts-catalogs')
|
||||
AND c.name_original IS NOT NULL
|
||||
AND c.name <> t.translated_name
|
||||
LIMIT ${chunkSize}
|
||||
)
|
||||
UPDATE categories
|
||||
SET name = targets.translated_name
|
||||
FROM targets
|
||||
WHERE categories.id = targets.id
|
||||
RETURNING categories.id
|
||||
`;
|
||||
const n = result.count ?? result.length;
|
||||
totalUpdated += n;
|
||||
console.log(` +${n} (total: ${totalUpdated})`);
|
||||
if (n === 0) break;
|
||||
}
|
||||
console.log(`Categories updated: ${totalUpdated}`);
|
||||
}
|
||||
|
||||
if (target === "all" || target === "parts") {
|
||||
console.log("\nUpdating parts...");
|
||||
let totalUpdated = 0;
|
||||
while (true) {
|
||||
const result = await sql`
|
||||
WITH targets AS (
|
||||
SELECT p.id, t.translated_name
|
||||
FROM parts p
|
||||
JOIN emex_category_translations t ON p.name_original = t.original_name
|
||||
WHERE p.source IN ('emex', 'parts-catalogs')
|
||||
AND p.name_original IS NOT NULL
|
||||
AND p.name <> t.translated_name
|
||||
LIMIT ${chunkSize}
|
||||
)
|
||||
UPDATE parts
|
||||
SET name = targets.translated_name
|
||||
FROM targets
|
||||
WHERE parts.id = targets.id
|
||||
RETURNING parts.id
|
||||
`;
|
||||
const n = result.count ?? result.length;
|
||||
totalUpdated += n;
|
||||
console.log(` +${n} (total: ${totalUpdated})`);
|
||||
if (n === 0) break;
|
||||
}
|
||||
console.log(`Parts updated: ${totalUpdated}`);
|
||||
}
|
||||
|
||||
// Step 4: Redis cache flush — both per-vehicle category trees and translation entries
|
||||
console.log("\nFlushing Redis caches...");
|
||||
const redis = new Redis({
|
||||
host: process.env.REDIS_HOST || "127.0.0.1",
|
||||
port: parseInt(process.env.REDIS_PORT || "6379", 10),
|
||||
password: process.env.REDIS_PASSWORD,
|
||||
lazyConnect: true,
|
||||
});
|
||||
try {
|
||||
await redis.connect();
|
||||
const patterns = ["cat:tree:*", "tr:*"];
|
||||
for (const pattern of patterns) {
|
||||
let cursor = "0";
|
||||
let totalDeleted = 0;
|
||||
do {
|
||||
const [next, keys] = await redis.scan(cursor, "MATCH", pattern, "COUNT", 500);
|
||||
cursor = next;
|
||||
if (keys.length) {
|
||||
await redis.del(...keys);
|
||||
totalDeleted += keys.length;
|
||||
}
|
||||
} while (cursor !== "0");
|
||||
console.log(` ${pattern}: ${totalDeleted} keys deleted`);
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn(`Redis flush failed: ${(err as Error).message}`);
|
||||
} finally {
|
||||
redis.disconnect();
|
||||
}
|
||||
|
||||
await sql.end();
|
||||
console.log("\nDone.");
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
371
scripts/emex-translate-bootstrap.ts
Normal file
371
scripts/emex-translate-bootstrap.ts
Normal file
@@ -0,0 +1,371 @@
|
||||
/**
|
||||
* EMEX Türkçe Çeviri Bootstrap
|
||||
*
|
||||
* DB'deki tüm unique EMEX kategori (categories.name_original) ve parça
|
||||
* (parts.name_original) adlarını OpenRouter üzerinden DeepSeek V3 ile
|
||||
* Türkçe'ye çevirir ve `emex_category_translations` tablosuna yazar.
|
||||
*
|
||||
* Kullanım:
|
||||
* pnpm tsx scripts/emex-translate-bootstrap.ts --dry-run # kapsam + maliyet
|
||||
* pnpm tsx scripts/emex-translate-bootstrap.ts --limit=100 # küçük örnek
|
||||
* pnpm tsx scripts/emex-translate-bootstrap.ts # tam çalıştırma
|
||||
* pnpm tsx scripts/emex-translate-bootstrap.ts --resume # checkpoint'ten devam
|
||||
*/
|
||||
|
||||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import * as dotenv from "dotenv";
|
||||
import OpenAI from "openai";
|
||||
import { drizzle } from "drizzle-orm/postgres-js";
|
||||
import postgres from "postgres";
|
||||
|
||||
dotenv.config({ path: path.join(__dirname, "../apps/api/.env") });
|
||||
|
||||
import { emexCategoryTranslations } from "../apps/api/src/database/schema/core";
|
||||
|
||||
// ---------- CLI ----------
|
||||
const argv = process.argv.slice(2);
|
||||
const dryRun = argv.includes("--dry-run");
|
||||
const resume = argv.includes("--resume");
|
||||
|
||||
const limitArg = argv.find((a) => a.startsWith("--limit="));
|
||||
const limit = limitArg ? parseInt(limitArg.split("=")[1], 10) : null;
|
||||
|
||||
const batchSizeArg = argv.find((a) => a.startsWith("--batch-size="));
|
||||
const batchSize = batchSizeArg ? parseInt(batchSizeArg.split("=")[1], 10) : 50;
|
||||
|
||||
const concurrencyArg = argv.find((a) => a.startsWith("--concurrency="));
|
||||
const concurrency = concurrencyArg ? parseInt(concurrencyArg.split("=")[1], 10) : 5;
|
||||
|
||||
const CHECKPOINT_PATH = path.join(__dirname, ".emex-translate-checkpoint.json");
|
||||
const MODEL = "deepseek/deepseek-chat"; // OpenRouter slug for DeepSeek V3 (latest)
|
||||
const OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1";
|
||||
const MARKER = "Belirtilmemiş";
|
||||
|
||||
// ---------- Prompt ----------
|
||||
// DeepSeek V3 has automatic server-side prompt caching when the prefix is stable
|
||||
// (https://api-docs.deepseek.com/guides/kv_cache). Keeping this exact system
|
||||
// prompt across batches yields native cache hits without explicit cache_control.
|
||||
const SYSTEM_PROMPT = `Sen bir Türk otomotiv çevirmenisin. Görevin: EMEX otomobil yedek parça kataloğundan gelen kategori ve parça isimlerini İngilizce'den (zaman zaman Rusça'dan) Türkçe'ye çevirmek.
|
||||
|
||||
Kurallar:
|
||||
1. Türkiye yedek parça sektöründe kullanılan terminolojiyi kullan (örn. "Brake Pad" → "Fren Balatası", "Spark Plug" → "Buji").
|
||||
2. Türkçe karakterleri (ş, ı, ğ, ü, ö, ç) doğru kullan.
|
||||
3. Marka isimleri (BMW, VW, Toyota), model kodları, OEM parça kodları ve teknik kısaltmalar (ABS, ESP, OBD, ECU) çevirmeden olduğu gibi kalır.
|
||||
4. Belirsiz veya doğrudan karşılığı olmayan terim için en yakın TR karşılığını yaz; çok belirsizse orijinali koru.
|
||||
5. Kısa ve UI'da gösterilebilir olmalı (1-5 kelime ideal).
|
||||
6. Rusça girişler de TR'ye çevrilir.
|
||||
7. "Boot" otomotiv bağlamında "Bagaj" demektir, "Çizme" değil.
|
||||
8. Çıktı: girdi listesinin **aynı sırasında**, eşit uzunlukta JSON dizisi.
|
||||
|
||||
Örnekler (otomotiv bağlamı):
|
||||
- "Engine Oil Filter" → "Motor Yağ Filtresi"
|
||||
- "Front Brake Pad Set" → "Ön Fren Balata Seti"
|
||||
- "Cooling System" → "Soğutma Sistemi"
|
||||
- "Cylinder Head Gasket" → "Silindir Kapağı Contası"
|
||||
- "Suspension" → "Süspansiyon"
|
||||
- "Combination Rearlight/-Parts" → "Stop Lambası / Parçaları"
|
||||
- "Air Filter, passenger compartment" → "Polen Filtresi"
|
||||
- "Bumper/ Parts" → "Tampon / Parçaları"
|
||||
- "Fuel Tank / Parts" → "Yakıt Deposu / Parçaları"
|
||||
- "Gaskets / Seals" → "Contalar / Keçeler"
|
||||
- "Radiator /Parts" → "Radyatör / Parçaları"
|
||||
- "Valves/ Parts" → "Supaplar / Parçaları"
|
||||
- "Indicator/ Parts" → "Sinyal Lambası / Parçaları"
|
||||
- "Headlight/ Insert" → "Far / İç Parçalar"
|
||||
- "Alternator" → "Alternatör"
|
||||
- "Battery" → "Akü"
|
||||
- "Boot" → "Bagaj"
|
||||
- "Hood" → "Kaput"
|
||||
- "Bonnet" → "Kaput"
|
||||
- "SCREW" → "Vida"
|
||||
- "BOLT" → "Cıvata"
|
||||
- "NUT" → "Somun"
|
||||
- "CLIP" → "Klips"
|
||||
- "Cover" → "Kapak"
|
||||
- "Bracket" → "Braket"
|
||||
- "Spring" → "Yay"
|
||||
- "Six point socket screw" → "Altıgen İçten Vidalı"
|
||||
- "Plane washer" → "Düz Pul"
|
||||
- "Lock washer" → "Yaylı Pul"
|
||||
- "Flange screw" → "Flanşlı Vida"
|
||||
- "БОЛТ" → "Cıvata"
|
||||
- "ВТУЛКА" → "Burç"
|
||||
- "КОЛЛЕКТОР ВПУСКНОЙ" → "Emme Manifoldu"
|
||||
- "ГОЛОВКА БЛОКА ЦИЛИНДРОВ" → "Silindir Kapağı"
|
||||
- "Наименование не указано" → "Belirtilmemiş"
|
||||
|
||||
Yanıt formatı KESİN olarak şu JSON şeklinde olmalı, başka hiçbir metin ekleme:
|
||||
{"translations": ["çeviri1", "çeviri2", ...]}`;
|
||||
|
||||
// ---------- Checkpoint ----------
|
||||
interface Checkpoint {
|
||||
completed: Record<string, string>;
|
||||
failed: string[];
|
||||
}
|
||||
|
||||
function loadCheckpoint(): Checkpoint {
|
||||
if (!fs.existsSync(CHECKPOINT_PATH)) return { completed: {}, failed: [] };
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(CHECKPOINT_PATH, "utf-8"));
|
||||
} catch {
|
||||
return { completed: {}, failed: [] };
|
||||
}
|
||||
}
|
||||
|
||||
function saveCheckpoint(cp: Checkpoint) {
|
||||
fs.writeFileSync(CHECKPOINT_PATH, JSON.stringify(cp, null, 2));
|
||||
}
|
||||
|
||||
// ---------- Filtering ----------
|
||||
function classify(text: string): "skip" | "marker" | "ok" {
|
||||
const trimmed = text.trim();
|
||||
if (!trimmed) return "skip";
|
||||
if (trimmed === "Наименование не указано") return "marker";
|
||||
// Tek karakter veya sadece sayılar (1, 12, 100)
|
||||
if (/^\d+$/.test(trimmed)) return "marker";
|
||||
// OEM kod gibi (5 karaktere kadar A-Z0-9 + en az 1 rakam)
|
||||
if (/^[A-Z0-9-]{1,5}$/i.test(trimmed) && /\d/.test(trimmed)) return "marker";
|
||||
return "ok";
|
||||
}
|
||||
|
||||
// ---------- LLM batch ----------
|
||||
async function translateBatch(
|
||||
ai: OpenAI,
|
||||
terms: string[],
|
||||
): Promise<string[]> {
|
||||
let lastErr: unknown = null;
|
||||
for (let attempt = 0; attempt < 5; attempt++) {
|
||||
try {
|
||||
const response = await ai.chat.completions.create({
|
||||
model: MODEL,
|
||||
max_tokens: 4096,
|
||||
// DeepSeek V3 supports JSON mode; this guarantees parseable output.
|
||||
response_format: { type: "json_object" },
|
||||
messages: [
|
||||
{ role: "system", content: SYSTEM_PROMPT },
|
||||
{ role: "user", content: JSON.stringify({ terms }) },
|
||||
],
|
||||
});
|
||||
|
||||
const content = response.choices[0]?.message?.content;
|
||||
if (!content) throw new Error("Empty response");
|
||||
|
||||
const text = content.trim();
|
||||
const match = text.match(/\{[\s\S]*?"translations"[\s\S]*?\}/);
|
||||
if (!match) throw new Error(`No JSON in response: ${text.slice(0, 200)}`);
|
||||
const parsed = JSON.parse(match[0]) as { translations: string[] };
|
||||
if (!Array.isArray(parsed.translations)) {
|
||||
throw new Error("translations is not an array");
|
||||
}
|
||||
if (parsed.translations.length !== terms.length) {
|
||||
throw new Error(
|
||||
`Length mismatch: expected ${terms.length}, got ${parsed.translations.length}`,
|
||||
);
|
||||
}
|
||||
return parsed.translations;
|
||||
} catch (err) {
|
||||
lastErr = err;
|
||||
const status =
|
||||
(err as { status?: number })?.status ??
|
||||
(err as { response?: { status?: number } })?.response?.status;
|
||||
const retriable =
|
||||
status === 429 ||
|
||||
status === 529 ||
|
||||
(typeof status === "number" && status >= 500 && status < 600);
|
||||
if (retriable && attempt < 4) {
|
||||
const backoff = Math.min(2 ** attempt * 1000 + Math.random() * 500, 30_000);
|
||||
console.warn(
|
||||
`Retry attempt ${attempt + 1} after ${Math.round(backoff)}ms (status=${status})`,
|
||||
);
|
||||
await new Promise((r) => setTimeout(r, backoff));
|
||||
continue;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
throw lastErr;
|
||||
}
|
||||
|
||||
// ---------- Main ----------
|
||||
async function main() {
|
||||
if (!process.env.DATABASE_URL) {
|
||||
throw new Error("DATABASE_URL env var is required");
|
||||
}
|
||||
if (!dryRun && !process.env.OPENROUTER_API_KEY) {
|
||||
throw new Error("OPENROUTER_API_KEY env var is required (set in apps/api/.env)");
|
||||
}
|
||||
|
||||
const sql = postgres(process.env.DATABASE_URL);
|
||||
const db = drizzle(sql);
|
||||
const ai = !dryRun
|
||||
? new OpenAI({
|
||||
apiKey: process.env.OPENROUTER_API_KEY,
|
||||
baseURL: OPENROUTER_BASE_URL,
|
||||
defaultHeaders: {
|
||||
// OpenRouter recommends these for ranking/abuse detection
|
||||
"HTTP-Referer": "https://sase.tr",
|
||||
"X-Title": "Sase EMEX Translation Bootstrap",
|
||||
},
|
||||
})
|
||||
: null;
|
||||
|
||||
console.log("Fetching unique EMEX category and part names from DB...");
|
||||
const catNames = await sql<{ name_original: string }[]>`
|
||||
SELECT DISTINCT name_original FROM categories
|
||||
WHERE source IN ('emex', 'parts-catalogs')
|
||||
AND name_original IS NOT NULL
|
||||
AND length(trim(name_original)) > 0
|
||||
`;
|
||||
const partNames = await sql<{ name_original: string }[]>`
|
||||
SELECT DISTINCT name_original FROM parts
|
||||
WHERE source IN ('emex', 'parts-catalogs')
|
||||
AND name_original IS NOT NULL
|
||||
AND length(trim(name_original)) > 0
|
||||
`;
|
||||
console.log(`Categories: ${catNames.length} unique, Parts: ${partNames.length} unique`);
|
||||
|
||||
const allUnique = [
|
||||
...new Set([
|
||||
...catNames.map((r) => r.name_original),
|
||||
...partNames.map((r) => r.name_original),
|
||||
]),
|
||||
].sort();
|
||||
console.log(`Combined unique: ${allUnique.length}`);
|
||||
|
||||
console.log("Loading existing translations...");
|
||||
const existing = await db
|
||||
.select({ originalName: emexCategoryTranslations.originalName })
|
||||
.from(emexCategoryTranslations);
|
||||
const existingSet = new Set(existing.map((r) => r.originalName));
|
||||
|
||||
const checkpoint = resume ? loadCheckpoint() : { completed: {}, failed: [] };
|
||||
const completedSet = new Set(Object.keys(checkpoint.completed));
|
||||
|
||||
const skipTerms: string[] = [];
|
||||
const markerTerms: string[] = [];
|
||||
const llmTerms: string[] = [];
|
||||
|
||||
for (const term of allUnique) {
|
||||
if (existingSet.has(term)) continue;
|
||||
if (completedSet.has(term)) continue;
|
||||
const decision = classify(term);
|
||||
if (decision === "skip") skipTerms.push(term);
|
||||
else if (decision === "marker") markerTerms.push(term);
|
||||
else llmTerms.push(term);
|
||||
}
|
||||
|
||||
const finalLlmTerms = limit ? llmTerms.slice(0, limit) : llmTerms;
|
||||
const batches: string[][] = [];
|
||||
for (let i = 0; i < finalLlmTerms.length; i += batchSize) {
|
||||
batches.push(finalLlmTerms.slice(i, i + batchSize));
|
||||
}
|
||||
|
||||
// Cost estimate (DeepSeek V3 via OpenRouter, current rates ~ $0.27/M in, $1.10/M out)
|
||||
// System prompt is ~2K tokens; DeepSeek auto-caches stable prefixes (cache hit ~10x cheaper).
|
||||
const inputTokens = batches.length * 2000 + finalLlmTerms.length * 8;
|
||||
const outputTokens = finalLlmTerms.length * 6;
|
||||
const estCost =
|
||||
(inputTokens / 1_000_000) * 0.27 + (outputTokens / 1_000_000) * 1.1;
|
||||
|
||||
console.log("\n=== Translation Plan ===");
|
||||
console.log(`In DB already: ${existingSet.size}`);
|
||||
console.log(`In checkpoint: ${completedSet.size}`);
|
||||
console.log(`Skip (empty): ${skipTerms.length}`);
|
||||
console.log(`Marker only: ${markerTerms.length} ("${MARKER}")`);
|
||||
console.log(`LLM translate: ${finalLlmTerms.length} (model: ${MODEL})`);
|
||||
console.log(`Batches: ${batches.length} × ${batchSize}, concurrency=${concurrency}`);
|
||||
console.log(`Est. cost: ~$${estCost.toFixed(2)} (DeepSeek V3 via OpenRouter)`);
|
||||
|
||||
if (dryRun) {
|
||||
console.log("\nDRY RUN — no API calls or DB writes. Sample terms:");
|
||||
finalLlmTerms.slice(0, 20).forEach((t) => console.log(` - ${t}`));
|
||||
await sql.end();
|
||||
return;
|
||||
}
|
||||
|
||||
// Marker bulk insert
|
||||
if (markerTerms.length) {
|
||||
console.log(`\nInserting ${markerTerms.length} marker rows...`);
|
||||
for (let i = 0; i < markerTerms.length; i += 1000) {
|
||||
const chunk = markerTerms.slice(i, i + 1000);
|
||||
await db
|
||||
.insert(emexCategoryTranslations)
|
||||
.values(
|
||||
chunk.map((t) => ({
|
||||
originalName: t,
|
||||
translatedName: MARKER,
|
||||
isManual: false,
|
||||
})),
|
||||
)
|
||||
.onConflictDoNothing();
|
||||
}
|
||||
}
|
||||
|
||||
// LLM batches with bounded concurrency
|
||||
let success = 0;
|
||||
let failed = 0;
|
||||
let inFlight = 0;
|
||||
let nextIdx = 0;
|
||||
|
||||
await new Promise<void>((resolve) => {
|
||||
const launch = () => {
|
||||
if (nextIdx >= batches.length && inFlight === 0) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
while (inFlight < concurrency && nextIdx < batches.length) {
|
||||
const idx = nextIdx++;
|
||||
const batch = batches[idx];
|
||||
inFlight++;
|
||||
(async () => {
|
||||
try {
|
||||
if (!ai) throw new Error("AI client not initialized");
|
||||
const translations = await translateBatch(ai, batch);
|
||||
await db
|
||||
.insert(emexCategoryTranslations)
|
||||
.values(
|
||||
batch.map((orig, i) => ({
|
||||
originalName: orig,
|
||||
translatedName: translations[i] || orig,
|
||||
isManual: false,
|
||||
})),
|
||||
)
|
||||
.onConflictDoNothing();
|
||||
for (let i = 0; i < batch.length; i++) {
|
||||
checkpoint.completed[batch[i]] = translations[i] || batch[i];
|
||||
}
|
||||
success += batch.length;
|
||||
if (idx % 5 === 0) saveCheckpoint(checkpoint);
|
||||
console.log(
|
||||
`Batch ${idx + 1}/${batches.length} ok (${batch.length}). Total ok: ${success}`,
|
||||
);
|
||||
} catch (err) {
|
||||
failed += batch.length;
|
||||
checkpoint.failed.push(...batch);
|
||||
console.error(`Batch ${idx + 1} failed: ${(err as Error).message}`);
|
||||
} finally {
|
||||
inFlight--;
|
||||
launch();
|
||||
}
|
||||
})();
|
||||
}
|
||||
};
|
||||
launch();
|
||||
});
|
||||
|
||||
saveCheckpoint(checkpoint);
|
||||
|
||||
console.log("\n=== Done ===");
|
||||
console.log(`LLM translated: ${success}`);
|
||||
console.log(`Markers: ${markerTerms.length}`);
|
||||
console.log(`Failed: ${failed}`);
|
||||
console.log(`Checkpoint: ${CHECKPOINT_PATH}`);
|
||||
|
||||
await sql.end();
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 115 KiB After Width: | Height: | Size: 112 KiB |
69
scripts/test-emex-http.mjs
Normal file
69
scripts/test-emex-http.mjs
Normal file
@@ -0,0 +1,69 @@
|
||||
/**
|
||||
* EMEX HTTP decode timing test
|
||||
* Usage: node scripts/test-emex-http.mjs
|
||||
*/
|
||||
import { ProxyAgent } from "/home/s/ss/node_modules/.pnpm/undici@7.22.0/node_modules/undici/index.js";
|
||||
|
||||
const VIN = "WDD1173431N193569";
|
||||
const EMEX_BASE_URL = "https://emexdwc.ae";
|
||||
const UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36";
|
||||
|
||||
// Same proxy config as emex.service.ts defaults
|
||||
const PROXY_HOST = "74.81.81.81";
|
||||
const PROXY_PORT_START = 10001;
|
||||
const PROXY_PORT_END = 10099;
|
||||
const PROXY_USER = "1726bbe361918676d44e";
|
||||
const PROXY_PASS = "f11c7b6128cc86c6";
|
||||
|
||||
const port = Math.floor(Math.random() * (PROXY_PORT_END - PROXY_PORT_START + 1)) + PROXY_PORT_START;
|
||||
const proxyUri = `http://${PROXY_USER}:${PROXY_PASS}@${PROXY_HOST}:${port}`;
|
||||
|
||||
console.log(`VIN : ${VIN}`);
|
||||
console.log(`Proxy : ${PROXY_HOST}:${port}`);
|
||||
console.log(`URL : ${EMEX_BASE_URL}/Vehicles.aspx?ft=findByVIN&c=&ssd=&vin=${VIN}`);
|
||||
console.log("─".repeat(50));
|
||||
|
||||
const agent = new ProxyAgent({
|
||||
uri: proxyUri,
|
||||
connect: { timeout: 30000 },
|
||||
requestTls: { timeout: 30000 },
|
||||
});
|
||||
|
||||
const url = `${EMEX_BASE_URL}/Vehicles.aspx?ft=findByVIN&c=&ssd=&vin=${VIN}`;
|
||||
|
||||
const t0 = Date.now();
|
||||
try {
|
||||
const res = await fetch(url, {
|
||||
headers: { "User-Agent": UA, Accept: "text/html,application/xhtml+xml" },
|
||||
signal: AbortSignal.timeout(60000),
|
||||
dispatcher: agent,
|
||||
});
|
||||
|
||||
const elapsed = Date.now() - t0;
|
||||
const html = await res.text();
|
||||
|
||||
console.log(`HTTP status : ${res.status}`);
|
||||
console.log(`Süre : ${elapsed} ms`);
|
||||
console.log(`Response : ${html.length} bytes`);
|
||||
|
||||
// Araç listesini parse et
|
||||
const linkRx = /href="(Vehicle\.aspx\?[^"]+)">([^<]+)<\/a>/g;
|
||||
const matches = [];
|
||||
let m;
|
||||
while ((m = linkRx.exec(html)) !== null) {
|
||||
matches.push(m[2].trim());
|
||||
}
|
||||
|
||||
if (matches.length > 0) {
|
||||
console.log(`\nBulunan araçlar (${matches.length}):`);
|
||||
matches.forEach((v, i) => console.log(` [${i}] ${v}`));
|
||||
} else {
|
||||
console.log("\nAraç bulunamadı (boş sonuç).");
|
||||
// Ham HTML'in ilk 500 karakterini göster
|
||||
console.log("\nHTML (ilk 500 kar):");
|
||||
console.log(html.slice(0, 500));
|
||||
}
|
||||
} catch (err) {
|
||||
const elapsed = Date.now() - t0;
|
||||
console.error(`HATA (${elapsed} ms): ${err.message}`);
|
||||
}
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 584 KiB After Width: | Height: | Size: 332 KiB |
Reference in New Issue
Block a user