f4fea1e429
Some checks failed
Sync dev → Gitea / Mirror dev to Gitea (push) Has been cancelled
- Added a comment line to main.ts for deployment verification purposes
372 lines
14 KiB
TypeScript
372 lines
14 KiB
TypeScript
/**
|
||
* EMEX Türkçe Çeviri Bootstrap
|
||
*
|
||
* DB'deki tüm unique EMEX kategori (categories.name_original) ve parça
|
||
* (parts.name_original) adlarını OpenRouter üzerinden DeepSeek V3 ile
|
||
* Türkçe'ye çevirir ve `emex_category_translations` tablosuna yazar.
|
||
*
|
||
* Kullanım:
|
||
* pnpm tsx scripts/emex-translate-bootstrap.ts --dry-run # kapsam + maliyet
|
||
* pnpm tsx scripts/emex-translate-bootstrap.ts --limit=100 # küçük örnek
|
||
* pnpm tsx scripts/emex-translate-bootstrap.ts # tam çalıştırma
|
||
* pnpm tsx scripts/emex-translate-bootstrap.ts --resume # checkpoint'ten devam
|
||
*/
|
||
|
||
import * as fs from "node:fs";
|
||
import * as path from "node:path";
|
||
import * as dotenv from "dotenv";
|
||
import OpenAI from "openai";
|
||
import { drizzle } from "drizzle-orm/postgres-js";
|
||
import postgres from "postgres";
|
||
|
||
dotenv.config({ path: path.join(__dirname, "../apps/api/.env") });
|
||
|
||
import { emexCategoryTranslations } from "../apps/api/src/database/schema/core";
|
||
|
||
// ---------- CLI ----------
|
||
const argv = process.argv.slice(2);
|
||
const dryRun = argv.includes("--dry-run");
|
||
const resume = argv.includes("--resume");
|
||
|
||
const limitArg = argv.find((a) => a.startsWith("--limit="));
|
||
const limit = limitArg ? parseInt(limitArg.split("=")[1], 10) : null;
|
||
|
||
const batchSizeArg = argv.find((a) => a.startsWith("--batch-size="));
|
||
const batchSize = batchSizeArg ? parseInt(batchSizeArg.split("=")[1], 10) : 50;
|
||
|
||
const concurrencyArg = argv.find((a) => a.startsWith("--concurrency="));
|
||
const concurrency = concurrencyArg ? parseInt(concurrencyArg.split("=")[1], 10) : 5;
|
||
|
||
const CHECKPOINT_PATH = path.join(__dirname, ".emex-translate-checkpoint.json");
|
||
const MODEL = "deepseek/deepseek-chat"; // OpenRouter slug for DeepSeek V3 (latest)
|
||
const OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1";
|
||
const MARKER = "Belirtilmemiş";
|
||
|
||
// ---------- Prompt ----------
|
||
// DeepSeek V3 has automatic server-side prompt caching when the prefix is stable
|
||
// (https://api-docs.deepseek.com/guides/kv_cache). Keeping this exact system
|
||
// prompt across batches yields native cache hits without explicit cache_control.
|
||
const SYSTEM_PROMPT = `Sen bir Türk otomotiv çevirmenisin. Görevin: EMEX otomobil yedek parça kataloğundan gelen kategori ve parça isimlerini İngilizce'den (zaman zaman Rusça'dan) Türkçe'ye çevirmek.
|
||
|
||
Kurallar:
|
||
1. Türkiye yedek parça sektöründe kullanılan terminolojiyi kullan (örn. "Brake Pad" → "Fren Balatası", "Spark Plug" → "Buji").
|
||
2. Türkçe karakterleri (ş, ı, ğ, ü, ö, ç) doğru kullan.
|
||
3. Marka isimleri (BMW, VW, Toyota), model kodları, OEM parça kodları ve teknik kısaltmalar (ABS, ESP, OBD, ECU) çevirmeden olduğu gibi kalır.
|
||
4. Belirsiz veya doğrudan karşılığı olmayan terim için en yakın TR karşılığını yaz; çok belirsizse orijinali koru.
|
||
5. Kısa ve UI'da gösterilebilir olmalı (1-5 kelime ideal).
|
||
6. Rusça girişler de TR'ye çevrilir.
|
||
7. "Boot" otomotiv bağlamında "Bagaj" demektir, "Çizme" değil.
|
||
8. Çıktı: girdi listesinin **aynı sırasında**, eşit uzunlukta JSON dizisi.
|
||
|
||
Örnekler (otomotiv bağlamı):
|
||
- "Engine Oil Filter" → "Motor Yağ Filtresi"
|
||
- "Front Brake Pad Set" → "Ön Fren Balata Seti"
|
||
- "Cooling System" → "Soğutma Sistemi"
|
||
- "Cylinder Head Gasket" → "Silindir Kapağı Contası"
|
||
- "Suspension" → "Süspansiyon"
|
||
- "Combination Rearlight/-Parts" → "Stop Lambası / Parçaları"
|
||
- "Air Filter, passenger compartment" → "Polen Filtresi"
|
||
- "Bumper/ Parts" → "Tampon / Parçaları"
|
||
- "Fuel Tank / Parts" → "Yakıt Deposu / Parçaları"
|
||
- "Gaskets / Seals" → "Contalar / Keçeler"
|
||
- "Radiator /Parts" → "Radyatör / Parçaları"
|
||
- "Valves/ Parts" → "Supaplar / Parçaları"
|
||
- "Indicator/ Parts" → "Sinyal Lambası / Parçaları"
|
||
- "Headlight/ Insert" → "Far / İç Parçalar"
|
||
- "Alternator" → "Alternatör"
|
||
- "Battery" → "Akü"
|
||
- "Boot" → "Bagaj"
|
||
- "Hood" → "Kaput"
|
||
- "Bonnet" → "Kaput"
|
||
- "SCREW" → "Vida"
|
||
- "BOLT" → "Cıvata"
|
||
- "NUT" → "Somun"
|
||
- "CLIP" → "Klips"
|
||
- "Cover" → "Kapak"
|
||
- "Bracket" → "Braket"
|
||
- "Spring" → "Yay"
|
||
- "Six point socket screw" → "Altıgen İçten Vidalı"
|
||
- "Plane washer" → "Düz Pul"
|
||
- "Lock washer" → "Yaylı Pul"
|
||
- "Flange screw" → "Flanşlı Vida"
|
||
- "БОЛТ" → "Cıvata"
|
||
- "ВТУЛКА" → "Burç"
|
||
- "КОЛЛЕКТОР ВПУСКНОЙ" → "Emme Manifoldu"
|
||
- "ГОЛОВКА БЛОКА ЦИЛИНДРОВ" → "Silindir Kapağı"
|
||
- "Наименование не указано" → "Belirtilmemiş"
|
||
|
||
Yanıt formatı KESİN olarak şu JSON şeklinde olmalı, başka hiçbir metin ekleme:
|
||
{"translations": ["çeviri1", "çeviri2", ...]}`;
|
||
|
||
// ---------- Checkpoint ----------
|
||
interface Checkpoint {
|
||
completed: Record<string, string>;
|
||
failed: string[];
|
||
}
|
||
|
||
function loadCheckpoint(): Checkpoint {
|
||
if (!fs.existsSync(CHECKPOINT_PATH)) return { completed: {}, failed: [] };
|
||
try {
|
||
return JSON.parse(fs.readFileSync(CHECKPOINT_PATH, "utf-8"));
|
||
} catch {
|
||
return { completed: {}, failed: [] };
|
||
}
|
||
}
|
||
|
||
function saveCheckpoint(cp: Checkpoint) {
|
||
fs.writeFileSync(CHECKPOINT_PATH, JSON.stringify(cp, null, 2));
|
||
}
|
||
|
||
// ---------- Filtering ----------
|
||
function classify(text: string): "skip" | "marker" | "ok" {
|
||
const trimmed = text.trim();
|
||
if (!trimmed) return "skip";
|
||
if (trimmed === "Наименование не указано") return "marker";
|
||
// Tek karakter veya sadece sayılar (1, 12, 100)
|
||
if (/^\d+$/.test(trimmed)) return "marker";
|
||
// OEM kod gibi (5 karaktere kadar A-Z0-9 + en az 1 rakam)
|
||
if (/^[A-Z0-9-]{1,5}$/i.test(trimmed) && /\d/.test(trimmed)) return "marker";
|
||
return "ok";
|
||
}
|
||
|
||
// ---------- LLM batch ----------
|
||
async function translateBatch(
|
||
ai: OpenAI,
|
||
terms: string[],
|
||
): Promise<string[]> {
|
||
let lastErr: unknown = null;
|
||
for (let attempt = 0; attempt < 5; attempt++) {
|
||
try {
|
||
const response = await ai.chat.completions.create({
|
||
model: MODEL,
|
||
max_tokens: 4096,
|
||
// DeepSeek V3 supports JSON mode; this guarantees parseable output.
|
||
response_format: { type: "json_object" },
|
||
messages: [
|
||
{ role: "system", content: SYSTEM_PROMPT },
|
||
{ role: "user", content: JSON.stringify({ terms }) },
|
||
],
|
||
});
|
||
|
||
const content = response.choices[0]?.message?.content;
|
||
if (!content) throw new Error("Empty response");
|
||
|
||
const text = content.trim();
|
||
const match = text.match(/\{[\s\S]*?"translations"[\s\S]*?\}/);
|
||
if (!match) throw new Error(`No JSON in response: ${text.slice(0, 200)}`);
|
||
const parsed = JSON.parse(match[0]) as { translations: string[] };
|
||
if (!Array.isArray(parsed.translations)) {
|
||
throw new Error("translations is not an array");
|
||
}
|
||
if (parsed.translations.length !== terms.length) {
|
||
throw new Error(
|
||
`Length mismatch: expected ${terms.length}, got ${parsed.translations.length}`,
|
||
);
|
||
}
|
||
return parsed.translations;
|
||
} catch (err) {
|
||
lastErr = err;
|
||
const status =
|
||
(err as { status?: number })?.status ??
|
||
(err as { response?: { status?: number } })?.response?.status;
|
||
const retriable =
|
||
status === 429 ||
|
||
status === 529 ||
|
||
(typeof status === "number" && status >= 500 && status < 600);
|
||
if (retriable && attempt < 4) {
|
||
const backoff = Math.min(2 ** attempt * 1000 + Math.random() * 500, 30_000);
|
||
console.warn(
|
||
`Retry attempt ${attempt + 1} after ${Math.round(backoff)}ms (status=${status})`,
|
||
);
|
||
await new Promise((r) => setTimeout(r, backoff));
|
||
continue;
|
||
}
|
||
throw err;
|
||
}
|
||
}
|
||
throw lastErr;
|
||
}
|
||
|
||
// ---------- Main ----------
|
||
async function main() {
|
||
if (!process.env.DATABASE_URL) {
|
||
throw new Error("DATABASE_URL env var is required");
|
||
}
|
||
if (!dryRun && !process.env.OPENROUTER_API_KEY) {
|
||
throw new Error("OPENROUTER_API_KEY env var is required (set in apps/api/.env)");
|
||
}
|
||
|
||
const sql = postgres(process.env.DATABASE_URL);
|
||
const db = drizzle(sql);
|
||
const ai = !dryRun
|
||
? new OpenAI({
|
||
apiKey: process.env.OPENROUTER_API_KEY,
|
||
baseURL: OPENROUTER_BASE_URL,
|
||
defaultHeaders: {
|
||
// OpenRouter recommends these for ranking/abuse detection
|
||
"HTTP-Referer": "https://sase.tr",
|
||
"X-Title": "Sase EMEX Translation Bootstrap",
|
||
},
|
||
})
|
||
: null;
|
||
|
||
console.log("Fetching unique EMEX category and part names from DB...");
|
||
const catNames = await sql<{ name_original: string }[]>`
|
||
SELECT DISTINCT name_original FROM categories
|
||
WHERE source IN ('emex', 'parts-catalogs')
|
||
AND name_original IS NOT NULL
|
||
AND length(trim(name_original)) > 0
|
||
`;
|
||
const partNames = await sql<{ name_original: string }[]>`
|
||
SELECT DISTINCT name_original FROM parts
|
||
WHERE source IN ('emex', 'parts-catalogs')
|
||
AND name_original IS NOT NULL
|
||
AND length(trim(name_original)) > 0
|
||
`;
|
||
console.log(`Categories: ${catNames.length} unique, Parts: ${partNames.length} unique`);
|
||
|
||
const allUnique = [
|
||
...new Set([
|
||
...catNames.map((r) => r.name_original),
|
||
...partNames.map((r) => r.name_original),
|
||
]),
|
||
].sort();
|
||
console.log(`Combined unique: ${allUnique.length}`);
|
||
|
||
console.log("Loading existing translations...");
|
||
const existing = await db
|
||
.select({ originalName: emexCategoryTranslations.originalName })
|
||
.from(emexCategoryTranslations);
|
||
const existingSet = new Set(existing.map((r) => r.originalName));
|
||
|
||
const checkpoint = resume ? loadCheckpoint() : { completed: {}, failed: [] };
|
||
const completedSet = new Set(Object.keys(checkpoint.completed));
|
||
|
||
const skipTerms: string[] = [];
|
||
const markerTerms: string[] = [];
|
||
const llmTerms: string[] = [];
|
||
|
||
for (const term of allUnique) {
|
||
if (existingSet.has(term)) continue;
|
||
if (completedSet.has(term)) continue;
|
||
const decision = classify(term);
|
||
if (decision === "skip") skipTerms.push(term);
|
||
else if (decision === "marker") markerTerms.push(term);
|
||
else llmTerms.push(term);
|
||
}
|
||
|
||
const finalLlmTerms = limit ? llmTerms.slice(0, limit) : llmTerms;
|
||
const batches: string[][] = [];
|
||
for (let i = 0; i < finalLlmTerms.length; i += batchSize) {
|
||
batches.push(finalLlmTerms.slice(i, i + batchSize));
|
||
}
|
||
|
||
// Cost estimate (DeepSeek V3 via OpenRouter, current rates ~ $0.27/M in, $1.10/M out)
|
||
// System prompt is ~2K tokens; DeepSeek auto-caches stable prefixes (cache hit ~10x cheaper).
|
||
const inputTokens = batches.length * 2000 + finalLlmTerms.length * 8;
|
||
const outputTokens = finalLlmTerms.length * 6;
|
||
const estCost =
|
||
(inputTokens / 1_000_000) * 0.27 + (outputTokens / 1_000_000) * 1.1;
|
||
|
||
console.log("\n=== Translation Plan ===");
|
||
console.log(`In DB already: ${existingSet.size}`);
|
||
console.log(`In checkpoint: ${completedSet.size}`);
|
||
console.log(`Skip (empty): ${skipTerms.length}`);
|
||
console.log(`Marker only: ${markerTerms.length} ("${MARKER}")`);
|
||
console.log(`LLM translate: ${finalLlmTerms.length} (model: ${MODEL})`);
|
||
console.log(`Batches: ${batches.length} × ${batchSize}, concurrency=${concurrency}`);
|
||
console.log(`Est. cost: ~$${estCost.toFixed(2)} (DeepSeek V3 via OpenRouter)`);
|
||
|
||
if (dryRun) {
|
||
console.log("\nDRY RUN — no API calls or DB writes. Sample terms:");
|
||
finalLlmTerms.slice(0, 20).forEach((t) => console.log(` - ${t}`));
|
||
await sql.end();
|
||
return;
|
||
}
|
||
|
||
// Marker bulk insert
|
||
if (markerTerms.length) {
|
||
console.log(`\nInserting ${markerTerms.length} marker rows...`);
|
||
for (let i = 0; i < markerTerms.length; i += 1000) {
|
||
const chunk = markerTerms.slice(i, i + 1000);
|
||
await db
|
||
.insert(emexCategoryTranslations)
|
||
.values(
|
||
chunk.map((t) => ({
|
||
originalName: t,
|
||
translatedName: MARKER,
|
||
isManual: false,
|
||
})),
|
||
)
|
||
.onConflictDoNothing();
|
||
}
|
||
}
|
||
|
||
// LLM batches with bounded concurrency
|
||
let success = 0;
|
||
let failed = 0;
|
||
let inFlight = 0;
|
||
let nextIdx = 0;
|
||
|
||
await new Promise<void>((resolve) => {
|
||
const launch = () => {
|
||
if (nextIdx >= batches.length && inFlight === 0) {
|
||
resolve();
|
||
return;
|
||
}
|
||
while (inFlight < concurrency && nextIdx < batches.length) {
|
||
const idx = nextIdx++;
|
||
const batch = batches[idx];
|
||
inFlight++;
|
||
(async () => {
|
||
try {
|
||
if (!ai) throw new Error("AI client not initialized");
|
||
const translations = await translateBatch(ai, batch);
|
||
await db
|
||
.insert(emexCategoryTranslations)
|
||
.values(
|
||
batch.map((orig, i) => ({
|
||
originalName: orig,
|
||
translatedName: translations[i] || orig,
|
||
isManual: false,
|
||
})),
|
||
)
|
||
.onConflictDoNothing();
|
||
for (let i = 0; i < batch.length; i++) {
|
||
checkpoint.completed[batch[i]] = translations[i] || batch[i];
|
||
}
|
||
success += batch.length;
|
||
if (idx % 5 === 0) saveCheckpoint(checkpoint);
|
||
console.log(
|
||
`Batch ${idx + 1}/${batches.length} ok (${batch.length}). Total ok: ${success}`,
|
||
);
|
||
} catch (err) {
|
||
failed += batch.length;
|
||
checkpoint.failed.push(...batch);
|
||
console.error(`Batch ${idx + 1} failed: ${(err as Error).message}`);
|
||
} finally {
|
||
inFlight--;
|
||
launch();
|
||
}
|
||
})();
|
||
}
|
||
};
|
||
launch();
|
||
});
|
||
|
||
saveCheckpoint(checkpoint);
|
||
|
||
console.log("\n=== Done ===");
|
||
console.log(`LLM translated: ${success}`);
|
||
console.log(`Markers: ${markerTerms.length}`);
|
||
console.log(`Failed: ${failed}`);
|
||
console.log(`Checkpoint: ${CHECKPOINT_PATH}`);
|
||
|
||
await sql.end();
|
||
}
|
||
|
||
main().catch((err) => {
|
||
console.error(err);
|
||
process.exit(1);
|
||
});
|