feat(sase): VIN decode anomaly detection + Telegram alerts

Adds the active half of Phase 8: dashboard was passive, this pushes
when something breaks. Targets MTTD < 10 minutes from the PRD.

Detection (panel-web)
- detectVinAnomalies() compares a 15-minute current window against a
  baseline built from the same 15-minute slot across the previous 7
  days (same-hour-of-day, average across days that had ≥ 1 sample).
- Five hits, two severities:
  * success_rate_drop — high at ≥ 5pp drop, critical at ≥ 15pp
  * p95_latency_spike — high at 2× baseline (and > 1s), critical at 3×
  * volume_drop — high at ≥ 80% below baseline, critical at ≥ 95%
  * volume_spike — high at ≥ 5× baseline
  * timeout_dominance — high when ≥ 50% of failures are timeouts
- Minimum-volume guards on both current and baseline so quiet hours
  don't generate noise (MIN_CURRENT_VOLUME=5, MIN_BASELINE_VOLUME=10).
- Each anomaly carries a 15-min-bucket dedupe key — same anomaly type
  fires at most once per bucket regardless of cron cadence.

Endpoint
- GET /api/internal/vin-anomaly-check, gated by x-internal-worker-token
  header (constant-time compare against INTERNAL_WORKER_TOKEN env).
  Returns { current, baseline, anomalies[] }.

Worker
- New job vin-anomaly-detect, BullMQ scheduler */5 * * * *.
  Fetches the panel endpoint, then for each anomaly calls
  alertVinAnomaly() — sendTelegram with the bucket dedupe key, so
  Redis SETEX NX dedupes across the 1h cooldown window.
- alertVinAnomaly() in worker/lib/telegram.ts formats the message with
  severity icon + baseline/observed/volume context + dashboard link.
- runVinAnomalyDetect returns { ok, checked, anomalies, alertsFired,
  alertsDeduped }; pipeline logs only when something happened or the
  check failed.

Env
- INTERNAL_WORKER_TOKEN set on both panel-web and panel-worker
  (32-byte hex, generated in Coolify).
- PANEL_INTERNAL_URL on panel-worker → coolify-network UUID hostname
  for panel-web, no Cloudflare/Tailscale hop on internal calls.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

apps/worker/src/jobs/vin-anomaly.ts

@@ -0,0 +1,100 @@
+import { alertVinAnomaly, isTelegramConfigured } from "../lib/telegram";
+
+const PANEL_BASE =
+  process.env.PANEL_INTERNAL_URL ??
+  process.env.PANEL_PUBLIC_URL ??
+  "http://panel-web:3000";
+
+const WORKER_TOKEN = process.env.INTERNAL_WORKER_TOKEN ?? "";
+
+type AnomalyHit = {
+  type: string;
+  severity: "high" | "critical";
+  message: string;
+  baseline: number;
+  observed: number;
+  current_volume: number;
+  baseline_volume: number;
+  detected_at: string;
+  dedupe_key: string;
+};
+
+type CheckResponse = {
+  ok: boolean;
+  error?: string;
+  current?: { total: number; successRate: number };
+  baseline?: { total: number; successRate: number };
+  anomalies?: AnomalyHit[];
+};
+
+export async function runVinAnomalyDetect(): Promise<{
+  ok: boolean;
+  checked: boolean;
+  anomalies: number;
+  alertsFired: number;
+  alertsDeduped: number;
+  reason?: string;
+}> {
+  if (!WORKER_TOKEN) {
+    return { ok: false, checked: false, anomalies: 0, alertsFired: 0, alertsDeduped: 0, reason: "INTERNAL_WORKER_TOKEN not set" };
+  }
+  if (!isTelegramConfigured()) {
+    return { ok: false, checked: false, anomalies: 0, alertsFired: 0, alertsDeduped: 0, reason: "telegram not configured" };
+  }
+
+  let res: Response;
+  try {
+    res = await fetch(`${PANEL_BASE}/api/internal/vin-anomaly-check`, {
+      headers: {
+        "x-internal-worker-token": WORKER_TOKEN,
+        "cache-control": "no-store",
+      },
+    });
+  } catch (e) {
+    return {
+      ok: false,
+      checked: false,
+      anomalies: 0,
+      alertsFired: 0,
+      alertsDeduped: 0,
+      reason: `fetch failed: ${(e as Error).message}`,
+    };
+  }
+  if (!res.ok) {
+    return {
+      ok: false,
+      checked: false,
+      anomalies: 0,
+      alertsFired: 0,
+      alertsDeduped: 0,
+      reason: `panel returned ${res.status}`,
+    };
+  }
+  const data = (await res.json()) as CheckResponse;
+  if (!data.ok || !data.anomalies) {
+    return {
+      ok: !!data.ok,
+      checked: true,
+      anomalies: 0,
+      alertsFired: 0,
+      alertsDeduped: 0,
+      reason: data.error,
+    };
+  }
+
+  const panelPublic = process.env.PANEL_PUBLIC_URL ?? "https://sp.semih.ai";
+  let fired = 0;
+  let deduped = 0;
+  for (const a of data.anomalies) {
+    const r = await alertVinAnomaly({ panelUrl: panelPublic, ...a });
+    if (r.ok && r.deduped) deduped++;
+    else if (r.ok) fired++;
+  }
+  return {
+    ok: true,
+    checked: true,
+    anomalies: data.anomalies.length,
+    alertsFired: fired,
+    alertsDeduped: deduped,
+  };
+}

apps/worker/src/lib/telegram.ts

@@ -119,6 +119,32 @@ export function alertBudgetCap(opts: {
   return sendTelegram({ text, dedupeKey: `budget:${opts.state}:${day}` });
 }
 
+export type VinAnomalyAlert = {
+  type: string;
+  severity: "high" | "critical";
+  message: string;
+  baseline: number;
+  observed: number;
+  current_volume: number;
+  baseline_volume: number;
+  detected_at: string;
+  dedupe_key: string;
+};
+
+export function alertVinAnomaly(
+  opts: { panelUrl: string } & VinAnomalyAlert,
+): Promise<TelegramSendResult> {
+  const icon = opts.severity === "critical" ? "🔴" : "🟡";
+  const text = [
+    `${icon} <b>VIN decode anomaly</b>: ${opts.type}`,
+    escapeHtml(opts.message),
+    `Volume: now ${opts.current_volume} · baseline ${opts.baseline_volume}`,
+    ``,
+    `<a href="${opts.panelUrl}/projects/sase/vin-decode">VIN Decode dashboard</a>`,
+  ].join("\n");
+  return sendTelegram({ text, dedupeKey: opts.dedupe_key });
+}
+
 function escapeHtml(s: string): string {
   return s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
 }

apps/worker/src/schedulers/pipeline.ts

@@ -9,6 +9,7 @@ import { runGithubSync } from "../jobs/github-sync";
 import { runRetention } from "../jobs/retention";
 import { runEvalSet } from "../jobs/eval-run";
 import { runDailyBrief } from "../jobs/daily-brief";
+import { runVinAnomalyDetect } from "../jobs/vin-anomaly";
 
 const QUEUE = "insight-pipeline";
 
@@ -77,6 +78,15 @@ async function runJob(job: Job) {
       console.log(`[pipeline] daily-brief sent=${res.sent}${res.reason ? ` reason=${res.reason}` : ""}`);
       return res;
     }
+    case "vin-anomaly-detect": {
+      const res = await runVinAnomalyDetect();
+      if (res.anomalies > 0 || !res.ok) {
+        console.log(
+          `[pipeline] vin-anomaly checked=${res.checked} anomalies=${res.anomalies} fired=${res.alertsFired} deduped=${res.alertsDeduped}${res.reason ? ` reason=${res.reason}` : ""}`,
+        );
+      }
+      return res;
+    }
     default:
       return { ok: false, error: `unknown job ${job.name}` };
   }
@@ -123,6 +133,11 @@ export async function startInsightPipeline() {
     { pattern: "0 5 * * *" }, // 08:00 Europe/Istanbul = 05:00 UTC
     { name: "daily-brief", data: {}, opts: { removeOnComplete: 30, removeOnFail: 15 } },
   );
+  await queue.upsertJobScheduler(
+    "vin-anomaly-detect",
+    { pattern: "*/5 * * * *" },
+    { name: "vin-anomaly-detect", data: {}, opts: { removeOnComplete: 30, removeOnFail: 15 } },
+  );
 
   new Worker(QUEUE, runJob, {
     connection: redis,
@@ -131,6 +146,6 @@ export async function startInsightPipeline() {
     stalledInterval: 60_000,
   });
   console.log(
-    "[pipeline] armed: posthog-ingest@*/5min, tag-sessions@*/2min, compress-sessions@*/3min, analyze@*/4min, validation@05:00, github-sync@*/10min, retention@04:15, daily-brief@05:00",
+    "[pipeline] armed: posthog-ingest@*/5min, tag-sessions@*/2min, compress-sessions@*/3min, analyze@*/4min, validation@05:00, github-sync@*/10min, retention@04:15, daily-brief@05:00, vin-anomaly-detect@*/5min",
   );
 }