feat(sase): deploy regression Telegram alert

Closes the alert side of the deploy-regression view added in Faz 2b.
The dashboard table already flagged regressed deploys; this commit
pushes a Telegram when one happens, so MTTD doesn't depend on the
founder checking the dashboard.

Detection (panel)
- detectVinRegressions() pulls the last 20 Coolify deploys for the
  Sase.tr app, filters to those whose post-window has elapsed (≥30min
  since finishedAt) and isn't too old (≤180min since finishedAt), and
  reuses analyzeDeployRegressions to compute the 30min before/after
  success-rate slices. A row is flagged when:
    - both before and after have ≥5 samples, and
    - success rate dropped ≥10pp (severity 'high'; ≥15pp → 'critical').
- Returns a RegressionHit per flagged deploy with a 24h dedupe TTL
  keyed on deploymentUuid so each deploy alerts exactly once ever
  (regardless of how often the 5-min cron checks).

Endpoint
- GET /api/internal/vin-anomaly-check now returns
  { ok, current, baseline, anomalies, regressions }.

Worker
- sendTelegram() accepts an optional dedupeTtlSeconds override so
  per-call long-TTL dedupes (like deploy alerts) don't have to go
  through the global env default.
- New alertVinRegression() formats severity icon + before/after %
  + deploy commit/timestamp + dashboard link.
- runVinAnomalyDetect now also walks the regressions array and
  fires Telegram for each. Returns { anomalies, regressions,
  alertsFired, alertsDeduped }; pipeline log prints when either
  count is non-zero.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

apps/web/src/app/api/internal/vin-anomaly-check/route.ts

@@ -1,6 +1,6 @@
 import { NextResponse } from "next/server";
 import { timingSafeEqual } from "node:crypto";
-import { detectVinAnomalies } from "@/lib/sase/vin-anomaly";
+import { detectVinAnomalies, detectVinRegressions } from "@/lib/sase/vin-anomaly";
 
 export const dynamic = "force-dynamic";
 
@@ -20,6 +20,9 @@ export async function GET(req: Request) {
   if (!validToken(req)) {
     return NextResponse.json({ ok: false, error: "unauthorized" }, { status: 401 });
   }
-  const result = await detectVinAnomalies();
-  return NextResponse.json({ ok: true, ...result });
+  const [anomalyResult, regressions] = await Promise.all([
+    detectVinAnomalies(),
+    detectVinRegressions(),
+  ]);
+  return NextResponse.json({ ok: true, ...anomalyResult, regressions });
 }

apps/web/src/lib/sase/vin-anomaly.ts

@@ -1,4 +1,5 @@
 import { saseDb } from "@/lib/db-sase";
+import { listSaseDeploys, analyzeDeployRegressions } from "./deploy-timeline";
 
 const CURRENT_WINDOW_MIN = 15;
 const BASELINE_DAYS = 7;
@@ -248,3 +249,72 @@ export async function detectVinAnomalies(): Promise<{
 function pct(v: number): string {
   return `${(v * 100).toFixed(1)}%`;
 }
+
+// ─── Deploy regression detection ──────────────────────────────────────────
+// Per-deploy comparison of success rate in the 30min window before vs after
+// the deploy finished. A regression is a ≥10pp drop with ≥5 samples on
+// both sides. Each deploy fires at most one Telegram (dedupe by uuid +
+// long TTL on the worker side).
+
+const REGRESSION_DROP_PP = 10;
+const CRITICAL_DROP_PP = 15;
+// Only inspect deploys whose post-window has fully elapsed.
+const POST_WINDOW_MIN = 30;
+// Only inspect deploys finished in the last LOOKBACK_MIN minutes — older ones
+// have either already alerted or won't be useful (sustained issues will be
+// caught by the baseline-anomaly detector anyway).
+const REGRESSION_LOOKBACK_MIN = 180;
+
+export type RegressionHit = {
+  type: "deploy_regression";
+  severity: "high" | "critical";
+  message: string;
+  deploymentUuid: string;
+  commit: string | null;
+  deployStartedAt: string;
+  before: { successRate: number; total: number };
+  after: { successRate: number; total: number };
+  deltaPp: number;
+  detected_at: string;
+  dedupe_key: string;
+  dedupe_ttl_seconds: number;
+};
+
+export async function detectVinRegressions(): Promise<RegressionHit[]> {
+  const deploys = await listSaseDeploys(20);
+  if (deploys.length === 0) return [];
+
+  const now = Date.now();
+  const candidate = deploys.filter((d) => {
+    const finishedAt = d.finishedAt ?? d.startedAt;
+    const age = now - finishedAt.getTime();
+    return (
+      age >= POST_WINDOW_MIN * 60_000 && age <= REGRESSION_LOOKBACK_MIN * 60_000
+    );
+  });
+  if (candidate.length === 0) return [];
+
+  const analyses = await analyzeDeployRegressions(candidate, REGRESSION_DROP_PP);
+  return analyses
+    .filter((a) => a.regressed)
+    .map<RegressionHit>((a) => {
+      const commit = a.deploy.commit?.slice(0, 8) ?? "?";
+      const severity: "high" | "critical" =
+        a.successRateDeltaPp <= -CRITICAL_DROP_PP ? "critical" : "high";
+      return {
+        type: "deploy_regression",
+        severity,
+        message: `Deploy ${commit}: ${pct(a.before.successRate)} → ${pct(a.after.successRate)} (${a.successRateDeltaPp.toFixed(1)}pp, n=${a.before.total}/${a.after.total})`,
+        deploymentUuid: a.deploy.deploymentUuid,
+        commit: a.deploy.commit,
+        deployStartedAt: a.deploy.startedAt.toISOString(),
+        before: { successRate: a.before.successRate, total: a.before.total },
+        after: { successRate: a.after.successRate, total: a.after.total },
+        deltaPp: a.successRateDeltaPp,
+        detected_at: new Date().toISOString(),
+        // Per-deploy dedupe with 24h TTL — one alert per deploy ever.
+        dedupe_key: `vin:regression:${a.deploy.deploymentUuid}`,
+        dedupe_ttl_seconds: 86_400,
+      };
+    });
+}

apps/worker/src/jobs/vin-anomaly.ts

@@ -1,4 +1,8 @@
-import { alertVinAnomaly, isTelegramConfigured } from "../lib/telegram";
+import {
+  alertVinAnomaly,
+  alertVinRegression,
+  isTelegramConfigured,
+} from "../lib/telegram";
 
 const PANEL_BASE =
   process.env.PANEL_INTERNAL_URL ??
@@ -19,27 +23,60 @@ type AnomalyHit = {
   dedupe_key: string;
 };
 
+type RegressionHit = {
+  type: "deploy_regression";
+  severity: "high" | "critical";
+  message: string;
+  deploymentUuid: string;
+  commit: string | null;
+  deployStartedAt: string;
+  before: { successRate: number; total: number };
+  after: { successRate: number; total: number };
+  deltaPp: number;
+  detected_at: string;
+  dedupe_key: string;
+  dedupe_ttl_seconds: number;
+};
+
 type CheckResponse = {
   ok: boolean;
   error?: string;
   current?: { total: number; successRate: number };
   baseline?: { total: number; successRate: number };
   anomalies?: AnomalyHit[];
+  regressions?: RegressionHit[];
 };
 
 export async function runVinAnomalyDetect(): Promise<{
   ok: boolean;
   checked: boolean;
   anomalies: number;
+  regressions: number;
   alertsFired: number;
   alertsDeduped: number;
   reason?: string;
 }> {
   if (!WORKER_TOKEN) {
-    return { ok: false, checked: false, anomalies: 0, alertsFired: 0, alertsDeduped: 0, reason: "INTERNAL_WORKER_TOKEN not set" };
+    return {
+      ok: false,
+      checked: false,
+      anomalies: 0,
+      regressions: 0,
+      alertsFired: 0,
+      alertsDeduped: 0,
+      reason: "INTERNAL_WORKER_TOKEN not set",
+    };
   }
   if (!isTelegramConfigured()) {
-    return { ok: false, checked: false, anomalies: 0, alertsFired: 0, alertsDeduped: 0, reason: "telegram not configured" };
+    return {
+      ok: false,
+      checked: false,
+      anomalies: 0,
+      regressions: 0,
+      alertsFired: 0,
+      alertsDeduped: 0,
+      reason: "telegram not configured",
+    };
   }
 
   let res: Response;
@@ -55,6 +92,7 @@ export async function runVinAnomalyDetect(): Promise<{
       ok: false,
       checked: false,
       anomalies: 0,
+      regressions: 0,
       alertsFired: 0,
       alertsDeduped: 0,
       reason: `fetch failed: ${(e as Error).message}`,
@@ -65,17 +103,19 @@ export async function runVinAnomalyDetect(): Promise<{
       ok: false,
       checked: false,
       anomalies: 0,
+      regressions: 0,
       alertsFired: 0,
       alertsDeduped: 0,
       reason: `panel returned ${res.status}`,
     };
   }
   const data = (await res.json()) as CheckResponse;
-  if (!data.ok || !data.anomalies) {
+  if (!data.ok) {
     return {
-      ok: !!data.ok,
+      ok: false,
       checked: true,
       anomalies: 0,
+      regressions: 0,
       alertsFired: 0,
       alertsDeduped: 0,
       reason: data.error,
@@ -83,17 +123,25 @@ export async function runVinAnomalyDetect(): Promise<{
   }
 
   const panelPublic = process.env.PANEL_PUBLIC_URL ?? "https://sp.semih.ai";
+  const anomalies = data.anomalies ?? [];
+  const regressions = data.regressions ?? [];
   let fired = 0;
   let deduped = 0;
-  for (const a of data.anomalies) {
+  for (const a of anomalies) {
     const r = await alertVinAnomaly({ panelUrl: panelPublic, ...a });
     if (r.ok && r.deduped) deduped++;
     else if (r.ok) fired++;
   }
+  for (const r of regressions) {
+    const out = await alertVinRegression({ panelUrl: panelPublic, ...r });
+    if (out.ok && out.deduped) deduped++;
+    else if (out.ok) fired++;
+  }
   return {
     ok: true,
     checked: true,
-    anomalies: data.anomalies.length,
+    anomalies: anomalies.length,
+    regressions: regressions.length,
     alertsFired: fired,
     alertsDeduped: deduped,
   };

apps/worker/src/lib/telegram.ts

@@ -16,6 +16,7 @@ export type TelegramSendResult = { ok: boolean; deduped?: boolean; error?: strin
 export async function sendTelegram(opts: {
   text: string;
   dedupeKey?: string; // optional — skip if same key fired in last TTL window
+  dedupeTtlSeconds?: number; // override the env default for this send
   silent?: boolean;
   parseMode?: "HTML" | "Markdown";
 }): Promise<TelegramSendResult> {
@@ -24,7 +25,8 @@ export async function sendTelegram(opts: {
   // Dedupe check
   if (opts.dedupeKey) {
     const key = `tg:dedupe:${opts.dedupeKey}`;
-    const set = await redis.set(key, "1", "EX", DEDUPE_TTL_SECONDS, "NX").catch(() => null);
+    const ttl = opts.dedupeTtlSeconds ?? DEDUPE_TTL_SECONDS;
+    const set = await redis.set(key, "1", "EX", ttl, "NX").catch(() => null);
     if (set === null) return { ok: true, deduped: true };
   }
 
@@ -145,6 +147,40 @@ export function alertVinAnomaly(
   return sendTelegram({ text, dedupeKey: opts.dedupe_key });
 }
 
+export type VinRegressionAlert = {
+  type: "deploy_regression";
+  severity: "high" | "critical";
+  message: string;
+  deploymentUuid: string;
+  commit: string | null;
+  deployStartedAt: string;
+  before: { successRate: number; total: number };
+  after: { successRate: number; total: number };
+  deltaPp: number;
+  detected_at: string;
+  dedupe_key: string;
+  dedupe_ttl_seconds: number;
+};
+
+export function alertVinRegression(
+  opts: { panelUrl: string } & VinRegressionAlert,
+): Promise<TelegramSendResult> {
+  const icon = opts.severity === "critical" ? "🔴" : "🟠";
+  const commit = opts.commit?.slice(0, 8) ?? "?";
+  const text = [
+    `${icon} <b>VIN regression after deploy</b>`,
+    escapeHtml(opts.message),
+    `Deploy: <code>${commit}</code> · ${opts.deployStartedAt.slice(0, 16).replace("T", " ")}`,
+    ``,
+    `<a href="${opts.panelUrl}/projects/sase/vin-decode">VIN Decode dashboard</a>`,
+  ].join("\n");
+  return sendTelegram({
+    text,
+    dedupeKey: opts.dedupe_key,
+    dedupeTtlSeconds: opts.dedupe_ttl_seconds,
+  });
+}
+
 function escapeHtml(s: string): string {
   return s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
 }

apps/worker/src/schedulers/pipeline.ts

@@ -80,9 +80,9 @@ async function runJob(job: Job) {
     }
     case "vin-anomaly-detect": {
       const res = await runVinAnomalyDetect();
-      if (res.anomalies > 0 || !res.ok) {
+      if (res.anomalies > 0 || res.regressions > 0 || !res.ok) {
         console.log(
-          `[pipeline] vin-anomaly checked=${res.checked} anomalies=${res.anomalies} fired=${res.alertsFired} deduped=${res.alertsDeduped}${res.reason ? ` reason=${res.reason}` : ""}`,
+          `[pipeline] vin-anomaly checked=${res.checked} anomalies=${res.anomalies} regressions=${res.regressions} fired=${res.alertsFired} deduped=${res.alertsDeduped}${res.reason ? ` reason=${res.reason}` : ""}`,
         );
       }
       return res;