feat(sase): deploy regression Telegram alert

Closes the alert side of the deploy-regression view added in Faz 2b.
The dashboard table already flagged regressed deploys; this commit
pushes a Telegram when one happens, so MTTD doesn't depend on the
founder checking the dashboard.

Detection (panel)
- detectVinRegressions() pulls the last 20 Coolify deploys for the
  Sase.tr app, filters to those whose post-window has elapsed (≥30min
  since finishedAt) and isn't too old (≤180min since finishedAt), and
  reuses analyzeDeployRegressions to compute the 30min before/after
  success-rate slices. A row is flagged when:
    - both before and after have ≥5 samples, and
    - success rate dropped ≥10pp (severity 'high'; ≥15pp → 'critical').
- Returns a RegressionHit per flagged deploy with a 24h dedupe TTL
  keyed on deploymentUuid so each deploy alerts exactly once ever
  (regardless of how often the 5-min cron checks).

Endpoint
- GET /api/internal/vin-anomaly-check now returns
  { ok, current, baseline, anomalies, regressions }.

Worker
- sendTelegram() accepts an optional dedupeTtlSeconds override so
  per-call long-TTL dedupes (like deploy alerts) don't have to go
  through the global env default.
- New alertVinRegression() formats severity icon + before/after %
  + deploy commit/timestamp + dashboard link.
- runVinAnomalyDetect now also walks the regressions array and
  fires Telegram for each. Returns { anomalies, regressions,
  alertsFired, alertsDeduped }; pipeline log prints when either
  count is non-zero.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Semih
2026-05-18 13:33:07 +03:00
parent b5ba8919b8
commit a10996f6c5
5 changed files with 170 additions and 13 deletions

View File

@@ -1,6 +1,6 @@
import { NextResponse } from "next/server";
import { timingSafeEqual } from "node:crypto";
import { detectVinAnomalies } from "@/lib/sase/vin-anomaly";
import { detectVinAnomalies, detectVinRegressions } from "@/lib/sase/vin-anomaly";
export const dynamic = "force-dynamic";
@@ -20,6 +20,9 @@ export async function GET(req: Request) {
if (!validToken(req)) {
return NextResponse.json({ ok: false, error: "unauthorized" }, { status: 401 });
}
const result = await detectVinAnomalies();
return NextResponse.json({ ok: true, ...result });
const [anomalyResult, regressions] = await Promise.all([
detectVinAnomalies(),
detectVinRegressions(),
]);
return NextResponse.json({ ok: true, ...anomalyResult, regressions });
}

View File

@@ -1,4 +1,5 @@
import { saseDb } from "@/lib/db-sase";
import { listSaseDeploys, analyzeDeployRegressions } from "./deploy-timeline";
const CURRENT_WINDOW_MIN = 15;
const BASELINE_DAYS = 7;
@@ -248,3 +249,72 @@ export async function detectVinAnomalies(): Promise<{
function pct(v: number): string {
return `${(v * 100).toFixed(1)}%`;
}
// ─── Deploy regression detection ──────────────────────────────────────────
// Per-deploy comparison of success rate in the 30min window before vs after
// the deploy finished. A regression is a ≥10pp drop with ≥5 samples on
// both sides. Each deploy fires at most one Telegram (dedupe by uuid +
// long TTL on the worker side).
const REGRESSION_DROP_PP = 10;
const CRITICAL_DROP_PP = 15;
// Only inspect deploys whose post-window has fully elapsed.
const POST_WINDOW_MIN = 30;
// Only inspect deploys finished in the last LOOKBACK_MIN minutes — older ones
// have either already alerted or won't be useful (sustained issues will be
// caught by the baseline-anomaly detector anyway).
const REGRESSION_LOOKBACK_MIN = 180;
export type RegressionHit = {
type: "deploy_regression";
severity: "high" | "critical";
message: string;
deploymentUuid: string;
commit: string | null;
deployStartedAt: string;
before: { successRate: number; total: number };
after: { successRate: number; total: number };
deltaPp: number;
detected_at: string;
dedupe_key: string;
dedupe_ttl_seconds: number;
};
export async function detectVinRegressions(): Promise<RegressionHit[]> {
const deploys = await listSaseDeploys(20);
if (deploys.length === 0) return [];
const now = Date.now();
const candidate = deploys.filter((d) => {
const finishedAt = d.finishedAt ?? d.startedAt;
const age = now - finishedAt.getTime();
return (
age >= POST_WINDOW_MIN * 60_000 && age <= REGRESSION_LOOKBACK_MIN * 60_000
);
});
if (candidate.length === 0) return [];
const analyses = await analyzeDeployRegressions(candidate, REGRESSION_DROP_PP);
return analyses
.filter((a) => a.regressed)
.map<RegressionHit>((a) => {
const commit = a.deploy.commit?.slice(0, 8) ?? "?";
const severity: "high" | "critical" =
a.successRateDeltaPp <= -CRITICAL_DROP_PP ? "critical" : "high";
return {
type: "deploy_regression",
severity,
message: `Deploy ${commit}: ${pct(a.before.successRate)}${pct(a.after.successRate)} (${a.successRateDeltaPp.toFixed(1)}pp, n=${a.before.total}/${a.after.total})`,
deploymentUuid: a.deploy.deploymentUuid,
commit: a.deploy.commit,
deployStartedAt: a.deploy.startedAt.toISOString(),
before: { successRate: a.before.successRate, total: a.before.total },
after: { successRate: a.after.successRate, total: a.after.total },
deltaPp: a.successRateDeltaPp,
detected_at: new Date().toISOString(),
// Per-deploy dedupe with 24h TTL — one alert per deploy ever.
dedupe_key: `vin:regression:${a.deploy.deploymentUuid}`,
dedupe_ttl_seconds: 86_400,
};
});
}