feat(sase): deploy regression Telegram alert
Closes the alert side of the deploy-regression view added in Faz 2b.
The dashboard table already flagged regressed deploys; this commit
pushes a Telegram when one happens, so MTTD doesn't depend on the
founder checking the dashboard.
Detection (panel)
- detectVinRegressions() pulls the last 20 Coolify deploys for the
Sase.tr app, filters to those whose post-window has elapsed (≥30min
since finishedAt) and isn't too old (≤180min since finishedAt), and
reuses analyzeDeployRegressions to compute the 30min before/after
success-rate slices. A row is flagged when:
- both before and after have ≥5 samples, and
- success rate dropped ≥10pp (severity 'high'; ≥15pp → 'critical').
- Returns a RegressionHit per flagged deploy with a 24h dedupe TTL
keyed on deploymentUuid so each deploy alerts exactly once ever
(regardless of how often the 5-min cron checks).
Endpoint
- GET /api/internal/vin-anomaly-check now returns
{ ok, current, baseline, anomalies, regressions }.
Worker
- sendTelegram() accepts an optional dedupeTtlSeconds override so
per-call long-TTL dedupes (like deploy alerts) don't have to go
through the global env default.
- New alertVinRegression() formats severity icon + before/after %
+ deploy commit/timestamp + dashboard link.
- runVinAnomalyDetect now also walks the regressions array and
fires Telegram for each. Returns { anomalies, regressions,
alertsFired, alertsDeduped }; pipeline log prints when either
count is non-zero.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { timingSafeEqual } from "node:crypto";
|
||||
import { detectVinAnomalies } from "@/lib/sase/vin-anomaly";
|
||||
import { detectVinAnomalies, detectVinRegressions } from "@/lib/sase/vin-anomaly";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
@@ -20,6 +20,9 @@ export async function GET(req: Request) {
|
||||
if (!validToken(req)) {
|
||||
return NextResponse.json({ ok: false, error: "unauthorized" }, { status: 401 });
|
||||
}
|
||||
const result = await detectVinAnomalies();
|
||||
return NextResponse.json({ ok: true, ...result });
|
||||
const [anomalyResult, regressions] = await Promise.all([
|
||||
detectVinAnomalies(),
|
||||
detectVinRegressions(),
|
||||
]);
|
||||
return NextResponse.json({ ok: true, ...anomalyResult, regressions });
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { saseDb } from "@/lib/db-sase";
|
||||
import { listSaseDeploys, analyzeDeployRegressions } from "./deploy-timeline";
|
||||
|
||||
const CURRENT_WINDOW_MIN = 15;
|
||||
const BASELINE_DAYS = 7;
|
||||
@@ -248,3 +249,72 @@ export async function detectVinAnomalies(): Promise<{
|
||||
function pct(v: number): string {
|
||||
return `${(v * 100).toFixed(1)}%`;
|
||||
}
|
||||
|
||||
// ─── Deploy regression detection ──────────────────────────────────────────
|
||||
// Per-deploy comparison of success rate in the 30min window before vs after
|
||||
// the deploy finished. A regression is a ≥10pp drop with ≥5 samples on
|
||||
// both sides. Each deploy fires at most one Telegram (dedupe by uuid +
|
||||
// long TTL on the worker side).
|
||||
|
||||
const REGRESSION_DROP_PP = 10;
|
||||
const CRITICAL_DROP_PP = 15;
|
||||
// Only inspect deploys whose post-window has fully elapsed.
|
||||
const POST_WINDOW_MIN = 30;
|
||||
// Only inspect deploys finished in the last LOOKBACK_MIN minutes — older ones
|
||||
// have either already alerted or won't be useful (sustained issues will be
|
||||
// caught by the baseline-anomaly detector anyway).
|
||||
const REGRESSION_LOOKBACK_MIN = 180;
|
||||
|
||||
export type RegressionHit = {
|
||||
type: "deploy_regression";
|
||||
severity: "high" | "critical";
|
||||
message: string;
|
||||
deploymentUuid: string;
|
||||
commit: string | null;
|
||||
deployStartedAt: string;
|
||||
before: { successRate: number; total: number };
|
||||
after: { successRate: number; total: number };
|
||||
deltaPp: number;
|
||||
detected_at: string;
|
||||
dedupe_key: string;
|
||||
dedupe_ttl_seconds: number;
|
||||
};
|
||||
|
||||
export async function detectVinRegressions(): Promise<RegressionHit[]> {
|
||||
const deploys = await listSaseDeploys(20);
|
||||
if (deploys.length === 0) return [];
|
||||
|
||||
const now = Date.now();
|
||||
const candidate = deploys.filter((d) => {
|
||||
const finishedAt = d.finishedAt ?? d.startedAt;
|
||||
const age = now - finishedAt.getTime();
|
||||
return (
|
||||
age >= POST_WINDOW_MIN * 60_000 && age <= REGRESSION_LOOKBACK_MIN * 60_000
|
||||
);
|
||||
});
|
||||
if (candidate.length === 0) return [];
|
||||
|
||||
const analyses = await analyzeDeployRegressions(candidate, REGRESSION_DROP_PP);
|
||||
return analyses
|
||||
.filter((a) => a.regressed)
|
||||
.map<RegressionHit>((a) => {
|
||||
const commit = a.deploy.commit?.slice(0, 8) ?? "?";
|
||||
const severity: "high" | "critical" =
|
||||
a.successRateDeltaPp <= -CRITICAL_DROP_PP ? "critical" : "high";
|
||||
return {
|
||||
type: "deploy_regression",
|
||||
severity,
|
||||
message: `Deploy ${commit}: ${pct(a.before.successRate)} → ${pct(a.after.successRate)} (${a.successRateDeltaPp.toFixed(1)}pp, n=${a.before.total}/${a.after.total})`,
|
||||
deploymentUuid: a.deploy.deploymentUuid,
|
||||
commit: a.deploy.commit,
|
||||
deployStartedAt: a.deploy.startedAt.toISOString(),
|
||||
before: { successRate: a.before.successRate, total: a.before.total },
|
||||
after: { successRate: a.after.successRate, total: a.after.total },
|
||||
deltaPp: a.successRateDeltaPp,
|
||||
detected_at: new Date().toISOString(),
|
||||
// Per-deploy dedupe with 24h TTL — one alert per deploy ever.
|
||||
dedupe_key: `vin:regression:${a.deploy.deploymentUuid}`,
|
||||
dedupe_ttl_seconds: 86_400,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user