root/sp
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

fix(sase): p95_latency_spike — fresh-only p95 + min fresh volume + 8s floor

Browse Source 
p95 was computed over ALL successful rows: cache hits (~10-30ms) set the
baseline, a single lock_wait row (duplicate click waiting on the in-flight
decode of the same VIN) set the spike. On a healthy 3-query window this read
as 'P95 15ms → 2058ms (137× baseline)' (2026-06-11, 4th such false alert).

- p95 now over FRESH decodes only (source='cache', db_hit/redis_positive and
  lock_wait rows excluded; null-safe via coalesce)
- both windows must have ≥4 fresh decodes — a 'p95' of fewer is one slow
  request, not a tail
- absolute floor raised 1s → 8s (fresh decodes legitimately take seconds;
  matches the dashboard's fresh-P95 ok≤8s tone)

Simulated on today's prod data: alert window had 0 fresh decodes → silent.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Semih
2026-06-11 18:21:20 +03:00
parent b9ea801d0f
commit 5dde24855c
1 changed files with 50 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
apps/web/src/lib/sase/vin-anomaly.ts
View File

@@ -24,6 +24,16 @@ const MIN_VOLUME_SPIKE_ABS = 25;
// unknown_vin_spike needs enough DISTINCT truly-unrecognized lookups that it
// reads as a decoder problem, not a couple users querying uncovered cars.
const MIN_UNKNOWN_LOOKUPS = 4;
// p95_latency_spike: p95 is computed over FRESH decodes only (cache hits answer
// in ~10-30ms and lock_wait rows measure a duplicate-click queue, not decode
// speed — mixing them made p95 ≈ max of 3 values: 15ms baselines vs one 2s
// lock_wait fired "137× baseline" on a healthy window, 2026-06-11). A p95 over
// fewer than this many fresh decodes is just one slow request, not a tail.
const MIN_FRESH_P95 = 4;
// Fresh decodes legitimately take seconds (multi-source race through residential
// proxies); only alert when the fresh tail is genuinely bad in absolute terms.
// Matches the dashboard's "P95 yanıt (fresh)" ok≤8s tone.
const P95_FRESH_FLOOR_MS = 8_000;
export type AnomalyType =
| "success_rate_drop"
@@ -53,7 +63,11 @@ type WindowStats = {
failed: number;
successRate: number;
errorRate: number;
// p95 over FRESH decodes only (cache hits + lock_wait rows excluded) — see
// MIN_FRESH_P95. Null when the window had no fresh decodes.
p95: number | null;
// How many fresh decodes the p95 stands on — the spike check gates on this.
freshCount: number;
avgMs: number | null;
timeouts: number;
unknownVins: number;
@@ -80,6 +94,7 @@ async function windowStats(start: Date, end: Date): Promise<WindowStats> {
succeeded: bigint;
failed: bigint;
p95: number | null;
fresh_count: bigint;
avg_ms: number | null;
timeouts: bigint;
unknown_vins: bigint;
@@ -96,8 +111,19 @@ async function windowStats(start: Date, end: Date): Promise<WindowStats> {
count(*) AS total,
count(*) FILTER (WHERE success = true) AS succeeded,
count(*) FILTER (WHERE success = false) AS failed,
-- Fresh decodes only: cache hits answer in ~10-30ms and lock_wait rows
-- measure how long a duplicate request waited for the in-flight decode of
-- the same VIN — neither says anything about decode speed. coalesce keeps
-- the predicate null-safe for rows without timings.
percentile_cont(0.95) WITHIN GROUP (ORDER BY response_time_ms)
FILTER (WHERE success = true AND response_time_ms IS NOT NULL)::int AS p95,
FILTER (WHERE success = true AND response_time_ms IS NOT NULL
AND coalesce(source, '') <> 'cache'
AND coalesce(timings->>'cache_source', 'miss')
NOT IN ('db_hit', 'redis_positive', 'lock_wait'))::int AS p95,
count(*) FILTER (WHERE success = true AND response_time_ms IS NOT NULL
AND coalesce(source, '') <> 'cache'
AND coalesce(timings->>'cache_source', 'miss')
NOT IN ('db_hit', 'redis_positive', 'lock_wait')) AS fresh_count,
avg(response_time_ms) FILTER (WHERE success = true AND response_time_ms IS NOT NULL)::int AS avg_ms,
count(*) FILTER (
WHERE success = false
@@ -187,6 +213,7 @@ async function windowStats(start: Date, end: Date): Promise<WindowStats> {
succeeded: 0n,
failed: 0n,
p95: null,
fresh_count: 0n,
avg_ms: null,
timeouts: 0n,
unknown_vins: 0n,
@@ -208,6 +235,7 @@ async function windowStats(start: Date, end: Date): Promise<WindowStats> {
successRate: total > 0 ? succeeded / total : 0,
errorRate: total > 0 ? failed / total : 0,
p95: r.p95,
freshCount: Number(r.fresh_count),
avgMs: r.avg_ms,
timeouts: Number(r.timeouts),
unknownVins: Number(r.unknown_vins),
@@ -243,6 +271,7 @@ async function baselineStats(currentEnd: Date): Promise<WindowStats> {
successRate: 0,
errorRate: 0,
p95: null,
freshCount: 0,
avgMs: null,
timeouts: 0,
unknownVins: 0,
@@ -258,6 +287,7 @@ async function baselineStats(currentEnd: Date): Promise<WindowStats> {
const total = samples.reduce((a, b) => a + b.total, 0);
const succeeded = samples.reduce((a, b) => a + b.succeeded, 0);
const failed = samples.reduce((a, b) => a + b.failed, 0);
const freshCount = samples.reduce((a, b) => a + b.freshCount, 0);
const p95Values = samples.map((s) => s.p95).filter((v): v is number => v != null);
const avgValues = samples.map((s) => s.avgMs).filter((v): v is number => v != null);
const timeouts = samples.reduce((a, b) => a + b.timeouts, 0);
@@ -269,6 +299,7 @@ async function baselineStats(currentEnd: Date): Promise<WindowStats> {
successRate: total > 0 ? succeeded / total : 0,
errorRate: total > 0 ? failed / total : 0,
p95: p95Values.length ? Math.round(p95Values.reduce((a, b) => a + b, 0) / p95Values.length) : null,
freshCount,
avgMs: avgValues.length ? Math.round(avgValues.reduce((a, b) => a + b, 0) / avgValues.length) : null,
timeouts,
unknownVins,
@@ -366,18 +397,31 @@ export async function detectVinAnomalies(): Promise<{
}
}
// 2. P95 latency spike: 2x baseline + over an absolute floor (1s)
if (current.p95 != null && baseline.p95 != null && current.p95 > 1000) {
// 2. P95 latency spike — FRESH decodes only (cache hits + lock_wait excluded
// in the SQL), and only when both windows have enough fresh decodes that a
// p95 is a tail and not one slow request. The absolute floor is fresh-
// calibrated: a 2-3s fresh p95 is a normal residential-proxy decode.
// (Pre-fix: p95 over ALL rows → 15ms cache baseline vs one 2.3s lock_wait
// row read as "137× baseline" on a healthy 3-query window, 2026-06-11.)
if (
current.p95 != null &&
baseline.p95 != null &&
current.freshCount >= MIN_FRESH_P95 &&
baseline.freshCount >= MIN_FRESH_P95 &&
current.p95 > P95_FRESH_FLOOR_MS
) {
const ratio = current.p95 / baseline.p95;
if (ratio >= 2) {
hits.push({
type: "p95_latency_spike",
severity: ratio >= 3 ? "critical" : "high",
message: `P95 yanıt ${baseline.p95}ms → ${current.p95}ms (${ratio.toFixed(1)}× baseline)`,
message:
`P95 yanıt (fresh) ${baseline.p95}ms → ${current.p95}ms ` +
`(${ratio.toFixed(1)}× baseline · ${current.freshCount} fresh decode · cache/lock_wait hariç)`,
baseline: baseline.p95,
observed: current.p95,
current_volume: current.total,
baseline_volume: baseline.total,
current_volume: current.freshCount,
baseline_volume: baseline.freshCount,
detected_at: ts,
dedupe_key: `vin:p95_spike:${bucket}`,
});