fix(sase): p95_latency_spike — fresh-only p95 + min fresh volume + 8s floor
p95 was computed over ALL successful rows: cache hits (~10-30ms) set the baseline, a single lock_wait row (duplicate click waiting on the in-flight decode of the same VIN) set the spike. On a healthy 3-query window this read as 'P95 15ms → 2058ms (137× baseline)' (2026-06-11, 4th such false alert). - p95 now over FRESH decodes only (source='cache', db_hit/redis_positive and lock_wait rows excluded; null-safe via coalesce) - both windows must have ≥4 fresh decodes — a 'p95' of fewer is one slow request, not a tail - absolute floor raised 1s → 8s (fresh decodes legitimately take seconds; matches the dashboard's fresh-P95 ok≤8s tone) Simulated on today's prod data: alert window had 0 fresh decodes → silent. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -24,6 +24,16 @@ const MIN_VOLUME_SPIKE_ABS = 25;
|
|||||||
// unknown_vin_spike needs enough DISTINCT truly-unrecognized lookups that it
|
// unknown_vin_spike needs enough DISTINCT truly-unrecognized lookups that it
|
||||||
// reads as a decoder problem, not a couple users querying uncovered cars.
|
// reads as a decoder problem, not a couple users querying uncovered cars.
|
||||||
const MIN_UNKNOWN_LOOKUPS = 4;
|
const MIN_UNKNOWN_LOOKUPS = 4;
|
||||||
|
// p95_latency_spike: p95 is computed over FRESH decodes only (cache hits answer
|
||||||
|
// in ~10-30ms and lock_wait rows measure a duplicate-click queue, not decode
|
||||||
|
// speed — mixing them made p95 ≈ max of 3 values: 15ms baselines vs one 2s
|
||||||
|
// lock_wait fired "137× baseline" on a healthy window, 2026-06-11). A p95 over
|
||||||
|
// fewer than this many fresh decodes is just one slow request, not a tail.
|
||||||
|
const MIN_FRESH_P95 = 4;
|
||||||
|
// Fresh decodes legitimately take seconds (multi-source race through residential
|
||||||
|
// proxies); only alert when the fresh tail is genuinely bad in absolute terms.
|
||||||
|
// Matches the dashboard's "P95 yanıt (fresh)" ok≤8s tone.
|
||||||
|
const P95_FRESH_FLOOR_MS = 8_000;
|
||||||
|
|
||||||
export type AnomalyType =
|
export type AnomalyType =
|
||||||
| "success_rate_drop"
|
| "success_rate_drop"
|
||||||
@@ -53,7 +63,11 @@ type WindowStats = {
|
|||||||
failed: number;
|
failed: number;
|
||||||
successRate: number;
|
successRate: number;
|
||||||
errorRate: number;
|
errorRate: number;
|
||||||
|
// p95 over FRESH decodes only (cache hits + lock_wait rows excluded) — see
|
||||||
|
// MIN_FRESH_P95. Null when the window had no fresh decodes.
|
||||||
p95: number | null;
|
p95: number | null;
|
||||||
|
// How many fresh decodes the p95 stands on — the spike check gates on this.
|
||||||
|
freshCount: number;
|
||||||
avgMs: number | null;
|
avgMs: number | null;
|
||||||
timeouts: number;
|
timeouts: number;
|
||||||
unknownVins: number;
|
unknownVins: number;
|
||||||
@@ -80,6 +94,7 @@ async function windowStats(start: Date, end: Date): Promise<WindowStats> {
|
|||||||
succeeded: bigint;
|
succeeded: bigint;
|
||||||
failed: bigint;
|
failed: bigint;
|
||||||
p95: number | null;
|
p95: number | null;
|
||||||
|
fresh_count: bigint;
|
||||||
avg_ms: number | null;
|
avg_ms: number | null;
|
||||||
timeouts: bigint;
|
timeouts: bigint;
|
||||||
unknown_vins: bigint;
|
unknown_vins: bigint;
|
||||||
@@ -96,8 +111,19 @@ async function windowStats(start: Date, end: Date): Promise<WindowStats> {
|
|||||||
count(*) AS total,
|
count(*) AS total,
|
||||||
count(*) FILTER (WHERE success = true) AS succeeded,
|
count(*) FILTER (WHERE success = true) AS succeeded,
|
||||||
count(*) FILTER (WHERE success = false) AS failed,
|
count(*) FILTER (WHERE success = false) AS failed,
|
||||||
|
-- Fresh decodes only: cache hits answer in ~10-30ms and lock_wait rows
|
||||||
|
-- measure how long a duplicate request waited for the in-flight decode of
|
||||||
|
-- the same VIN — neither says anything about decode speed. coalesce keeps
|
||||||
|
-- the predicate null-safe for rows without timings.
|
||||||
percentile_cont(0.95) WITHIN GROUP (ORDER BY response_time_ms)
|
percentile_cont(0.95) WITHIN GROUP (ORDER BY response_time_ms)
|
||||||
FILTER (WHERE success = true AND response_time_ms IS NOT NULL)::int AS p95,
|
FILTER (WHERE success = true AND response_time_ms IS NOT NULL
|
||||||
|
AND coalesce(source, '') <> 'cache'
|
||||||
|
AND coalesce(timings->>'cache_source', 'miss')
|
||||||
|
NOT IN ('db_hit', 'redis_positive', 'lock_wait'))::int AS p95,
|
||||||
|
count(*) FILTER (WHERE success = true AND response_time_ms IS NOT NULL
|
||||||
|
AND coalesce(source, '') <> 'cache'
|
||||||
|
AND coalesce(timings->>'cache_source', 'miss')
|
||||||
|
NOT IN ('db_hit', 'redis_positive', 'lock_wait')) AS fresh_count,
|
||||||
avg(response_time_ms) FILTER (WHERE success = true AND response_time_ms IS NOT NULL)::int AS avg_ms,
|
avg(response_time_ms) FILTER (WHERE success = true AND response_time_ms IS NOT NULL)::int AS avg_ms,
|
||||||
count(*) FILTER (
|
count(*) FILTER (
|
||||||
WHERE success = false
|
WHERE success = false
|
||||||
@@ -187,6 +213,7 @@ async function windowStats(start: Date, end: Date): Promise<WindowStats> {
|
|||||||
succeeded: 0n,
|
succeeded: 0n,
|
||||||
failed: 0n,
|
failed: 0n,
|
||||||
p95: null,
|
p95: null,
|
||||||
|
fresh_count: 0n,
|
||||||
avg_ms: null,
|
avg_ms: null,
|
||||||
timeouts: 0n,
|
timeouts: 0n,
|
||||||
unknown_vins: 0n,
|
unknown_vins: 0n,
|
||||||
@@ -208,6 +235,7 @@ async function windowStats(start: Date, end: Date): Promise<WindowStats> {
|
|||||||
successRate: total > 0 ? succeeded / total : 0,
|
successRate: total > 0 ? succeeded / total : 0,
|
||||||
errorRate: total > 0 ? failed / total : 0,
|
errorRate: total > 0 ? failed / total : 0,
|
||||||
p95: r.p95,
|
p95: r.p95,
|
||||||
|
freshCount: Number(r.fresh_count),
|
||||||
avgMs: r.avg_ms,
|
avgMs: r.avg_ms,
|
||||||
timeouts: Number(r.timeouts),
|
timeouts: Number(r.timeouts),
|
||||||
unknownVins: Number(r.unknown_vins),
|
unknownVins: Number(r.unknown_vins),
|
||||||
@@ -243,6 +271,7 @@ async function baselineStats(currentEnd: Date): Promise<WindowStats> {
|
|||||||
successRate: 0,
|
successRate: 0,
|
||||||
errorRate: 0,
|
errorRate: 0,
|
||||||
p95: null,
|
p95: null,
|
||||||
|
freshCount: 0,
|
||||||
avgMs: null,
|
avgMs: null,
|
||||||
timeouts: 0,
|
timeouts: 0,
|
||||||
unknownVins: 0,
|
unknownVins: 0,
|
||||||
@@ -258,6 +287,7 @@ async function baselineStats(currentEnd: Date): Promise<WindowStats> {
|
|||||||
const total = samples.reduce((a, b) => a + b.total, 0);
|
const total = samples.reduce((a, b) => a + b.total, 0);
|
||||||
const succeeded = samples.reduce((a, b) => a + b.succeeded, 0);
|
const succeeded = samples.reduce((a, b) => a + b.succeeded, 0);
|
||||||
const failed = samples.reduce((a, b) => a + b.failed, 0);
|
const failed = samples.reduce((a, b) => a + b.failed, 0);
|
||||||
|
const freshCount = samples.reduce((a, b) => a + b.freshCount, 0);
|
||||||
const p95Values = samples.map((s) => s.p95).filter((v): v is number => v != null);
|
const p95Values = samples.map((s) => s.p95).filter((v): v is number => v != null);
|
||||||
const avgValues = samples.map((s) => s.avgMs).filter((v): v is number => v != null);
|
const avgValues = samples.map((s) => s.avgMs).filter((v): v is number => v != null);
|
||||||
const timeouts = samples.reduce((a, b) => a + b.timeouts, 0);
|
const timeouts = samples.reduce((a, b) => a + b.timeouts, 0);
|
||||||
@@ -269,6 +299,7 @@ async function baselineStats(currentEnd: Date): Promise<WindowStats> {
|
|||||||
successRate: total > 0 ? succeeded / total : 0,
|
successRate: total > 0 ? succeeded / total : 0,
|
||||||
errorRate: total > 0 ? failed / total : 0,
|
errorRate: total > 0 ? failed / total : 0,
|
||||||
p95: p95Values.length ? Math.round(p95Values.reduce((a, b) => a + b, 0) / p95Values.length) : null,
|
p95: p95Values.length ? Math.round(p95Values.reduce((a, b) => a + b, 0) / p95Values.length) : null,
|
||||||
|
freshCount,
|
||||||
avgMs: avgValues.length ? Math.round(avgValues.reduce((a, b) => a + b, 0) / avgValues.length) : null,
|
avgMs: avgValues.length ? Math.round(avgValues.reduce((a, b) => a + b, 0) / avgValues.length) : null,
|
||||||
timeouts,
|
timeouts,
|
||||||
unknownVins,
|
unknownVins,
|
||||||
@@ -366,18 +397,31 @@ export async function detectVinAnomalies(): Promise<{
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2. P95 latency spike: 2x baseline + over an absolute floor (1s)
|
// 2. P95 latency spike — FRESH decodes only (cache hits + lock_wait excluded
|
||||||
if (current.p95 != null && baseline.p95 != null && current.p95 > 1000) {
|
// in the SQL), and only when both windows have enough fresh decodes that a
|
||||||
|
// p95 is a tail and not one slow request. The absolute floor is fresh-
|
||||||
|
// calibrated: a 2-3s fresh p95 is a normal residential-proxy decode.
|
||||||
|
// (Pre-fix: p95 over ALL rows → 15ms cache baseline vs one 2.3s lock_wait
|
||||||
|
// row read as "137× baseline" on a healthy 3-query window, 2026-06-11.)
|
||||||
|
if (
|
||||||
|
current.p95 != null &&
|
||||||
|
baseline.p95 != null &&
|
||||||
|
current.freshCount >= MIN_FRESH_P95 &&
|
||||||
|
baseline.freshCount >= MIN_FRESH_P95 &&
|
||||||
|
current.p95 > P95_FRESH_FLOOR_MS
|
||||||
|
) {
|
||||||
const ratio = current.p95 / baseline.p95;
|
const ratio = current.p95 / baseline.p95;
|
||||||
if (ratio >= 2) {
|
if (ratio >= 2) {
|
||||||
hits.push({
|
hits.push({
|
||||||
type: "p95_latency_spike",
|
type: "p95_latency_spike",
|
||||||
severity: ratio >= 3 ? "critical" : "high",
|
severity: ratio >= 3 ? "critical" : "high",
|
||||||
message: `P95 yanıt ${baseline.p95}ms → ${current.p95}ms (${ratio.toFixed(1)}× baseline)`,
|
message:
|
||||||
|
`P95 yanıt (fresh) ${baseline.p95}ms → ${current.p95}ms ` +
|
||||||
|
`(${ratio.toFixed(1)}× baseline · ${current.freshCount} fresh decode · cache/lock_wait hariç)`,
|
||||||
baseline: baseline.p95,
|
baseline: baseline.p95,
|
||||||
observed: current.p95,
|
observed: current.p95,
|
||||||
current_volume: current.total,
|
current_volume: current.freshCount,
|
||||||
baseline_volume: baseline.total,
|
baseline_volume: baseline.freshCount,
|
||||||
detected_at: ts,
|
detected_at: ts,
|
||||||
dedupe_key: `vin:p95_spike:${bucket}`,
|
dedupe_key: `vin:p95_spike:${bucket}`,
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user