root/sp
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

fix(analytics): VIN success_rate_drop — exclude coverage gaps, dedupe retries

Browse Source 
Completes acce1fc (which gated volume_spike/unknown_vin_spike for retry noise but
left success_rate_drop on a raw row-rate). Today two 🔴 success_rate_drop alerts
fired (88.9%→60%, 61.9%→33.3%) on a single user retrying one catalogless Fiat VIN
6× in a low-volume window — 0 real decode failures.

Root cause: success_rate_drop used succeeded/total over raw rows with only a
MIN_CURRENT_VOLUME=5 gate — no distinct-user gate and no coverage-gap exclusion,
unlike the sibling checks. At ~2.6 decodes/15min, one user's retries collapse the
"rate".

Fix — bring it to the same standard:
- Measure REAL (infra) failures only: exclude coverage gaps (no-catalog / unknown
  / unsupported VIN; ~20-35% of traffic, driven by which cars users query, not
  decode health — already tracked by unknown_vin_spike).
- Dedupe by DISTINCT (user,vin) lookup so retries count once.
- Gate on ≥2 distinct users + ≥6 decodable lookups; fire on ≥10pp rise in real
  failure rate AND ≥20% absolute floor.
- Also added an absolute-volume floor (MIN_VOLUME_SPIKE_ABS=25) to volume_spike,
  which fired at now=8 yesterday (below baseline).

Validated against prod query_logs: both alert windows now GATED (real_failed=0);
the 06-10 09:30 real-incident window (7 users, 5/8 real failures, DataImpulse
degradation) still FIRES. typecheck clean.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Semih
2026-06-11 10:39:38 +03:00
parent acce1fcc8e
commit 1d9de6d8e6
1 changed files with 100 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

116
apps/web/src/lib/sase/vin-anomaly.ts
View File

@@ -12,6 +12,15 @@ const MIN_DISTINCT_USERS = 2;
// volume_drop needs a meaningful expected volume — at ~2.5 expected/window
// (night hours) a single quiet window reads as a −100% outage.
const MIN_EXPECTED_PER_WINDOW = 4;
// success_rate_drop is a RATE over a tiny window (~2.6 decodes/15min). It must
// see enough DISTINCT (user,vin) lookups that aren't coverage gaps before a
// "rate" means anything — otherwise one user retrying one catalogless VIN reads
// as a 30pp collapse (seen 2026-06-11: 6 retries of one Fiat VIN → 88.9%→60%).
const MIN_DECODABLE_LOOKUPS = 6;
// volume_spike is meant to catch fraud/viral surges, not a normal busy 15min.
// With a ~1.4 expected/window baseline, 8 decodes already trips 5×; require a
// real absolute surge too (seen 2026-06-10: fired at now=8, below baseline).
const MIN_VOLUME_SPIKE_ABS = 25;
export type AnomalyType =
| "success_rate_drop"
@@ -48,6 +57,14 @@ type WindowStats = {
distinctUsers: number;
distinctLookups: number;
unknownVinUsers: number;
// Real (infra) failures = failures that are NOT coverage gaps (no-catalog /
// unknown / unsupported VIN). These are the decode-health signal.
realFailed: number;
// Distinct (user,vin) lookups excluding coverage gaps — the denominator for a
// meaningful real-failure rate (dedupes a retry-happy user).
decodableLookups: number;
// Distinct (user,vin) lookups that hit a real (infra) failure.
realFailedLookups: number;
};
async function windowStats(start: Date, end: Date): Promise<WindowStats> {
@@ -63,6 +80,9 @@ async function windowStats(start: Date, end: Date): Promise<WindowStats> {
distinct_users: bigint;
distinct_lookups: bigint;
unknown_vin_users: bigint;
real_failed: bigint;
decodable_lookups: bigint;
real_failed_lookups: bigint;
}>
>`
SELECT
@@ -94,12 +114,47 @@ async function windowStats(start: Date, end: Date): Promise<WindowStats> {
count(DISTINCT user_id) FILTER (
WHERE success = false
AND (
error_message ILIKE '%unknown vin%'
error_message ILIKE '%catalog%'
OR error_message ILIKE '%unknown vin%'
OR error_message ILIKE '%tanınamad%'
OR error_message ILIKE '%destekl%'
OR (timings->>'result_kind') = 'unknown'
)
) AS unknown_vin_users
) AS unknown_vin_users,
-- Real (infra) failures: failed AND not a coverage gap (no-catalog /
-- unknown / unsupported VIN). These are the decode-health signal.
count(*) FILTER (
WHERE success = false
AND NOT (
error_message ILIKE '%catalog%'
OR error_message ILIKE '%unknown vin%'
OR error_message ILIKE '%tanınamad%'
OR error_message ILIKE '%destekl%'
OR (timings->>'result_kind') = 'unknown'
)
) AS real_failed,
-- Distinct (user,vin) lookups that are NOT coverage gaps = the meaningful
-- denominator (dedupes one user hammering one VIN).
count(DISTINCT (user_id, vin)) FILTER (
WHERE success = true
OR NOT (
error_message ILIKE '%catalog%'
OR error_message ILIKE '%unknown vin%'
OR error_message ILIKE '%tanınamad%'
OR error_message ILIKE '%destekl%'
OR (timings->>'result_kind') = 'unknown'
)
) AS decodable_lookups,
count(DISTINCT (user_id, vin)) FILTER (
WHERE success = false
AND NOT (
error_message ILIKE '%catalog%'
OR error_message ILIKE '%unknown vin%'
OR error_message ILIKE '%tanınamad%'
OR error_message ILIKE '%destekl%'
OR (timings->>'result_kind') = 'unknown'
)
) AS real_failed_lookups
FROM query_logs
WHERE created_at >= ${start} AND created_at < ${end}
`;
@@ -114,6 +169,9 @@ async function windowStats(start: Date, end: Date): Promise<WindowStats> {
distinct_users: 0n,
distinct_lookups: 0n,
unknown_vin_users: 0n,
real_failed: 0n,
decodable_lookups: 0n,
real_failed_lookups: 0n,
};
const total = Number(r.total);
const succeeded = Number(r.succeeded);
@@ -131,6 +189,9 @@ async function windowStats(start: Date, end: Date): Promise<WindowStats> {
distinctUsers: Number(r.distinct_users),
distinctLookups: Number(r.distinct_lookups),
unknownVinUsers: Number(r.unknown_vin_users),
realFailed: Number(r.real_failed),
decodableLookups: Number(r.decodable_lookups),
realFailedLookups: Number(r.real_failed_lookups),
};
}
@@ -162,6 +223,9 @@ async function baselineStats(currentEnd: Date): Promise<WindowStats> {
distinctUsers: 0,
distinctLookups: 0,
unknownVinUsers: 0,
realFailed: 0,
decodableLookups: 0,
realFailedLookups: 0,
};
}
const total = samples.reduce((a, b) => a + b.total, 0);
@@ -182,10 +246,14 @@ async function baselineStats(currentEnd: Date): Promise<WindowStats> {
timeouts,
unknownVins,
// Distinct counts don't aggregate across day-samples; summed here only to
// satisfy the shape — gating uses the current window's values.
// satisfy the shape — gating uses the current window's values. The pooled
// realFailedLookups/decodableLookups still give a usable baseline fail RATE.
distinctUsers: samples.reduce((a, b) => a + b.distinctUsers, 0),
distinctLookups: samples.reduce((a, b) => a + b.distinctLookups, 0),
unknownVinUsers: samples.reduce((a, b) => a + b.unknownVinUsers, 0),
realFailed: samples.reduce((a, b) => a + b.realFailed, 0),
decodableLookups: samples.reduce((a, b) => a + b.decodableLookups, 0),
realFailedLookups: samples.reduce((a, b) => a + b.realFailedLookups, 0),
};
}
@@ -233,22 +301,37 @@ export async function detectVinAnomalies(): Promise<{
const ts = now.toISOString();
const bucket = Math.floor(now.getTime() / (CURRENT_WINDOW_MIN * 60_000));
// 1. Success rate drop: >= 5 percentage points below baseline (high), >= 15pp (critical)
// 1. Success rate drop — REAL decode failures only (proxy/timeout/transport).
// Coverage gaps ("no catalog"/unknown VIN; ~20-35% of traffic and driven by
// which cars users happen to query, not decode health) are EXCLUDED — they're
// tracked by unknown_vin_spike. Measured over DISTINCT (user,vin) lookups so a
// retry-happy user can't move it, and gated on ≥2 distinct users + enough
// decodable lookups that a rate is statistically meaningful. (Pre-fix this was
// a raw row-rate over 5 rows with no user/coverage gating → one user retrying
// one catalogless VIN read as a 30pp collapse.)
if (
current.total >= MIN_CURRENT_VOLUME &&
baseline.total >= MIN_BASELINE_VOLUME &&
baseline.successRate > 0
current.decodableLookups >= MIN_DECODABLE_LOOKUPS &&
current.distinctUsers >= MIN_DISTINCT_USERS &&
baseline.decodableLookups >= MIN_BASELINE_VOLUME
) {
const dropPp = (baseline.successRate - current.successRate) * 100;
if (dropPp >= 5) {
const curFailRate = current.realFailedLookups / current.decodableLookups;
const baseFailRate = baseline.realFailedLookups / baseline.decodableLookups;
const risePp = (curFailRate - baseFailRate) * 100;
// Need both a real rise vs baseline AND an absolute floor — a jump from 1% to
// 12% on a quiet window isn't worth a 🔴.
if (risePp >= 10 && curFailRate >= 0.2) {
const curSucc = 1 - curFailRate;
const baseSucc = 1 - baseFailRate;
hits.push({
type: "success_rate_drop",
severity: dropPp >= 15 ? "critical" : "high",
message: `Decode başarı oranı ${pct(baseline.successRate)} → ${pct(current.successRate)} (−${dropPp.toFixed(1)}pp, son ${CURRENT_WINDOW_MIN}dk)`,
baseline: baseline.successRate,
observed: current.successRate,
current_volume: current.total,
baseline_volume: baseline.total,
severity: risePp >= 25 ? "critical" : "high",
message:
`Decode başarı oranı ${pct(baseSucc)} → ${pct(curSucc)} ` +
`(−${risePp.toFixed(1)}pp gerçek hata · ${current.realFailedLookups}/${current.decodableLookups} lookup · katalog-boşluğu hariç · son ${CURRENT_WINDOW_MIN}dk)`,
baseline: baseSucc,
observed: curSucc,
current_volume: current.decodableLookups,
baseline_volume: baseline.decodableLookups,
detected_at: ts,
dedupe_key: `vin:success_rate_drop:${bucket}`,
});
@@ -300,8 +383,9 @@ export async function detectVinAnomalies(): Promise<{
// that the volume isn't just the same VIN re-queried.
if (
baseline.total >= MIN_BASELINE_VOLUME &&
current.total >= MIN_VOLUME_SPIKE_ABS &&
current.distinctUsers >= MIN_DISTINCT_USERS &&
current.distinctLookups >= MIN_CURRENT_VOLUME
current.distinctLookups >= MIN_VOLUME_SPIKE_ABS
) {
const expectedPerWindow = baseline.total / BASELINE_DAYS;
if (expectedPerWindow >= 1 && current.total / expectedPerWindow >= 5) {