fix(analytics): VIN anomaly detector — suppress single-user retry noise
Night of 2026-06-11 one user retrying a catalogless Toyota VIN tripped volume_spike + unknown_vin_spike, and ~2.5/window night baselines made quiet windows fire volume_drop at -100%. - volume_spike: require >=2 distinct users and >=5 distinct (user,vin) lookups so same-VIN retries don't count as a spike - unknown_vin_spike: require unknown failures from >=2 distinct users - volume_drop: raise expected-per-window floor from 1 to 4 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -5,6 +5,13 @@ const CURRENT_WINDOW_MIN = 15;
|
|||||||
const BASELINE_DAYS = 7;
|
const BASELINE_DAYS = 7;
|
||||||
const MIN_BASELINE_VOLUME = 10; // skip if baseline volume below this — too noisy
|
const MIN_BASELINE_VOLUME = 10; // skip if baseline volume below this — too noisy
|
||||||
const MIN_CURRENT_VOLUME = 5; // skip checks that need volume to be meaningful
|
const MIN_CURRENT_VOLUME = 5; // skip checks that need volume to be meaningful
|
||||||
|
// A single retry-happy user can produce 6+ rows in a window and trip both
|
||||||
|
// volume_spike and unknown_vin_spike (seen 2026-06-11: one user retrying one
|
||||||
|
// catalogless Toyota VIN). Require at least this many distinct users.
|
||||||
|
const MIN_DISTINCT_USERS = 2;
|
||||||
|
// volume_drop needs a meaningful expected volume — at ~2.5 expected/window
|
||||||
|
// (night hours) a single quiet window reads as a −100% outage.
|
||||||
|
const MIN_EXPECTED_PER_WINDOW = 4;
|
||||||
|
|
||||||
export type AnomalyType =
|
export type AnomalyType =
|
||||||
| "success_rate_drop"
|
| "success_rate_drop"
|
||||||
@@ -38,6 +45,9 @@ type WindowStats = {
|
|||||||
avgMs: number | null;
|
avgMs: number | null;
|
||||||
timeouts: number;
|
timeouts: number;
|
||||||
unknownVins: number;
|
unknownVins: number;
|
||||||
|
distinctUsers: number;
|
||||||
|
distinctLookups: number;
|
||||||
|
unknownVinUsers: number;
|
||||||
};
|
};
|
||||||
|
|
||||||
async function windowStats(start: Date, end: Date): Promise<WindowStats> {
|
async function windowStats(start: Date, end: Date): Promise<WindowStats> {
|
||||||
@@ -50,6 +60,9 @@ async function windowStats(start: Date, end: Date): Promise<WindowStats> {
|
|||||||
avg_ms: number | null;
|
avg_ms: number | null;
|
||||||
timeouts: bigint;
|
timeouts: bigint;
|
||||||
unknown_vins: bigint;
|
unknown_vins: bigint;
|
||||||
|
distinct_users: bigint;
|
||||||
|
distinct_lookups: bigint;
|
||||||
|
unknown_vin_users: bigint;
|
||||||
}>
|
}>
|
||||||
>`
|
>`
|
||||||
SELECT
|
SELECT
|
||||||
@@ -75,7 +88,18 @@ async function windowStats(start: Date, end: Date): Promise<WindowStats> {
|
|||||||
OR error_message ILIKE '%destekl%'
|
OR error_message ILIKE '%destekl%'
|
||||||
OR (timings->>'result_kind') = 'unknown'
|
OR (timings->>'result_kind') = 'unknown'
|
||||||
)
|
)
|
||||||
) AS unknown_vins
|
) AS unknown_vins,
|
||||||
|
count(DISTINCT user_id) AS distinct_users,
|
||||||
|
count(DISTINCT (user_id, vin)) AS distinct_lookups,
|
||||||
|
count(DISTINCT user_id) FILTER (
|
||||||
|
WHERE success = false
|
||||||
|
AND (
|
||||||
|
error_message ILIKE '%unknown vin%'
|
||||||
|
OR error_message ILIKE '%tanınamad%'
|
||||||
|
OR error_message ILIKE '%destekl%'
|
||||||
|
OR (timings->>'result_kind') = 'unknown'
|
||||||
|
)
|
||||||
|
) AS unknown_vin_users
|
||||||
FROM query_logs
|
FROM query_logs
|
||||||
WHERE created_at >= ${start} AND created_at < ${end}
|
WHERE created_at >= ${start} AND created_at < ${end}
|
||||||
`;
|
`;
|
||||||
@@ -87,6 +111,9 @@ async function windowStats(start: Date, end: Date): Promise<WindowStats> {
|
|||||||
avg_ms: null,
|
avg_ms: null,
|
||||||
timeouts: 0n,
|
timeouts: 0n,
|
||||||
unknown_vins: 0n,
|
unknown_vins: 0n,
|
||||||
|
distinct_users: 0n,
|
||||||
|
distinct_lookups: 0n,
|
||||||
|
unknown_vin_users: 0n,
|
||||||
};
|
};
|
||||||
const total = Number(r.total);
|
const total = Number(r.total);
|
||||||
const succeeded = Number(r.succeeded);
|
const succeeded = Number(r.succeeded);
|
||||||
@@ -101,6 +128,9 @@ async function windowStats(start: Date, end: Date): Promise<WindowStats> {
|
|||||||
avgMs: r.avg_ms,
|
avgMs: r.avg_ms,
|
||||||
timeouts: Number(r.timeouts),
|
timeouts: Number(r.timeouts),
|
||||||
unknownVins: Number(r.unknown_vins),
|
unknownVins: Number(r.unknown_vins),
|
||||||
|
distinctUsers: Number(r.distinct_users),
|
||||||
|
distinctLookups: Number(r.distinct_lookups),
|
||||||
|
unknownVinUsers: Number(r.unknown_vin_users),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -129,6 +159,9 @@ async function baselineStats(currentEnd: Date): Promise<WindowStats> {
|
|||||||
avgMs: null,
|
avgMs: null,
|
||||||
timeouts: 0,
|
timeouts: 0,
|
||||||
unknownVins: 0,
|
unknownVins: 0,
|
||||||
|
distinctUsers: 0,
|
||||||
|
distinctLookups: 0,
|
||||||
|
unknownVinUsers: 0,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
const total = samples.reduce((a, b) => a + b.total, 0);
|
const total = samples.reduce((a, b) => a + b.total, 0);
|
||||||
@@ -148,6 +181,11 @@ async function baselineStats(currentEnd: Date): Promise<WindowStats> {
|
|||||||
avgMs: avgValues.length ? Math.round(avgValues.reduce((a, b) => a + b, 0) / avgValues.length) : null,
|
avgMs: avgValues.length ? Math.round(avgValues.reduce((a, b) => a + b, 0) / avgValues.length) : null,
|
||||||
timeouts,
|
timeouts,
|
||||||
unknownVins,
|
unknownVins,
|
||||||
|
// Distinct counts don't aggregate across day-samples; summed here only to
|
||||||
|
// satisfy the shape — gating uses the current window's values.
|
||||||
|
distinctUsers: samples.reduce((a, b) => a + b.distinctUsers, 0),
|
||||||
|
distinctLookups: samples.reduce((a, b) => a + b.distinctLookups, 0),
|
||||||
|
unknownVinUsers: samples.reduce((a, b) => a + b.unknownVinUsers, 0),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -238,7 +276,10 @@ export async function detectVinAnomalies(): Promise<{
|
|||||||
// 3. Volume drop: >= 80% below baseline (high), >= 95% (critical) — outage signal
|
// 3. Volume drop: >= 80% below baseline (high), >= 95% (critical) — outage signal
|
||||||
if (baseline.total >= MIN_BASELINE_VOLUME) {
|
if (baseline.total >= MIN_BASELINE_VOLUME) {
|
||||||
const expectedPerWindow = baseline.total / BASELINE_DAYS;
|
const expectedPerWindow = baseline.total / BASELINE_DAYS;
|
||||||
if (expectedPerWindow >= 1 && current.total / expectedPerWindow <= 0.2) {
|
if (
|
||||||
|
expectedPerWindow >= MIN_EXPECTED_PER_WINDOW &&
|
||||||
|
current.total / expectedPerWindow <= 0.2
|
||||||
|
) {
|
||||||
const dropPct = (1 - current.total / expectedPerWindow) * 100;
|
const dropPct = (1 - current.total / expectedPerWindow) * 100;
|
||||||
hits.push({
|
hits.push({
|
||||||
type: "volume_drop",
|
type: "volume_drop",
|
||||||
@@ -254,8 +295,14 @@ export async function detectVinAnomalies(): Promise<{
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 4. Volume spike: 5x baseline — fraud or viral
|
// 4. Volume spike: 5x baseline — fraud or viral. One user's retry burst is
|
||||||
if (baseline.total >= MIN_BASELINE_VOLUME) {
|
// neither; require multiple users AND enough unique user×VIN lookups
|
||||||
|
// that the volume isn't just the same VIN re-queried.
|
||||||
|
if (
|
||||||
|
baseline.total >= MIN_BASELINE_VOLUME &&
|
||||||
|
current.distinctUsers >= MIN_DISTINCT_USERS &&
|
||||||
|
current.distinctLookups >= MIN_CURRENT_VOLUME
|
||||||
|
) {
|
||||||
const expectedPerWindow = baseline.total / BASELINE_DAYS;
|
const expectedPerWindow = baseline.total / BASELINE_DAYS;
|
||||||
if (expectedPerWindow >= 1 && current.total / expectedPerWindow >= 5) {
|
if (expectedPerWindow >= 1 && current.total / expectedPerWindow >= 5) {
|
||||||
hits.push({
|
hits.push({
|
||||||
@@ -291,7 +338,7 @@ export async function detectVinAnomalies(): Promise<{
|
|||||||
// generic success-rate-drop catches this too, but this names it explicitly
|
// generic success-rate-drop catches this too, but this names it explicitly
|
||||||
// and attributes the dominant failing source. Fire when the unknown-VIN
|
// and attributes the dominant failing source. Fire when the unknown-VIN
|
||||||
// share jumps ≥2× baseline, or surges from a near-zero baseline.
|
// share jumps ≥2× baseline, or surges from a near-zero baseline.
|
||||||
if (current.total >= MIN_CURRENT_VOLUME) {
|
if (current.total >= MIN_CURRENT_VOLUME && current.unknownVinUsers >= MIN_DISTINCT_USERS) {
|
||||||
const curRate = current.unknownVins / current.total;
|
const curRate = current.unknownVins / current.total;
|
||||||
const baseRate = baseline.total > 0 ? baseline.unknownVins / baseline.total : 0;
|
const baseRate = baseline.total > 0 ? baseline.unknownVins / baseline.total : 0;
|
||||||
const spiked =
|
const spiked =
|
||||||
|
|||||||
Reference in New Issue
Block a user