fix(analytics): unknown_vin_spike — exclude identified-but-no-catalog, dedupe
After the success_rate_drop fix, the SAME benign data (one Fiat VIN, "No catalog — identified as Fiat", retried across a couple users) just moved the false positive to unknown_vin_spike (live: "Tanınamadı 11.1%→62.5%, 5/8, critical"). A car we DID identify but lack a catalog for is a coverage gap, not an unrecognized VIN. - unknown_vins / unknown_vin_users now count only TRULY unrecognized VINs: exclude identified_no_catalog=1 (the "No catalog — identified as X" case). - Add distinct_unknown_lookups (deduped) + gate the check on ≥4 distinct unrecognized lookups so a couple users' uncovered cars can't trip it. Validated vs prod query_logs: all recent windows truly_unknown=0 → GATED; a real "tanınamadı" surge (≥2 users, ≥4 distinct unrecognized VINs, rate ≥2× baseline) still fires. typecheck clean. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Semih
@@ -21,6 +21,9 @@ const MIN_DECODABLE_LOOKUPS = 6;
|
||||
// With a ~1.4 expected/window baseline, 8 decodes already trips 5×; require a
|
||||
// real absolute surge too (seen 2026-06-10: fired at now=8, below baseline).
|
||||
const MIN_VOLUME_SPIKE_ABS = 25;
|
||||
// unknown_vin_spike needs enough DISTINCT truly-unrecognized lookups that it
|
||||
// reads as a decoder problem, not a couple users querying uncovered cars.
|
||||
const MIN_UNKNOWN_LOOKUPS = 4;
|
||||
|
||||
export type AnomalyType =
|
||||
| "success_rate_drop"
|
||||
@@ -57,6 +60,9 @@ type WindowStats = {
|
||||
distinctUsers: number;
|
||||
distinctLookups: number;
|
||||
unknownVinUsers: number;
|
||||
// Distinct (user,vin) lookups that were TRULY unrecognized (excl. identified-
|
||||
// but-no-catalog) — the deduped denominator for the unknown-VIN-spike check.
|
||||
distinctUnknownLookups: number;
|
||||
// Real (infra) failures = failures that are NOT coverage gaps (no-catalog /
|
||||
// unknown / unsupported VIN). These are the decode-health signal.
|
||||
realFailed: number;
|
||||
@@ -80,6 +86,7 @@ async function windowStats(start: Date, end: Date): Promise<WindowStats> {
|
||||
distinct_users: bigint;
|
||||
distinct_lookups: bigint;
|
||||
unknown_vin_users: bigint;
|
||||
distinct_unknown_lookups: bigint;
|
||||
real_failed: bigint;
|
||||
decodable_lookups: bigint;
|
||||
real_failed_lookups: bigint;
|
||||
@@ -100,8 +107,13 @@ async function windowStats(start: Date, end: Date): Promise<WindowStats> {
|
||||
OR (timings->>'aborted')::boolean = true
|
||||
)
|
||||
) AS timeouts,
|
||||
-- TRULY unrecognized VINs (possible decoder/WMI problem). A car we DID
|
||||
-- identify but have no catalog for (identified_no_catalog=1, "No catalog —
|
||||
-- identified as X") is a coverage gap, NOT an unknown VIN — exclude it, or
|
||||
-- a couple users querying uncovered cars reads as a decoder outage.
|
||||
count(*) FILTER (
|
||||
WHERE success = false
|
||||
AND (timings->>'identified_no_catalog') IS DISTINCT FROM '1'
|
||||
AND (
|
||||
error_message ILIKE '%unknown vin%'
|
||||
OR error_message ILIKE '%tanınamad%'
|
||||
@@ -113,14 +125,26 @@ async function windowStats(start: Date, end: Date): Promise<WindowStats> {
|
||||
count(DISTINCT (user_id, vin)) AS distinct_lookups,
|
||||
count(DISTINCT user_id) FILTER (
|
||||
WHERE success = false
|
||||
AND (timings->>'identified_no_catalog') IS DISTINCT FROM '1'
|
||||
AND (
|
||||
error_message ILIKE '%catalog%'
|
||||
OR error_message ILIKE '%unknown vin%'
|
||||
error_message ILIKE '%unknown vin%'
|
||||
OR error_message ILIKE '%tanınamad%'
|
||||
OR error_message ILIKE '%destekl%'
|
||||
OR (timings->>'result_kind') = 'unknown'
|
||||
)
|
||||
) AS unknown_vin_users,
|
||||
-- Distinct (user,vin) truly-unrecognized lookups — dedupes a user retrying
|
||||
-- one unrecognized VIN so it can't fake a spike on its own.
|
||||
count(DISTINCT (user_id, vin)) FILTER (
|
||||
WHERE success = false
|
||||
AND (timings->>'identified_no_catalog') IS DISTINCT FROM '1'
|
||||
AND (
|
||||
error_message ILIKE '%unknown vin%'
|
||||
OR error_message ILIKE '%tanınamad%'
|
||||
OR error_message ILIKE '%destekl%'
|
||||
OR (timings->>'result_kind') = 'unknown'
|
||||
)
|
||||
) AS distinct_unknown_lookups,
|
||||
-- Real (infra) failures: failed AND not a coverage gap (no-catalog /
|
||||
-- unknown / unsupported VIN). These are the decode-health signal.
|
||||
count(*) FILTER (
|
||||
@@ -169,6 +193,7 @@ async function windowStats(start: Date, end: Date): Promise<WindowStats> {
|
||||
distinct_users: 0n,
|
||||
distinct_lookups: 0n,
|
||||
unknown_vin_users: 0n,
|
||||
distinct_unknown_lookups: 0n,
|
||||
real_failed: 0n,
|
||||
decodable_lookups: 0n,
|
||||
real_failed_lookups: 0n,
|
||||
@@ -189,6 +214,7 @@ async function windowStats(start: Date, end: Date): Promise<WindowStats> {
|
||||
distinctUsers: Number(r.distinct_users),
|
||||
distinctLookups: Number(r.distinct_lookups),
|
||||
unknownVinUsers: Number(r.unknown_vin_users),
|
||||
distinctUnknownLookups: Number(r.distinct_unknown_lookups),
|
||||
realFailed: Number(r.real_failed),
|
||||
decodableLookups: Number(r.decodable_lookups),
|
||||
realFailedLookups: Number(r.real_failed_lookups),
|
||||
@@ -223,6 +249,7 @@ async function baselineStats(currentEnd: Date): Promise<WindowStats> {
|
||||
distinctUsers: 0,
|
||||
distinctLookups: 0,
|
||||
unknownVinUsers: 0,
|
||||
distinctUnknownLookups: 0,
|
||||
realFailed: 0,
|
||||
decodableLookups: 0,
|
||||
realFailedLookups: 0,
|
||||
@@ -251,6 +278,7 @@ async function baselineStats(currentEnd: Date): Promise<WindowStats> {
|
||||
distinctUsers: samples.reduce((a, b) => a + b.distinctUsers, 0),
|
||||
distinctLookups: samples.reduce((a, b) => a + b.distinctLookups, 0),
|
||||
unknownVinUsers: samples.reduce((a, b) => a + b.unknownVinUsers, 0),
|
||||
distinctUnknownLookups: samples.reduce((a, b) => a + b.distinctUnknownLookups, 0),
|
||||
realFailed: samples.reduce((a, b) => a + b.realFailed, 0),
|
||||
decodableLookups: samples.reduce((a, b) => a + b.decodableLookups, 0),
|
||||
realFailedLookups: samples.reduce((a, b) => a + b.realFailedLookups, 0),
|
||||
@@ -418,11 +446,17 @@ export async function detectVinAnomalies(): Promise<{
|
||||
});
|
||||
}
|
||||
|
||||
// 6. Unknown-VIN spike ("tanınamadı" patlaması) — proxy/decoder outage. The
|
||||
// generic success-rate-drop catches this too, but this names it explicitly
|
||||
// and attributes the dominant failing source. Fire when the unknown-VIN
|
||||
// share jumps ≥2× baseline, or surges from a near-zero baseline.
|
||||
if (current.total >= MIN_CURRENT_VOLUME && current.unknownVinUsers >= MIN_DISTINCT_USERS) {
|
||||
// 6. Unknown-VIN spike ("tanınamadı" patlaması) — a decoder/WMI problem where
|
||||
// VINs we normally recognize suddenly can't be identified. Counts only TRULY
|
||||
// unrecognized VINs (identified-but-no-catalog coverage gaps are excluded in
|
||||
// the SQL), deduped by lookup, and requires ≥2 distinct users + enough
|
||||
// distinct unrecognized lookups so a couple users' uncovered cars don't fire
|
||||
// it. Fire when the share jumps ≥2× baseline or surges from a near-zero base.
|
||||
if (
|
||||
current.total >= MIN_CURRENT_VOLUME &&
|
||||
current.unknownVinUsers >= MIN_DISTINCT_USERS &&
|
||||
current.distinctUnknownLookups >= MIN_UNKNOWN_LOOKUPS
|
||||
) {
|
||||
const curRate = current.unknownVins / current.total;
|
||||
const baseRate = baseline.total > 0 ? baseline.unknownVins / baseline.total : 0;
|
||||
const spiked =
|
||||
|
||||
Reference in New Issue
Block a user