root/sp
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

fix(analytics): VIN decode dashboard — honest KPI definitions

Browse Source 
Raw succeeded/total read 61% on a healthy pipeline because coverage
gaps (no-catalog, unknown VIN) counted as failures, and latency
percentiles mixed ~10ms cache hits with ~5-40s provider chains
(P50 28ms next to P95 25.8s meant nothing).

- partition failures exactly: no_catalog / unknown / real (timeout,
  provider) via one CASE bucket
- Başarı = decode health (coverage gaps excluded), Gerçek hata real
  failures only, new Kapsama dışı card (katalog yok + unknown VIN)
- P50/P95/P99 now fresh decodes only, cache hits excluded
- evening brief gains realErrorRatePct/decodeHealthPct/noCatalog24h

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Semih
2026-06-11 11:12:33 +03:00
parent 8e6c0d0131
commit 9c10513fde
3 changed files with 95 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
apps/web/src/app/projects/sase/vin-decode/page.tsx
View File

@@ -133,39 +133,57 @@ export default async function VinDecodePage({
<div className="grid grid-cols-2 gap-3 md:grid-cols-3 lg:grid-cols-6">
<Kpi
label="Başarı"
value={pct(health.successRate)}
hint={`${health.successCount.toLocaleString("tr-TR")} ok`}
label="Başarı (decode)"
value={pct(health.decodeHealthRate)}
hint={`${health.successCount.toLocaleString("tr-TR")} ok / ${(
health.successCount + health.realFailureCount
).toLocaleString("tr-TR")} · kapsama dışı hariç`}
tone={
health.successRate >= 0.9
health.decodeHealthRate >= 0.9
? "ok"
: health.successRate >= 0.75
: health.decodeHealthRate >= 0.75
? "warn"
: "bad"
}
/>
<Kpi
label="Hata"
value={pct(health.errorRate)}
hint={`${health.failureCount.toLocaleString("tr-TR")} fail`}
label="Gerçek hata"
value={pct(health.realFailureRate)}
hint={`${health.realFailureCount.toLocaleString("tr-TR")} fail (timeout/provider)`}
tone={
health.errorRate <= 0.05
health.realFailureRate <= 0.05
? "ok"
: health.errorRate <= 0.15
: health.realFailureRate <= 0.15
? "warn"
: "bad"
}
/>
<Kpi
label="P95 yanıt"
label="Kapsama dışı"
value={pct(
health.totalCount > 0
? (health.noCatalogCount + health.unknownVinCount) / health.totalCount
: 0,
)}
hint={`${health.noCatalogCount.toLocaleString("tr-TR")} katalog yok · ${health.unknownVinCount.toLocaleString("tr-TR")} unknown VIN`}
tone={
(health.noCatalogCount + health.unknownVinCount) / Math.max(health.totalCount, 1) <= 0.1
? "ok"
: (health.noCatalogCount + health.unknownVinCount) / Math.max(health.totalCount, 1) <= 0.3
? "warn"
: "bad"
}
/>
<Kpi
label="P95 yanıt (fresh)"
value={ms(health.p95ResponseMs)}
hint={`P50 ${ms(health.p50ResponseMs)} · P99 ${ms(health.p99ResponseMs)}`}
hint={`P50 ${ms(health.p50ResponseMs)} · P99 ${ms(health.p99ResponseMs)} · cache hariç`}
tone={
!health.p95ResponseMs
? undefined
: health.p95ResponseMs <= 3000
: health.p95ResponseMs <= 8000
? "ok"
: health.p95ResponseMs <= 8000
: health.p95ResponseMs <= 25000
? "warn"
: "bad"
}
@@ -178,14 +196,9 @@ export default async function VinDecodePage({
<Kpi
label="Timeout"
value={pct(health.timeoutRate)}
hint={`${health.timeoutCount.toLocaleString("tr-TR")}`}
hint={`${health.timeoutCount.toLocaleString("tr-TR")} / tüm sorgular`}
tone={health.timeoutRate <= 0.05 ? "ok" : "warn"}
/>
<Kpi
label="Unknown VIN"
value={pct(health.unknownVinRate)}
hint={`${health.unknownVinCount.toLocaleString("tr-TR")}`}
/>
</div>
<div className="grid grid-cols-1 gap-3 lg:grid-cols-2">

6
apps/web/src/lib/sase/brief-data.ts
View File

@@ -41,6 +41,12 @@ export async function getSaseBriefMetrics() {
successDeltaPp: round(health.successDeltaPp, 1),
volumeDeltaPct: round(health.volumeDeltaPct * 100, 0),
errorRatePct: round(op.errorRate * 100, 1),
// real = timeout/provider failures only; coverage gaps (no-catalog /
// unknown VIN) are reported separately so a healthy pipeline with
// uncovered-car demand doesn't read as an error spike.
realErrorRatePct: round(op.realFailureRate * 100, 1),
decodeHealthPct: round(op.decodeHealthRate * 100, 1),
noCatalog24h: op.noCatalogCount,
timeouts24h: op.timeoutCount,
unknownVins24h: op.unknownVinCount,
cacheHitRatePct: round(op.cacheHitRate * 100, 0),

73
apps/web/src/lib/sase/vin-decode.ts
View File

@@ -16,6 +16,12 @@ function rangeStart(range: TimeRange): Date {
}
// ─── Operasyonel sağlık ───────────────────────────────────────────────────
// Failures partition into three mutually exclusive buckets (see vin-anomaly.ts):
// no_catalog — car identified but no parts catalog mapped (coverage gap)
// unknown — VIN/WMI truly unrecognized (coverage gap, decoder breadth)
// real — infra failures: timeout/budget/provider (decode health)
// "Başarı" as raw succeeded/total reads 60% when users query uncovered cars on a
// perfectly healthy pipeline — decodeHealthRate excludes coverage gaps.
export type OperationalHealth = {
range: TimeRange;
rangeStart: Date;
@@ -24,12 +30,20 @@ export type OperationalHealth = {
failureCount: number;
successRate: number;
errorRate: number;
// succeeded / (succeeded + real failures) — pipeline health w/o coverage gaps
decodeHealthRate: number;
realFailureCount: number;
realFailureRate: number; // real failures / (succeeded + real failures)
noCatalogCount: number;
noCatalogRate: number; // share of ALL queries
cacheHitCount: number;
cacheHitRate: number;
timeoutCount: number;
timeoutRate: number;
unknownVinCount: number;
unknownVinRate: number;
// Latency percentiles are FRESH decodes only (cache hits excluded) — mixing
// ~10ms cache hits with ~5-40s provider chains made P50/P95 meaningless.
p50ResponseMs: number | null;
p95ResponseMs: number | null;
p99ResponseMs: number | null;
@@ -39,28 +53,48 @@ export type OperationalHealth = {
export async function getOperationalHealth(range: TimeRange): Promise<OperationalHealth> {
const start = rangeStart(range);
// fail_bucket is a single CASE so the three failure classes partition exactly:
// no_catalog + unknown + real == failed, no overlap, no gap.
const rows = await saseDb.$queryRaw<
Array<{
total: bigint;
succeeded: bigint;
failed: bigint;
no_catalog: bigint;
unknown_vins: bigint;
real_failed: bigint;
cache_hits: bigint;
timeouts: bigint;
unknown_vins: bigint;
p50: number | null;
p95: number | null;
p99: number | null;
avg_ms: number | null;
}>
>`
WITH q AS (
SELECT *,
(source = 'cache' OR (timings->>'cache_source') IN ('db_hit', 'redis_positive')) AS is_cache_hit,
CASE
WHEN success THEN NULL
WHEN error_message ILIKE 'no catalog%'
OR (timings->>'identified_no_catalog') = '1' THEN 'no_catalog'
WHEN error_message ILIKE '%unknown vin%'
OR error_message ILIKE '%tanınamad%'
OR error_message ILIKE '%destekl%'
OR (timings->>'result_kind') = 'unknown' THEN 'unknown'
ELSE 'real'
END AS fail_bucket
FROM query_logs
WHERE created_at >= ${start}
)
SELECT
count(*) AS total,
count(*) FILTER (WHERE success = true) AS succeeded,
count(*) FILTER (WHERE success = false) AS failed,
count(*) FILTER (
WHERE source = 'cache'
OR (timings->>'cache_source') IN ('db_hit', 'redis_positive')
) AS cache_hits,
count(*) FILTER (WHERE fail_bucket = 'no_catalog') AS no_catalog,
count(*) FILTER (WHERE fail_bucket = 'unknown') AS unknown_vins,
count(*) FILTER (WHERE fail_bucket = 'real') AS real_failed,
count(*) FILTER (WHERE is_cache_hit) AS cache_hits,
count(*) FILTER (
WHERE success = false
AND (
@@ -69,28 +103,25 @@ export async function getOperationalHealth(range: TimeRange): Promise<Operationa
OR (timings->>'aborted')::boolean = true
)
) AS timeouts,
count(*) FILTER (
WHERE success = false
AND (error_message ILIKE '%Unknown VIN%' OR error_message ILIKE '%tanınamadı%')
) AS unknown_vins,
percentile_cont(0.50) WITHIN GROUP (ORDER BY response_time_ms)
FILTER (WHERE success = true AND response_time_ms IS NOT NULL)::int AS p50,
FILTER (WHERE success = true AND response_time_ms IS NOT NULL AND NOT is_cache_hit)::int AS p50,
percentile_cont(0.95) WITHIN GROUP (ORDER BY response_time_ms)
FILTER (WHERE success = true AND response_time_ms IS NOT NULL)::int AS p95,
FILTER (WHERE success = true AND response_time_ms IS NOT NULL AND NOT is_cache_hit)::int AS p95,
percentile_cont(0.99) WITHIN GROUP (ORDER BY response_time_ms)
FILTER (WHERE success = true AND response_time_ms IS NOT NULL)::int AS p99,
avg(response_time_ms) FILTER (WHERE success = true AND response_time_ms IS NOT NULL)::int AS avg_ms
FROM query_logs
WHERE created_at >= ${start}
FILTER (WHERE success = true AND response_time_ms IS NOT NULL AND NOT is_cache_hit)::int AS p99,
avg(response_time_ms) FILTER (WHERE success = true AND response_time_ms IS NOT NULL AND NOT is_cache_hit)::int AS avg_ms
FROM q
`;
const r = rows[0] ?? {
total: 0n,
succeeded: 0n,
failed: 0n,
no_catalog: 0n,
unknown_vins: 0n,
real_failed: 0n,
cache_hits: 0n,
timeouts: 0n,
unknown_vins: 0n,
p50: null,
p95: null,
p99: null,
@@ -100,9 +131,12 @@ export async function getOperationalHealth(range: TimeRange): Promise<Operationa
const total = Number(r.total);
const succeeded = Number(r.succeeded);
const failed = Number(r.failed);
const noCatalog = Number(r.no_catalog);
const unknownVins = Number(r.unknown_vins);
const realFailed = Number(r.real_failed);
const cacheHits = Number(r.cache_hits);
const timeouts = Number(r.timeouts);
const unknownVins = Number(r.unknown_vins);
const decodable = succeeded + realFailed;
return {
range,
@@ -112,6 +146,11 @@ export async function getOperationalHealth(range: TimeRange): Promise<Operationa
failureCount: failed,
successRate: total > 0 ? succeeded / total : 0,
errorRate: total > 0 ? failed / total : 0,
decodeHealthRate: decodable > 0 ? succeeded / decodable : 0,
realFailureCount: realFailed,
realFailureRate: decodable > 0 ? realFailed / decodable : 0,
noCatalogCount: noCatalog,
noCatalogRate: total > 0 ? noCatalog / total : 0,
cacheHitCount: cacheHits,
cacheHitRate: total > 0 ? cacheHits / total : 0,
timeoutCount: timeouts,