plugin(telemetry-watcher): collapse to single index.ts for ESM load
Fusion's plugin loader imports source files directly via Node 22 ESM without a TS loader, so relative imports like "./internal/dedup.js" fail at runtime: there's no compiled .js artifact and Node won't fall back to .ts. Inline severity classifier, dedup cache, rate limiter, and Grafana parser into the single entry file. Tests now import named exports from "../index.js" directly. The Phase-2 split into separate source files can come back once fusion adds a TS-aware plugin loader. All 14 unit tests still pass. Behavior unchanged.
This commit is contained in:
Semih
@@ -1,8 +1,12 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { classifySeverity, type SeverityThresholds, type SignalContext } from "../internal/severity.js";
|
||||
import { DedupCache } from "../internal/dedup.js";
|
||||
import { IncidentRateLimiter } from "../internal/rate-limit.js";
|
||||
import { parseGrafanaPayload } from "../sources/grafana.js";
|
||||
import {
|
||||
classifySeverity,
|
||||
type SeverityThresholds,
|
||||
type SignalContext,
|
||||
DedupCache,
|
||||
IncidentRateLimiter,
|
||||
parseGrafanaPayload,
|
||||
} from "../index.js";
|
||||
|
||||
const thresholds: SeverityThresholds = {
|
||||
latencyMs: 500,
|
||||
@@ -94,7 +98,7 @@ describe("IncidentRateLimiter", () => {
|
||||
lim.record();
|
||||
lim.record();
|
||||
expect(lim.shouldThrottle().throttle).toBe(true);
|
||||
now += 4_000_000; // > 1h
|
||||
now += 4_000_000;
|
||||
expect(lim.shouldThrottle().throttle).toBe(false);
|
||||
});
|
||||
|
||||
@@ -103,7 +107,7 @@ describe("IncidentRateLimiter", () => {
|
||||
const lim = new IncidentRateLimiter({ hourly: 100, daily: 2, now: () => now });
|
||||
lim.record();
|
||||
lim.record();
|
||||
now += 2 * 3_600_000; // hour passed but day hasn't
|
||||
now += 2 * 3_600_000;
|
||||
expect(lim.shouldThrottle().throttle).toBe(true);
|
||||
expect(lim.shouldThrottle().reason).toBe("daily");
|
||||
});
|
||||
|
||||
@@ -12,8 +12,17 @@
|
||||
*
|
||||
* Phase 1 scope: Grafana webhook only. Plugin tools and additional sources
|
||||
* land in Phase 2 with the same dedup/severity primitives.
|
||||
*
|
||||
* NOTE on layout: Fusion's runtime imports plugin source files directly via
|
||||
* Node ESM with no transpilation. Relative imports of sibling .ts files
|
||||
* therefore fail at load time (Node doesn't resolve "./foo.js" against an
|
||||
* actual "./foo.ts" without a TS loader). We keep the entire plugin in this
|
||||
* single file to sidestep that. Tests live alongside it and import named
|
||||
* exports directly. When fusion adds a TS-aware loader for plugins, we can
|
||||
* split this back into multiple files.
|
||||
*/
|
||||
|
||||
import { createHash } from "node:crypto";
|
||||
import { definePlugin } from "@fusion/plugin-sdk";
|
||||
import type {
|
||||
FusionPlugin,
|
||||
@@ -22,12 +31,289 @@ import type {
|
||||
PluginSettingSchema,
|
||||
} from "@fusion/plugin-sdk";
|
||||
|
||||
import { DedupCache } from "./internal/dedup.js";
|
||||
import { IncidentRateLimiter } from "./internal/rate-limit.js";
|
||||
import { classifySeverity, type SeverityThresholds, type Severity } from "./internal/severity.js";
|
||||
import { parseGrafanaPayload, type ParsedGrafanaSignal } from "./sources/grafana.js";
|
||||
// ── Severity classifier ─────────────────────────────────────────────────────
|
||||
|
||||
// ── Settings Schema ─────────────────────────────────────────────────────────
|
||||
export type Severity = "P0" | "P1" | "P2" | "P3";
|
||||
|
||||
export interface SeverityThresholds {
|
||||
/** Latency P95 floor (ms) above which a request is "slow". Default: 500ms. */
|
||||
latencyMs: number;
|
||||
/** Error rate floor (fraction 0..1) at which we open an incident. Default: 0.05 = 5%. */
|
||||
errorRate: number;
|
||||
/** Funnel drop floor (percentage points off baseline) for product incidents. Default: 10. */
|
||||
funnelDropPp: number;
|
||||
/** Sustained traffic floor (req/min) — below this we don't trust noisy ratios. Default: 60. */
|
||||
trafficFloorRpm: number;
|
||||
}
|
||||
|
||||
export interface SignalContext {
|
||||
/** "grafana" | "posthog" | "sentry" | "user-feedback" */
|
||||
source: string;
|
||||
/** Domain hint from upstream alert: "backend" | "frontend" | "product" | "infra" | "unknown". */
|
||||
domain: string;
|
||||
/** Numeric measurement that triggered the alert, if applicable. */
|
||||
value?: number;
|
||||
/** Free-form alert metadata. */
|
||||
meta?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Classify a signal into a severity bucket. The rules below are intentionally
|
||||
* conservative — they prefer P1/P2 to P0 unless the evidence is overwhelming.
|
||||
* Critical-path keywords ("payment", "auth", "billing", "subscription") escalate
|
||||
* one bucket because those surfaces have outsized user impact even on small
|
||||
* regressions.
|
||||
*/
|
||||
export function classifySeverity(
|
||||
signal: SignalContext,
|
||||
thresholds: SeverityThresholds,
|
||||
): Severity {
|
||||
const meta = signal.meta ?? {};
|
||||
const summary = `${meta.summary ?? ""} ${meta.title ?? ""} ${meta.alertname ?? ""}`.toLowerCase();
|
||||
const isCriticalPath =
|
||||
/\b(payment|iyzico|eft|auth|sign-?in|sign-?up|billing|subscription)\b/.test(summary);
|
||||
|
||||
let bucket: Severity = "P3";
|
||||
|
||||
if (signal.source === "grafana") {
|
||||
const latency = Number(meta.latencyMs ?? signal.value ?? 0);
|
||||
const errorRate = Number(meta.errorRate ?? 0);
|
||||
const traffic = Number(meta.trafficRpm ?? thresholds.trafficFloorRpm);
|
||||
|
||||
if (errorRate >= 0.5 && traffic >= thresholds.trafficFloorRpm) bucket = "P0";
|
||||
else if (errorRate >= thresholds.errorRate * 4 && traffic >= thresholds.trafficFloorRpm) bucket = "P1";
|
||||
else if (latency >= thresholds.latencyMs * 4) bucket = "P1";
|
||||
else if (latency >= thresholds.latencyMs * 2 || errorRate >= thresholds.errorRate) bucket = "P2";
|
||||
else bucket = "P3";
|
||||
} else if (signal.source === "posthog") {
|
||||
const dropPp = Math.abs(Number(meta.funnelDropPp ?? 0));
|
||||
if (dropPp >= thresholds.funnelDropPp * 4) bucket = "P0";
|
||||
else if (dropPp >= thresholds.funnelDropPp * 2) bucket = "P1";
|
||||
else if (dropPp >= thresholds.funnelDropPp) bucket = "P2";
|
||||
else bucket = "P3";
|
||||
} else if (signal.source === "sentry") {
|
||||
const eventCount = Number(meta.eventCount ?? 0);
|
||||
if (eventCount >= 200) bucket = "P0";
|
||||
else if (eventCount >= 50) bucket = "P1";
|
||||
else if (eventCount >= 10) bucket = "P2";
|
||||
else bucket = "P3";
|
||||
} else if (signal.source === "user-feedback") {
|
||||
bucket = "P2";
|
||||
}
|
||||
|
||||
if (isCriticalPath && bucket !== "P0") {
|
||||
bucket = bucket === "P1" ? "P0" : bucket === "P2" ? "P1" : "P2";
|
||||
}
|
||||
|
||||
return bucket;
|
||||
}
|
||||
|
||||
// ── Dedup cache ──────────────────────────────────────────────────────────────
|
||||
|
||||
export interface DedupEntry {
|
||||
taskId: string;
|
||||
firstSeenAt: number;
|
||||
lastSeenAt: number;
|
||||
hitCount: number;
|
||||
}
|
||||
|
||||
export interface DedupCacheOptions {
|
||||
/** Window size (ms). Default: 4 hours. */
|
||||
windowMs?: number;
|
||||
/** Wall clock — injectable for tests. */
|
||||
now?: () => number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fingerprint-based dedup for incoming signals. State is in-memory only by
|
||||
* design — on container restart we lose the cache. Re-opening the same
|
||||
* incident is at worst a duplicate task that the agents collapse on next
|
||||
* triage.
|
||||
*/
|
||||
export class DedupCache {
|
||||
private readonly entries = new Map<string, DedupEntry>();
|
||||
private readonly windowMs: number;
|
||||
private readonly now: () => number;
|
||||
|
||||
constructor(options: DedupCacheOptions = {}) {
|
||||
this.windowMs = options.windowMs ?? 4 * 60 * 60 * 1000;
|
||||
this.now = options.now ?? Date.now;
|
||||
}
|
||||
|
||||
static fingerprint(parts: ReadonlyArray<string>): string {
|
||||
return createHash("sha1").update(parts.join("\0")).digest("hex").slice(0, 16);
|
||||
}
|
||||
|
||||
hit(fingerprint: string): DedupEntry | null {
|
||||
this.evictExpired();
|
||||
const existing = this.entries.get(fingerprint);
|
||||
if (!existing) return null;
|
||||
existing.lastSeenAt = this.now();
|
||||
existing.hitCount += 1;
|
||||
return existing;
|
||||
}
|
||||
|
||||
register(fingerprint: string, taskId: string): DedupEntry {
|
||||
const ts = this.now();
|
||||
const entry: DedupEntry = { taskId, firstSeenAt: ts, lastSeenAt: ts, hitCount: 1 };
|
||||
this.entries.set(fingerprint, entry);
|
||||
return entry;
|
||||
}
|
||||
|
||||
evictExpired(): void {
|
||||
const cutoff = this.now() - this.windowMs;
|
||||
for (const [k, v] of this.entries) {
|
||||
if (v.lastSeenAt < cutoff) this.entries.delete(k);
|
||||
}
|
||||
}
|
||||
|
||||
size(): number {
|
||||
this.evictExpired();
|
||||
return this.entries.size;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Rate limiter ────────────────────────────────────────────────────────────
|
||||
|
||||
export interface RateLimitOptions {
|
||||
hourly: number;
|
||||
daily: number;
|
||||
now?: () => number;
|
||||
}
|
||||
|
||||
export class IncidentRateLimiter {
|
||||
private readonly hourly: number;
|
||||
private readonly daily: number;
|
||||
private readonly now: () => number;
|
||||
private readonly events: number[] = [];
|
||||
|
||||
constructor(options: RateLimitOptions) {
|
||||
this.hourly = options.hourly;
|
||||
this.daily = options.daily;
|
||||
this.now = options.now ?? Date.now;
|
||||
}
|
||||
|
||||
shouldThrottle(): { throttle: boolean; reason?: "hourly" | "daily" } {
|
||||
this.prune();
|
||||
const now = this.now();
|
||||
const hourlyCount = this.events.filter((t) => t >= now - 3_600_000).length;
|
||||
if (hourlyCount >= this.hourly) return { throttle: true, reason: "hourly" };
|
||||
if (this.events.length >= this.daily) return { throttle: true, reason: "daily" };
|
||||
return { throttle: false };
|
||||
}
|
||||
|
||||
record(): void {
|
||||
this.events.push(this.now());
|
||||
}
|
||||
|
||||
private prune(): void {
|
||||
const cutoff = this.now() - 24 * 3_600_000;
|
||||
while (this.events.length && this.events[0]! < cutoff) {
|
||||
this.events.shift();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Grafana payload parser ──────────────────────────────────────────────────
|
||||
|
||||
interface GrafanaWebhookAlert {
|
||||
status?: string;
|
||||
labels?: Record<string, string>;
|
||||
annotations?: Record<string, string>;
|
||||
startsAt?: string;
|
||||
endsAt?: string;
|
||||
generatorURL?: string;
|
||||
fingerprint?: string;
|
||||
values?: Record<string, number>;
|
||||
}
|
||||
|
||||
interface GrafanaWebhookPayload {
|
||||
status?: string;
|
||||
alerts?: GrafanaWebhookAlert[];
|
||||
commonLabels?: Record<string, string>;
|
||||
commonAnnotations?: Record<string, string>;
|
||||
title?: string;
|
||||
message?: string;
|
||||
externalURL?: string;
|
||||
}
|
||||
|
||||
export interface ParsedGrafanaSignal {
|
||||
signal: SignalContext;
|
||||
grafanaFingerprint: string;
|
||||
alertname: string;
|
||||
description: string;
|
||||
firing: boolean;
|
||||
}
|
||||
|
||||
function classifyDomain(alertname: string, labels: Record<string, string>): string {
|
||||
const haystack = `${alertname} ${Object.values(labels).join(" ")}`.toLowerCase();
|
||||
if (/\b(payment|iyzico|eft|billing|subscription)\b/.test(haystack)) return "backend";
|
||||
if (/\b(auth|sign-?in|sign-?up|session|cookie)\b/.test(haystack)) return "backend";
|
||||
if (/\b(vehicle|vin|catalog|emex|pl24|partscatalog)\b/.test(haystack)) return "backend";
|
||||
if (/\b(rum|web-vitals|lcp|cls|inp|faro|frontend)\b/.test(haystack)) return "frontend";
|
||||
if (/\b(latency|p95|p99|error.?rate|http|api)\b/.test(haystack)) return "backend";
|
||||
if (/\b(worker|bull|queue|job)\b/.test(haystack)) return "backend";
|
||||
if (/\b(funnel|conversion|signup|posthog)\b/.test(haystack)) return "product";
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
/**
|
||||
* Translate a Grafana webhook payload into one signal per firing alert.
|
||||
* Resolved alerts are dropped — recovery is handled separately by post-deploy
|
||||
* verification.
|
||||
*/
|
||||
export function parseGrafanaPayload(payload: unknown): ParsedGrafanaSignal[] {
|
||||
if (!payload || typeof payload !== "object") return [];
|
||||
const data = payload as GrafanaWebhookPayload;
|
||||
const alerts = Array.isArray(data.alerts) ? data.alerts : [];
|
||||
const out: ParsedGrafanaSignal[] = [];
|
||||
|
||||
for (const alert of alerts) {
|
||||
const firing = (alert.status ?? data.status ?? "").toLowerCase() === "firing";
|
||||
const labels = alert.labels ?? {};
|
||||
const annotations = alert.annotations ?? {};
|
||||
const alertname = labels.alertname ?? data.title ?? "unknown-alert";
|
||||
const summary = annotations.summary ?? annotations.description ?? data.message ?? "";
|
||||
|
||||
const valueEntries = Object.entries(alert.values ?? {});
|
||||
const primaryValue = valueEntries.length > 0 ? Number(valueEntries[0]![1]) : undefined;
|
||||
|
||||
const meta: Record<string, unknown> = {
|
||||
...labels,
|
||||
summary,
|
||||
title: alertname,
|
||||
alertname,
|
||||
generatorURL: alert.generatorURL,
|
||||
startsAt: alert.startsAt,
|
||||
};
|
||||
if (typeof labels.endpoint === "string") meta.endpoint = labels.endpoint;
|
||||
if (typeof labels.severity === "string") meta.alertSeverity = labels.severity;
|
||||
if (typeof primaryValue === "number" && !Number.isNaN(primaryValue)) {
|
||||
if (/latency|duration|p95|p99/i.test(alertname)) meta.latencyMs = primaryValue;
|
||||
else if (/error|fail|5xx/i.test(alertname)) meta.errorRate = primaryValue;
|
||||
}
|
||||
|
||||
const signal: SignalContext = {
|
||||
source: "grafana",
|
||||
domain: classifyDomain(alertname, labels),
|
||||
value: primaryValue,
|
||||
meta,
|
||||
};
|
||||
|
||||
out.push({
|
||||
signal,
|
||||
grafanaFingerprint:
|
||||
alert.fingerprint || `${alertname}|${labels.endpoint ?? "*"}|${labels.severity ?? "*"}`,
|
||||
alertname,
|
||||
description: summary,
|
||||
firing,
|
||||
});
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
// ── Plugin manifest + handlers ──────────────────────────────────────────────
|
||||
|
||||
const settingsSchema: Record<string, PluginSettingSchema> = {
|
||||
grafanaWebhookSecret: {
|
||||
@@ -40,7 +326,7 @@ const settingsSchema: Record<string, PluginSettingSchema> = {
|
||||
type: "string",
|
||||
label: "Triage Agent ID",
|
||||
description:
|
||||
"Agent that incident tasks are auto-assigned to. Defaults to the first agent named 'Triage Agent' or, when absent, to the configured CTO so heartbeats still pick it up.",
|
||||
"Agent that incident tasks are auto-assigned to. When blank, tasks land unassigned in the triage column for the agent's heartbeat to pick up.",
|
||||
},
|
||||
latencyMs: {
|
||||
type: "number",
|
||||
@@ -88,8 +374,6 @@ const settingsSchema: Record<string, PluginSettingSchema> = {
|
||||
},
|
||||
};
|
||||
|
||||
// ── Plugin-scoped state (created per `onLoad`) ──────────────────────────────
|
||||
|
||||
interface PluginState {
|
||||
dedup: DedupCache;
|
||||
limiter: IncidentRateLimiter;
|
||||
@@ -144,7 +428,7 @@ function buildIncidentDescription(
|
||||
``,
|
||||
`### Triage Agent: next steps`,
|
||||
`1. Read \`.fusion/memory/team-charter.md\` for routing rules.`,
|
||||
`2. Verify the alert with \`fn_query_grafana_tempo\` / \`fn_query_grafana_loki\` (Phase 2 tools — for now, inspect via \`fn_task_log\`).`,
|
||||
`2. Verify the alert with \`fn_query_grafana_tempo\` / \`fn_query_grafana_loki\` (Phase 2 tools).`,
|
||||
`3. Per-severity routing:`,
|
||||
` - **P0** → open council task (CEO + CTO + CPO).`,
|
||||
` - **P1** → delegate directly to CTO with \`cto/brief\`.`,
|
||||
@@ -160,18 +444,6 @@ function buildIncidentDescription(
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
async function resolveTriageAgentId(ctx: PluginContext): Promise<string | undefined> {
|
||||
const explicit = ctx.settings.triageAgentId as string | undefined;
|
||||
if (explicit) return explicit;
|
||||
|
||||
// Fall back to discovery via taskStore's project root: peek at agents/ on disk.
|
||||
// The plugin context has a read-oriented taskStore; AgentStore is not directly
|
||||
// exposed, so we leave this best-effort and let the task be unassigned when
|
||||
// discovery fails. Triage Agent will still pick the task up on its next
|
||||
// heartbeat scan if it polls open triage-column tasks (per its instructions).
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const plugin: FusionPlugin = definePlugin({
|
||||
manifest: {
|
||||
id: "fusion-plugin-telemetry-watcher",
|
||||
@@ -195,7 +467,6 @@ const plugin: FusionPlugin = definePlugin({
|
||||
};
|
||||
const headers = req.headers ?? {};
|
||||
|
||||
// Bearer auth check (when secret configured).
|
||||
const expectedSecret = ctx.settings.grafanaWebhookSecret as string | undefined;
|
||||
if (expectedSecret && expectedSecret.length > 0) {
|
||||
const auth = (headers.authorization || headers.Authorization) as string | undefined;
|
||||
@@ -218,7 +489,7 @@ const plugin: FusionPlugin = definePlugin({
|
||||
}
|
||||
|
||||
const thresholds = readThresholds(ctx);
|
||||
const triageAgentId = await resolveTriageAgentId(ctx);
|
||||
const triageAgentId = (ctx.settings.triageAgentId as string | undefined) || undefined;
|
||||
|
||||
let opened = 0;
|
||||
let deduped = 0;
|
||||
@@ -226,7 +497,7 @@ const plugin: FusionPlugin = definePlugin({
|
||||
const taskIds: string[] = [];
|
||||
|
||||
for (const item of parsed) {
|
||||
if (!item.firing) continue; // resolved alerts handled by post-deploy verification
|
||||
if (!item.firing) continue;
|
||||
|
||||
const severity = classifySeverity(item.signal, thresholds);
|
||||
const fingerprint = DedupCache.fingerprint([
|
||||
@@ -238,8 +509,6 @@ const plugin: FusionPlugin = definePlugin({
|
||||
const dedupHit = state.dedup.hit(fingerprint);
|
||||
if (dedupHit) {
|
||||
deduped += 1;
|
||||
// Best-effort dedup note. We avoid using non-public store mutators
|
||||
// and rely on the agent's heartbeat to absorb hits via task log.
|
||||
ctx.logger.info(
|
||||
`Deduped Grafana signal: fingerprint=${fingerprint} hits=${dedupHit.hitCount} taskId=${dedupHit.taskId}`,
|
||||
);
|
||||
@@ -276,27 +545,20 @@ const plugin: FusionPlugin = definePlugin({
|
||||
} as never,
|
||||
});
|
||||
|
||||
// We rely on Triage Agent's heartbeat to assign + delegate per
|
||||
// team-charter; a dashboard-side process can also explicitly
|
||||
// assign here when triageAgentId is configured.
|
||||
if (triageAgentId) {
|
||||
try {
|
||||
// taskStore exposes updateTask in core but the plugin SDK
|
||||
// declares it read-only. Falls through silently when the
|
||||
// method is unavailable; Triage Agent will still pick up
|
||||
// unassigned triage-column tasks on its next heartbeat.
|
||||
const store = ctx.taskStore as unknown as {
|
||||
updateTask?: (id: string, patch: Record<string, unknown>) => Promise<unknown>;
|
||||
};
|
||||
if (typeof store.updateTask === "function") {
|
||||
const store = ctx.taskStore as unknown as {
|
||||
updateTask?: (id: string, patch: Record<string, unknown>) => Promise<unknown>;
|
||||
};
|
||||
if (typeof store.updateTask === "function") {
|
||||
try {
|
||||
await store.updateTask(task.id, { assignedAgentId: triageAgentId });
|
||||
} catch (assignErr) {
|
||||
ctx.logger.warn(
|
||||
`Could not auto-assign task ${task.id}: ${
|
||||
assignErr instanceof Error ? assignErr.message : String(assignErr)
|
||||
}`,
|
||||
);
|
||||
}
|
||||
} catch (assignErr) {
|
||||
ctx.logger.warn(
|
||||
`Could not auto-assign task ${task.id} to triage agent: ${
|
||||
assignErr instanceof Error ? assignErr.message : String(assignErr)
|
||||
}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -316,14 +578,7 @@ const plugin: FusionPlugin = definePlugin({
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
accepted: parsed.length,
|
||||
opened,
|
||||
deduped,
|
||||
throttled,
|
||||
taskIds,
|
||||
};
|
||||
return { ok: true, accepted: parsed.length, opened, deduped, throttled, taskIds };
|
||||
},
|
||||
} satisfies PluginRouteDefinition,
|
||||
],
|
||||
|
||||
@@ -1,87 +0,0 @@
|
||||
/**
|
||||
* Fingerprint-based dedup for incoming signals.
|
||||
*
|
||||
* Two signals fingerprint identically when their (source, primaryDimension,
|
||||
* severity) tuple matches. The first occurrence in a rolling window opens an
|
||||
* incident task; subsequent occurrences are recorded against the same task
|
||||
* via task log. Once the window expires, a fresh fingerprint can re-open.
|
||||
*
|
||||
* State is in-memory only by design. On container restart we lose the dedup
|
||||
* cache; that's acceptable for Phase 1 — re-opening the same incident is at
|
||||
* worst a duplicate task that the agents will collapse on next triage. A
|
||||
* future phase can persist the cache alongside the plugin's settings.
|
||||
*/
|
||||
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
export interface DedupEntry {
|
||||
/** Existing task that absorbs duplicates. */
|
||||
taskId: string;
|
||||
/** First-seen unix ms. */
|
||||
firstSeenAt: number;
|
||||
/** Most-recent occurrence unix ms. */
|
||||
lastSeenAt: number;
|
||||
/** Total occurrences observed in this window. */
|
||||
hitCount: number;
|
||||
}
|
||||
|
||||
export interface DedupCacheOptions {
|
||||
/** Window size (ms). Default: 4 hours. */
|
||||
windowMs?: number;
|
||||
/** Wall clock — injectable for tests. */
|
||||
now?: () => number;
|
||||
}
|
||||
|
||||
export class DedupCache {
|
||||
private readonly entries = new Map<string, DedupEntry>();
|
||||
private readonly windowMs: number;
|
||||
private readonly now: () => number;
|
||||
|
||||
constructor(options: DedupCacheOptions = {}) {
|
||||
this.windowMs = options.windowMs ?? 4 * 60 * 60 * 1000;
|
||||
this.now = options.now ?? Date.now;
|
||||
}
|
||||
|
||||
static fingerprint(parts: ReadonlyArray<string>): string {
|
||||
return createHash("sha1").update(parts.join("\0")).digest("hex").slice(0, 16);
|
||||
}
|
||||
|
||||
/**
|
||||
* Look up a fingerprint and either return the existing entry (and tick
|
||||
* lastSeenAt + hitCount) or signal that the caller should open a fresh
|
||||
* task and call `register()` afterwards.
|
||||
*/
|
||||
hit(fingerprint: string): DedupEntry | null {
|
||||
this.evictExpired();
|
||||
const existing = this.entries.get(fingerprint);
|
||||
if (!existing) return null;
|
||||
existing.lastSeenAt = this.now();
|
||||
existing.hitCount += 1;
|
||||
return existing;
|
||||
}
|
||||
|
||||
register(fingerprint: string, taskId: string): DedupEntry {
|
||||
const ts = this.now();
|
||||
const entry: DedupEntry = {
|
||||
taskId,
|
||||
firstSeenAt: ts,
|
||||
lastSeenAt: ts,
|
||||
hitCount: 1,
|
||||
};
|
||||
this.entries.set(fingerprint, entry);
|
||||
return entry;
|
||||
}
|
||||
|
||||
/** Drop entries whose lastSeenAt is older than windowMs ago. */
|
||||
evictExpired(): void {
|
||||
const cutoff = this.now() - this.windowMs;
|
||||
for (const [k, v] of this.entries) {
|
||||
if (v.lastSeenAt < cutoff) this.entries.delete(k);
|
||||
}
|
||||
}
|
||||
|
||||
size(): number {
|
||||
this.evictExpired();
|
||||
return this.entries.size;
|
||||
}
|
||||
}
|
||||
@@ -1,50 +0,0 @@
|
||||
/**
|
||||
* Sliding-window rate limiter for incident task creation.
|
||||
*
|
||||
* Two windows: hourly and daily. When either is breached, callers should
|
||||
* coalesce overflow signals into a single "telemetry storm" task instead of
|
||||
* opening N individual incidents.
|
||||
*/
|
||||
|
||||
export interface RateLimitOptions {
|
||||
hourly: number;
|
||||
daily: number;
|
||||
now?: () => number;
|
||||
}
|
||||
|
||||
export class IncidentRateLimiter {
|
||||
private readonly hourly: number;
|
||||
private readonly daily: number;
|
||||
private readonly now: () => number;
|
||||
private readonly events: number[] = [];
|
||||
|
||||
constructor(options: RateLimitOptions) {
|
||||
this.hourly = options.hourly;
|
||||
this.daily = options.daily;
|
||||
this.now = options.now ?? Date.now;
|
||||
}
|
||||
|
||||
/**
|
||||
* Test whether opening a fresh incident now would violate either limit.
|
||||
* Does NOT record the event — call `record()` once you actually open one.
|
||||
*/
|
||||
shouldThrottle(): { throttle: boolean; reason?: "hourly" | "daily" } {
|
||||
this.prune();
|
||||
const now = this.now();
|
||||
const hourlyCount = this.events.filter((t) => t >= now - 3_600_000).length;
|
||||
if (hourlyCount >= this.hourly) return { throttle: true, reason: "hourly" };
|
||||
if (this.events.length >= this.daily) return { throttle: true, reason: "daily" };
|
||||
return { throttle: false };
|
||||
}
|
||||
|
||||
record(): void {
|
||||
this.events.push(this.now());
|
||||
}
|
||||
|
||||
private prune(): void {
|
||||
const cutoff = this.now() - 24 * 3_600_000;
|
||||
while (this.events.length && this.events[0]! < cutoff) {
|
||||
this.events.shift();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,84 +0,0 @@
|
||||
/**
|
||||
* Severity classifier for incoming telemetry signals.
|
||||
*
|
||||
* Plugin settings expose four threshold rules (P0/P1/P2/P3 boundaries) but
|
||||
* the actual classification is deterministic and pure: given a signal, return
|
||||
* one of the four buckets. Settings are passed in as `thresholds` so the same
|
||||
* function works in tests without a live PluginContext.
|
||||
*/
|
||||
|
||||
export type Severity = "P0" | "P1" | "P2" | "P3";
|
||||
|
||||
export interface SeverityThresholds {
|
||||
/** Latency P95 floor (ms) above which a request is "slow". Default: 500ms. */
|
||||
latencyMs: number;
|
||||
/** Error rate floor (fraction 0..1) at which we open an incident. Default: 0.05 = 5%. */
|
||||
errorRate: number;
|
||||
/** Funnel drop floor (percentage points off baseline) for product incidents. Default: 10. */
|
||||
funnelDropPp: number;
|
||||
/** Sustained traffic floor (req/min) — below this we don't trust noisy ratios. Default: 60. */
|
||||
trafficFloorRpm: number;
|
||||
}
|
||||
|
||||
export interface SignalContext {
|
||||
/** "grafana" | "posthog" | "sentry" | "user-feedback" */
|
||||
source: string;
|
||||
/** Domain hint from upstream alert (when available): "backend" | "frontend" | "product" | "infra" | "unknown". */
|
||||
domain: string;
|
||||
/** Numeric measurement that triggered the alert, if applicable. */
|
||||
value?: number;
|
||||
/** Free-form alert metadata. */
|
||||
meta?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Classify a signal into a severity bucket.
|
||||
*
|
||||
* The rules below are intentionally conservative — they prefer P1/P2 to
|
||||
* P0 unless the evidence is overwhelming. Critical-path keywords ("payment",
|
||||
* "auth", "billing", "subscription") escalate one bucket because those
|
||||
* surfaces have outsized user impact even on small regressions.
|
||||
*/
|
||||
export function classifySeverity(
|
||||
signal: SignalContext,
|
||||
thresholds: SeverityThresholds,
|
||||
): Severity {
|
||||
const meta = signal.meta ?? {};
|
||||
const summary = `${meta.summary ?? ""} ${meta.title ?? ""} ${meta.alertname ?? ""}`.toLowerCase();
|
||||
const isCriticalPath =
|
||||
/\b(payment|iyzico|eft|auth|sign-?in|sign-?up|billing|subscription)\b/.test(summary);
|
||||
|
||||
let bucket: Severity = "P3";
|
||||
|
||||
if (signal.source === "grafana") {
|
||||
const latency = Number(meta.latencyMs ?? signal.value ?? 0);
|
||||
const errorRate = Number(meta.errorRate ?? 0);
|
||||
const traffic = Number(meta.trafficRpm ?? thresholds.trafficFloorRpm);
|
||||
|
||||
if (errorRate >= 0.5 && traffic >= thresholds.trafficFloorRpm) bucket = "P0";
|
||||
else if (errorRate >= thresholds.errorRate * 4 && traffic >= thresholds.trafficFloorRpm) bucket = "P1";
|
||||
else if (latency >= thresholds.latencyMs * 4) bucket = "P1";
|
||||
else if (latency >= thresholds.latencyMs * 2 || errorRate >= thresholds.errorRate) bucket = "P2";
|
||||
else bucket = "P3";
|
||||
} else if (signal.source === "posthog") {
|
||||
const dropPp = Math.abs(Number(meta.funnelDropPp ?? 0));
|
||||
if (dropPp >= thresholds.funnelDropPp * 4) bucket = "P0";
|
||||
else if (dropPp >= thresholds.funnelDropPp * 2) bucket = "P1";
|
||||
else if (dropPp >= thresholds.funnelDropPp) bucket = "P2";
|
||||
else bucket = "P3";
|
||||
} else if (signal.source === "sentry") {
|
||||
const eventCount = Number(meta.eventCount ?? 0);
|
||||
if (eventCount >= 200) bucket = "P0";
|
||||
else if (eventCount >= 50) bucket = "P1";
|
||||
else if (eventCount >= 10) bucket = "P2";
|
||||
else bucket = "P3";
|
||||
} else if (signal.source === "user-feedback") {
|
||||
bucket = "P2";
|
||||
}
|
||||
|
||||
if (isCriticalPath && bucket !== "P0") {
|
||||
bucket = bucket === "P1" ? "P0" : bucket === "P2" ? "P1" : "P2";
|
||||
}
|
||||
|
||||
return bucket;
|
||||
}
|
||||
@@ -1,118 +0,0 @@
|
||||
/**
|
||||
* Grafana Alerting webhook ingestor.
|
||||
*
|
||||
* Grafana's contact-point webhook posts a JSON envelope with one or more
|
||||
* alerts. We translate each firing alert into a `SignalContext` that the
|
||||
* plugin core can dedup, classify, and turn into incident tasks.
|
||||
*
|
||||
* Reference payload: https://grafana.com/docs/grafana/latest/alerting/configure-notifications/manage-contact-points/integrations/webhook-notifier/
|
||||
*/
|
||||
|
||||
import type { SignalContext } from "../internal/severity.js";
|
||||
|
||||
interface GrafanaWebhookAlert {
|
||||
status?: string;
|
||||
labels?: Record<string, string>;
|
||||
annotations?: Record<string, string>;
|
||||
startsAt?: string;
|
||||
endsAt?: string;
|
||||
generatorURL?: string;
|
||||
fingerprint?: string;
|
||||
values?: Record<string, number>;
|
||||
}
|
||||
|
||||
interface GrafanaWebhookPayload {
|
||||
status?: string;
|
||||
alerts?: GrafanaWebhookAlert[];
|
||||
commonLabels?: Record<string, string>;
|
||||
commonAnnotations?: Record<string, string>;
|
||||
title?: string;
|
||||
message?: string;
|
||||
externalURL?: string;
|
||||
}
|
||||
|
||||
export interface ParsedGrafanaSignal {
|
||||
signal: SignalContext;
|
||||
/** Stable fingerprint Grafana already computed (use as primary dimension). */
|
||||
grafanaFingerprint: string;
|
||||
/** Inline summary for the incident task title. */
|
||||
alertname: string;
|
||||
/** Human-readable description for the incident task body. */
|
||||
description: string;
|
||||
/** Was the alert state "firing" when the webhook was sent? */
|
||||
firing: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Decide which sase domain the alert belongs to from labels and alertname.
|
||||
* Conservative: when uncertain, return "unknown" so triage classifies further.
|
||||
*/
|
||||
function classifyDomain(alertname: string, labels: Record<string, string>): string {
|
||||
const haystack = `${alertname} ${Object.values(labels).join(" ")}`.toLowerCase();
|
||||
if (/\b(payment|iyzico|eft|billing|subscription)\b/.test(haystack)) return "backend";
|
||||
if (/\b(auth|sign-?in|sign-?up|session|cookie)\b/.test(haystack)) return "backend";
|
||||
if (/\b(vehicle|vin|catalog|emex|pl24|partscatalog)\b/.test(haystack)) return "backend";
|
||||
if (/\b(rum|web-vitals|lcp|cls|inp|faro|frontend)\b/.test(haystack)) return "frontend";
|
||||
if (/\b(latency|p95|p99|error.?rate|http|api)\b/.test(haystack)) return "backend";
|
||||
if (/\b(worker|bull|queue|job)\b/.test(haystack)) return "backend";
|
||||
if (/\b(funnel|conversion|signup|posthog)\b/.test(haystack)) return "product";
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
/**
|
||||
* Translate a Grafana webhook payload into one signal per firing alert.
|
||||
* Resolved/normal alerts are dropped — telemetry-watcher only opens tasks
|
||||
* for new fires; recovery is handled separately by post-deploy verification.
|
||||
*/
|
||||
export function parseGrafanaPayload(payload: unknown): ParsedGrafanaSignal[] {
|
||||
if (!payload || typeof payload !== "object") return [];
|
||||
const data = payload as GrafanaWebhookPayload;
|
||||
const alerts = Array.isArray(data.alerts) ? data.alerts : [];
|
||||
const out: ParsedGrafanaSignal[] = [];
|
||||
|
||||
for (const alert of alerts) {
|
||||
const firing = (alert.status ?? data.status ?? "").toLowerCase() === "firing";
|
||||
const labels = alert.labels ?? {};
|
||||
const annotations = alert.annotations ?? {};
|
||||
const alertname = labels.alertname ?? data.title ?? "unknown-alert";
|
||||
const summary = annotations.summary ?? annotations.description ?? data.message ?? "";
|
||||
|
||||
const valueEntries = Object.entries(alert.values ?? {});
|
||||
const primaryValue = valueEntries.length > 0 ? Number(valueEntries[0]![1]) : undefined;
|
||||
|
||||
// Map common Prometheus/Tempo label names into our meta surface.
|
||||
const meta: Record<string, unknown> = {
|
||||
...labels,
|
||||
summary,
|
||||
title: alertname,
|
||||
alertname,
|
||||
generatorURL: alert.generatorURL,
|
||||
startsAt: alert.startsAt,
|
||||
};
|
||||
if (typeof labels.endpoint === "string") meta.endpoint = labels.endpoint;
|
||||
if (typeof labels.severity === "string") meta.alertSeverity = labels.severity;
|
||||
if (typeof primaryValue === "number" && !Number.isNaN(primaryValue)) {
|
||||
// Heuristic: latency-named metrics use ms; rate-named metrics 0..1.
|
||||
if (/latency|duration|p95|p99/i.test(alertname)) meta.latencyMs = primaryValue;
|
||||
else if (/error|fail|5xx/i.test(alertname)) meta.errorRate = primaryValue;
|
||||
}
|
||||
|
||||
const signal: SignalContext = {
|
||||
source: "grafana",
|
||||
domain: classifyDomain(alertname, labels),
|
||||
value: primaryValue,
|
||||
meta,
|
||||
};
|
||||
|
||||
out.push({
|
||||
signal,
|
||||
grafanaFingerprint:
|
||||
alert.fingerprint || `${alertname}|${labels.endpoint ?? "*"}|${labels.severity ?? "*"}`,
|
||||
alertname,
|
||||
description: summary,
|
||||
firing,
|
||||
});
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
Reference in New Issue
Block a user