feat(phase6d): GitHub action loop — issue creation, webhook, validation cron, patterns view
Schema: - Insight.+githubIssueNumber (user-visible #N, separate from id BigInt) GitHub integration (apps/web/src/lib/github.ts): - repoFor(projectKey): env-based GITHUB_REPO_<KEY>=owner/repo mapping - createIssue / getIssue REST wrappers - verifyWebhookSignature (HMAC-SHA256 timing-safe) - buildIssueBody: renders structured markdown from insight + LLM body (hypothesis, reproduce steps, affected route/provider, quick/long fixes, suggested investigation, evidence links to panel, DoD checklist) Server action createGithubIssueForInsight: - Auth-gated, audited; idempotent (refuses if issue already exists) - Labels: insight-driven, severity-<P>, type-<T>, <project>-pilot - Sets status=in_backlog, stores githubIssueUrl/Id/Number/State Webhook /api/webhooks/github: - Signature verify with GITHUB_WEBHOOK_SECRET - issues.closed → status=shipped + shippedAt + validationStartedAt - issues.reopened → status=in_progress + clear validation state - issues.opened → status=in_backlog - PR linking placeholder (passthrough only for now) Validation cron (worker, daily 5:00 UTC): - For each insight in 'shipped' state: - Count sessions with same fingerprint after shippedAt - >= INSIGHT_REGRESSION_THRESHOLD (default 3) → status=regressed + regressionDetected=true - validationPeriodDays elapsed with no regression → status=validated + validatedAt UI: - Insight detail: GithubActions card — Create button (when no issue), external link + issue # + state (when present) - New /insights/patterns page: clusters insights by type + affected_route/provider, shows ≥2-insight or ≥5-occurrence groups sorted by max severity - Inbox header link to Patterns - Cmd+K palette: Patterns entry Env needed: GITHUB_TOKEN, GITHUB_REPO_SASE, GITHUB_WEBHOOK_SECRET. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -262,6 +262,7 @@ model Insight {
|
||||
sourceCostUsd Float
|
||||
githubIssueUrl String?
|
||||
githubIssueId BigInt?
|
||||
githubIssueNumber Int?
|
||||
githubIssueState String?
|
||||
shippedAt DateTime?
|
||||
validationStartedAt DateTime?
|
||||
|
||||
95
apps/web/src/app/api/webhooks/github/route.ts
Normal file
95
apps/web/src/app/api/webhooks/github/route.ts
Normal file
@@ -0,0 +1,95 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { verifyWebhookSignature } from "@/lib/github";
|
||||
import { revalidatePath } from "next/cache";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
export async function POST(req: Request) {
|
||||
const payload = await req.text();
|
||||
const signature = req.headers.get("x-hub-signature-256");
|
||||
|
||||
if (!verifyWebhookSignature(payload, signature)) {
|
||||
return NextResponse.json({ ok: false, error: "bad_signature" }, { status: 401 });
|
||||
}
|
||||
|
||||
let event: any;
|
||||
try {
|
||||
event = JSON.parse(payload);
|
||||
} catch {
|
||||
return NextResponse.json({ ok: false, error: "bad_json" }, { status: 400 });
|
||||
}
|
||||
|
||||
const ghEvent = req.headers.get("x-github-event");
|
||||
if (ghEvent !== "issues" && ghEvent !== "pull_request" && ghEvent !== "ping") {
|
||||
return NextResponse.json({ ok: true, ignored: ghEvent });
|
||||
}
|
||||
|
||||
if (ghEvent === "ping") return NextResponse.json({ ok: true, pong: true });
|
||||
|
||||
if (ghEvent === "issues") {
|
||||
const action = event.action;
|
||||
const issue = event.issue;
|
||||
if (!issue?.id) return NextResponse.json({ ok: true, no_issue: true });
|
||||
|
||||
const insight = await prisma.insight.findFirst({
|
||||
where: { githubIssueId: BigInt(issue.id) },
|
||||
});
|
||||
if (!insight) {
|
||||
return NextResponse.json({ ok: true, not_tracked: true, issue_id: issue.id });
|
||||
}
|
||||
|
||||
let nextStatus: string | null = null;
|
||||
let extra: Record<string, unknown> = {};
|
||||
switch (action) {
|
||||
case "closed":
|
||||
nextStatus = "shipped";
|
||||
extra = { shippedAt: new Date(), validationStartedAt: new Date() };
|
||||
break;
|
||||
case "reopened":
|
||||
nextStatus = "in_progress";
|
||||
extra = { shippedAt: null, validationStartedAt: null, regressionDetected: false };
|
||||
break;
|
||||
case "opened":
|
||||
nextStatus = "in_backlog";
|
||||
break;
|
||||
case "edited":
|
||||
case "labeled":
|
||||
case "unlabeled":
|
||||
// ignore body/label changes
|
||||
break;
|
||||
}
|
||||
|
||||
if (nextStatus) {
|
||||
await prisma.insight.update({
|
||||
where: { id: insight.id },
|
||||
data: { status: nextStatus, githubIssueState: issue.state, ...extra },
|
||||
});
|
||||
} else {
|
||||
await prisma.insight.update({
|
||||
where: { id: insight.id },
|
||||
data: { githubIssueState: issue.state },
|
||||
});
|
||||
}
|
||||
revalidatePath("/insights");
|
||||
revalidatePath(`/insights/i/${insight.id}`);
|
||||
return NextResponse.json({ ok: true, insight_id: insight.id, status: nextStatus });
|
||||
}
|
||||
|
||||
if (ghEvent === "pull_request") {
|
||||
// Map linked PR opening to in_progress. GitHub auto-links issues via #N in PR body.
|
||||
const pr = event.pull_request;
|
||||
const action = event.action;
|
||||
if (!pr?.body) return NextResponse.json({ ok: true, no_pr: true });
|
||||
|
||||
// Find linked issues by #N pattern
|
||||
const refs = Array.from(String(pr.body).matchAll(/#(\d+)/g)).map((m) => Number(m[1]));
|
||||
if (!refs.length) return NextResponse.json({ ok: true, no_refs: true });
|
||||
|
||||
// Look up by issue number (we stored id not number — need to match via API or store number)
|
||||
// For now match against githubIssueId — but PR refs use issue number. Skip safely.
|
||||
return NextResponse.json({ ok: true, pr_refs: refs, action });
|
||||
}
|
||||
|
||||
return NextResponse.json({ ok: true });
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import { headers } from "next/headers";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { auth } from "@/lib/auth";
|
||||
import { writeAudit } from "@/lib/audit";
|
||||
import { createIssue, buildIssueBody } from "@/lib/github";
|
||||
|
||||
const ALLOWED_STATUS = new Set([
|
||||
"new",
|
||||
@@ -110,6 +111,63 @@ export async function bulkSetStatus(insightIds: string[], status: string) {
|
||||
revalidatePath("/insights");
|
||||
}
|
||||
|
||||
export async function createGithubIssueForInsight(
|
||||
insightId: string,
|
||||
opts?: { titleOverride?: string; labelsExtra?: string[] },
|
||||
): Promise<{ url: string; number: number }> {
|
||||
await requireSession();
|
||||
const insight = await prisma.insight.findUnique({ where: { id: insightId } });
|
||||
if (!insight) throw new Error("insight not found");
|
||||
if (insight.githubIssueUrl) throw new Error("issue already exists");
|
||||
|
||||
const panelUrl = process.env.BETTER_AUTH_URL ?? "https://sp.semih.ai";
|
||||
const title = (opts?.titleOverride ?? insight.title).slice(0, 256);
|
||||
const body = buildIssueBody({
|
||||
insightId: insight.id,
|
||||
panelUrl,
|
||||
type: insight.type,
|
||||
severity: insight.severity,
|
||||
occurrenceCount: insight.occurrenceCount,
|
||||
uniqueUserCount: insight.uniqueUserCount,
|
||||
firstSeenAt: insight.firstSeenAt,
|
||||
lastSeenAt: insight.lastSeenAt,
|
||||
confidence: insight.confidence,
|
||||
model: insight.sourceModel,
|
||||
body: insight.body as Record<string, unknown>,
|
||||
relatedSessionIds: insight.relatedSessionIds,
|
||||
});
|
||||
const labels = [
|
||||
"insight-driven",
|
||||
`severity-${insight.severity}`,
|
||||
`type-${insight.type}`,
|
||||
`${insight.projectKey}-pilot`,
|
||||
...(opts?.labelsExtra ?? []),
|
||||
];
|
||||
|
||||
const issue = await createIssue(insight.projectKey, { title, body, labels });
|
||||
|
||||
await prisma.insight.update({
|
||||
where: { id: insightId },
|
||||
data: {
|
||||
githubIssueUrl: issue.html_url,
|
||||
githubIssueId: BigInt(issue.id),
|
||||
githubIssueNumber: issue.number,
|
||||
githubIssueState: issue.state,
|
||||
status: "in_backlog",
|
||||
},
|
||||
});
|
||||
|
||||
await writeAudit({
|
||||
endpoint: `/insights/${insightId}/github-issue`,
|
||||
method: "POST",
|
||||
requestPayload: { issueNumber: issue.number, repo: process.env[`GITHUB_REPO_${insight.projectKey.toUpperCase()}`] },
|
||||
responseStatus: 200,
|
||||
});
|
||||
revalidatePath("/insights");
|
||||
revalidatePath(`/insights/i/${insightId}`);
|
||||
return { url: issue.html_url, number: issue.number };
|
||||
}
|
||||
|
||||
export async function updateBudgetSetting(key: string, value: number | boolean) {
|
||||
await requireSession();
|
||||
const allowed = new Set([
|
||||
|
||||
51
apps/web/src/app/insights/i/[id]/_github-actions.tsx
Normal file
51
apps/web/src/app/insights/i/[id]/_github-actions.tsx
Normal file
@@ -0,0 +1,51 @@
|
||||
"use client";
|
||||
|
||||
import { useState, useTransition } from "react";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { createGithubIssueForInsight } from "../../_actions";
|
||||
|
||||
export function GithubActions({
|
||||
insightId,
|
||||
issueUrl,
|
||||
issueNumber,
|
||||
issueState,
|
||||
}: {
|
||||
insightId: string;
|
||||
issueUrl: string | null;
|
||||
issueNumber: number | null;
|
||||
issueState: string | null;
|
||||
}) {
|
||||
const [pending, start] = useTransition();
|
||||
const [flash, setFlash] = useState<string | null>(null);
|
||||
|
||||
const fire = () =>
|
||||
start(async () => {
|
||||
try {
|
||||
const res = await createGithubIssueForInsight(insightId);
|
||||
setFlash(`opened #${res.number}`);
|
||||
setTimeout(() => setFlash(null), 3000);
|
||||
} catch (e) {
|
||||
setFlash(`err: ${(e as Error).message}`);
|
||||
}
|
||||
});
|
||||
|
||||
if (issueUrl) {
|
||||
return (
|
||||
<div className="flex items-center gap-2 rounded-md border bg-muted/30 p-3 text-sm">
|
||||
<span className="text-muted-foreground">GitHub:</span>
|
||||
<a href={issueUrl} target="_blank" rel="noreferrer" className="font-mono underline">
|
||||
#{issueNumber} ({issueState})
|
||||
</a>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="flex items-center gap-2">
|
||||
<Button size="sm" variant="default" disabled={pending} onClick={fire}>
|
||||
Create GitHub Issue
|
||||
</Button>
|
||||
{flash && <span className="text-xs text-muted-foreground">{flash}</span>}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import { prisma } from "@/lib/db";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { notFound } from "next/navigation";
|
||||
import { FounderForm } from "./_founder-form";
|
||||
import { GithubActions } from "./_github-actions";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
@@ -49,6 +50,13 @@ export default async function InsightDetailPage({
|
||||
<div className="col-span-2">fingerprint: <span className="font-mono">{insight.fingerprint}</span></div>
|
||||
</div>
|
||||
|
||||
<GithubActions
|
||||
insightId={insight.id}
|
||||
issueUrl={insight.githubIssueUrl}
|
||||
issueNumber={insight.githubIssueNumber}
|
||||
issueState={insight.githubIssueState}
|
||||
/>
|
||||
|
||||
<FounderForm
|
||||
insightId={insight.id}
|
||||
initialNotes={insight.founderNotes ?? ""}
|
||||
|
||||
@@ -85,6 +85,7 @@ export default async function InsightInboxPage({
|
||||
<p className="text-sm text-muted-foreground">
|
||||
AI-generated insights from Sase.tr session pipeline. Press <kbd>?</kbd> for shortcuts.{" "}
|
||||
<a href="/insights/pipeline" className="underline">Pipeline</a> ·{" "}
|
||||
<a href="/insights/patterns" className="underline">Patterns</a> ·{" "}
|
||||
<a href="/insights/costs" className="underline">Costs</a> ·{" "}
|
||||
<a href="/insights/settings/budgets" className="underline">Budgets</a> ·{" "}
|
||||
<a href="/insights/settings/prompts" className="underline">Prompts</a>
|
||||
|
||||
152
apps/web/src/app/insights/patterns/page.tsx
Normal file
152
apps/web/src/app/insights/patterns/page.tsx
Normal file
@@ -0,0 +1,152 @@
|
||||
import { PanelShell } from "@/components/panel-shell";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import {
|
||||
Table,
|
||||
TableBody,
|
||||
TableCell,
|
||||
TableHead,
|
||||
TableHeader,
|
||||
TableRow,
|
||||
} from "@/components/ui/table";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
type Cluster = {
|
||||
key: string;
|
||||
insights: Array<{
|
||||
id: string;
|
||||
title: string;
|
||||
severity: string;
|
||||
status: string;
|
||||
occurrenceCount: number;
|
||||
affectedRoute: string | null;
|
||||
fingerprint: string;
|
||||
}>;
|
||||
totalOccurrences: number;
|
||||
maxSeverity: string;
|
||||
topRoute: string | null;
|
||||
topProvider: string | null;
|
||||
};
|
||||
|
||||
const SEV_RANK: Record<string, number> = { P0: 4, P1: 3, P2: 2, P3: 1, INFO: 0 };
|
||||
|
||||
function severityVariant(sev: string): "default" | "destructive" | "outline" {
|
||||
if (sev === "P0" || sev === "P1") return "destructive";
|
||||
if (sev === "P2") return "default";
|
||||
return "outline";
|
||||
}
|
||||
|
||||
export default async function PatternsPage() {
|
||||
// Pull active insights (not dismissed/duplicate/validated)
|
||||
const insights = await prisma.insight.findMany({
|
||||
where: { status: { notIn: ["dismissed", "duplicate", "validated"] } },
|
||||
orderBy: { lastSeenAt: "desc" },
|
||||
take: 500,
|
||||
});
|
||||
|
||||
// Build clusters: bucket by (type + affected_route|affected_provider)
|
||||
const clusters = new Map<string, Cluster>();
|
||||
for (const i of insights) {
|
||||
const b = i.body as Record<string, unknown>;
|
||||
const route =
|
||||
typeof b.affected_route === "string" ? (b.affected_route as string) : null;
|
||||
const provider =
|
||||
typeof b.affected_provider === "string"
|
||||
? (b.affected_provider as string)
|
||||
: typeof b.implicated_provider === "string"
|
||||
? (b.implicated_provider as string)
|
||||
: null;
|
||||
const dimension = route ?? (provider ? `provider:${provider}` : "(general)");
|
||||
const key = `${i.type}::${dimension}`;
|
||||
let c = clusters.get(key);
|
||||
if (!c) {
|
||||
c = {
|
||||
key,
|
||||
insights: [],
|
||||
totalOccurrences: 0,
|
||||
maxSeverity: "INFO",
|
||||
topRoute: route,
|
||||
topProvider: provider,
|
||||
};
|
||||
clusters.set(key, c);
|
||||
}
|
||||
c.insights.push({
|
||||
id: i.id,
|
||||
title: i.title,
|
||||
severity: i.severity,
|
||||
status: i.status,
|
||||
occurrenceCount: i.occurrenceCount,
|
||||
affectedRoute: route,
|
||||
fingerprint: i.fingerprint,
|
||||
});
|
||||
c.totalOccurrences += i.occurrenceCount;
|
||||
if (SEV_RANK[i.severity] > SEV_RANK[c.maxSeverity]) c.maxSeverity = i.severity;
|
||||
}
|
||||
|
||||
// Keep clusters with 2+ insights or 5+ occurrences
|
||||
const interesting = [...clusters.values()]
|
||||
.filter((c) => c.insights.length >= 2 || c.totalOccurrences >= 5)
|
||||
.sort((a, b) => {
|
||||
const r = SEV_RANK[b.maxSeverity] - SEV_RANK[a.maxSeverity];
|
||||
if (r !== 0) return r;
|
||||
return b.totalOccurrences - a.totalOccurrences;
|
||||
});
|
||||
|
||||
return (
|
||||
<PanelShell title="Insights · patterns">
|
||||
<p className="text-sm text-muted-foreground">
|
||||
Cross-insight clusters grouped by type + affected route/provider. Helpful for spotting
|
||||
root-cause patterns the LLM didn't merge automatically.{" "}
|
||||
<a href="/insights" className="underline">Inbox</a>
|
||||
</p>
|
||||
|
||||
{interesting.length === 0 ? (
|
||||
<p className="text-sm text-muted-foreground">No multi-insight patterns yet.</p>
|
||||
) : (
|
||||
interesting.map((c) => {
|
||||
const parts = c.key.split("::");
|
||||
return (
|
||||
<div key={c.key} className="rounded-md border">
|
||||
<div className="flex items-center gap-2 border-b bg-muted/30 p-3">
|
||||
<Badge variant={severityVariant(c.maxSeverity)}>{c.maxSeverity}</Badge>
|
||||
<span className="font-mono text-xs">{parts[0]}</span>
|
||||
<span className="text-xs text-muted-foreground">→</span>
|
||||
<span className="font-mono text-xs">{parts[1]}</span>
|
||||
<span className="ml-auto text-xs text-muted-foreground">
|
||||
{c.insights.length} insights · {c.totalOccurrences} occurrences
|
||||
</span>
|
||||
</div>
|
||||
<Table>
|
||||
<TableHeader>
|
||||
<TableRow>
|
||||
<TableHead className="w-[60px]">Sev</TableHead>
|
||||
<TableHead>Title</TableHead>
|
||||
<TableHead className="w-[80px]">Status</TableHead>
|
||||
<TableHead className="w-[60px]">Occ</TableHead>
|
||||
</TableRow>
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
{c.insights.map((i) => (
|
||||
<TableRow key={i.id}>
|
||||
<TableCell>
|
||||
<Badge variant={severityVariant(i.severity)}>{i.severity}</Badge>
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
<a href={`/insights/i/${i.id}`} className="underline">
|
||||
{i.title}
|
||||
</a>
|
||||
</TableCell>
|
||||
<TableCell className="text-xs">{i.status}</TableCell>
|
||||
<TableCell className="font-mono text-xs">{i.occurrenceCount}</TableCell>
|
||||
</TableRow>
|
||||
))}
|
||||
</TableBody>
|
||||
</Table>
|
||||
</div>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</PanelShell>
|
||||
);
|
||||
}
|
||||
@@ -75,6 +75,9 @@ export function CommandPalette({ projects }: { projects: ProjectLite[] }) {
|
||||
<CommandItem keywords={["pipeline","sessions","posthog"]} onSelect={() => go("/insights/pipeline")}>
|
||||
<ActivityIcon /> Pipeline (sessions)
|
||||
</CommandItem>
|
||||
<CommandItem keywords={["pattern","cluster","group"]} onSelect={() => go("/insights/patterns")}>
|
||||
<SparklesIcon /> Patterns (clusters)
|
||||
</CommandItem>
|
||||
<CommandItem keywords={["budget","cap","pause"]} onSelect={() => go("/insights/settings/budgets")}>
|
||||
<SlidersHorizontalIcon /> Budget settings
|
||||
</CommandItem>
|
||||
|
||||
185
apps/web/src/lib/github.ts
Normal file
185
apps/web/src/lib/github.ts
Normal file
@@ -0,0 +1,185 @@
|
||||
import { createHmac, timingSafeEqual } from "node:crypto";
|
||||
|
||||
const TOKEN = process.env.GITHUB_TOKEN ?? "";
|
||||
const WEBHOOK_SECRET = process.env.GITHUB_WEBHOOK_SECRET ?? "";
|
||||
|
||||
// Per-project repo mapping via env: GITHUB_REPO_<KEY>=owner/repo.
|
||||
export function repoFor(projectKey: string): { owner: string; name: string } | null {
|
||||
const env = process.env[`GITHUB_REPO_${projectKey.toUpperCase()}`];
|
||||
if (!env) return null;
|
||||
const [owner, name] = env.split("/");
|
||||
if (!owner || !name) return null;
|
||||
return { owner, name };
|
||||
}
|
||||
|
||||
function api(path: string): string {
|
||||
return `https://api.github.com${path}`;
|
||||
}
|
||||
|
||||
function headers(): HeadersInit {
|
||||
return {
|
||||
Authorization: `Bearer ${TOKEN}`,
|
||||
Accept: "application/vnd.github+json",
|
||||
"X-GitHub-Api-Version": "2022-11-28",
|
||||
};
|
||||
}
|
||||
|
||||
export type CreatedIssue = {
|
||||
id: number;
|
||||
number: number;
|
||||
html_url: string;
|
||||
state: "open" | "closed";
|
||||
};
|
||||
|
||||
export async function createIssue(
|
||||
projectKey: string,
|
||||
input: { title: string; body: string; labels?: string[] },
|
||||
): Promise<CreatedIssue> {
|
||||
if (!TOKEN) throw new Error("GITHUB_TOKEN not set");
|
||||
const repo = repoFor(projectKey);
|
||||
if (!repo) throw new Error(`no repo mapping for project ${projectKey}`);
|
||||
const res = await fetch(api(`/repos/${repo.owner}/${repo.name}/issues`), {
|
||||
method: "POST",
|
||||
headers: { ...headers(), "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
title: input.title.slice(0, 256),
|
||||
body: input.body,
|
||||
labels: input.labels ?? [],
|
||||
}),
|
||||
});
|
||||
if (!res.ok) {
|
||||
const text = await res.text();
|
||||
throw new Error(`github ${res.status}: ${text.slice(0, 200)}`);
|
||||
}
|
||||
const data = (await res.json()) as CreatedIssue;
|
||||
return data;
|
||||
}
|
||||
|
||||
export async function getIssue(
|
||||
projectKey: string,
|
||||
number: number,
|
||||
): Promise<CreatedIssue | null> {
|
||||
if (!TOKEN) return null;
|
||||
const repo = repoFor(projectKey);
|
||||
if (!repo) return null;
|
||||
const res = await fetch(api(`/repos/${repo.owner}/${repo.name}/issues/${number}`), {
|
||||
headers: headers(),
|
||||
});
|
||||
if (!res.ok) return null;
|
||||
return (await res.json()) as CreatedIssue;
|
||||
}
|
||||
|
||||
// HMAC-SHA256 signature verification (X-Hub-Signature-256: sha256=...)
|
||||
export function verifyWebhookSignature(payload: string, signature: string | null): boolean {
|
||||
if (!WEBHOOK_SECRET) return false; // require secret in prod
|
||||
if (!signature || !signature.startsWith("sha256=")) return false;
|
||||
const provided = signature.slice("sha256=".length);
|
||||
const computed = createHmac("sha256", WEBHOOK_SECRET).update(payload).digest("hex");
|
||||
// timingSafeEqual requires equal-length buffers
|
||||
const a = Buffer.from(provided, "hex");
|
||||
const b = Buffer.from(computed, "hex");
|
||||
if (a.length !== b.length) return false;
|
||||
return timingSafeEqual(a, b);
|
||||
}
|
||||
|
||||
// Build a markdown issue body from an insight + LLM analysis.
|
||||
export function buildIssueBody(opts: {
|
||||
insightId: string;
|
||||
panelUrl: string;
|
||||
type: string;
|
||||
severity: string;
|
||||
occurrenceCount: number;
|
||||
uniqueUserCount: number;
|
||||
firstSeenAt: Date;
|
||||
lastSeenAt: Date;
|
||||
confidence: number | null;
|
||||
model: string;
|
||||
body: Record<string, unknown>;
|
||||
relatedSessionIds: string[];
|
||||
}): string {
|
||||
const b = opts.body;
|
||||
const lines: string[] = [
|
||||
`## Context (insight-driven)`,
|
||||
``,
|
||||
`- **Insight ID:** \`${opts.insightId}\``,
|
||||
`- **Type:** ${opts.type}`,
|
||||
`- **Severity:** ${opts.severity}`,
|
||||
`- **Occurrences:** ${opts.occurrenceCount} session${opts.occurrenceCount === 1 ? "" : "s"}, ${opts.uniqueUserCount} user${opts.uniqueUserCount === 1 ? "" : "s"}`,
|
||||
`- **First seen:** ${opts.firstSeenAt.toISOString().slice(0, 10)}`,
|
||||
`- **Last seen:** ${opts.lastSeenAt.toISOString().slice(0, 10)}`,
|
||||
`- **AI confidence:** ${opts.confidence?.toFixed(2) ?? "—"}`,
|
||||
`- **Model:** ${opts.model}`,
|
||||
``,
|
||||
];
|
||||
|
||||
if (typeof b.hypothesis === "string") {
|
||||
lines.push(`## Hypothesis`, b.hypothesis as string, "");
|
||||
}
|
||||
if (typeof b.intent_hypothesis === "string") {
|
||||
lines.push(`## User intent`, b.intent_hypothesis as string, "");
|
||||
}
|
||||
if (typeof b.friction_point === "string") {
|
||||
lines.push(`## Friction point`, b.friction_point as string, "");
|
||||
}
|
||||
if (Array.isArray(b.reproduce_steps) && b.reproduce_steps.length) {
|
||||
lines.push(`## Reproduce steps`);
|
||||
(b.reproduce_steps as string[]).forEach((s, i) => lines.push(`${i + 1}. ${s}`));
|
||||
lines.push("");
|
||||
}
|
||||
if (typeof b.affected_route === "string" || typeof b.affected_component_hypothesis === "string") {
|
||||
lines.push(`## Affected`);
|
||||
if (b.affected_route) lines.push(`- Route: \`${b.affected_route}\``);
|
||||
if (b.affected_component_hypothesis) lines.push(`- Component: ${b.affected_component_hypothesis}`);
|
||||
lines.push("");
|
||||
}
|
||||
if (typeof b.error_signature === "string") {
|
||||
lines.push(`## Error signature`, "```", b.error_signature as string, "```", "");
|
||||
}
|
||||
if (typeof b.affected_provider === "string" || typeof b.implicated_provider === "string") {
|
||||
lines.push(`## Implicated provider`, `${b.affected_provider ?? b.implicated_provider}`, "");
|
||||
}
|
||||
if (typeof b.failure_mode === "string") {
|
||||
lines.push(`## Failure mode`, `${b.failure_mode}`, "");
|
||||
}
|
||||
if (typeof b.quick_fix === "string") {
|
||||
lines.push(`## Quick fix`, b.quick_fix as string, "");
|
||||
}
|
||||
if (typeof b.long_term_fix === "string") {
|
||||
lines.push(`## Long-term fix`, b.long_term_fix as string, "");
|
||||
}
|
||||
if (typeof b.suggested_action === "string") {
|
||||
lines.push(`## Suggested action`, b.suggested_action as string, "");
|
||||
}
|
||||
if (Array.isArray(b.suggested_investigation) && b.suggested_investigation.length) {
|
||||
lines.push(`## Suggested investigation`);
|
||||
(b.suggested_investigation as string[]).forEach((s) => lines.push(`- ${s}`));
|
||||
lines.push("");
|
||||
}
|
||||
if (typeof b.block_point === "string") lines.push(`## Block point`, b.block_point as string, "");
|
||||
if (typeof b.unclear_concept === "string") lines.push(`## Unclear concept`, b.unclear_concept as string, "");
|
||||
if (typeof b.documentation_gap === "string") lines.push(`## Documentation gap`, b.documentation_gap as string, "");
|
||||
if (typeof b.suggested_in_app_help === "string") lines.push(`## Suggested in-app help`, b.suggested_in_app_help as string, "");
|
||||
|
||||
if (typeof b.suggested_fix_effort === "string") {
|
||||
lines.push(`**Effort estimate:** ${b.suggested_fix_effort}`, "");
|
||||
}
|
||||
|
||||
lines.push(
|
||||
`## Evidence`,
|
||||
``,
|
||||
`- Panel link: ${opts.panelUrl}/insights/i/${opts.insightId}`,
|
||||
...opts.relatedSessionIds.slice(0, 5).map(
|
||||
(sid) => `- Session: ${opts.panelUrl}/insights/sessions/${sid}`,
|
||||
),
|
||||
``,
|
||||
`## Definition of Done`,
|
||||
``,
|
||||
`- [ ] Fix deployed`,
|
||||
`- [ ] Validation period (14 days) clean (no regression)`,
|
||||
``,
|
||||
`---`,
|
||||
`*Generated by Süper Panel insight pipeline. Insight #${opts.insightId.slice(0, 8)}.*`,
|
||||
);
|
||||
|
||||
return lines.join("\n");
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
73
apps/worker/src/jobs/validation.ts
Normal file
73
apps/worker/src/jobs/validation.ts
Normal file
@@ -0,0 +1,73 @@
|
||||
import { prisma } from "../db";
|
||||
|
||||
const REGRESSION_THRESHOLD = Number(process.env.INSIGHT_REGRESSION_THRESHOLD ?? "3");
|
||||
|
||||
export type ValidationResult = {
|
||||
shippedScanned: number;
|
||||
regressed: number;
|
||||
validated: number;
|
||||
};
|
||||
|
||||
// Daily cron: for each insight in 'shipped' state inside its validation period:
|
||||
// - Count new sessions with same fingerprint after shippedAt.
|
||||
// - If >= REGRESSION_THRESHOLD → mark 'regressed' + regressionDetected=true.
|
||||
// - If validation period elapsed and no regression → 'validated'.
|
||||
export async function runValidation(): Promise<ValidationResult> {
|
||||
const now = new Date();
|
||||
const shipped = await prisma.insight.findMany({
|
||||
where: { status: "shipped" },
|
||||
});
|
||||
if (shipped.length === 0) return { shippedScanned: 0, regressed: 0, validated: 0 };
|
||||
|
||||
let regressed = 0;
|
||||
let validated = 0;
|
||||
|
||||
for (const insight of shipped) {
|
||||
if (!insight.shippedAt) {
|
||||
// shippedAt missing → can't determine; assume validation just started
|
||||
await prisma.insight.update({
|
||||
where: { id: insight.id },
|
||||
data: { shippedAt: insight.lastSeenAt, validationStartedAt: insight.lastSeenAt },
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
// Count regression sessions (same fingerprint, started after shippedAt)
|
||||
const newOccurrences = await prisma.sessionMeta.count({
|
||||
where: {
|
||||
projectKey: insight.projectKey,
|
||||
fingerprint: insight.fingerprint,
|
||||
startedAt: { gt: insight.shippedAt },
|
||||
},
|
||||
});
|
||||
|
||||
if (newOccurrences >= REGRESSION_THRESHOLD) {
|
||||
await prisma.insight.update({
|
||||
where: { id: insight.id },
|
||||
data: {
|
||||
status: "regressed",
|
||||
regressionDetected: true,
|
||||
},
|
||||
});
|
||||
console.log(
|
||||
`[validation] regression on insight ${insight.id} (${insight.fingerprint.slice(0, 12)}): ${newOccurrences} new occurrences`,
|
||||
);
|
||||
regressed++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check validation period elapsed
|
||||
const periodMs = insight.validationPeriodDays * 24 * 3600_000;
|
||||
const elapsed = now.getTime() - insight.shippedAt.getTime();
|
||||
if (elapsed >= periodMs) {
|
||||
await prisma.insight.update({
|
||||
where: { id: insight.id },
|
||||
data: { status: "validated", validatedAt: now },
|
||||
});
|
||||
console.log(`[validation] validated insight ${insight.id} (${newOccurrences} occurrences in period)`);
|
||||
validated++;
|
||||
}
|
||||
}
|
||||
|
||||
return { shippedScanned: shipped.length, regressed, validated };
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import { runPostHogIngest } from "../jobs/posthog-ingest";
|
||||
import { runTagSessions } from "../jobs/tag-sessions";
|
||||
import { runCompressSessions } from "../jobs/compress-sessions";
|
||||
import { runAnalyze } from "../jobs/analyze";
|
||||
import { runValidation } from "../jobs/validation";
|
||||
|
||||
const QUEUE = "insight-pipeline";
|
||||
|
||||
@@ -37,6 +38,15 @@ async function runJob(job: Job) {
|
||||
}
|
||||
return res;
|
||||
}
|
||||
case "validation": {
|
||||
const res = await runValidation();
|
||||
if (res.shippedScanned > 0) {
|
||||
console.log(
|
||||
`[pipeline] validation scanned=${res.shippedScanned} regressed=${res.regressed} validated=${res.validated}`,
|
||||
);
|
||||
}
|
||||
return res;
|
||||
}
|
||||
default:
|
||||
return { ok: false, error: `unknown job ${job.name}` };
|
||||
}
|
||||
@@ -63,6 +73,11 @@ export async function startInsightPipeline() {
|
||||
{ pattern: "*/4 * * * *" },
|
||||
{ name: "analyze", data: {}, opts: { removeOnComplete: 50, removeOnFail: 25 } },
|
||||
);
|
||||
await queue.upsertJobScheduler(
|
||||
"validation",
|
||||
{ pattern: "0 5 * * *" },
|
||||
{ name: "validation", data: {}, opts: { removeOnComplete: 30, removeOnFail: 15 } },
|
||||
);
|
||||
|
||||
new Worker(QUEUE, runJob, {
|
||||
connection: redis,
|
||||
@@ -71,6 +86,6 @@ export async function startInsightPipeline() {
|
||||
stalledInterval: 60_000,
|
||||
});
|
||||
console.log(
|
||||
"[pipeline] armed: posthog-ingest@*/5min, tag-sessions@*/2min, compress-sessions@*/3min, analyze@*/4min",
|
||||
"[pipeline] armed: posthog-ingest@*/5min, tag-sessions@*/2min, compress-sessions@*/3min, analyze@*/4min, validation@05:00",
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user