feat(phase6a): behavioral insight pipeline ingestion + tagging + compression
- prisma: sessions_meta, compressed_sessions, ingestion_watermarks - worker: PostHog client (eu.i.posthog.com), heuristic filter - worker: BullMQ insight-pipeline queue (ingest 5min / tag 2min / compress 3min) - worker: tagger (bug_suspected, ux_friction, upgrade_hesitation, etc.) + severity scoring - worker: rrweb -> semantic timeline transform + 8-pattern PII sanitization - worker: fingerprint hash + MinIO upload (bucket: insight-compressed) - web: /insights pipeline dashboard + session timeline viewer - sidebar: Insights nav entry LLM (Phase 6b), insight inbox, cost dashboard, GitHub loop deferred to later phases. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Semih
126
apps/web/src/app/insights/page.tsx
Normal file
126
apps/web/src/app/insights/page.tsx
Normal file
@@ -0,0 +1,126 @@
|
||||
import { PanelShell } from "@/components/panel-shell";
|
||||
import { prisma } from "@/lib/db";
|
||||
import {
|
||||
Table,
|
||||
TableBody,
|
||||
TableCell,
|
||||
TableHead,
|
||||
TableHeader,
|
||||
TableRow,
|
||||
} from "@/components/ui/table";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
export default async function InsightsPage() {
|
||||
const [byStatus, recent, watermark, compressed] = await Promise.all([
|
||||
prisma.sessionMeta.groupBy({
|
||||
by: ["status"],
|
||||
_count: { status: true },
|
||||
}),
|
||||
prisma.sessionMeta.findMany({
|
||||
where: { status: { in: ["tagged", "compressed"] } },
|
||||
orderBy: { startedAt: "desc" },
|
||||
take: 50,
|
||||
}),
|
||||
prisma.ingestionWatermark.findUnique({ where: { projectKey: "sase" } }),
|
||||
prisma.compressedSession.count(),
|
||||
]);
|
||||
|
||||
const counts: Record<string, number> = {};
|
||||
for (const row of byStatus) counts[row.status] = row._count.status;
|
||||
const total = Object.values(counts).reduce((a, b) => a + b, 0);
|
||||
|
||||
return (
|
||||
<PanelShell title="Insights · Sase.tr (pilot)">
|
||||
<p className="text-sm text-muted-foreground">
|
||||
Phase 6a — ingestion · heuristic filter · tagging · semantic compression. LLM stage gelmedi.
|
||||
</p>
|
||||
|
||||
<div className="grid grid-cols-2 gap-4 md:grid-cols-5">
|
||||
<KPI label="Total sessions" value={total.toString()} />
|
||||
<KPI label="Pending" value={(counts.pending_signal ?? 0).toString()} />
|
||||
<KPI label="Tagged" value={(counts.tagged ?? 0).toString()} />
|
||||
<KPI label="Compressed" value={compressed.toString()} />
|
||||
<KPI label="Discarded" value={(counts.discarded ?? 0).toString()} />
|
||||
</div>
|
||||
|
||||
<div className="rounded-md border p-3 text-xs text-muted-foreground">
|
||||
Watermark:{" "}
|
||||
<span className="font-mono">
|
||||
{watermark ? watermark.lastPolledAt.toISOString() : "never polled"}
|
||||
</span>{" "}
|
||||
· ingest cadence: 5 min
|
||||
</div>
|
||||
|
||||
<h2 className="mt-2 text-sm font-medium text-muted-foreground">Recent tagged sessions</h2>
|
||||
<div className="rounded-md border">
|
||||
<Table>
|
||||
<TableHeader>
|
||||
<TableRow>
|
||||
<TableHead>Session</TableHead>
|
||||
<TableHead>Status</TableHead>
|
||||
<TableHead>Severity</TableHead>
|
||||
<TableHead>Score</TableHead>
|
||||
<TableHead>Tags</TableHead>
|
||||
<TableHead>Started</TableHead>
|
||||
<TableHead>Duration</TableHead>
|
||||
<TableHead>URL</TableHead>
|
||||
</TableRow>
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
{recent.length === 0 ? (
|
||||
<TableRow>
|
||||
<TableCell colSpan={8} className="text-center text-xs text-muted-foreground">
|
||||
No tagged sessions yet — pipeline may still be warming up.
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
) : (
|
||||
recent.map((s) => (
|
||||
<TableRow key={s.id}>
|
||||
<TableCell className="font-mono text-xs">
|
||||
<a className="underline" href={`/insights/sessions/${s.id}`}>
|
||||
{s.id.slice(0, 12)}…
|
||||
</a>
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
<Badge variant={s.status === "compressed" ? "default" : "outline"}>
|
||||
{s.status}
|
||||
</Badge>
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
{s.severity ? <Badge variant="outline">{s.severity}</Badge> : "—"}
|
||||
</TableCell>
|
||||
<TableCell className="font-mono text-xs">{s.score ?? "—"}</TableCell>
|
||||
<TableCell className="text-xs">
|
||||
{s.tags.length ? s.tags.join(", ") : "—"}
|
||||
</TableCell>
|
||||
<TableCell className="text-xs text-muted-foreground">
|
||||
{s.startedAt.toISOString().slice(0, 19).replace("T", " ")}
|
||||
</TableCell>
|
||||
<TableCell className="text-xs">{Math.round(s.durationMs / 1000)}s</TableCell>
|
||||
<TableCell className="max-w-[200px] truncate font-mono text-xs">
|
||||
{s.startUrl ?? "—"}
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
))
|
||||
)}
|
||||
</TableBody>
|
||||
</Table>
|
||||
</div>
|
||||
</PanelShell>
|
||||
);
|
||||
}
|
||||
|
||||
function KPI({ label, value }: { label: string; value: string }) {
|
||||
return (
|
||||
<Card>
|
||||
<CardHeader className="pb-2">
|
||||
<CardDescription>{label}</CardDescription>
|
||||
<CardTitle className="text-2xl">{value}</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent className="h-1" />
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
93
apps/web/src/app/insights/sessions/[id]/page.tsx
Normal file
93
apps/web/src/app/insights/sessions/[id]/page.tsx
Normal file
@@ -0,0 +1,93 @@
|
||||
import { PanelShell } from "@/components/panel-shell";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { notFound } from "next/navigation";
|
||||
import { Client as MinioClient } from "minio";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
async function fetchTimeline(bucket: string, key: string): Promise<string | null> {
|
||||
const access = process.env.MINIO_ACCESS_KEY ?? "";
|
||||
const secret = process.env.MINIO_SECRET_KEY ?? "";
|
||||
if (!access || !secret) return null;
|
||||
const c = new MinioClient({
|
||||
endPoint: process.env.MINIO_ENDPOINT ?? "minio-global",
|
||||
port: Number(process.env.MINIO_PORT ?? 9000),
|
||||
useSSL: (process.env.MINIO_USE_SSL ?? "false") === "true",
|
||||
accessKey: access,
|
||||
secretKey: secret,
|
||||
});
|
||||
try {
|
||||
const stream = await c.getObject(bucket, key);
|
||||
return await new Promise<string>((resolve, reject) => {
|
||||
const chunks: Buffer[] = [];
|
||||
stream.on("data", (d) => chunks.push(d as Buffer));
|
||||
stream.on("end", () => resolve(Buffer.concat(chunks).toString("utf-8")));
|
||||
stream.on("error", reject);
|
||||
});
|
||||
} catch (e) {
|
||||
return `(unable to load timeline: ${(e as Error).message})`;
|
||||
}
|
||||
}
|
||||
|
||||
export default async function SessionDetailPage({
|
||||
params,
|
||||
}: {
|
||||
params: Promise<{ id: string }>;
|
||||
}) {
|
||||
const { id } = await params;
|
||||
const s = await prisma.sessionMeta.findUnique({
|
||||
where: { id },
|
||||
include: { compressed: true },
|
||||
});
|
||||
if (!s) notFound();
|
||||
|
||||
const bucket = process.env.INSIGHT_COMPRESSED_BUCKET ?? "insight-compressed";
|
||||
const timeline = s.compressed
|
||||
? await fetchTimeline(bucket, s.compressed.semanticTimelineMinioKey)
|
||||
: null;
|
||||
|
||||
return (
|
||||
<PanelShell title="Session detail">
|
||||
<a href="/insights" className="text-xs underline text-muted-foreground">
|
||||
← Back to insights
|
||||
</a>
|
||||
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<h2 className="text-lg font-mono">{s.id}</h2>
|
||||
<Badge variant="outline">{s.status}</Badge>
|
||||
{s.severity && <Badge variant="outline">{s.severity}</Badge>}
|
||||
{s.tags.map((t) => (
|
||||
<Badge key={t} variant="secondary">{t}</Badge>
|
||||
))}
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-2 gap-x-6 gap-y-1 text-xs">
|
||||
<div>started: <span className="font-mono">{s.startedAt.toISOString()}</span></div>
|
||||
<div>duration: {Math.round(s.durationMs / 1000)}s</div>
|
||||
<div>auth: {s.isAuthenticated ? "yes" : "no"}</div>
|
||||
<div>tier: {s.subscriptionTier ?? "—"}</div>
|
||||
<div>errors: {s.errorCount}</div>
|
||||
<div>clicks: {s.clickCount}</div>
|
||||
<div>start_url: <span className="font-mono">{s.startUrl ?? "—"}</span></div>
|
||||
<div>score: {s.score ?? "—"}</div>
|
||||
<div className="col-span-2">promotion reasons: {s.promotionReasons.join(", ") || "—"}</div>
|
||||
{s.fingerprint && <div className="col-span-2">fingerprint: <span className="font-mono">{s.fingerprint}</span></div>}
|
||||
</div>
|
||||
|
||||
{s.compressed && (
|
||||
<div className="text-xs text-muted-foreground">
|
||||
tokens: ~{s.compressed.tokenCountInput} · PII matches: {s.compressed.sanitizationMatchCount}
|
||||
{s.compressed.sanitizationMatchCount === 0 && s.compressed.tokenCountInput > 500 && (
|
||||
<span className="ml-2 text-amber-600">⚠ zero PII matches with large output</span>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<h3 className="mt-4 text-sm font-medium">Semantic Timeline</h3>
|
||||
<pre className="rounded-md border bg-muted/30 p-3 text-xs whitespace-pre-wrap font-mono">
|
||||
{timeline ?? "(no compressed timeline yet)"}
|
||||
</pre>
|
||||
</PanelShell>
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user