feat(insights): compress.ts noise filter + per-session reprocess
Drop successful (2xx/3xx) asset and telemetry GETs from the semantic
timeline — these were eating MAX_LINES (80) before any user-behavior
signal could surface. In a 9m32s session we observed only the first
1:18 reached the LLM input because PostHog /collect/* pings filled
the budget. Failures (4xx/5xx) are still emitted as signal.
Filtered prefixes: /collect/, /flags, /array/, /static/, /assets/,
/api/surveys, /css2. Also raise MAX_LINES 80→120.
Add optional { sessionId } to POST /api/insights/reprocess so a single
session can be re-run end-to-end without resetting a whole batch.
This commit is contained in:
@@ -9,29 +9,43 @@ export async function POST(req: Request) {
|
||||
const session = await auth.api.getSession({ headers: await headers() });
|
||||
if (!session) return NextResponse.json({ ok: false, error: "unauthenticated" }, { status: 401 });
|
||||
|
||||
const body = (await req.json().catch(() => ({}))) as { scope?: string; limit?: number };
|
||||
const body = (await req.json().catch(() => ({}))) as {
|
||||
scope?: string;
|
||||
limit?: number;
|
||||
sessionId?: string;
|
||||
};
|
||||
const scope = body.scope ?? "tagged-and-discarded";
|
||||
const limit = Math.min(Math.max(body.limit ?? 200, 1), 1000);
|
||||
|
||||
// Reset session status so the pipeline picks them up again with the new v1.2 logic.
|
||||
const where =
|
||||
scope === "all"
|
||||
? { status: { in: ["tagged", "compressed", "discarded", "analyzed"] as string[] } }
|
||||
: scope === "compressed"
|
||||
? { status: "compressed" }
|
||||
: scope === "discarded"
|
||||
? { status: "discarded" }
|
||||
: scope === "analyzed"
|
||||
? { status: "analyzed" }
|
||||
: { status: { in: ["tagged", "discarded"] as string[] } };
|
||||
let ids: string[];
|
||||
if (body.sessionId) {
|
||||
const found = await prisma.sessionMeta.findUnique({
|
||||
where: { id: body.sessionId },
|
||||
select: { id: true },
|
||||
});
|
||||
if (!found) return NextResponse.json({ ok: false, error: "not_found" }, { status: 404 });
|
||||
ids = [found.id];
|
||||
} else {
|
||||
// Reset session status so the pipeline picks them up again with the new v1.2 logic.
|
||||
const where =
|
||||
scope === "all"
|
||||
? { status: { in: ["tagged", "compressed", "discarded", "analyzed"] as string[] } }
|
||||
: scope === "compressed"
|
||||
? { status: "compressed" }
|
||||
: scope === "discarded"
|
||||
? { status: "discarded" }
|
||||
: scope === "analyzed"
|
||||
? { status: "analyzed" }
|
||||
: { status: { in: ["tagged", "discarded"] as string[] } };
|
||||
|
||||
const candidates = await prisma.sessionMeta.findMany({
|
||||
where,
|
||||
select: { id: true },
|
||||
orderBy: { startedAt: "desc" },
|
||||
take: limit,
|
||||
});
|
||||
const ids = candidates.map((c) => c.id);
|
||||
const candidates = await prisma.sessionMeta.findMany({
|
||||
where,
|
||||
select: { id: true },
|
||||
orderBy: { startedAt: "desc" },
|
||||
take: limit,
|
||||
});
|
||||
ids = candidates.map((c) => c.id);
|
||||
}
|
||||
if (!ids.length) return NextResponse.json({ ok: true, reset: 0 });
|
||||
|
||||
const updated = await prisma.sessionMeta.updateMany({
|
||||
|
||||
@@ -32,7 +32,26 @@ type SessionHeader = {
|
||||
companyContext?: string | null;
|
||||
};
|
||||
|
||||
const MAX_LINES = 80;
|
||||
const MAX_LINES = 120;
|
||||
|
||||
// Asset and telemetry GETs that successful (2xx/3xx) don't carry user-behavior
|
||||
// signal. We still emit them when they fail (4xx/5xx) — those indicate real
|
||||
// problems. Dropping these reclaims MAX_LINES budget for clicks, inputs, custom
|
||||
// events, and API failures that actually explain the session.
|
||||
function isNoiseUrl(pathname: string): boolean {
|
||||
return (
|
||||
pathname.startsWith("/collect/") ||
|
||||
pathname.startsWith("/flags") ||
|
||||
pathname.startsWith("/array/") ||
|
||||
pathname.startsWith("/static/") ||
|
||||
pathname.startsWith("/assets/") ||
|
||||
pathname.startsWith("/api/surveys") ||
|
||||
pathname === "/css2" ||
|
||||
pathname.startsWith("/css2/") ||
|
||||
pathname.endsWith(".js.map") ||
|
||||
pathname.endsWith(".css.map")
|
||||
);
|
||||
}
|
||||
|
||||
export function compressSnapshots(
|
||||
events: RREvent[],
|
||||
@@ -138,6 +157,8 @@ export function compressSnapshots(
|
||||
const method = r.method ?? "GET";
|
||||
const rurl = stripQuery(String(r.name ?? r.url ?? "/"));
|
||||
if (typeof status === "number") {
|
||||
// Skip successful asset/telemetry GETs — keep failures (signal).
|
||||
if (status < 400 && isNoiseUrl(rurl)) continue;
|
||||
const tag = status >= 500 ? "❌" : status >= 400 ? "⚠" : "→";
|
||||
lines.push(`${ts} → net ${method} ${rurl} ${tag} ${status}`);
|
||||
if (status >= 500) {
|
||||
|
||||
Reference in New Issue
Block a user