feat(archive): add groups + recordings to the PostHog archive

- groups: posthog-resource-archive now also snapshots group types + group
  instances (groups_types/ + groups/?group_type_index) into
  posthog_resource_snapshots (change-detected).
- recordings: new posthog-recording-archive job (@*/6h) captures complete
  metadata for EVERY recording into posthog_recordings (skipDuplicates over a
  rolling 90d window) + MinIO dump — a superset of the insight pipeline's
  promoted SessionMeta subset. rrweb blobs still archived by archive-recordings.
- /posthog-archive browser: Groups / Group Types tabs + a Recordings tab.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Semih
2026-06-10 21:19:54 +03:00
parent e953f5fc34
commit 400ed224c5
7 changed files with 373 additions and 45 deletions

View File

@@ -561,6 +561,35 @@ model PosthogResourceSnapshot {
@@map("posthog_resource_snapshots")
}
// Complete metadata for EVERY session recording (not just the insight-pipeline's
// promoted subset in SessionMeta). One row per recording; full list-item payload
// kept lossless. rrweb blobs themselves are archived separately to MinIO by the
// archive-recordings job for ingested sessions.
model PosthogRecording {
id String @id // PostHog recording / session id
projectKey String
distinctId String?
personName String?
startTime DateTime?
endTime DateTime?
durationSec Int?
activeSeconds Int?
clickCount Int?
keypressCount Int?
mouseActivityCount Int?
consoleErrorCount Int?
startUrl String?
ongoing Boolean @default(false)
properties Json // full recording list item, lossless
capturedAt DateTime @default(now())
dumpedAt DateTime?
@@index([projectKey, startTime])
@@index([distinctId])
@@index([dumpedAt])
@@map("posthog_recordings")
}
// ---------- Phase C: Sentry archive ----------
// Sentry issue (group) aggregate state. Upserted to the latest snapshot — the

View File

@@ -13,6 +13,8 @@ import {
import {
getArchiveCounts,
getLatestResources,
getRecordingCount,
getRecentRecordings,
RESOURCE_TYPES,
TYPE_LABEL,
type ResourceType,
@@ -26,24 +28,34 @@ function isType(v: string | undefined): v is ResourceType {
return !!v && (RESOURCE_TYPES as readonly string[]).includes(v);
}
function fmtDur(sec: number | null): string {
if (!sec || sec <= 0) return "—";
const m = Math.floor(sec / 60);
const s = sec % 60;
return `${m}:${String(s).padStart(2, "0")}`;
}
export default async function PosthogArchivePage({
searchParams,
}: {
searchParams: Promise<{ type?: string }>;
}) {
const sp = await searchParams;
const isRecording = sp.type === "recording";
const type: ResourceType = isType(sp.type) ? sp.type : "feature_flag";
const [counts, resources] = await Promise.all([
const [counts, recCount, resources, recordings] = await Promise.all([
getArchiveCounts(PROJECT),
getLatestResources(PROJECT, type),
getRecordingCount(PROJECT),
isRecording ? Promise.resolve([]) : getLatestResources(PROJECT, type),
isRecording ? getRecentRecordings(PROJECT, 150) : Promise.resolve([]),
]);
return (
<PanelShell title="PostHog Arşiv">
<p className="text-sm text-muted-foreground">
PostHog&apos;daki tüm kaynak modülleri (survey, feature flag, experiment, dashboard, insight,
annotation, action) değişiklik geçmişiyle birlikte arşivleniyor (~6 saat tazelik, DB + MinIO).
PostHog&apos;daki tüm modüller feature flag, experiment, survey, insight, dashboard, annotation,
action, group ve recording değişiklik geçmişiyle birlikte arşivleniyor (~6 saat tazelik, DB + MinIO).
</p>
<div className="flex flex-wrap gap-1.5">
@@ -51,7 +63,7 @@ export default async function PosthogArchivePage({
<Link
key={c.type}
href={`/posthog-archive?type=${c.type}`}
className={buttonVariants({ variant: c.type === type ? "default" : "outline", size: "sm" })}
className={buttonVariants({ variant: !isRecording && c.type === type ? "default" : "outline", size: "sm" })}
>
{TYPE_LABEL[c.type] ?? c.type}
<Badge variant="secondary" className="ml-1.5">
@@ -59,48 +71,97 @@ export default async function PosthogArchivePage({
</Badge>
</Link>
))}
<Link
href="/posthog-archive?type=recording"
className={buttonVariants({ variant: isRecording ? "default" : "outline", size: "sm" })}
>
{TYPE_LABEL.recording}
<Badge variant="secondary" className="ml-1.5">
{recCount.toLocaleString("tr-TR")}
</Badge>
</Link>
</div>
<div className="rounded-md border">
<Table>
<TableHeader>
<TableRow>
<TableHead>Ad</TableHead>
<TableHead className="w-28">ID</TableHead>
<TableHead className="w-24 text-right">Versiyon</TableHead>
<TableHead className="w-36 text-right">Son yakalama</TableHead>
</TableRow>
</TableHeader>
<TableBody>
{resources.map((r) => (
<TableRow key={r.resourceId}>
<TableCell className="max-w-[460px] truncate font-medium">
<Link
href={`/posthog-archive/${type}/${encodeURIComponent(r.resourceId)}`}
className="underline"
>
{r.name || `(adsız ${type})`}
</Link>
</TableCell>
<TableCell className="font-mono text-xs text-muted-foreground">{r.resourceId}</TableCell>
<TableCell className="text-right tabular-nums">
{r.versions > 1 ? <Badge variant="outline">{r.versions}</Badge> : r.versions}
</TableCell>
<TableCell className="text-right font-mono text-xs text-muted-foreground">
{r.capturedAt.toISOString().slice(0, 16).replace("T", " ")}
</TableCell>
</TableRow>
))}
{resources.length === 0 && (
{isRecording ? (
<div className="rounded-md border">
<Table>
<TableHeader>
<TableRow>
<TableCell colSpan={4} className="text-center text-sm text-muted-foreground">
Bu tip için henüz arşiv yok.
</TableCell>
<TableHead>Kullanıcı</TableHead>
<TableHead className="max-w-[360px]">Başlangıç URL</TableHead>
<TableHead className="w-20 text-right">Süre</TableHead>
<TableHead className="w-16 text-right">Tık</TableHead>
<TableHead className="w-16 text-right">Hata</TableHead>
<TableHead className="w-36 text-right">Başlangıç</TableHead>
</TableRow>
)}
</TableBody>
</Table>
</div>
</TableHeader>
<TableBody>
{recordings.map((r) => (
<TableRow key={r.id}>
<TableCell className="font-mono text-xs">{r.personName || r.distinctId?.slice(0, 14) || "anon"}</TableCell>
<TableCell className="max-w-[360px] truncate text-xs text-muted-foreground">{r.startUrl || "—"}</TableCell>
<TableCell className="text-right tabular-nums">{fmtDur(r.durationSec)}</TableCell>
<TableCell className="text-right tabular-nums">{r.clickCount ?? 0}</TableCell>
<TableCell className="text-right tabular-nums">
{r.consoleErrorCount ? <span className="text-red-600">{r.consoleErrorCount}</span> : 0}
</TableCell>
<TableCell className="text-right font-mono text-xs text-muted-foreground">
{r.startTime ? r.startTime.toISOString().slice(0, 16).replace("T", " ") : "—"}
</TableCell>
</TableRow>
))}
{recordings.length === 0 && (
<TableRow>
<TableCell colSpan={6} className="text-center text-sm text-muted-foreground">
Henüz recording arşivlenmedi (ilk çalışma sonrası dolar).
</TableCell>
</TableRow>
)}
</TableBody>
</Table>
</div>
) : (
<div className="rounded-md border">
<Table>
<TableHeader>
<TableRow>
<TableHead>Ad</TableHead>
<TableHead className="w-28">ID</TableHead>
<TableHead className="w-24 text-right">Versiyon</TableHead>
<TableHead className="w-36 text-right">Son yakalama</TableHead>
</TableRow>
</TableHeader>
<TableBody>
{resources.map((r) => (
<TableRow key={r.resourceId}>
<TableCell className="max-w-[460px] truncate font-medium">
<Link
href={`/posthog-archive/${type}/${encodeURIComponent(r.resourceId)}`}
className="underline"
>
{r.name || `(adsız ${type})`}
</Link>
</TableCell>
<TableCell className="font-mono text-xs text-muted-foreground">{r.resourceId}</TableCell>
<TableCell className="text-right tabular-nums">
{r.versions > 1 ? <Badge variant="outline">{r.versions}</Badge> : r.versions}
</TableCell>
<TableCell className="text-right font-mono text-xs text-muted-foreground">
{r.capturedAt.toISOString().slice(0, 16).replace("T", " ")}
</TableCell>
</TableRow>
))}
{resources.length === 0 && (
<TableRow>
<TableCell colSpan={4} className="text-center text-sm text-muted-foreground">
Bu tip için henüz arşiv yok.
</TableCell>
</TableRow>
)}
</TableBody>
</Table>
</div>
)}
</PanelShell>
);
}

View File

@@ -8,6 +8,8 @@ export const RESOURCE_TYPES = [
"dashboard",
"annotation",
"action",
"group",
"group_type",
] as const;
export type ResourceType = (typeof RESOURCE_TYPES)[number];
@@ -19,6 +21,9 @@ export const TYPE_LABEL: Record<string, string> = {
dashboard: "Dashboards",
annotation: "Annotations",
action: "Actions",
group: "Groups",
group_type: "Group Types",
recording: "Recordings",
};
export type ArchiveCount = { type: string; resources: number; snapshots: number; lastCaptured: Date | null };
@@ -97,3 +102,40 @@ export async function getResourceHistory(
});
return rows;
}
// ── Recordings (separate table: posthog_recordings — full recording metadata) ──
export async function getRecordingCount(projectKey: string): Promise<number> {
const rows = await prisma.$queryRaw<Array<{ c: bigint }>>`
SELECT count(*)::bigint AS c FROM posthog_recordings WHERE "projectKey" = ${projectKey}
`;
return Number(rows[0]?.c ?? 0);
}
export type RecordingRow = {
id: string;
distinctId: string | null;
personName: string | null;
durationSec: number | null;
clickCount: number | null;
consoleErrorCount: number | null;
startUrl: string | null;
startTime: Date | null;
};
export async function getRecentRecordings(projectKey: string, limit = 100): Promise<RecordingRow[]> {
return prisma.posthogRecording.findMany({
where: { projectKey },
orderBy: { startTime: "desc" },
take: limit,
select: {
id: true,
distinctId: true,
personName: true,
durationSec: true,
clickCount: true,
consoleErrorCount: true,
startUrl: true,
startTime: true,
},
});
}

View File

@@ -0,0 +1,86 @@
import { gzipSync } from "node:zlib";
import { prisma } from "../db";
import { listRecordings, isConfigured } from "../lib/posthog";
import { putBuffer } from "../lib/minio";
// Complete metadata archive of EVERY session recording — superset of the insight
// pipeline's promoted SessionMeta rows. Pages the recording list over a rolling
// window and inserts only new ids (skipDuplicates). rrweb blobs are archived
// separately by archive-recordings for ingested sessions.
const PROJECT_KEY = process.env.INSIGHT_PROJECT_KEY ?? "sase";
const ARCHIVE_BUCKET = process.env.INSIGHT_ARCHIVE_BUCKET ?? "posthog-archive";
const WINDOW_DAYS = Number(process.env.INSIGHT_RECORDING_META_WINDOW_DAYS ?? "90");
const MAX_PAGES = Number(process.env.INSIGHT_RECORDING_META_MAX_PAGES ?? "40");
const PAGE = 100;
export type RecordingMetaArchiveResult = {
ok: boolean;
fetched: number;
inserted: number;
pages: number;
dumped: boolean;
reason?: string;
};
const int = (v: unknown): number | null => {
const n = Math.round(Number(v));
return Number.isFinite(n) ? n : null;
};
export async function runArchivePosthogRecordings(): Promise<RecordingMetaArchiveResult> {
if (!isConfigured()) {
return { ok: false, fetched: 0, inserted: 0, pages: 0, dumped: false, reason: "posthog_not_configured" };
}
const dateFrom = new Date(Date.now() - WINDOW_DAYS * 864e5).toISOString();
const dateTo = new Date(Date.now() - 3_600_000).toISOString(); // 1h buffer → finalized recordings only
let fetched = 0;
let inserted = 0;
let pages = 0;
const all: Array<Record<string, unknown>> = [];
for (let offset = 0; pages < MAX_PAGES; offset += PAGE, pages++) {
let resp;
try {
resp = await listRecordings({ dateFrom, dateTo, limit: PAGE, offset });
} catch (e) {
if (pages === 0) return { ok: false, fetched, inserted, pages, dumped: false, reason: (e as Error).message };
break; // mid-run page failure → keep what we have, retry next cycle
}
const items = resp.results ?? [];
if (items.length === 0) break;
fetched += items.length;
all.push(...(items as unknown as Array<Record<string, unknown>>));
const rows = items.map((r) => ({
id: r.id,
projectKey: PROJECT_KEY,
distinctId: r.distinct_id ?? null,
personName: r.person?.name ?? null,
startTime: r.start_time ? new Date(r.start_time) : null,
endTime: r.end_time ? new Date(r.end_time) : null,
durationSec: int(r.recording_duration),
activeSeconds: int(r.active_seconds),
clickCount: int(r.click_count),
keypressCount: int(r.keypress_count),
mouseActivityCount: int(r.mouse_activity_count),
consoleErrorCount: int(r.console_error_count),
startUrl: r.start_url ?? null,
ongoing: Boolean(r.ongoing),
properties: r as object,
}));
const res = await prisma.posthogRecording.createMany({ data: rows as never, skipDuplicates: true });
inserted += res.count;
if (!resp.has_next && !resp.next) break;
}
let dumped = false;
if (all.length) {
const day = new Date().toISOString().slice(0, 10);
await putBuffer(ARCHIVE_BUCKET, `recordings/${day}.json.gz`, gzipSync(Buffer.from(JSON.stringify(all))), "application/gzip");
dumped = true;
}
return { ok: true, fetched, inserted, pages, dumped };
}

View File

@@ -1,6 +1,6 @@
import { gzipSync } from "node:zlib";
import { prisma } from "../db";
import { listResource, isConfigured } from "../lib/posthog";
import { listResource, listGroupTypes, listGroups, isConfigured } from "../lib/posthog";
import { stableHash } from "../lib/hash";
import { putBuffer } from "../lib/minio";
@@ -35,6 +35,23 @@ function hashConfig(item: Record<string, unknown>): string {
return stableHash(clean);
}
type SnapRow = { projectKey: string; resourceType: string; resourceId: string; name: string | null; stateHash: string; data: object };
// Insert only rows whose config changed since the latest stored snapshot.
async function persistChanged(type: string, rows: SnapRow[]): Promise<number> {
const latest = await prisma.$queryRaw<Array<{ resourceId: string; stateHash: string }>>`
SELECT DISTINCT ON ("resourceId") "resourceId", "stateHash"
FROM posthog_resource_snapshots
WHERE "projectKey" = ${PROJECT_KEY} AND "resourceType" = ${type}
ORDER BY "resourceId", "capturedAt" DESC
`;
const latestHash = new Map(latest.map((x) => [x.resourceId, x.stateHash]));
const changed = rows.filter((row) => row.resourceId && latestHash.get(row.resourceId) !== row.stateHash);
if (!changed.length) return 0;
const res = await prisma.posthogResourceSnapshot.createMany({ data: changed as never });
return res.count;
}
export type ResourceArchiveResult = {
ok: boolean;
perType: Record<string, { scanned: number; snapshots: number; dumped: boolean; error?: string }>;
@@ -87,5 +104,53 @@ export async function runArchivePosthogResources(): Promise<ResourceArchiveResul
}
}
// ── Groups: group types + instances (separate API shape) ──
try {
const types = await listGroupTypes();
const typeRows: SnapRow[] = types
.map((t) => ({
projectKey: PROJECT_KEY,
resourceType: "group_type",
resourceId: String(t.group_type_index ?? t.group_type ?? ""),
name: String(t.name_singular ?? t.group_type ?? ""),
stateHash: hashConfig(t),
data: t as object,
}))
.filter((row) => row.resourceId);
perType["group_type"] = { scanned: typeRows.length, snapshots: await persistChanged("group_type", typeRows), dumped: false };
if (types.length) {
await putBuffer(ARCHIVE_BUCKET, `resources/group_type/${day}.json.gz`, gzipSync(Buffer.from(JSON.stringify(types))), "application/gzip");
perType["group_type"].dumped = true;
}
let allGroups: Array<Record<string, unknown>> = [];
const groupRows: SnapRow[] = [];
for (const t of types) {
const idx = Number(t.group_type_index ?? -1);
if (idx < 0) continue;
const groups = await listGroups(idx);
allGroups = allGroups.concat(groups);
for (const g of groups) {
const key = String(g.group_key ?? "");
if (!key) continue;
groupRows.push({
projectKey: PROJECT_KEY,
resourceType: "group",
resourceId: `${idx}:${key}`,
name: key,
stateHash: hashConfig(g),
data: g as object,
});
}
}
perType["group"] = { scanned: groupRows.length, snapshots: await persistChanged("group", groupRows), dumped: false };
if (allGroups.length) {
await putBuffer(ARCHIVE_BUCKET, `resources/group/${day}.json.gz`, gzipSync(Buffer.from(JSON.stringify(allGroups))), "application/gzip");
perType["group"].dumped = true;
}
} catch (e) {
perType["group"] = { scanned: 0, snapshots: 0, dumped: false, error: (e as Error).message };
}
return { ok: true, perType };
}

View File

@@ -141,6 +141,36 @@ export async function listResource(
return out;
}
// Group types (e.g. "company") + group instances. Groups use a separate API
// shape (group_type_index), so they can't go through listResource.
export async function listGroupTypes(): Promise<Array<Record<string, unknown>>> {
const res = await fetch(`${HOST}/api/projects/${PROJECT_ID}/groups_types/`, { headers: headers() });
if (!res.ok) throw new Error(`posthog groups_types ${res.status}: ${(await res.text().catch(() => "")).slice(0, 120)}`);
const data = (await res.json()) as unknown;
if (Array.isArray(data)) return data as Array<Record<string, unknown>>;
const r = (data as { results?: Array<Record<string, unknown>> }).results;
return Array.isArray(r) ? r : [];
}
export async function listGroups(
groupTypeIndex: number,
opts: { maxPages?: number } = {},
): Promise<Array<Record<string, unknown>>> {
const maxPages = opts.maxPages ?? 50;
const out: Array<Record<string, unknown>> = [];
let url: string | null = `${HOST}/api/projects/${PROJECT_ID}/groups/?group_type_index=${groupTypeIndex}&limit=100`;
let pages = 0;
while (url && pages < maxPages) {
const res: Response = await fetch(url, { headers: headers() });
if (!res.ok) throw new Error(`posthog groups ${res.status}: ${(await res.text().catch(() => "")).slice(0, 120)}`);
const data = (await res.json()) as { results?: Array<Record<string, unknown>>; next?: string | null };
if (Array.isArray(data.results)) out.push(...data.results);
url = data.next ?? null;
pages++;
}
return out;
}
// ---------- Custom events ----------
export type PHCustomEvent = {

View File

@@ -16,6 +16,7 @@ import { runArchivePosthogEvents } from "../jobs/posthog-event-archive";
import { runArchiveRecordings } from "../jobs/archive-recordings";
import { runArchiveIdentity } from "../jobs/posthog-identity-archive";
import { runArchivePosthogResources } from "../jobs/posthog-resource-archive";
import { runArchivePosthogRecordings } from "../jobs/posthog-recording-archive";
import { runArchiveSentry } from "../jobs/sentry-archive";
const QUEUE = "insight-pipeline";
@@ -146,6 +147,15 @@ async function runJob(job: Job) {
}
return res;
}
case "posthog-recording-archive": {
const res = await runArchivePosthogRecordings();
if (res.inserted > 0 || !res.ok) {
console.log(
`[pipeline] recording-archive fetched=${res.fetched} inserted=${res.inserted} pages=${res.pages}${res.reason ? ` reason=${res.reason}` : ""}`,
);
}
return res;
}
case "sentry-archive": {
const res = await runArchiveSentry();
if (res.eventsInserted > 0 || res.issuesSynced > 0 || res.dumped > 0 || res.error) {
@@ -239,6 +249,11 @@ export async function startInsightPipeline() {
{ pattern: "15 */6 * * *" },
{ name: "posthog-resource-archive", data: {}, opts: { removeOnComplete: 30, removeOnFail: 15 } },
);
await queue.upsertJobScheduler(
"posthog-recording-archive",
{ pattern: "45 */6 * * *" },
{ name: "posthog-recording-archive", data: {}, opts: { removeOnComplete: 30, removeOnFail: 15 } },
);
await queue.upsertJobScheduler(
"sentry-archive",
{ pattern: "0 * * * *" },
@@ -252,7 +267,7 @@ export async function startInsightPipeline() {
stalledInterval: 60_000,
});
console.log(
"[pipeline] armed: posthog-ingest@*/5min, tag-sessions@*/2min, compress-sessions@*/3min, analyze@*/4min, validation@05:00, github-sync@*/10min, retention@04:15, daily-brief@05:00, vin-anomaly-detect@*/5min, catalog-gap-detect@*/6h, sase-evening-brief@17:00UTC, posthog-event-archive@*/15min, archive-recordings@*/6h, posthog-identity-archive@03:00, posthog-resource-archive@*/6h, sentry-archive@hourly",
"[pipeline] armed: posthog-ingest@*/5min, tag-sessions@*/2min, compress-sessions@*/3min, analyze@*/4min, validation@05:00, github-sync@*/10min, retention@04:15, daily-brief@05:00, vin-anomaly-detect@*/5min, catalog-gap-detect@*/6h, sase-evening-brief@17:00UTC, posthog-event-archive@*/15min, archive-recordings@*/6h, posthog-identity-archive@03:00, posthog-resource-archive@*/6h, posthog-recording-archive@*/6h, sentry-archive@hourly",
);
return worker;
}