feat(archive): add groups + recordings to the PostHog archive
- groups: posthog-resource-archive now also snapshots group types + group instances (groups_types/ + groups/?group_type_index) into posthog_resource_snapshots (change-detected). - recordings: new posthog-recording-archive job (@*/6h) captures complete metadata for EVERY recording into posthog_recordings (skipDuplicates over a rolling 90d window) + MinIO dump — a superset of the insight pipeline's promoted SessionMeta subset. rrweb blobs still archived by archive-recordings. - /posthog-archive browser: Groups / Group Types tabs + a Recordings tab. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -561,6 +561,35 @@ model PosthogResourceSnapshot {
|
||||
@@map("posthog_resource_snapshots")
|
||||
}
|
||||
|
||||
// Complete metadata for EVERY session recording (not just the insight-pipeline's
|
||||
// promoted subset in SessionMeta). One row per recording; full list-item payload
|
||||
// kept lossless. rrweb blobs themselves are archived separately to MinIO by the
|
||||
// archive-recordings job for ingested sessions.
|
||||
model PosthogRecording {
|
||||
id String @id // PostHog recording / session id
|
||||
projectKey String
|
||||
distinctId String?
|
||||
personName String?
|
||||
startTime DateTime?
|
||||
endTime DateTime?
|
||||
durationSec Int?
|
||||
activeSeconds Int?
|
||||
clickCount Int?
|
||||
keypressCount Int?
|
||||
mouseActivityCount Int?
|
||||
consoleErrorCount Int?
|
||||
startUrl String?
|
||||
ongoing Boolean @default(false)
|
||||
properties Json // full recording list item, lossless
|
||||
capturedAt DateTime @default(now())
|
||||
dumpedAt DateTime?
|
||||
|
||||
@@index([projectKey, startTime])
|
||||
@@index([distinctId])
|
||||
@@index([dumpedAt])
|
||||
@@map("posthog_recordings")
|
||||
}
|
||||
|
||||
// ---------- Phase C: Sentry archive ----------
|
||||
|
||||
// Sentry issue (group) aggregate state. Upserted to the latest snapshot — the
|
||||
|
||||
@@ -13,6 +13,8 @@ import {
|
||||
import {
|
||||
getArchiveCounts,
|
||||
getLatestResources,
|
||||
getRecordingCount,
|
||||
getRecentRecordings,
|
||||
RESOURCE_TYPES,
|
||||
TYPE_LABEL,
|
||||
type ResourceType,
|
||||
@@ -26,24 +28,34 @@ function isType(v: string | undefined): v is ResourceType {
|
||||
return !!v && (RESOURCE_TYPES as readonly string[]).includes(v);
|
||||
}
|
||||
|
||||
function fmtDur(sec: number | null): string {
|
||||
if (!sec || sec <= 0) return "—";
|
||||
const m = Math.floor(sec / 60);
|
||||
const s = sec % 60;
|
||||
return `${m}:${String(s).padStart(2, "0")}`;
|
||||
}
|
||||
|
||||
export default async function PosthogArchivePage({
|
||||
searchParams,
|
||||
}: {
|
||||
searchParams: Promise<{ type?: string }>;
|
||||
}) {
|
||||
const sp = await searchParams;
|
||||
const isRecording = sp.type === "recording";
|
||||
const type: ResourceType = isType(sp.type) ? sp.type : "feature_flag";
|
||||
|
||||
const [counts, resources] = await Promise.all([
|
||||
const [counts, recCount, resources, recordings] = await Promise.all([
|
||||
getArchiveCounts(PROJECT),
|
||||
getLatestResources(PROJECT, type),
|
||||
getRecordingCount(PROJECT),
|
||||
isRecording ? Promise.resolve([]) : getLatestResources(PROJECT, type),
|
||||
isRecording ? getRecentRecordings(PROJECT, 150) : Promise.resolve([]),
|
||||
]);
|
||||
|
||||
return (
|
||||
<PanelShell title="PostHog Arşiv">
|
||||
<p className="text-sm text-muted-foreground">
|
||||
PostHog'daki tüm kaynak modülleri (survey, feature flag, experiment, dashboard, insight,
|
||||
annotation, action) — değişiklik geçmişiyle birlikte arşivleniyor (~6 saat tazelik, DB + MinIO).
|
||||
PostHog'daki tüm modüller — feature flag, experiment, survey, insight, dashboard, annotation,
|
||||
action, group ve recording — değişiklik geçmişiyle birlikte arşivleniyor (~6 saat tazelik, DB + MinIO).
|
||||
</p>
|
||||
|
||||
<div className="flex flex-wrap gap-1.5">
|
||||
@@ -51,7 +63,7 @@ export default async function PosthogArchivePage({
|
||||
<Link
|
||||
key={c.type}
|
||||
href={`/posthog-archive?type=${c.type}`}
|
||||
className={buttonVariants({ variant: c.type === type ? "default" : "outline", size: "sm" })}
|
||||
className={buttonVariants({ variant: !isRecording && c.type === type ? "default" : "outline", size: "sm" })}
|
||||
>
|
||||
{TYPE_LABEL[c.type] ?? c.type}
|
||||
<Badge variant="secondary" className="ml-1.5">
|
||||
@@ -59,48 +71,97 @@ export default async function PosthogArchivePage({
|
||||
</Badge>
|
||||
</Link>
|
||||
))}
|
||||
<Link
|
||||
href="/posthog-archive?type=recording"
|
||||
className={buttonVariants({ variant: isRecording ? "default" : "outline", size: "sm" })}
|
||||
>
|
||||
{TYPE_LABEL.recording}
|
||||
<Badge variant="secondary" className="ml-1.5">
|
||||
{recCount.toLocaleString("tr-TR")}
|
||||
</Badge>
|
||||
</Link>
|
||||
</div>
|
||||
|
||||
<div className="rounded-md border">
|
||||
<Table>
|
||||
<TableHeader>
|
||||
<TableRow>
|
||||
<TableHead>Ad</TableHead>
|
||||
<TableHead className="w-28">ID</TableHead>
|
||||
<TableHead className="w-24 text-right">Versiyon</TableHead>
|
||||
<TableHead className="w-36 text-right">Son yakalama</TableHead>
|
||||
</TableRow>
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
{resources.map((r) => (
|
||||
<TableRow key={r.resourceId}>
|
||||
<TableCell className="max-w-[460px] truncate font-medium">
|
||||
<Link
|
||||
href={`/posthog-archive/${type}/${encodeURIComponent(r.resourceId)}`}
|
||||
className="underline"
|
||||
>
|
||||
{r.name || `(adsız ${type})`}
|
||||
</Link>
|
||||
</TableCell>
|
||||
<TableCell className="font-mono text-xs text-muted-foreground">{r.resourceId}</TableCell>
|
||||
<TableCell className="text-right tabular-nums">
|
||||
{r.versions > 1 ? <Badge variant="outline">{r.versions}</Badge> : r.versions}
|
||||
</TableCell>
|
||||
<TableCell className="text-right font-mono text-xs text-muted-foreground">
|
||||
{r.capturedAt.toISOString().slice(0, 16).replace("T", " ")}
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
))}
|
||||
{resources.length === 0 && (
|
||||
{isRecording ? (
|
||||
<div className="rounded-md border">
|
||||
<Table>
|
||||
<TableHeader>
|
||||
<TableRow>
|
||||
<TableCell colSpan={4} className="text-center text-sm text-muted-foreground">
|
||||
Bu tip için henüz arşiv yok.
|
||||
</TableCell>
|
||||
<TableHead>Kullanıcı</TableHead>
|
||||
<TableHead className="max-w-[360px]">Başlangıç URL</TableHead>
|
||||
<TableHead className="w-20 text-right">Süre</TableHead>
|
||||
<TableHead className="w-16 text-right">Tık</TableHead>
|
||||
<TableHead className="w-16 text-right">Hata</TableHead>
|
||||
<TableHead className="w-36 text-right">Başlangıç</TableHead>
|
||||
</TableRow>
|
||||
)}
|
||||
</TableBody>
|
||||
</Table>
|
||||
</div>
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
{recordings.map((r) => (
|
||||
<TableRow key={r.id}>
|
||||
<TableCell className="font-mono text-xs">{r.personName || r.distinctId?.slice(0, 14) || "anon"}</TableCell>
|
||||
<TableCell className="max-w-[360px] truncate text-xs text-muted-foreground">{r.startUrl || "—"}</TableCell>
|
||||
<TableCell className="text-right tabular-nums">{fmtDur(r.durationSec)}</TableCell>
|
||||
<TableCell className="text-right tabular-nums">{r.clickCount ?? 0}</TableCell>
|
||||
<TableCell className="text-right tabular-nums">
|
||||
{r.consoleErrorCount ? <span className="text-red-600">{r.consoleErrorCount}</span> : 0}
|
||||
</TableCell>
|
||||
<TableCell className="text-right font-mono text-xs text-muted-foreground">
|
||||
{r.startTime ? r.startTime.toISOString().slice(0, 16).replace("T", " ") : "—"}
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
))}
|
||||
{recordings.length === 0 && (
|
||||
<TableRow>
|
||||
<TableCell colSpan={6} className="text-center text-sm text-muted-foreground">
|
||||
Henüz recording arşivlenmedi (ilk çalışma sonrası dolar).
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
)}
|
||||
</TableBody>
|
||||
</Table>
|
||||
</div>
|
||||
) : (
|
||||
<div className="rounded-md border">
|
||||
<Table>
|
||||
<TableHeader>
|
||||
<TableRow>
|
||||
<TableHead>Ad</TableHead>
|
||||
<TableHead className="w-28">ID</TableHead>
|
||||
<TableHead className="w-24 text-right">Versiyon</TableHead>
|
||||
<TableHead className="w-36 text-right">Son yakalama</TableHead>
|
||||
</TableRow>
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
{resources.map((r) => (
|
||||
<TableRow key={r.resourceId}>
|
||||
<TableCell className="max-w-[460px] truncate font-medium">
|
||||
<Link
|
||||
href={`/posthog-archive/${type}/${encodeURIComponent(r.resourceId)}`}
|
||||
className="underline"
|
||||
>
|
||||
{r.name || `(adsız ${type})`}
|
||||
</Link>
|
||||
</TableCell>
|
||||
<TableCell className="font-mono text-xs text-muted-foreground">{r.resourceId}</TableCell>
|
||||
<TableCell className="text-right tabular-nums">
|
||||
{r.versions > 1 ? <Badge variant="outline">{r.versions}</Badge> : r.versions}
|
||||
</TableCell>
|
||||
<TableCell className="text-right font-mono text-xs text-muted-foreground">
|
||||
{r.capturedAt.toISOString().slice(0, 16).replace("T", " ")}
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
))}
|
||||
{resources.length === 0 && (
|
||||
<TableRow>
|
||||
<TableCell colSpan={4} className="text-center text-sm text-muted-foreground">
|
||||
Bu tip için henüz arşiv yok.
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
)}
|
||||
</TableBody>
|
||||
</Table>
|
||||
</div>
|
||||
)}
|
||||
</PanelShell>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -8,6 +8,8 @@ export const RESOURCE_TYPES = [
|
||||
"dashboard",
|
||||
"annotation",
|
||||
"action",
|
||||
"group",
|
||||
"group_type",
|
||||
] as const;
|
||||
export type ResourceType = (typeof RESOURCE_TYPES)[number];
|
||||
|
||||
@@ -19,6 +21,9 @@ export const TYPE_LABEL: Record<string, string> = {
|
||||
dashboard: "Dashboards",
|
||||
annotation: "Annotations",
|
||||
action: "Actions",
|
||||
group: "Groups",
|
||||
group_type: "Group Types",
|
||||
recording: "Recordings",
|
||||
};
|
||||
|
||||
export type ArchiveCount = { type: string; resources: number; snapshots: number; lastCaptured: Date | null };
|
||||
@@ -97,3 +102,40 @@ export async function getResourceHistory(
|
||||
});
|
||||
return rows;
|
||||
}
|
||||
|
||||
// ── Recordings (separate table: posthog_recordings — full recording metadata) ──
|
||||
export async function getRecordingCount(projectKey: string): Promise<number> {
|
||||
const rows = await prisma.$queryRaw<Array<{ c: bigint }>>`
|
||||
SELECT count(*)::bigint AS c FROM posthog_recordings WHERE "projectKey" = ${projectKey}
|
||||
`;
|
||||
return Number(rows[0]?.c ?? 0);
|
||||
}
|
||||
|
||||
export type RecordingRow = {
|
||||
id: string;
|
||||
distinctId: string | null;
|
||||
personName: string | null;
|
||||
durationSec: number | null;
|
||||
clickCount: number | null;
|
||||
consoleErrorCount: number | null;
|
||||
startUrl: string | null;
|
||||
startTime: Date | null;
|
||||
};
|
||||
|
||||
export async function getRecentRecordings(projectKey: string, limit = 100): Promise<RecordingRow[]> {
|
||||
return prisma.posthogRecording.findMany({
|
||||
where: { projectKey },
|
||||
orderBy: { startTime: "desc" },
|
||||
take: limit,
|
||||
select: {
|
||||
id: true,
|
||||
distinctId: true,
|
||||
personName: true,
|
||||
durationSec: true,
|
||||
clickCount: true,
|
||||
consoleErrorCount: true,
|
||||
startUrl: true,
|
||||
startTime: true,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
86
apps/worker/src/jobs/posthog-recording-archive.ts
Normal file
86
apps/worker/src/jobs/posthog-recording-archive.ts
Normal file
@@ -0,0 +1,86 @@
|
||||
import { gzipSync } from "node:zlib";
|
||||
import { prisma } from "../db";
|
||||
import { listRecordings, isConfigured } from "../lib/posthog";
|
||||
import { putBuffer } from "../lib/minio";
|
||||
|
||||
// Complete metadata archive of EVERY session recording — superset of the insight
|
||||
// pipeline's promoted SessionMeta rows. Pages the recording list over a rolling
|
||||
// window and inserts only new ids (skipDuplicates). rrweb blobs are archived
|
||||
// separately by archive-recordings for ingested sessions.
|
||||
const PROJECT_KEY = process.env.INSIGHT_PROJECT_KEY ?? "sase";
|
||||
const ARCHIVE_BUCKET = process.env.INSIGHT_ARCHIVE_BUCKET ?? "posthog-archive";
|
||||
const WINDOW_DAYS = Number(process.env.INSIGHT_RECORDING_META_WINDOW_DAYS ?? "90");
|
||||
const MAX_PAGES = Number(process.env.INSIGHT_RECORDING_META_MAX_PAGES ?? "40");
|
||||
const PAGE = 100;
|
||||
|
||||
export type RecordingMetaArchiveResult = {
|
||||
ok: boolean;
|
||||
fetched: number;
|
||||
inserted: number;
|
||||
pages: number;
|
||||
dumped: boolean;
|
||||
reason?: string;
|
||||
};
|
||||
|
||||
const int = (v: unknown): number | null => {
|
||||
const n = Math.round(Number(v));
|
||||
return Number.isFinite(n) ? n : null;
|
||||
};
|
||||
|
||||
export async function runArchivePosthogRecordings(): Promise<RecordingMetaArchiveResult> {
|
||||
if (!isConfigured()) {
|
||||
return { ok: false, fetched: 0, inserted: 0, pages: 0, dumped: false, reason: "posthog_not_configured" };
|
||||
}
|
||||
|
||||
const dateFrom = new Date(Date.now() - WINDOW_DAYS * 864e5).toISOString();
|
||||
const dateTo = new Date(Date.now() - 3_600_000).toISOString(); // 1h buffer → finalized recordings only
|
||||
let fetched = 0;
|
||||
let inserted = 0;
|
||||
let pages = 0;
|
||||
const all: Array<Record<string, unknown>> = [];
|
||||
|
||||
for (let offset = 0; pages < MAX_PAGES; offset += PAGE, pages++) {
|
||||
let resp;
|
||||
try {
|
||||
resp = await listRecordings({ dateFrom, dateTo, limit: PAGE, offset });
|
||||
} catch (e) {
|
||||
if (pages === 0) return { ok: false, fetched, inserted, pages, dumped: false, reason: (e as Error).message };
|
||||
break; // mid-run page failure → keep what we have, retry next cycle
|
||||
}
|
||||
const items = resp.results ?? [];
|
||||
if (items.length === 0) break;
|
||||
fetched += items.length;
|
||||
all.push(...(items as unknown as Array<Record<string, unknown>>));
|
||||
|
||||
const rows = items.map((r) => ({
|
||||
id: r.id,
|
||||
projectKey: PROJECT_KEY,
|
||||
distinctId: r.distinct_id ?? null,
|
||||
personName: r.person?.name ?? null,
|
||||
startTime: r.start_time ? new Date(r.start_time) : null,
|
||||
endTime: r.end_time ? new Date(r.end_time) : null,
|
||||
durationSec: int(r.recording_duration),
|
||||
activeSeconds: int(r.active_seconds),
|
||||
clickCount: int(r.click_count),
|
||||
keypressCount: int(r.keypress_count),
|
||||
mouseActivityCount: int(r.mouse_activity_count),
|
||||
consoleErrorCount: int(r.console_error_count),
|
||||
startUrl: r.start_url ?? null,
|
||||
ongoing: Boolean(r.ongoing),
|
||||
properties: r as object,
|
||||
}));
|
||||
const res = await prisma.posthogRecording.createMany({ data: rows as never, skipDuplicates: true });
|
||||
inserted += res.count;
|
||||
|
||||
if (!resp.has_next && !resp.next) break;
|
||||
}
|
||||
|
||||
let dumped = false;
|
||||
if (all.length) {
|
||||
const day = new Date().toISOString().slice(0, 10);
|
||||
await putBuffer(ARCHIVE_BUCKET, `recordings/${day}.json.gz`, gzipSync(Buffer.from(JSON.stringify(all))), "application/gzip");
|
||||
dumped = true;
|
||||
}
|
||||
|
||||
return { ok: true, fetched, inserted, pages, dumped };
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
import { gzipSync } from "node:zlib";
|
||||
import { prisma } from "../db";
|
||||
import { listResource, isConfigured } from "../lib/posthog";
|
||||
import { listResource, listGroupTypes, listGroups, isConfigured } from "../lib/posthog";
|
||||
import { stableHash } from "../lib/hash";
|
||||
import { putBuffer } from "../lib/minio";
|
||||
|
||||
@@ -35,6 +35,23 @@ function hashConfig(item: Record<string, unknown>): string {
|
||||
return stableHash(clean);
|
||||
}
|
||||
|
||||
type SnapRow = { projectKey: string; resourceType: string; resourceId: string; name: string | null; stateHash: string; data: object };
|
||||
|
||||
// Insert only rows whose config changed since the latest stored snapshot.
|
||||
async function persistChanged(type: string, rows: SnapRow[]): Promise<number> {
|
||||
const latest = await prisma.$queryRaw<Array<{ resourceId: string; stateHash: string }>>`
|
||||
SELECT DISTINCT ON ("resourceId") "resourceId", "stateHash"
|
||||
FROM posthog_resource_snapshots
|
||||
WHERE "projectKey" = ${PROJECT_KEY} AND "resourceType" = ${type}
|
||||
ORDER BY "resourceId", "capturedAt" DESC
|
||||
`;
|
||||
const latestHash = new Map(latest.map((x) => [x.resourceId, x.stateHash]));
|
||||
const changed = rows.filter((row) => row.resourceId && latestHash.get(row.resourceId) !== row.stateHash);
|
||||
if (!changed.length) return 0;
|
||||
const res = await prisma.posthogResourceSnapshot.createMany({ data: changed as never });
|
||||
return res.count;
|
||||
}
|
||||
|
||||
export type ResourceArchiveResult = {
|
||||
ok: boolean;
|
||||
perType: Record<string, { scanned: number; snapshots: number; dumped: boolean; error?: string }>;
|
||||
@@ -87,5 +104,53 @@ export async function runArchivePosthogResources(): Promise<ResourceArchiveResul
|
||||
}
|
||||
}
|
||||
|
||||
// ── Groups: group types + instances (separate API shape) ──
|
||||
try {
|
||||
const types = await listGroupTypes();
|
||||
const typeRows: SnapRow[] = types
|
||||
.map((t) => ({
|
||||
projectKey: PROJECT_KEY,
|
||||
resourceType: "group_type",
|
||||
resourceId: String(t.group_type_index ?? t.group_type ?? ""),
|
||||
name: String(t.name_singular ?? t.group_type ?? ""),
|
||||
stateHash: hashConfig(t),
|
||||
data: t as object,
|
||||
}))
|
||||
.filter((row) => row.resourceId);
|
||||
perType["group_type"] = { scanned: typeRows.length, snapshots: await persistChanged("group_type", typeRows), dumped: false };
|
||||
if (types.length) {
|
||||
await putBuffer(ARCHIVE_BUCKET, `resources/group_type/${day}.json.gz`, gzipSync(Buffer.from(JSON.stringify(types))), "application/gzip");
|
||||
perType["group_type"].dumped = true;
|
||||
}
|
||||
|
||||
let allGroups: Array<Record<string, unknown>> = [];
|
||||
const groupRows: SnapRow[] = [];
|
||||
for (const t of types) {
|
||||
const idx = Number(t.group_type_index ?? -1);
|
||||
if (idx < 0) continue;
|
||||
const groups = await listGroups(idx);
|
||||
allGroups = allGroups.concat(groups);
|
||||
for (const g of groups) {
|
||||
const key = String(g.group_key ?? "");
|
||||
if (!key) continue;
|
||||
groupRows.push({
|
||||
projectKey: PROJECT_KEY,
|
||||
resourceType: "group",
|
||||
resourceId: `${idx}:${key}`,
|
||||
name: key,
|
||||
stateHash: hashConfig(g),
|
||||
data: g as object,
|
||||
});
|
||||
}
|
||||
}
|
||||
perType["group"] = { scanned: groupRows.length, snapshots: await persistChanged("group", groupRows), dumped: false };
|
||||
if (allGroups.length) {
|
||||
await putBuffer(ARCHIVE_BUCKET, `resources/group/${day}.json.gz`, gzipSync(Buffer.from(JSON.stringify(allGroups))), "application/gzip");
|
||||
perType["group"].dumped = true;
|
||||
}
|
||||
} catch (e) {
|
||||
perType["group"] = { scanned: 0, snapshots: 0, dumped: false, error: (e as Error).message };
|
||||
}
|
||||
|
||||
return { ok: true, perType };
|
||||
}
|
||||
|
||||
@@ -141,6 +141,36 @@ export async function listResource(
|
||||
return out;
|
||||
}
|
||||
|
||||
// Group types (e.g. "company") + group instances. Groups use a separate API
|
||||
// shape (group_type_index), so they can't go through listResource.
|
||||
export async function listGroupTypes(): Promise<Array<Record<string, unknown>>> {
|
||||
const res = await fetch(`${HOST}/api/projects/${PROJECT_ID}/groups_types/`, { headers: headers() });
|
||||
if (!res.ok) throw new Error(`posthog groups_types ${res.status}: ${(await res.text().catch(() => "")).slice(0, 120)}`);
|
||||
const data = (await res.json()) as unknown;
|
||||
if (Array.isArray(data)) return data as Array<Record<string, unknown>>;
|
||||
const r = (data as { results?: Array<Record<string, unknown>> }).results;
|
||||
return Array.isArray(r) ? r : [];
|
||||
}
|
||||
|
||||
export async function listGroups(
|
||||
groupTypeIndex: number,
|
||||
opts: { maxPages?: number } = {},
|
||||
): Promise<Array<Record<string, unknown>>> {
|
||||
const maxPages = opts.maxPages ?? 50;
|
||||
const out: Array<Record<string, unknown>> = [];
|
||||
let url: string | null = `${HOST}/api/projects/${PROJECT_ID}/groups/?group_type_index=${groupTypeIndex}&limit=100`;
|
||||
let pages = 0;
|
||||
while (url && pages < maxPages) {
|
||||
const res: Response = await fetch(url, { headers: headers() });
|
||||
if (!res.ok) throw new Error(`posthog groups ${res.status}: ${(await res.text().catch(() => "")).slice(0, 120)}`);
|
||||
const data = (await res.json()) as { results?: Array<Record<string, unknown>>; next?: string | null };
|
||||
if (Array.isArray(data.results)) out.push(...data.results);
|
||||
url = data.next ?? null;
|
||||
pages++;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---------- Custom events ----------
|
||||
|
||||
export type PHCustomEvent = {
|
||||
|
||||
@@ -16,6 +16,7 @@ import { runArchivePosthogEvents } from "../jobs/posthog-event-archive";
|
||||
import { runArchiveRecordings } from "../jobs/archive-recordings";
|
||||
import { runArchiveIdentity } from "../jobs/posthog-identity-archive";
|
||||
import { runArchivePosthogResources } from "../jobs/posthog-resource-archive";
|
||||
import { runArchivePosthogRecordings } from "../jobs/posthog-recording-archive";
|
||||
import { runArchiveSentry } from "../jobs/sentry-archive";
|
||||
|
||||
const QUEUE = "insight-pipeline";
|
||||
@@ -146,6 +147,15 @@ async function runJob(job: Job) {
|
||||
}
|
||||
return res;
|
||||
}
|
||||
case "posthog-recording-archive": {
|
||||
const res = await runArchivePosthogRecordings();
|
||||
if (res.inserted > 0 || !res.ok) {
|
||||
console.log(
|
||||
`[pipeline] recording-archive fetched=${res.fetched} inserted=${res.inserted} pages=${res.pages}${res.reason ? ` reason=${res.reason}` : ""}`,
|
||||
);
|
||||
}
|
||||
return res;
|
||||
}
|
||||
case "sentry-archive": {
|
||||
const res = await runArchiveSentry();
|
||||
if (res.eventsInserted > 0 || res.issuesSynced > 0 || res.dumped > 0 || res.error) {
|
||||
@@ -239,6 +249,11 @@ export async function startInsightPipeline() {
|
||||
{ pattern: "15 */6 * * *" },
|
||||
{ name: "posthog-resource-archive", data: {}, opts: { removeOnComplete: 30, removeOnFail: 15 } },
|
||||
);
|
||||
await queue.upsertJobScheduler(
|
||||
"posthog-recording-archive",
|
||||
{ pattern: "45 */6 * * *" },
|
||||
{ name: "posthog-recording-archive", data: {}, opts: { removeOnComplete: 30, removeOnFail: 15 } },
|
||||
);
|
||||
await queue.upsertJobScheduler(
|
||||
"sentry-archive",
|
||||
{ pattern: "0 * * * *" },
|
||||
@@ -252,7 +267,7 @@ export async function startInsightPipeline() {
|
||||
stalledInterval: 60_000,
|
||||
});
|
||||
console.log(
|
||||
"[pipeline] armed: posthog-ingest@*/5min, tag-sessions@*/2min, compress-sessions@*/3min, analyze@*/4min, validation@05:00, github-sync@*/10min, retention@04:15, daily-brief@05:00, vin-anomaly-detect@*/5min, catalog-gap-detect@*/6h, sase-evening-brief@17:00UTC, posthog-event-archive@*/15min, archive-recordings@*/6h, posthog-identity-archive@03:00, posthog-resource-archive@*/6h, sentry-archive@hourly",
|
||||
"[pipeline] armed: posthog-ingest@*/5min, tag-sessions@*/2min, compress-sessions@*/3min, analyze@*/4min, validation@05:00, github-sync@*/10min, retention@04:15, daily-brief@05:00, vin-anomaly-detect@*/5min, catalog-gap-detect@*/6h, sase-evening-brief@17:00UTC, posthog-event-archive@*/15min, archive-recordings@*/6h, posthog-identity-archive@03:00, posthog-resource-archive@*/6h, posthog-recording-archive@*/6h, sentry-archive@hourly",
|
||||
);
|
||||
return worker;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user