root/sp
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

fix(archive): recording archive — pace pages + raise cap for full backfill

Browse Source 
40-page cap captured only the most recent ~13 days. Add a 150ms inter-page
delay (PostHog list API rate-limits a tight loop) and raise the default cap
to 200 pages / 120d window so a run can sweep the full retention.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Semih
2026-06-10 21:27:43 +03:00
parent 400ed224c5
commit 4eec4e7faf
1 changed files with 6 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
apps/worker/src/jobs/posthog-recording-archive.ts
View File

@@ -9,9 +9,12 @@ import { putBuffer } from "../lib/minio";
// separately by archive-recordings for ingested sessions. // separately by archive-recordings for ingested sessions.
const PROJECT_KEY = process.env.INSIGHT_PROJECT_KEY ?? "sase"; const PROJECT_KEY = process.env.INSIGHT_PROJECT_KEY ?? "sase";
const ARCHIVE_BUCKET = process.env.INSIGHT_ARCHIVE_BUCKET ?? "posthog-archive"; const ARCHIVE_BUCKET = process.env.INSIGHT_ARCHIVE_BUCKET ?? "posthog-archive";
const WINDOW_DAYS = Number(process.env.INSIGHT_RECORDING_META_WINDOW_DAYS ?? "90"); const WINDOW_DAYS = Number(process.env.INSIGHT_RECORDING_META_WINDOW_DAYS ?? "120");
const MAX_PAGES = Number(process.env.INSIGHT_RECORDING_META_MAX_PAGES ?? "40"); const MAX_PAGES = Number(process.env.INSIGHT_RECORDING_META_MAX_PAGES ?? "200");
const PAGE = 100; const PAGE = 100;
// Inter-page pace — PostHog's recording list API rate-limits a tight loop.
const DELAY_MS = Number(process.env.INSIGHT_RECORDING_META_DELAY_MS ?? "150");
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
export type RecordingMetaArchiveResult = { export type RecordingMetaArchiveResult = {
ok: boolean; ok: boolean;
@@ -40,6 +43,7 @@ export async function runArchivePosthogRecordings(): Promise<RecordingMetaArchiv
const all: Array<Record<string, unknown>> = []; const all: Array<Record<string, unknown>> = [];
for (let offset = 0; pages < MAX_PAGES; offset += PAGE, pages++) { for (let offset = 0; pages < MAX_PAGES; offset += PAGE, pages++) {
if (pages > 0) await sleep(DELAY_MS);
let resp; let resp;
try { try {
resp = await listRecordings({ dateFrom, dateTo, limit: PAGE, offset }); resp = await listRecordings({ dateFrom, dateTo, limit: PAGE, offset });