From ae8a8046bf4c8722486d0e4b235d0a06e25d3798 Mon Sep 17 00:00:00 2001 From: Semih Yesilyurt Date: Sun, 20 Sep 2026 08:31:55 +0300 Subject: [PATCH] =?UTF-8?q?refactor(prefetch):=20kap=C4=B1=20s=C4=B1ras?= =?UTF-8?q?=C4=B1n=C4=B1=20en=20ucuzdan=20pahal=C4=B1ya=20diz?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pencere kontrolü saf saat aritmetiği, hiç I/O yapmıyor ve pencere dışındaki iş zaten hiçbir faydalı iş yapamıyor — dolayısıyla dakikalık Redis sayacından da önce gelmeli. Yeni sıra: pencere → cooldown → dakikalık tavan → günlük bütçe. Böylece pencere dışında uyanan iş hiçbir sayacı kirletmiyor. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/jobs/prefetch-window-budget.spec.ts | 3 ++ apps/api/src/jobs/prefetch-worker.service.ts | 35 +++++++++++-------- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/apps/api/src/jobs/prefetch-window-budget.spec.ts b/apps/api/src/jobs/prefetch-window-budget.spec.ts index 44b82e6..a9702e0 100644 --- a/apps/api/src/jobs/prefetch-window-budget.spec.ts +++ b/apps/api/src/jobs/prefetch-window-budget.spec.ts @@ -164,6 +164,9 @@ describe("process() gate order — window before daily budget", () => { expect(job.moveToDelayed).toHaveBeenCalled(); expect(dailyIncrs(incrCalls)).toHaveLength(0); + // The per-minute counter is not charged either: the window gate is pure + // clock arithmetic and runs before any Redis write. + expect(incrCalls).toHaveLength(0); // …and the handler never ran, so nothing was fetched upstream either. expect(categoriesService.getCategoryWithParts).not.toHaveBeenCalled(); }); diff --git a/apps/api/src/jobs/prefetch-worker.service.ts b/apps/api/src/jobs/prefetch-worker.service.ts index fb854e8..fcde616 100644 --- a/apps/api/src/jobs/prefetch-worker.service.ts +++ b/apps/api/src/jobs/prefetch-worker.service.ts @@ -355,23 +355,30 @@ export class PrefetchWorkerService implements OnModuleInit, OnModuleDestroy { job.name === "prefetch-parts") ) { const lane = (job.data as { fast?: boolean }).fast ? "fast" : "main"; - await this.checkSourceRate(data.source, lane); - // Cooldown + business-hours window BEFORE the daily charge. These two - // gates used to live at the top of each handler, i.e. AFTER the budget - // was already debited, so every job that woke outside the window paid a + // GATE ORDER IS LOAD-BEARING — cheapest first, and every gate that can + // reject the job must run BEFORE any counter is debited. + // + // The business-hours window and the cooldown used to sit at the top of + // each handler, i.e. AFTER the per-minute counter and the daily budget + // had already been charged. So a job that woke outside the window paid a // budget unit to do nothing. Combined with a deferral target of "next UTC // midnight" (= 03:00 Europe/Istanbul, six hours before a 09:00 window - // opens) that formed a closed loop: the whole daily allowance was burned - // by no-op wake-ups before the window ever opened, so the source never - // ran again. Measured on prod 2026-09-20: pl24 at 600/600 with 0 catalog - // requests and 0 new categories for the day. Order matters here. - await checkCooldown(this.redis, data.source); + // opens) that closed a loop: the whole daily allowance was burned by + // no-op wake-ups before the window ever opened, so the source never ran + // again. Measured on prod 2026-09-20 — pl24 at 600/600 with 0 catalog + // requests and 0 new categories for the day. + // + // 1. Window: pure clock arithmetic, no I/O, and an out-of-window job can + // never do useful work — so nothing else is worth spending on it. checkTimeWindow(data.source); - // Daily budget AFTER the per-minute gate: a job deferred on the minute - // ceiling above never reaches here, so rate-limited retries don't inflate - // the daily counter — only jobs about to do real work are counted. The - // lane decides which threshold applies (backfill stops at the main limit, - // the user's fast lane may use the full budget). + // 2. Cooldown: one Redis TTL read. Pauses the whole worker (see catch). + await checkCooldown(this.redis, data.source); + // 3. Per-minute ceiling. + await this.checkSourceRate(data.source, lane); + // 4. Daily budget last: a job deferred on any gate above never reaches + // here, so only jobs about to do real work are counted. The lane + // decides which threshold applies (backfill stops at the main limit, + // the user's fast lane may use the full budget). await this.checkSourceDailyBudget(data.source, lane); } if (data.source === "parts-catalogs" && PCAT_PACE_MS > 0) {