feat(proxy): proxy_logs telemetry — per-attempt provider/ban/latency tracking
Some checks failed
QA Gate (P0/P1) / Test affected app (pull_request) Has been cancelled

One row per proxied upstream attempt (pcat call/capture/validate, emex http)
written fire-and-forget by the new ProxyTelemetryService (buffered, capped,
errors swallowed — telemetry can never hurt the request path).

- banned = upstream 403/429 (IP-block signal), distinct from auth/data errors
- sticky legs (pcat capture, emex floxy) carry a session_key; pcat capture
  also resolves the actual residential exit IP via a parallel ipify probe
  through the same sticky session → concrete banned-IP tracking
- rotating legs log provider + outcome (ban *rate* instead of per-IP)
- 30-day retention piggybacked on the query-cleanup job

Feeds the Süper Panel /analytics/proxy page (provider grading + banned IPs).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-06-11 17:27:48 +03:00
parent 9f47d15312
commit 45f7e3f5a4
12 changed files with 373 additions and 9 deletions

View File

@@ -1,8 +1,10 @@
import { Module } from "@nestjs/common";
import { ProxyTelemetryModule } from "../proxy-telemetry/proxy-telemetry.module";
import { EmexBrowserService } from "./emex.browser";
import { EmexService } from "./emex.service";
@Module({
imports: [ProxyTelemetryModule],
providers: [EmexBrowserService, EmexService],
exports: [EmexService],
})

View File

@@ -18,6 +18,7 @@ function makeConfig(overrides: Record<string, string>): ConfigService {
const browser = {} as never;
const redis = { set: vi.fn() } as never;
const posthog = {} as never;
const proxyTelemetry = { record: vi.fn() } as never;
describe("EmexService proxy-port coercion (regression)", () => {
it("does not throw 'Invalid URL' when ports arrive as strings over a real range", () => {
@@ -28,7 +29,7 @@ describe("EmexService proxy-port coercion (regression)", () => {
EMEX_PROXY_PORT_END: "10099",
});
// Pre-fix: threw "Invalid URL" here (port "4510001" > 65535).
expect(() => new EmexService(config, browser, redis, posthog)).not.toThrow();
expect(() => new EmexService(config, browser, redis, posthog, proxyTelemetry)).not.toThrow();
});
it("constructs with a single-port range (string env)", () => {
@@ -37,7 +38,7 @@ describe("EmexService proxy-port coercion (regression)", () => {
EMEX_PROXY_PORT_START: "823",
EMEX_PROXY_PORT_END: "823",
});
expect(() => new EmexService(config, browser, redis, posthog)).not.toThrow();
expect(() => new EmexService(config, browser, redis, posthog, proxyTelemetry)).not.toThrow();
});
it("falls back to a valid default when the port env is garbage", () => {
@@ -46,11 +47,11 @@ describe("EmexService proxy-port coercion (regression)", () => {
EMEX_PROXY_PORT_START: "not-a-number",
EMEX_PROXY_PORT_END: "999999",
});
expect(() => new EmexService(config, browser, redis, posthog)).not.toThrow();
expect(() => new EmexService(config, browser, redis, posthog, proxyTelemetry)).not.toThrow();
});
it("constructs cleanly with the proxy disabled", () => {
const config = makeConfig({ EMEX_USE_PROXY: "false" });
expect(() => new EmexService(config, browser, redis, posthog)).not.toThrow();
expect(() => new EmexService(config, browser, redis, posthog, proxyTelemetry)).not.toThrow();
});
});

View File

@@ -23,6 +23,10 @@ import { ProxyAgent } from "undici";
import { isBackfillContext } from "../../jobs/prefetch-context";
import { PostHogService } from "../../posthog/posthog.service";
import { RedisService } from "../../redis/redis.service";
import {
ProxyTelemetryService,
classifyTransportError,
} from "../proxy-telemetry/proxy-telemetry.service";
import { parseUnitLeaves, parseVehicleTree } from "./emex-tree.parser";
import { EmexBrowserService } from "./emex.browser";
import { createEmptyDecodedVehicle, mapEmexResponse } from "./emex.mapper";
@@ -142,6 +146,7 @@ export class EmexService {
private browserService: EmexBrowserService,
private redis: RedisService,
private posthog: PostHogService,
private proxyTelemetry: ProxyTelemetryService,
) {
// __dirname is apps/api/src/integrations/emex/ or dist/integrations/emex/
// Scraper lives at <monorepo-root>/scripts/emex-vin-scraper.js
@@ -389,12 +394,25 @@ export class EmexService {
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
const provider = schedule[attempt - 1]; // undefined when no proxy → direct
const agent = provider ? this.newProxyAgent(provider) : null;
const attemptStart = Date.now();
// Sticky Floxy sessions are the attributable identity here; DataImpulse
// rotates a random port per agent, so there is nothing stable to pin.
const sessionKey = provider === "floxy" ? `s-${this.floxySessionId}` : null;
try {
const res = await fetch(url, {
headers: { "User-Agent": EMEX_UA, Accept: "text/html,application/xhtml+xml" },
signal: AbortSignal.timeout(this.timeout),
...(agent ? { dispatcher: agent } : {}),
} as RequestInit);
this.proxyTelemetry.record({
service: "emex_http",
provider: provider ?? "none",
sessionKey,
targetHost: new URL(url).hostname,
statusCode: res.status,
success: res.ok,
durationMs: Date.now() - attemptStart,
});
if (!res.ok) {
throw new Error(`EMEX HTTP ${res.status} for ${url}`);
}
@@ -407,6 +425,19 @@ export class EmexService {
const e = err as Error & { cause?: unknown };
// A real HTTP response ("EMEX HTTP 404") is a definitive answer — never retry.
const httpAnswer = /^EMEX HTTP \d/.test(e.message);
// HTTP answers were already recorded right after fetch; only transport
// failures (no answer at all) still need a row.
if (!httpAnswer) {
this.proxyTelemetry.record({
service: "emex_http",
provider: provider ?? "none",
sessionKey,
targetHost: new URL(url).hostname,
errorKind: classifyTransportError(e),
success: false,
durationMs: Date.now() - attemptStart,
});
}
const transient =
!httpAnswer &&
(e.name === "TimeoutError" ||