feat(proxy): proxy_logs telemetry — per-attempt provider/ban/latency tracking
Some checks failed
QA Gate (P0/P1) / Test affected app (pull_request) Has been cancelled
Some checks failed
QA Gate (P0/P1) / Test affected app (pull_request) Has been cancelled
One row per proxied upstream attempt (pcat call/capture/validate, emex http) written fire-and-forget by the new ProxyTelemetryService (buffered, capped, errors swallowed — telemetry can never hurt the request path). - banned = upstream 403/429 (IP-block signal), distinct from auth/data errors - sticky legs (pcat capture, emex floxy) carry a session_key; pcat capture also resolves the actual residential exit IP via a parallel ipify probe through the same sticky session → concrete banned-IP tracking - rotating legs log provider + outcome (ban *rate* instead of per-IP) - 30-day retention piggybacked on the query-cleanup job Feeds the Süper Panel /analytics/proxy page (provider grading + banned IPs). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,8 +1,10 @@
|
||||
import { Module } from "@nestjs/common";
|
||||
import { ProxyTelemetryModule } from "../proxy-telemetry/proxy-telemetry.module";
|
||||
import { EmexBrowserService } from "./emex.browser";
|
||||
import { EmexService } from "./emex.service";
|
||||
|
||||
@Module({
|
||||
imports: [ProxyTelemetryModule],
|
||||
providers: [EmexBrowserService, EmexService],
|
||||
exports: [EmexService],
|
||||
})
|
||||
|
||||
@@ -18,6 +18,7 @@ function makeConfig(overrides: Record<string, string>): ConfigService {
|
||||
const browser = {} as never;
|
||||
const redis = { set: vi.fn() } as never;
|
||||
const posthog = {} as never;
|
||||
const proxyTelemetry = { record: vi.fn() } as never;
|
||||
|
||||
describe("EmexService proxy-port coercion (regression)", () => {
|
||||
it("does not throw 'Invalid URL' when ports arrive as strings over a real range", () => {
|
||||
@@ -28,7 +29,7 @@ describe("EmexService proxy-port coercion (regression)", () => {
|
||||
EMEX_PROXY_PORT_END: "10099",
|
||||
});
|
||||
// Pre-fix: threw "Invalid URL" here (port "4510001" > 65535).
|
||||
expect(() => new EmexService(config, browser, redis, posthog)).not.toThrow();
|
||||
expect(() => new EmexService(config, browser, redis, posthog, proxyTelemetry)).not.toThrow();
|
||||
});
|
||||
|
||||
it("constructs with a single-port range (string env)", () => {
|
||||
@@ -37,7 +38,7 @@ describe("EmexService proxy-port coercion (regression)", () => {
|
||||
EMEX_PROXY_PORT_START: "823",
|
||||
EMEX_PROXY_PORT_END: "823",
|
||||
});
|
||||
expect(() => new EmexService(config, browser, redis, posthog)).not.toThrow();
|
||||
expect(() => new EmexService(config, browser, redis, posthog, proxyTelemetry)).not.toThrow();
|
||||
});
|
||||
|
||||
it("falls back to a valid default when the port env is garbage", () => {
|
||||
@@ -46,11 +47,11 @@ describe("EmexService proxy-port coercion (regression)", () => {
|
||||
EMEX_PROXY_PORT_START: "not-a-number",
|
||||
EMEX_PROXY_PORT_END: "999999",
|
||||
});
|
||||
expect(() => new EmexService(config, browser, redis, posthog)).not.toThrow();
|
||||
expect(() => new EmexService(config, browser, redis, posthog, proxyTelemetry)).not.toThrow();
|
||||
});
|
||||
|
||||
it("constructs cleanly with the proxy disabled", () => {
|
||||
const config = makeConfig({ EMEX_USE_PROXY: "false" });
|
||||
expect(() => new EmexService(config, browser, redis, posthog)).not.toThrow();
|
||||
expect(() => new EmexService(config, browser, redis, posthog, proxyTelemetry)).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -23,6 +23,10 @@ import { ProxyAgent } from "undici";
|
||||
import { isBackfillContext } from "../../jobs/prefetch-context";
|
||||
import { PostHogService } from "../../posthog/posthog.service";
|
||||
import { RedisService } from "../../redis/redis.service";
|
||||
import {
|
||||
ProxyTelemetryService,
|
||||
classifyTransportError,
|
||||
} from "../proxy-telemetry/proxy-telemetry.service";
|
||||
import { parseUnitLeaves, parseVehicleTree } from "./emex-tree.parser";
|
||||
import { EmexBrowserService } from "./emex.browser";
|
||||
import { createEmptyDecodedVehicle, mapEmexResponse } from "./emex.mapper";
|
||||
@@ -142,6 +146,7 @@ export class EmexService {
|
||||
private browserService: EmexBrowserService,
|
||||
private redis: RedisService,
|
||||
private posthog: PostHogService,
|
||||
private proxyTelemetry: ProxyTelemetryService,
|
||||
) {
|
||||
// __dirname is apps/api/src/integrations/emex/ or dist/integrations/emex/
|
||||
// Scraper lives at <monorepo-root>/scripts/emex-vin-scraper.js
|
||||
@@ -389,12 +394,25 @@ export class EmexService {
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||
const provider = schedule[attempt - 1]; // undefined when no proxy → direct
|
||||
const agent = provider ? this.newProxyAgent(provider) : null;
|
||||
const attemptStart = Date.now();
|
||||
// Sticky Floxy sessions are the attributable identity here; DataImpulse
|
||||
// rotates a random port per agent, so there is nothing stable to pin.
|
||||
const sessionKey = provider === "floxy" ? `s-${this.floxySessionId}` : null;
|
||||
try {
|
||||
const res = await fetch(url, {
|
||||
headers: { "User-Agent": EMEX_UA, Accept: "text/html,application/xhtml+xml" },
|
||||
signal: AbortSignal.timeout(this.timeout),
|
||||
...(agent ? { dispatcher: agent } : {}),
|
||||
} as RequestInit);
|
||||
this.proxyTelemetry.record({
|
||||
service: "emex_http",
|
||||
provider: provider ?? "none",
|
||||
sessionKey,
|
||||
targetHost: new URL(url).hostname,
|
||||
statusCode: res.status,
|
||||
success: res.ok,
|
||||
durationMs: Date.now() - attemptStart,
|
||||
});
|
||||
if (!res.ok) {
|
||||
throw new Error(`EMEX HTTP ${res.status} for ${url}`);
|
||||
}
|
||||
@@ -407,6 +425,19 @@ export class EmexService {
|
||||
const e = err as Error & { cause?: unknown };
|
||||
// A real HTTP response ("EMEX HTTP 404") is a definitive answer — never retry.
|
||||
const httpAnswer = /^EMEX HTTP \d/.test(e.message);
|
||||
// HTTP answers were already recorded right after fetch; only transport
|
||||
// failures (no answer at all) still need a row.
|
||||
if (!httpAnswer) {
|
||||
this.proxyTelemetry.record({
|
||||
service: "emex_http",
|
||||
provider: provider ?? "none",
|
||||
sessionKey,
|
||||
targetHost: new URL(url).hostname,
|
||||
errorKind: classifyTransportError(e),
|
||||
success: false,
|
||||
durationMs: Date.now() - attemptStart,
|
||||
});
|
||||
}
|
||||
const transient =
|
||||
!httpAnswer &&
|
||||
(e.name === "TimeoutError" ||
|
||||
|
||||
Reference in New Issue
Block a user