root/sase.tr
1
0
Fork 0
Code Issues 5 Pull Requests 3 Actions 7 Packages Projects Releases Wiki Activity

perf(emex): Floxy residential primary transport (was DataImpulse-first)

Browse Source 
EMEX egress was DataImpulse-first: fetchEmexHtml tried 3 dead-prone DataImpulse
ports before falling to the Floxy fallback, and the browser scraper launched on a
random DataImpulse port. With ~50% of DataImpulse ports dead, this burned the 25s
decode budget before reaching Floxy (observed: a Ford VIN where pcat correctly
returned null fast, then EMEX's DataImpulse retries ate the budget).

ssd tokens are replayed from emex's own HTML and are NOT strictly IP-bound on the
.aspx endpoints (the prior random-port-per-call primary proved that), so a sticky
Floxy IP across the flow is strictly safer.

- EMEX_PROXY_PROVIDER (default "floxy" | "dataimpulse" | "none"). floxy →
  fetchEmexHtml schedule = [floxy, floxy, dataimpulse] (Floxy sticky primary,
  rolls IP on transport failure, single DataImpulse last-ditch). dataimpulse →
  legacy [di,di,di,floxy,floxy]. Default HTTP agent (image-dims) follows provider.
- EmexBrowserService launches chromium on a sticky Floxy session (one residential
  exit IP for the browser lifetime; fresh id per relaunch) under the same flag.
- Reuses the existing EMEX_FLOXY_* config (same Floxy account pcat now uses).

Rollback: EMEX_PROXY_PROVIDER=dataimpulse. typecheck+biome+11 emex tests green.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Semih Yesilyurt
2026-06-10 22:18:29 +03:00
parent 25ba37103d
commit ef4ab9bcd5
2 changed files with 81 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
apps/api/src/integrations/emex/emex.browser.ts
View File

@@ -61,22 +61,44 @@ export class EmexBrowserService implements OnModuleInit, OnModuleDestroy {
private launching: Promise<void> | null = null;
private readonly useProxy: boolean;
// "floxy" residential (default) | "dataimpulse" legacy | "none".
private readonly proxyProvider: "floxy" | "dataimpulse" | "none";
private readonly proxyHost: string;
private readonly proxyPortStart: number;
private readonly proxyPortEnd: number;
private readonly proxyUsername: string;
private readonly proxyPassword: string;
// Floxy residential — a sticky session pins ONE exit IP for the browser's whole
// lifetime (relaunch mints a fresh session id), which suits the chained scrape.
private readonly floxyHost: string;
private readonly floxyPort: number;
private readonly floxyUser: string;
private readonly floxyPass: string;
private readonly floxyLifetime: number;
private startedAt = 0;
constructor(private configService: ConfigService) {
this.semaphore = new Semaphore(MAX_CONCURRENT_PAGES);
this.useProxy = this.configService.get<string>("EMEX_USE_PROXY", "true") === "true";
const rawProvider = this.configService
.get<string>("EMEX_PROXY_PROVIDER", "floxy")
.toLowerCase();
this.proxyProvider = !this.useProxy
? "none"
: rawProvider === "dataimpulse" || rawProvider === "none"
? (rawProvider as "dataimpulse" | "none")
: "floxy";
this.proxyHost = this.configService.get<string>("EMEX_PROXY_HOST", "74.81.81.81");
this.proxyPortStart = this.configService.get<number>("EMEX_PROXY_PORT_START", 10001);
this.proxyPortEnd = this.configService.get<number>("EMEX_PROXY_PORT_END", 10099);
this.proxyUsername = this.configService.get<string>("EMEX_PROXY_USER", "1726bbe361918676d44e");
this.proxyPassword = this.configService.get<string>("EMEX_PROXY_PASS", "f11c7b6128cc86c6");
this.floxyHost = this.configService.get<string>("EMEX_FLOXY_HOST", "residential.floxy.io");
this.floxyPort = Number(this.configService.get("EMEX_FLOXY_PORT", 12321)) || 12321;
this.floxyUser = this.configService.get<string>("EMEX_FLOXY_USER", "d739255e819b");
this.floxyPass = this.configService.get<string>("EMEX_FLOXY_PASS", "9092873ba4e0");
this.floxyLifetime = Number(this.configService.get("EMEX_FLOXY_LIFETIME", 300)) || 300;
}
async onModuleInit(): Promise<void> {
@@ -184,14 +206,30 @@ export class EmexBrowserService implements OnModuleInit, OnModuleDestroy {
const executablePath = process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH;
if (executablePath) launchOptions.executablePath = executablePath;
if (this.useProxy) {
if (this.proxyProvider === "floxy") {
// Sticky Floxy session for the browser's lifetime → one residential exit IP
// for the whole chained scrape; a fresh id is minted on each (re)launch.
const sid = Math.random().toString(36).slice(2, 10);
const pass =
this.floxyLifetime > 0
? `${this.floxyPass}_session-${sid}_lifetime-${this.floxyLifetime}`
: this.floxyPass;
launchOptions.proxy = {
server: `http://${this.floxyHost}:${this.floxyPort}`,
username: this.floxyUser,
password: pass,
};
this.logger.log(
`Using Floxy residential proxy: ${this.floxyHost}:${this.floxyPort} (sticky ${this.floxyLifetime}s)`,
);
} else if (this.proxyProvider === "dataimpulse") {
const port = this.randomProxyPort();
launchOptions.proxy = {
server: `http://${this.proxyHost}:${port}`,
username: this.proxyUsername,
password: this.proxyPassword,
};
this.logger.log(`Using proxy: ${this.proxyHost}:${port}`);
this.logger.log(`Using DataImpulse proxy: ${this.proxyHost}:${port}`);
}
this.browser = await chromium.launch(launchOptions);

53
apps/api/src/integrations/emex/emex.service.ts
View File

@@ -132,6 +132,10 @@ export class EmexService {
private floxySessionId = Math.random().toString(36).slice(2, 10);
private floxySessionBornMs = Date.now();
private readonly emexDirectFallback: boolean;
// Transport provider for EMEX egress (HTTP + image-dims). "floxy" residential
// is primary by default (sticky session pins one exit IP across a chained flow),
// DataImpulse is the last-ditch fallback. See constructor.
private readonly emexProxyProvider: "floxy" | "dataimpulse" | "none";
constructor(
private configService: ConfigService,
@@ -169,15 +173,11 @@ export class EmexService {
portStart: toPort("EMEX_PROXY_PORT_START", 10001),
portEnd: toPort("EMEX_PROXY_PORT_END", 10099),
};
// Default agent for the low-stakes image-dims path; fetchEmexHtml builds a
// fresh agent per request so a flaky port can't pin every call.
this.proxyAgent = this.newProxyAgent();
this.logger.log(
`EMEX HTTP proxy enabled: ${this.emexProxy.host}:${this.emexProxy.portStart}-${this.emexProxy.portEnd}`,
`EMEX DataImpulse fallback enabled: ${this.emexProxy.host}:${this.emexProxy.portStart}-${this.emexProxy.portEnd}`,
);
} else {
this.emexProxy = null;
this.proxyAgent = null;
}
// Floxy residential fallback for when the DataImpulse pool flakes (connect
@@ -200,6 +200,28 @@ export class EmexService {
this.emexFloxy = null;
}
// Transport provider. EMEX_PROXY_PROVIDER: "floxy" (default) → Floxy residential
// sticky primary + a single DataImpulse last-ditch fallback; "dataimpulse" →
// legacy DataImpulse-first order; "none" → direct. ssd tokens are replayed from
// emex's own HTML and aren't strictly IP-bound on the .aspx endpoints (the old
// random-port-per-call primary proved that), but a sticky Floxy IP across the
// chained flow is still the safest choice — and avoids DataImpulse's ~50% dead
// ports that were eating the decode budget before reaching the Floxy fallback.
this.emexProxyProvider = !useProxy
? "none"
: (() => {
const p = this.configService.get<string>("EMEX_PROXY_PROVIDER", "floxy").toLowerCase();
return p === "dataimpulse" || p === "none" ? p : "floxy";
})();
// Default agent for the low-stakes image-dims path; fetchEmexHtml builds a
// fresh agent per request so a flaky exit can't pin every call.
this.proxyAgent =
this.emexProxyProvider === "none"
? null
: (this.newProxyAgent(this.emexProxyProvider === "dataimpulse" ? "dataimpulse" : "floxy") ??
this.newProxyAgent("dataimpulse"));
this.logger.log(`EMEX transport provider: ${this.emexProxyProvider}`);
this.logger.log(`EMEX Service initialized with scraper path: ${this.scraperPath}`);
}
@@ -347,14 +369,21 @@ export class EmexService {
}
private async fetchEmexHtml(url: string): Promise<string> {
// Attempt schedule: DataImpulse (rotating port, fresh agent each try to dodge
// a flaky sticky port — ~42% blip rate, undecoded-vin-rca.md EMEX #1) first,
// then the Floxy residential fallback when the DataImpulse pool throws
// transport errors. A definitive HTTP answer (e.g. 404) stops the schedule —
// it's a real result, and a different proxy IP must not "retry" it away.
// Attempt schedule by provider. floxy (default): Floxy residential sticky
// primary (one exit IP across the flow; rolls to a fresh IP on transport
// failure), then a single DataImpulse last-ditch try — this avoids DataImpulse's
// ~50% dead ports that used to burn the decode budget before reaching Floxy.
// dataimpulse (legacy): DataImpulse-first (rotating port each try), Floxy after.
// A definitive HTTP answer (e.g. 404) stops the schedule — it's a real result,
// and a different proxy IP must not "retry" it away.
const schedule: Array<"dataimpulse" | "floxy"> = [];
if (this.emexProxy) schedule.push("dataimpulse", "dataimpulse", "dataimpulse");
if (this.emexFloxy) schedule.push("floxy", "floxy");
if (this.emexProxyProvider === "floxy") {
if (this.emexFloxy) schedule.push("floxy", "floxy");
if (this.emexProxy) schedule.push("dataimpulse");
} else if (this.emexProxyProvider === "dataimpulse") {
if (this.emexProxy) schedule.push("dataimpulse", "dataimpulse", "dataimpulse");
if (this.emexFloxy) schedule.push("floxy", "floxy");
}
const maxAttempts = schedule.length || 1; // 0 → single proxy-less attempt
let lastErr: unknown;
for (let attempt = 1; attempt <= maxAttempts; attempt++) {