root/sase.tr
1
0
Fork 0
Code Issues 5 Pull Requests 3 Actions 7 Packages Projects Releases Wiki Activity

feat(emex): Floxy residential proxy fallback for the flaky DataImpulse pool

Browse Source 
DataImpulse (74.81.81.81, random port 10001-10099) intermittently throws
connect-timeouts / resets, dropping real emex hits. Add a Floxy residential
fallback (residential.floxy.io:12321): fetchEmexHtml now runs a provider
schedule — 3 DataImpulse attempts (rotating port), then 2 Floxy attempts —
and only falls back on transport errors (a definitive HTTP answer like 404
still stops the schedule). On by default; endpoint/creds overridable via
EMEX_FLOXY_* env. Direct proxy-less last resort still gated by
EMEX_DIRECT_FALLBACK.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Semih Yesilyurt
2026-06-05 15:54:16 +03:00
parent fd5c617252
commit ca9b11dff7
1 changed files with 54 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
apps/api/src/integrations/emex/emex.service.ts
View File

@@ -115,6 +115,14 @@ export class EmexService {
portStart: number;
portEnd: number;
} | null;
// Floxy residential fallback — used when the DataImpulse pool throws transport
// errors. Single rotating endpoint (one port; IP rotates per request).
private readonly emexFloxy: {
host: string;
port: number;
user: string;
pass: string;
} | null;
private readonly emexDirectFallback: boolean;
constructor(
@@ -163,6 +171,22 @@ export class EmexService {
this.proxyAgent = null;
}
// Floxy residential fallback for when the DataImpulse pool flakes (connect
// timeouts / resets). On by default; creds + endpoint overridable via env.
const floxyEnabled = this.configService.get<string>("EMEX_FLOXY_FALLBACK", "true") === "true";
if (floxyEnabled) {
const fport = Number(this.configService.get("EMEX_FLOXY_PORT", 12321));
this.emexFloxy = {
host: this.configService.get<string>("EMEX_FLOXY_HOST", "residential.floxy.io"),
port: Number.isInteger(fport) && fport >= 1 && fport <= 65535 ? fport : 12321,
user: this.configService.get<string>("EMEX_FLOXY_USER", "d739255e819b"),
pass: this.configService.get<string>("EMEX_FLOXY_PASS", "9092873ba4e0"),
};
this.logger.log(`EMEX Floxy fallback enabled: ${this.emexFloxy.host}:${this.emexFloxy.port}`);
} else {
this.emexFloxy = null;
}
this.logger.log(`EMEX Service initialized with scraper path: ${this.scraperPath}`);
}
@@ -269,7 +293,16 @@ export class EmexService {
* without requiring authentication cookies.
*/
/** Build a fresh proxy agent on a random port from the pool (null if proxy off). */
private newProxyAgent(): ProxyAgent | null {
private newProxyAgent(provider: "dataimpulse" | "floxy" = "dataimpulse"): ProxyAgent | null {
if (provider === "floxy") {
if (!this.emexFloxy) return null;
const { host, port, user, pass } = this.emexFloxy;
return new ProxyAgent({
uri: `http://${user}:${pass}@${host}:${port}`,
connect: { timeout: 30000 },
requestTls: { timeout: 30000 },
});
}
if (!this.emexProxy) return null;
const { host, user, pass, portStart, portEnd } = this.emexProxy;
const port = Math.floor(Math.random() * (portEnd - portStart + 1)) + portStart;
@@ -281,14 +314,19 @@ export class EmexService {
}
private async fetchEmexHtml(url: string): Promise<string> {
const maxAttempts = this.emexProxy ? 3 : 1;
// Attempt schedule: DataImpulse (rotating port, fresh agent each try to dodge
// a flaky sticky port — ~42% blip rate, undecoded-vin-rca.md EMEX #1) first,
// then the Floxy residential fallback when the DataImpulse pool throws
// transport errors. A definitive HTTP answer (e.g. 404) stops the schedule —
// it's a real result, and a different proxy IP must not "retry" it away.
const schedule: Array<"dataimpulse" | "floxy"> = [];
if (this.emexProxy) schedule.push("dataimpulse", "dataimpulse", "dataimpulse");
if (this.emexFloxy) schedule.push("floxy", "floxy");
const maxAttempts = schedule.length || 1; // 0 → single proxy-less attempt
let lastErr: unknown;
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
// Fresh agent every attempt: a single sticky DataImpulse port flakes
// intermittently ("fetch failed"/reset), silently dropping real emex hits
// (~42% blip rate observed — undecoded-vin-rca.md, EMEX #1). A new agent
// rotates the port and forces a new socket even on a single-port pool.
const agent = this.newProxyAgent();
const provider = schedule[attempt - 1]; // undefined when no proxy → direct
const agent = provider ? this.newProxyAgent(provider) : null;
try {
const res = await fetch(url, {
headers: { "User-Agent": EMEX_UA, Accept: "text/html,application/xhtml+xml" },
@@ -298,6 +336,9 @@ export class EmexService {
if (!res.ok) {
throw new Error(`EMEX HTTP ${res.status} for ${url}`);
}
if (provider === "floxy") {
this.logger.log(`EMEX fetch via Floxy fallback succeeded for ${url}`);
}
return await res.text();
} catch (err) {
lastErr = err;
@@ -313,15 +354,18 @@ export class EmexService {
`${e.message} ${String(e.cause ?? "")}`,
));
if (!transient || attempt === maxAttempts) break;
const next = schedule[attempt];
const via =
next === "floxy" && provider !== "floxy" ? "via Floxy fallback" : "with fresh proxy";
this.logger.warn(
`EMEX fetch transient error (attempt ${attempt}/${maxAttempts}) for ${url}: ${e.message} — retrying with fresh proxy`,
`EMEX fetch transient error (attempt ${attempt}/${maxAttempts}, ${provider}) for ${url}: ${e.message} — retrying ${via}`,
);
await new Promise((r) => setTimeout(r, 300 * attempt));
}
}
// Last resort (opt-in via EMEX_DIRECT_FALLBACK): one direct, proxy-less attempt
// for when the whole pool is down. Off by default — it exposes the origin IP.
if (this.emexProxy && this.emexDirectFallback) {
// for when both proxy pools are down. Off by default — it exposes the origin IP.
if ((this.emexProxy || this.emexFloxy) && this.emexDirectFallback) {
try {
const res = await fetch(url, {
headers: { "User-Agent": EMEX_UA, Accept: "text/html,application/xhtml+xml" },