mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-09-18 21:02:50 +03:00
fix(sse): rotate opencode accounts on geo-blocked 403 (#12941)
Validado numa worktree combinada com a onda de streaming desta leva sobre `release/v3.8.51`: typecheck:core limpo, check-api-typecheck OK (289), check-file-size OK após rebaseline, 88/88 nos testes focados. Estender a rotação que já existe para 429 ao 403 de bloqueio geográfico é a generalização certa, e manter a rejeição de fingerprint (Cloudflare 1010) fora dela é o que impede a rotação de queimar todas as contas contra uma recusa que não é de egresso. Nota: os checkboxes de validação do corpo ficaram em branco, mas o diff traz dois arquivos de teste — vale marcar da próxima para o revisor não precisar conferir.
This commit is contained in:
1
changelog.d/fixes/12941-opencode-geo-block-rotation.md
Normal file
1
changelog.d/fixes/12941-opencode-geo-block-rotation.md
Normal file
@@ -0,0 +1 @@
|
||||
- **fix(sse):** geo-blocked opencode requests rotate to the next account proxy instead of failing, so one refused egress no longer aborts the whole chain ([#12941](https://github.com/diegosouzapw/OmniRoute/pull/12941)) — thanks @maxmad64bis
|
||||
@@ -28,6 +28,7 @@ import {
|
||||
isEmptyUpstreamRejection,
|
||||
extractChatcmplId,
|
||||
} from "./accountRotation.ts";
|
||||
import { isOpencodeGeoBlocked, proxyKeyOf } from "./opencodeGeoBlock.ts";
|
||||
import { isNetworkRotationSharedEgressGuardEnabled } from "@/shared/utils/featureFlags";
|
||||
|
||||
/**
|
||||
@@ -278,8 +279,8 @@ export class OpencodeExecutor extends BaseExecutor {
|
||||
private accounts: OpencodeAccountState[] = [
|
||||
{ fingerprint: "", cooldownUntil: 0, consecutiveFails: 0, proxy: null },
|
||||
];
|
||||
// Not `private`: passed as the mutable rotation cursor to the shared
|
||||
// pickAccount() helper, which needs a plain `{ nextAccountIdx }` shape —
|
||||
// Not `private`: passed as the mutable rotation cursor to
|
||||
// pickRotatableAccount(), which needs a plain `{ nextAccountIdx }` shape —
|
||||
// TS's private-member nominal check rejects `this` there otherwise.
|
||||
nextAccountIdx = 0;
|
||||
|
||||
@@ -324,9 +325,11 @@ export class OpencodeExecutor extends BaseExecutor {
|
||||
if (this.nextAccountIdx >= this.accounts.length) this.nextAccountIdx = 0;
|
||||
}
|
||||
|
||||
/** Round-robin pick, skipping accounts in cooldown; falls back to the next index. */
|
||||
private pickAccount(): OpencodeAccountState {
|
||||
return pickRotatableAccount(this.accounts, this);
|
||||
/** Round-robin pick, skipping non-candidates; falls back to the next index. */
|
||||
private pickAccountWith(
|
||||
isReady: (account: OpencodeAccountState) => boolean
|
||||
): OpencodeAccountState {
|
||||
return pickRotatableAccount(this.accounts, this, isReady);
|
||||
}
|
||||
|
||||
private markCooldown(
|
||||
@@ -564,9 +567,48 @@ export class OpencodeExecutor extends BaseExecutor {
|
||||
// through the accounts is the retry). Avoids an unbounded loop on a
|
||||
// persistently malformed upstream.
|
||||
const emptyRejectionBudget = this.accounts.length === 1 ? 1 : 0;
|
||||
// 403-geo tried set: proxy keys already proven geo-blocked for this
|
||||
// request's model. Request-local only — nothing persists past execute().
|
||||
const geoTriedProxyKeys = new Set<string>();
|
||||
let directTried = false;
|
||||
|
||||
for (let attempt = 0; attempt < this.accounts.length + emptyRejectionBudget; attempt++) {
|
||||
const account = this.pickAccount();
|
||||
const isProxiedCandidate = (a: OpencodeAccountState): boolean => {
|
||||
if (a.cooldownUntil > Date.now()) return false;
|
||||
// Without any geo evidence this pass, every cooldown-ready account
|
||||
// stays eligible (preserves the plain round-robin first pick).
|
||||
if (a.proxy === null) return !directTried || geoTriedProxyKeys.size === 0;
|
||||
const k = proxyKeyOf(a.proxy);
|
||||
return k !== null && !geoTriedProxyKeys.has(k);
|
||||
};
|
||||
let account = this.pickAccountWith(isProxiedCandidate);
|
||||
// Last resort: a single direct attempt (distinct egress that may
|
||||
// succeed) once no proxied account is a candidate — never before.
|
||||
if (!isProxiedCandidate(account) && !directTried && geoTriedProxyKeys.size > 0) {
|
||||
const direct = this.accounts.find(
|
||||
(a) => a.proxy === null && a.cooldownUntil <= Date.now()
|
||||
);
|
||||
if (direct) {
|
||||
account = direct;
|
||||
}
|
||||
}
|
||||
const lastStatus = lastResult !== null ? lastResult.response.status : null;
|
||||
const lastWasGeo = lastStatus === 403 || lastStatus === 451;
|
||||
if (
|
||||
lastResult !== null &&
|
||||
geoTriedProxyKeys.size > 0 &&
|
||||
!isProxiedCandidate(account) &&
|
||||
!(account.proxy === null && !directTried)
|
||||
) {
|
||||
// Geo exhaustion (last was 403/451) → surface as-is, no success mark.
|
||||
// Any other last status (e.g. 429 after 403s) → skip without a call.
|
||||
if (lastWasGeo) break;
|
||||
continue;
|
||||
}
|
||||
// Commit the last-resort direct attempt so a later exclusion breaks
|
||||
// instead of retrying it. Set here (not at pick time) so the guard
|
||||
// above still lets this committed attempt through.
|
||||
if (account.proxy === null && geoTriedProxyKeys.size > 0) directTried = true;
|
||||
const masked = maskAccountId(account.fingerprint);
|
||||
|
||||
if (sharedEgressGuardEnabled && sharedEgressDown && !account.proxy) {
|
||||
@@ -641,6 +683,28 @@ export class OpencodeExecutor extends BaseExecutor {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (status === 403 || status === 451) {
|
||||
let bodyText: string | null = null;
|
||||
try {
|
||||
bodyText = await result.response.clone().text();
|
||||
} catch {
|
||||
log?.debug?.("OPENCODE", "body read failed on geo-block check");
|
||||
}
|
||||
if (bodyText !== null && isOpencodeGeoBlocked(status, bodyText)) {
|
||||
const key = proxyKeyOf(account.proxy);
|
||||
if (key !== null) geoTriedProxyKeys.add(key);
|
||||
else directTried = true;
|
||||
log?.warn?.(
|
||||
"OPENCODE",
|
||||
`geo-blocked on account ${masked} (proxy ${key ?? "direct"}), rotating to next…`
|
||||
);
|
||||
// Single account with a proxy: 0 retries (same egress = dead latency).
|
||||
// (The fast path above already covers single-without-proxy; here length===1 WITH proxy.)
|
||||
if (this.accounts.length === 1) return result;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Empty upstream rejection (malformed 400: no error field, no real
|
||||
// content, finish_reason null — see isEmptyUpstreamRejection). Rotate/
|
||||
// retry instead of propagating it as a fatal success: the observed
|
||||
|
||||
55
open-sse/executors/opencodeGeoBlock.ts
Normal file
55
open-sse/executors/opencodeGeoBlock.ts
Normal file
@@ -0,0 +1,55 @@
|
||||
/**
|
||||
* opencodeGeoBlock.ts — geo-block predicate for the opencode executor loop.
|
||||
*
|
||||
* Leaf module: zero internal imports (layering — errorClassifier pulls
|
||||
* accountFallback + registry + DB; this file must not). The 1010 check below
|
||||
* mirrors errorClassifier.isCloudflareFingerprintRejection semantics for the
|
||||
* tokens this path needs; any divergence is a bug — see the parity test.
|
||||
*/
|
||||
|
||||
// "not available in your country" is the observed opencode RegionError phrasing
|
||||
// (2026-09-07 — app.log: "This model is not available in your country.");
|
||||
// siblings cover the same class, not the single incident. No bare "in your
|
||||
// country/region": location text without the full prefix is not a geo signal.
|
||||
const GEO_SIGNALS = [
|
||||
"not available in your country",
|
||||
"not available in your region",
|
||||
"unsupported_country",
|
||||
"unsupported country",
|
||||
];
|
||||
|
||||
// `regionerror` word-bounded: bare substring would match region_error /
|
||||
// region-error variants, which are unobserved phrasings (fail closed).
|
||||
const REGION_ERROR_REGEX = /(?<![A-Za-z0-9_-])regionerror(?![A-Za-z0-9_-])/i;
|
||||
|
||||
// Fingerprint-first: a CDN 1010 rejection says nothing about account health —
|
||||
// it must never rotate as geo. Parity with errorClassifier
|
||||
// isCloudflareFingerprintRejection: the bare number 1010 alone is NOT a signal
|
||||
// (it occurs as port/count/model token) — only with an explicit Cloudflare key
|
||||
// or the unique tokens (mirrored vectors live in the parity test below).
|
||||
const CLOUDFLARE_1010_KEY_REGEX =
|
||||
/(?<![A-Za-z0-9_-])error[\s_-]?code[\\"':=\s]{0,12}1010(?!\w)|(?<![A-Za-z0-9_-])error[-_]\s?1010(?!\w)\/?/i;
|
||||
|
||||
function isFingerprintRejection(bodyText: string): boolean {
|
||||
const text = String(bodyText || "");
|
||||
const lower = text.toLowerCase();
|
||||
return (
|
||||
CLOUDFLARE_1010_KEY_REGEX.test(text) ||
|
||||
lower.includes("browser_signature_banned") ||
|
||||
lower.includes("fingerprint_rejection")
|
||||
);
|
||||
}
|
||||
|
||||
export function isOpencodeGeoBlocked(status: number, bodyText: string): boolean {
|
||||
if (status !== 403 && status !== 451) return false;
|
||||
const text = String(bodyText || "");
|
||||
if (isFingerprintRejection(text)) return false;
|
||||
const lower = text.toLowerCase();
|
||||
if (REGION_ERROR_REGEX.test(text)) return true;
|
||||
return GEO_SIGNALS.some((signal) => lower.includes(signal));
|
||||
}
|
||||
|
||||
export function proxyKeyOf(proxy: { host: string; port: number } | null): string | null {
|
||||
if (!proxy) return null;
|
||||
return `${proxy.host}:${proxy.port}`;
|
||||
}
|
||||
53
tests/unit/opencode-geo-block-predicate.test.ts
Normal file
53
tests/unit/opencode-geo-block-predicate.test.ts
Normal file
@@ -0,0 +1,53 @@
|
||||
import { describe, it } from "node:test";
|
||||
import assert from "node:assert";
|
||||
import { isOpencodeGeoBlocked, proxyKeyOf } from "../../open-sse/executors/opencodeGeoBlock.ts";
|
||||
|
||||
const GEO_BODY = JSON.stringify({
|
||||
error: { type: "RegionError", message: "This model is not available in your country." },
|
||||
});
|
||||
const FP_BODY = JSON.stringify({
|
||||
error: { error_code: 1010, message: "browser_signature_banned" },
|
||||
});
|
||||
const AUTH_BODY = JSON.stringify({ error: { message: "invalid api key", type: "auth_error" } });
|
||||
|
||||
describe("isOpencodeGeoBlocked", () => {
|
||||
it("matches 403 + country signal", () => {
|
||||
assert.strictEqual(isOpencodeGeoBlocked(403, GEO_BODY), true);
|
||||
});
|
||||
it("matches 451 geo by definition (signal present)", () => {
|
||||
assert.strictEqual(isOpencodeGeoBlocked(451, GEO_BODY.replace("country", "region")), true);
|
||||
});
|
||||
it("rejects fingerprint 1010 even with geo-looking text", () => {
|
||||
assert.strictEqual(isOpencodeGeoBlocked(403, FP_BODY), false);
|
||||
});
|
||||
it("parity with errorClassifier: keyed 1010 rejected, bare 1010 ignored", () => {
|
||||
assert.strictEqual(
|
||||
isOpencodeGeoBlocked(403, '{"error_code":1010,"message":"blocked"}'),
|
||||
false,
|
||||
"keyed 1010 = fingerprint, never geo"
|
||||
);
|
||||
assert.strictEqual(
|
||||
isOpencodeGeoBlocked(403, "retry after 1010 seconds, model is not available in your country"),
|
||||
true,
|
||||
"bare 1010 is not a fingerprint token; geo signal still matches"
|
||||
);
|
||||
});
|
||||
it("rejects 403 auth without signal", () => {
|
||||
assert.strictEqual(isOpencodeGeoBlocked(403, AUTH_BODY), false);
|
||||
});
|
||||
it("rejects non-403/451 statuses", () => {
|
||||
assert.strictEqual(isOpencodeGeoBlocked(429, GEO_BODY), false);
|
||||
assert.strictEqual(isOpencodeGeoBlocked(200, GEO_BODY), false);
|
||||
});
|
||||
it("rejects region_error / region-error adversarial variants without word boundary", () => {
|
||||
assert.strictEqual(isOpencodeGeoBlocked(403, "Region_error occurred"), false);
|
||||
assert.strictEqual(isOpencodeGeoBlocked(403, "region-error occurred"), false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("proxyKeyOf", () => {
|
||||
it("builds host:port key, null for null proxy", () => {
|
||||
assert.strictEqual(proxyKeyOf({ host: "proxy.example", port: 8080 }), "proxy.example:8080");
|
||||
assert.strictEqual(proxyKeyOf(null), null);
|
||||
});
|
||||
});
|
||||
471
tests/unit/opencode-geo-block-rotation.test.ts
Normal file
471
tests/unit/opencode-geo-block-rotation.test.ts
Normal file
@@ -0,0 +1,471 @@
|
||||
import { describe, it, beforeEach, afterEach, before, after } from "node:test";
|
||||
import assert from "node:assert";
|
||||
import net from "node:net";
|
||||
import { OpencodeExecutor } from "../../open-sse/executors/opencode.ts";
|
||||
import type { ExecutorLog, ProviderCredentials } from "../../open-sse/executors/base.ts";
|
||||
import { resolveProxyForRequest } from "../../open-sse/utils/proxyFetch.ts";
|
||||
|
||||
const log: ExecutorLog = { debug() {}, info() {}, warn() {}, error() {} };
|
||||
|
||||
const FP_A = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
|
||||
const FP_B = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb";
|
||||
const FP_C = "cccccccccccccccccccccccccccccccc";
|
||||
|
||||
const GEO_BODY = JSON.stringify({
|
||||
error: { type: "RegionError", message: "This model is not available in your country." },
|
||||
});
|
||||
|
||||
let serverA: net.Server;
|
||||
let serverB: net.Server;
|
||||
let serverC: net.Server;
|
||||
let serverD: net.Server;
|
||||
let portA = 0;
|
||||
let portB = 0;
|
||||
let portC = 0;
|
||||
let portD = 0;
|
||||
|
||||
function listen(server: net.Server): Promise<number> {
|
||||
return new Promise((resolve) => {
|
||||
server.listen(0, "127.0.0.1", () => {
|
||||
resolve((server.address() as net.AddressInfo).port);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
before(async () => {
|
||||
serverA = net.createServer((s) => s.destroy());
|
||||
serverB = net.createServer((s) => s.destroy());
|
||||
serverC = net.createServer((s) => s.destroy());
|
||||
serverD = net.createServer((s) => s.destroy());
|
||||
portA = await listen(serverA);
|
||||
portB = await listen(serverB);
|
||||
portC = await listen(serverC);
|
||||
portD = await listen(serverD);
|
||||
});
|
||||
|
||||
after(() => {
|
||||
serverA?.close();
|
||||
serverB?.close();
|
||||
serverC?.close();
|
||||
serverD?.close();
|
||||
});
|
||||
|
||||
const FP_D = "dddddddddddddddddddddddddddddddd";
|
||||
|
||||
function portFor(fp: string): number {
|
||||
if (fp === FP_A) return portA;
|
||||
if (fp === FP_B) return portB;
|
||||
if (fp === FP_C) return portC;
|
||||
return portD;
|
||||
}
|
||||
|
||||
function credentialsFor(fingerprints: string[]): ProviderCredentials {
|
||||
return {
|
||||
apiKey: null,
|
||||
accessToken: null,
|
||||
connectionId: "noauth",
|
||||
providerSpecificData: {
|
||||
fingerprints,
|
||||
accountProxies: fingerprints.map((fp) => ({
|
||||
fingerprint: fp,
|
||||
proxy: { type: "http", host: "127.0.0.1", port: portFor(fp) },
|
||||
})),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("OpencodeExecutor geo-block rotation", () => {
|
||||
let originalFetch: typeof globalThis.fetch;
|
||||
let observed: string[];
|
||||
|
||||
beforeEach(() => {
|
||||
originalFetch = globalThis.fetch;
|
||||
observed = [];
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
});
|
||||
|
||||
function installFetch(plan: Array<{ status: number; body?: string }>) {
|
||||
let call = 0;
|
||||
globalThis.fetch = (async (input: RequestInfo | URL) => {
|
||||
const url =
|
||||
typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
|
||||
const resolved = resolveProxyForRequest(url);
|
||||
observed.push(resolved.proxyUrl ? new URL(resolved.proxyUrl).port : "direct");
|
||||
const step = plan[Math.min(call, plan.length - 1)];
|
||||
call++;
|
||||
return new Response(step.body ?? JSON.stringify({ ok: step.status === 200 }), {
|
||||
status: step.status,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
}) as typeof globalThis.fetch;
|
||||
}
|
||||
|
||||
it("rotates past two geo-blocked proxies to the healthy third", async () => {
|
||||
const exec = new OpencodeExecutor("opencode-zen");
|
||||
installFetch([
|
||||
{ status: 403, body: GEO_BODY },
|
||||
{ status: 403, body: GEO_BODY },
|
||||
{ status: 200 },
|
||||
]);
|
||||
|
||||
const result = await exec.execute({
|
||||
model: "muse-spark-1.3-contributor-free",
|
||||
body: { messages: [{ role: "user", content: "hi" }], stream: false },
|
||||
stream: false,
|
||||
signal: null,
|
||||
credentials: credentialsFor([FP_A, FP_B, FP_C]),
|
||||
log,
|
||||
});
|
||||
|
||||
assert.strictEqual(
|
||||
(result as { response: Response }).response.status,
|
||||
200,
|
||||
"must rotate past geo-blocked proxies"
|
||||
);
|
||||
assert.strictEqual(observed.length, 3, "one call per distinct proxy");
|
||||
assert.ok(
|
||||
observed[0] === String(portA) &&
|
||||
observed.includes(String(portB)) &&
|
||||
observed.includes(String(portC)),
|
||||
"first attempt on A (fresh cursor), then rotation over untried proxies"
|
||||
);
|
||||
});
|
||||
|
||||
it("deduplicates fingerprints sharing one proxy (one call, not two)", async () => {
|
||||
const exec = new OpencodeExecutor("opencode-zen");
|
||||
// FP_A and FP_B share portA; FP_C is healthy on portC.
|
||||
const creds = credentialsFor([FP_A, FP_B, FP_C]);
|
||||
(creds.providerSpecificData as Record<string, unknown>).accountProxies = [
|
||||
{ fingerprint: FP_A, proxy: { type: "http", host: "127.0.0.1", port: portA } },
|
||||
{ fingerprint: FP_B, proxy: { type: "http", host: "127.0.0.1", port: portA } },
|
||||
{ fingerprint: FP_C, proxy: { type: "http", host: "127.0.0.1", port: portC } },
|
||||
];
|
||||
installFetch([{ status: 403, body: GEO_BODY }, { status: 200 }]);
|
||||
|
||||
const result = await exec.execute({
|
||||
model: "muse-spark-1.3-contributor-free",
|
||||
body: { messages: [{ role: "user", content: "hi" }], stream: false },
|
||||
stream: false,
|
||||
signal: null,
|
||||
credentials: creds,
|
||||
log,
|
||||
});
|
||||
|
||||
assert.strictEqual((result as { response: Response }).response.status, 200);
|
||||
assert.strictEqual(observed.length, 2, "shared proxy tried once, never re-called");
|
||||
assert.ok(observed.includes(String(portA)) && observed.includes(String(portC)));
|
||||
});
|
||||
|
||||
it("propagates a motif-less 403 immediately without rotation", async () => {
|
||||
const exec = new OpencodeExecutor("opencode-zen");
|
||||
installFetch([
|
||||
{ status: 403, body: JSON.stringify({ error: { message: "invalid api key" } }) },
|
||||
]);
|
||||
|
||||
const result = await exec.execute({
|
||||
model: "muse-spark-1.3-contributor-free",
|
||||
body: { messages: [{ role: "user", content: "hi" }], stream: false },
|
||||
stream: false,
|
||||
signal: null,
|
||||
credentials: credentialsFor([FP_A, FP_B]),
|
||||
log,
|
||||
});
|
||||
|
||||
assert.strictEqual((result as { response: Response }).response.status, 403);
|
||||
assert.strictEqual(observed.length, 1, "no retry on non-geo 403");
|
||||
});
|
||||
|
||||
it("propagates the last 403 after exhausting all proxies", async () => {
|
||||
const exec = new OpencodeExecutor("opencode-zen");
|
||||
installFetch([
|
||||
{ status: 403, body: GEO_BODY },
|
||||
{ status: 403, body: GEO_BODY },
|
||||
{ status: 403, body: GEO_BODY },
|
||||
// NOTE: 4th step (200) unreachable via break — documents intent, not a real call.
|
||||
{ status: 200 },
|
||||
]);
|
||||
|
||||
const result = await exec.execute({
|
||||
model: "muse-spark-1.3-contributor-free",
|
||||
body: { messages: [{ role: "user", content: "hi" }], stream: false },
|
||||
stream: false,
|
||||
signal: null,
|
||||
credentials: credentialsFor([FP_A, FP_B, FP_C]),
|
||||
log,
|
||||
});
|
||||
|
||||
assert.strictEqual((result as { response: Response }).response.status, 403);
|
||||
assert.strictEqual(observed.length, 3, "every proxy tried exactly once");
|
||||
for (const port of [portA, portB, portC]) {
|
||||
assert.ok(observed.includes(String(port)), `proxy ${port} tried`);
|
||||
}
|
||||
});
|
||||
|
||||
it("single proxied account: one call, immediate 403 (no dead retry)", async () => {
|
||||
const exec = new OpencodeExecutor("opencode-zen");
|
||||
installFetch([{ status: 403, body: GEO_BODY }, { status: 200 }]);
|
||||
|
||||
const result = await exec.execute({
|
||||
model: "muse-spark-1.3-contributor-free",
|
||||
body: { messages: [{ role: "user", content: "hi" }], stream: false },
|
||||
stream: false,
|
||||
signal: null,
|
||||
credentials: credentialsFor([FP_A]),
|
||||
log,
|
||||
});
|
||||
|
||||
assert.strictEqual((result as { response: Response }).response.status, 403);
|
||||
assert.strictEqual(observed.length, 1);
|
||||
});
|
||||
|
||||
it("geo rotation never cools the account down", async () => {
|
||||
// Warm-up materializes accounts; counters are preserved by fingerprint
|
||||
// across executes (opencode.ts:310-320), so pre-loading to 2 detects a
|
||||
// stray markCooldown (would raise to 3). The winning account's success
|
||||
// resets its own counter via markSuccess — only the winner resets.
|
||||
const exec2 = new OpencodeExecutor("opencode-zen");
|
||||
installFetch([{ status: 200 }]);
|
||||
await exec2.execute({
|
||||
model: "muse-spark-1.3-contributor-free",
|
||||
body: { messages: [{ role: "user", content: "hi" }], stream: false },
|
||||
stream: false,
|
||||
signal: null,
|
||||
credentials: credentialsFor([FP_A, FP_B]),
|
||||
log,
|
||||
});
|
||||
const mid = (
|
||||
exec2 as unknown as {
|
||||
accounts: Array<{ fingerprint: string; cooldownUntil: number; consecutiveFails: number }>;
|
||||
}
|
||||
).accounts;
|
||||
assert.strictEqual(mid.length, 2, "warm-up materialized both accounts");
|
||||
for (const a of mid) a.consecutiveFails = 2;
|
||||
installFetch([{ status: 403, body: GEO_BODY }, { status: 200 }]);
|
||||
await exec2.execute({
|
||||
model: "muse-spark-1.3-contributor-free",
|
||||
body: { messages: [{ role: "user", content: "hi" }], stream: false },
|
||||
stream: false,
|
||||
signal: null,
|
||||
credentials: credentialsFor([FP_A, FP_B]),
|
||||
log,
|
||||
});
|
||||
const after = (
|
||||
exec2 as unknown as {
|
||||
accounts: Array<{ fingerprint: string; cooldownUntil: number; consecutiveFails: number }>;
|
||||
}
|
||||
).accounts;
|
||||
for (const a of after) {
|
||||
assert.strictEqual(a.cooldownUntil, 0, "no cooldown from geo rotation");
|
||||
}
|
||||
assert.strictEqual(
|
||||
after.filter((a) => a.consecutiveFails === 0).length,
|
||||
1,
|
||||
"exactly one account reset to 0 (the winner via markSuccess)"
|
||||
);
|
||||
assert.strictEqual(
|
||||
after.filter((a) => a.consecutiveFails === 2).length,
|
||||
1,
|
||||
"the blocked account keeps its prior fails"
|
||||
);
|
||||
});
|
||||
|
||||
it("a geo-tried proxy is never re-called even after a 429", async () => {
|
||||
const exec = new OpencodeExecutor("opencode-zen");
|
||||
// A geo-blocked (tried), B rate-limited (cooled, untried-403), C geo-blocked
|
||||
// (tried), D healthy. The 4th pick must be D — never A or C again.
|
||||
const creds = credentialsFor([FP_A, FP_B, FP_C, FP_D]);
|
||||
installFetch([
|
||||
{ status: 403, body: GEO_BODY },
|
||||
{ status: 429 },
|
||||
{ status: 403, body: GEO_BODY },
|
||||
{ status: 200 },
|
||||
]);
|
||||
|
||||
const result = await exec.execute({
|
||||
model: "muse-spark-1.3-contributor-free",
|
||||
body: { messages: [{ role: "user", content: "hi" }], stream: false },
|
||||
stream: false,
|
||||
signal: null,
|
||||
credentials: creds,
|
||||
log,
|
||||
});
|
||||
|
||||
assert.strictEqual((result as { response: Response }).response.status, 200);
|
||||
assert.strictEqual(observed.length, 4, "no re-call of a tried proxy");
|
||||
assert.strictEqual(
|
||||
observed.filter((p) => p === String(portA)).length,
|
||||
1,
|
||||
"tried proxy A called exactly once"
|
||||
);
|
||||
assert.strictEqual(
|
||||
observed.filter((p) => p === String(portC)).length,
|
||||
1,
|
||||
"tried proxy C called exactly once"
|
||||
);
|
||||
assert.strictEqual(observed[3], String(portD), "healthy proxy D picked last");
|
||||
});
|
||||
|
||||
it("rotates on 451 geo like on 403", async () => {
|
||||
const exec = new OpencodeExecutor("opencode-zen");
|
||||
installFetch([{ status: 451, body: GEO_BODY }, { status: 200 }]);
|
||||
|
||||
const result = await exec.execute({
|
||||
model: "muse-spark-1.3-contributor-free",
|
||||
body: { messages: [{ role: "user", content: "hi" }], stream: false },
|
||||
stream: false,
|
||||
signal: null,
|
||||
credentials: credentialsFor([FP_A, FP_B]),
|
||||
log,
|
||||
});
|
||||
|
||||
assert.strictEqual((result as { response: Response }).response.status, 200);
|
||||
assert.strictEqual(observed.length, 2);
|
||||
});
|
||||
|
||||
it("does not retry a 1010 fingerprint rejection as geo", async () => {
|
||||
const exec = new OpencodeExecutor("opencode-zen");
|
||||
installFetch([
|
||||
{ status: 403, body: JSON.stringify({ error_code: 1010, message: "blocked" }) },
|
||||
// NOTE: 2nd step (200) unreachable (no 1010 retry) — documents intent, not a real call.
|
||||
{ status: 200 },
|
||||
]);
|
||||
|
||||
const result = await exec.execute({
|
||||
model: "muse-spark-1.3-contributor-free",
|
||||
body: { messages: [{ role: "user", content: "hi" }], stream: false },
|
||||
stream: false,
|
||||
signal: null,
|
||||
credentials: credentialsFor([FP_A, FP_B]),
|
||||
log,
|
||||
});
|
||||
|
||||
assert.strictEqual((result as { response: Response }).response.status, 403);
|
||||
assert.strictEqual(observed.length, 1, "fingerprint 1010 never rotates as geo");
|
||||
});
|
||||
|
||||
it("uses the direct account once, last, when all proxies are tried", async () => {
|
||||
const exec = new OpencodeExecutor("opencode-zen");
|
||||
const creds = credentialsFor([FP_A, FP_B]);
|
||||
(creds.providerSpecificData as Record<string, unknown>).accountProxies = [
|
||||
{ fingerprint: FP_A, proxy: { type: "http", host: "127.0.0.1", port: portA } },
|
||||
];
|
||||
installFetch([{ status: 403, body: GEO_BODY }, { status: 200 }]);
|
||||
|
||||
const result = await exec.execute({
|
||||
model: "muse-spark-1.3-contributor-free",
|
||||
body: { messages: [{ role: "user", content: "hi" }], stream: false },
|
||||
stream: false,
|
||||
signal: null,
|
||||
credentials: creds,
|
||||
log,
|
||||
});
|
||||
|
||||
assert.strictEqual((result as { response: Response }).response.status, 200);
|
||||
assert.strictEqual(observed.length, 2, "one proxied + one direct, direct last");
|
||||
assert.strictEqual(observed[0], String(portA));
|
||||
assert.strictEqual(observed[1], "direct");
|
||||
});
|
||||
|
||||
it("keeps the 200 success body intact after a geo rotation", async () => {
|
||||
const exec = new OpencodeExecutor("opencode-zen");
|
||||
installFetch([
|
||||
{ status: 403, body: GEO_BODY },
|
||||
{ status: 200, body: '{"ok":true}' },
|
||||
]);
|
||||
|
||||
const result = await exec.execute({
|
||||
model: "muse-spark-1.3-contributor-free",
|
||||
body: { messages: [{ role: "user", content: "hi" }], stream: false },
|
||||
stream: false,
|
||||
signal: null,
|
||||
credentials: credentialsFor([FP_A, FP_B]),
|
||||
log,
|
||||
});
|
||||
|
||||
const response = (result as { response: Response }).response;
|
||||
assert.strictEqual(response.status, 200);
|
||||
assert.strictEqual(await response.text(), '{"ok":true}');
|
||||
});
|
||||
|
||||
it("combined keyed-1010 plus geo phrase never rotates (fingerprint first)", async () => {
|
||||
const exec = new OpencodeExecutor("opencode-zen");
|
||||
installFetch([
|
||||
{
|
||||
status: 403,
|
||||
body: JSON.stringify({
|
||||
error_code: 1010,
|
||||
message: "not available in your country",
|
||||
}),
|
||||
},
|
||||
// NOTE: 2nd step (200) unreachable (1010 rejection = no retry) — documents intent, not a real call.
|
||||
{ status: 200 },
|
||||
]);
|
||||
|
||||
const result = await exec.execute({
|
||||
model: "muse-spark-1.3-contributor-free",
|
||||
body: { messages: [{ role: "user", content: "hi" }], stream: false },
|
||||
stream: false,
|
||||
signal: null,
|
||||
credentials: credentialsFor([FP_A, FP_B]),
|
||||
log,
|
||||
});
|
||||
|
||||
assert.strictEqual((result as { response: Response }).response.status, 403);
|
||||
assert.strictEqual(observed.length, 1, "fingerprint rejection wins over geo phrase");
|
||||
});
|
||||
|
||||
it("phrase-only geo signal without type still rotates", async () => {
|
||||
const exec = new OpencodeExecutor("opencode-zen");
|
||||
installFetch([
|
||||
{
|
||||
status: 403,
|
||||
body: JSON.stringify({
|
||||
error: { message: "This model is not available in your country." },
|
||||
}),
|
||||
},
|
||||
{ status: 200 },
|
||||
]);
|
||||
|
||||
const result = await exec.execute({
|
||||
model: "muse-spark-1.3-contributor-free",
|
||||
body: { messages: [{ role: "user", content: "hi" }], stream: false },
|
||||
stream: false,
|
||||
signal: null,
|
||||
credentials: credentialsFor([FP_A, FP_B]),
|
||||
log,
|
||||
});
|
||||
|
||||
assert.strictEqual((result as { response: Response }).response.status, 200);
|
||||
assert.strictEqual(observed.length, 2, "phrase match alone rotates, no type needed");
|
||||
});
|
||||
|
||||
it("fail-closed: bare country mention without the geo prefix does not rotate", async () => {
|
||||
// Bare location text is not a geo signal: only the full
|
||||
// "not available in your country/region" phrasing rotates.
|
||||
const exec = new OpencodeExecutor("opencode-zen");
|
||||
installFetch([
|
||||
{
|
||||
status: 403,
|
||||
body: JSON.stringify({
|
||||
error: { message: "quota for in your country dashboard exceeded" },
|
||||
}),
|
||||
},
|
||||
{ status: 200 },
|
||||
]);
|
||||
|
||||
const result = await exec.execute({
|
||||
model: "muse-spark-1.3-contributor-free",
|
||||
body: { messages: [{ role: "user", content: "hi" }], stream: false },
|
||||
stream: false,
|
||||
signal: null,
|
||||
credentials: credentialsFor([FP_A, FP_B]),
|
||||
log,
|
||||
});
|
||||
|
||||
assert.strictEqual((result as { response: Response }).response.status, 403);
|
||||
assert.strictEqual(observed.length, 1, "no retry without the full geo phrasing");
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user