From 4fc04a56045e0afb9b81b38ab8dc74fa9983e89d Mon Sep 17 00:00:00 2001 From: "Bob.Hou" Date: Sun, 9 Aug 2026 23:30:10 -0400 Subject: [PATCH] fix(combo): classify Cloudflare 1010 fingerprint rejection as non-auth (#9929) opencode.ai/zen/v1 rejects non-browser clients (urllib) with 403 error_code 1010 while curl on the same key succeeds. The 403 was treated as an auth-level failure and two of them crystallized a misleading ALL_ACCOUNTS_INACTIVE on the free pool. - errorClassifier: new FINGERPRINT_REJECTION type; a 403 carrying error_code 1010 / browser_signature_banned is the CDN refusing the client TLS/UA signature, not the account credentials. - combo/targetExhaustion: fingerprint rejections skip auth-level exhaustion so remaining targets stay eligible. - auth: resolveTerminalConnectionStatus no longer treats the fingerprint rejection as a terminal banned account state. UA passthrough is deliberately untouched: #5997/#5720 make the forward-only behavior load-bearing. Signed-off-by: Minxi Hou --- open-sse/services/combo/targetExhaustion.ts | 38 +++- open-sse/services/errorClassifier.ts | 36 ++++ src/sse/services/auth.ts | 8 +- .../combo/combo-target-exhaustion.test.ts | 155 ++++++++++++++ tests/unit/error-classifier.test.ts | 189 +++++++++++++++++- 5 files changed, 421 insertions(+), 5 deletions(-) diff --git a/open-sse/services/combo/targetExhaustion.ts b/open-sse/services/combo/targetExhaustion.ts index 85cc07dbb7..05d7335da2 100644 --- a/open-sse/services/combo/targetExhaustion.ts +++ b/open-sse/services/combo/targetExhaustion.ts @@ -22,6 +22,7 @@ import { } from "../accountFallback.ts"; import { RateLimitReason } from "../../config/constants.ts"; import { isProviderCircuitOpenResult, isRequestScopedUpstreamFailure } from "./comboPredicates.ts"; +import { isCloudflareFingerprintRejection } from "../errorClassifier.ts"; import type { ComboLogger, ResolvedComboTarget } from "./types.ts"; // Connection-level failure statuses: the provider connection itself is likely bad (upstream @@ -77,12 +78,45 @@ export function applyComboTargetExhaustion( target: ResolvedComboTarget, opts: ApplyComboTargetExhaustionOptions ): boolean { - const { result, sets, log, tag } = opts; + const { result, sets, log, tag, errorText, structuredError } = opts; const provider = target.provider; // #8133/#8137: auth-level failures (401/403) mean that connection's credentials are bad. // Split out to keep applyComboTargetExhaustion under the complexity ceiling. - if (AUTH_LEVEL_ERROR_STATUSES.includes(result.status) && provider && provider !== "unknown") { + // Cloudflare 1010 (a 403 carrying error_code 1010 / browser_signature_banned) is NOT an + // auth failure: the CDN in front of the upstream refused the client's TLS/UA signature, + // and a different client on the same key succeeds. Treating it as auth-level would mark + // every connection in the pool exhausted on the first 1010 and, with a multi-target combo, + // crystallize a misleading ALL_ACCOUNTS_INACTIVE after two such calls — see + // errorClassifier.isCloudflareFingerprintRejection. The signal may arrive via the + // upstream JSON's structuredError.message (nested "error_code":1010 / browser_signature_banned) + // when the raw errorText is generic, so inspect both. A normalized structuredError.code/type + // ("1010" / browser_signature_banned / fingerprint_rejection) is matched directly — it arrives + // without the error_code key that the text regex keys on. The comparison is case-insensitive + // (matching isCloudflareFingerprintRejection's lowercase) and exact: a numeric 10101 + // (port/count/request id) is a different token, never a 1010. + const fingerprintToken = [structuredError?.code, structuredError?.type].some((value) => + ["1010", "browser_signature_banned", "fingerprint_rejection"].includes( + value == null ? "" : String(value).toLowerCase() + ) + ); + // code/type can also carry the signal in a non-normalized form (e.g. a gateway stuffing + // "error_code: 1010" into the code field verbatim), so the shared text matcher sees every + // candidate string — the exact allowlist above is not the only path in. + const fingerprintText = isCloudflareFingerprintRejection( + [structuredError?.message, structuredError?.code, structuredError?.type, errorText] + .filter(Boolean) + .join(" ") + ); + if ( + AUTH_LEVEL_ERROR_STATUSES.includes(result.status) && + // Cloudflare 1010 is a 403-ONLY fingerprint rejection. A 401 that merely happens to + // mention "1010" or "fingerprint_rejection" in a port/count/model token must NOT skip + // auth-level exhaustion — only a 403 carrying the Cloudflare fingerprint signal does. + !(result.status === 403 && (fingerprintToken || fingerprintText)) && + provider && + provider !== "unknown" + ) { markAuthLevelExhaustion(target, { result, sets, log, tag }); return true; } diff --git a/open-sse/services/errorClassifier.ts b/open-sse/services/errorClassifier.ts index 71bc81f5eb..daa3bab657 100644 --- a/open-sse/services/errorClassifier.ts +++ b/open-sse/services/errorClassifier.ts @@ -78,6 +78,7 @@ export const PROVIDER_ERROR_TYPES = { OAUTH_INVALID_TOKEN: "oauth_invalid_token", EMPTY_CONTENT: "empty_content", MODEL_NOT_FOUND: "model_not_found", + FINGERPRINT_REJECTION: "fingerprint_rejection", }; export const CONTEXT_OVERFLOW_SIGNALS = [ @@ -113,6 +114,31 @@ export function containsModelUnavailableMessage(errorMessage: string): boolean { return MODEL_NAMED_UNSUPPORTED_REGEX.test(String(errorMessage || "").toLowerCase()); } +// Cloudflare 1010 "Access denied ... blocked based on your browser's signature" — +// a fingerprint/browser-like rejection issued by the CDN in front of an upstream +// (e.g. opencode.ai/zen/v1), carrying error_code 1010 or error_name +// "browser_signature_banned". Distinct from an auth 403: the account is healthy, +// the CLIENT's TLS/UA signature was refused. +// +// IMPORTANT: the bare number 1010 is NOT matched on its own — a 403 body can +// legitimately contain "1010" as a port, count, request id, or model token +// ("model foo-1010 is not supported", "retry after 1010 seconds"). 1010 is only +// treated as a fingerprint rejection when it appears with an explicit Cloudflare +// key (`error_code` / `error-code`) or the unique `browser_signature_banned` / +// `fingerprint_rejection` tokens. `\\?` tolerates the escaped-quote form that +// appears when the upstream body is nested inside the gateway's error.message JSON. +const CLOUDFLARE_1010_REGEX = + /(? { + // #1010 incident: urllib's Python-urllib UA is refused by Cloudflare in front of + // opencode.ai/zen/v1 with error_code 1010 while curl on the SAME key/body succeeds. + // Treating it as auth-level marks every connection exhausted and, after two such + // calls, flips the pool to ALL_ACCOUNTS_INACTIVE. It must fall through to the + // transient path (no exhaustion marking) so remaining targets still get tried. + const s = sets(); + const exhausted = applyComboTargetExhaustion(target(), { + ...baseOpts, + errorText: + '[openai/deepseek-v4-flash-free] [403]: {"type":"https://developers.cloudflare.com/support/troubleshooting/http-status-codes/cloudflare-1xxx-errors/error-1010/","title":"Error 1010: Access denied","status":403,"detail":"The site owner has blocked access based on your browser\'s signature.","instance":"a283cb68eb52bda8","error_code":1010,"error_name":"browser_signature_banned"}', + result: { status: 403 }, + fallbackResult: { creditsExhausted: false }, + sets: s, + }); + assert.equal(exhausted, false, "a fingerprint rejection must not exhaust the connection"); + assert.equal( + s.exhaustedConnections.size, + 0, + "a Cloudflare 1010 must not mark the connection exhausted for remaining targets" + ); + assert.equal( + s.exhaustedProviders.size, + 0, + "a Cloudflare 1010 must not exhaust the whole provider" + ); +}); + +test("plain 403 still marks auth-level exhaustion (1010 detection is specific)", () => { + const s = sets(); + const exhausted = applyComboTargetExhaustion(target(), { + ...baseOpts, + errorText: "you do not have permission to access this model", + result: { status: 403 }, + fallbackResult: { creditsExhausted: false }, + sets: s, + }); + assert.equal(exhausted, true); + assert.ok(s.exhaustedConnections.has("test-dedup-provider:conn-1")); +}); + +test("Cloudflare 1010 arriving via structuredError (not raw errorText) still avoids auth exhaustion", () => { + // The 1010 signal may surface in structuredError.message (nested JSON) while errorText + // stays generic. The auth-level guard must inspect both, or the #1010 failure recurs. + const s = sets(); + const exhausted = applyComboTargetExhaustion(target(), { + ...baseOpts, + errorText: "[403] forbidden", + structuredError: { code: "browser_signature_banned" }, + result: { status: 403 }, + fallbackResult: { creditsExhausted: false }, + sets: s, + }); + assert.equal(exhausted, false, "structuredError 1010 must not mark auth-level exhaustion"); + assert.equal(s.exhaustedConnections.size, 0); +}); + +test("Cloudflare 1010 in structuredError.code survives a generic structuredError.message (no || short-circuit)", () => { + // Review finding: structuredError.message being a generic value (e.g. "Forbidden") must + // NOT mask a 1010/browser_signature_banned signal carried in .code. The guard must check + // every candidate string, not short-circuit on the first truthy one. + const s = sets(); + const exhausted = applyComboTargetExhaustion(target(), { + ...baseOpts, + errorText: "You do not have permission", + structuredError: { message: "Forbidden", code: "browser_signature_banned" }, + result: { status: 403 }, + fallbackResult: { creditsExhausted: false }, + sets: s, + }); + assert.equal(exhausted, false, "a code-carried 1010 must not be masked by a generic message"); + assert.equal(s.exhaustedConnections.size, 0); +}); + +test("Cloudflare 1010 via structuredError.type still avoids auth exhaustion", () => { + // Round 6 finding: the guard promised a normalized structuredError.type of "1010" is + // matched directly, but only code/type named browser_signature_banned were checked. A 403 + // whose only fingerprint signal is type === "1010" fell through to auth-level exhaustion. + // combo.ts coerces upstream numeric codes via String() before building structuredError, so + // the string form is the runtime contract this path must honor. + const s = sets(); + const exhausted = applyComboTargetExhaustion(target(), { + ...baseOpts, + errorText: "Forbidden", + structuredError: { message: "generic", type: "1010" }, + result: { status: 403 }, + fallbackResult: { creditsExhausted: false }, + sets: s, + }); + assert.equal(exhausted, false, 'a 403 with structuredError.type "1010" must not mark auth-level'); + assert.equal(s.exhaustedConnections.size, 0); +}); + +test("structuredError code/type fingerprint match is case-insensitive like the text matcher", () => { + // Round 6 finding: fingerprintCode compared code/type case-sensitively while + // isCloudflareFingerprintRejection lowercases text — a 403 carrying + // "BROWSER_SIGNATURE_BANNED" (all-caps from a normalizing gateway) was treated as + // auth-level. Normalize code/type before comparing, mirroring the text matcher. + for (const structuredError of [ + { code: "BROWSER_SIGNATURE_BANNED" }, + { type: "Browser_Signature_Banned" }, + { code: "Fingerprint_Rejection" }, + ] as const) { + const s = sets(); + const exhausted = applyComboTargetExhaustion(target(), { + ...baseOpts, + errorText: "Forbidden", + structuredError, + result: { status: 403 }, + fallbackResult: { creditsExhausted: false }, + sets: s, + }); + assert.equal(exhausted, false, "a mixed-case fingerprint token must not mark auth-level"); + assert.equal(s.exhaustedConnections.size, 0); + } +}); + +test("Cloudflare 1010 inside a non-normalized structuredError.code still avoids auth exhaustion", () => { + // Round 7 finding: a gateway can stuff the raw Cloudflare body ("error_code: 1010") into + // the code field verbatim, so the exact token allowlist misses it and the shared text + // matcher never saw code/type. Feed every candidate string to the text matcher. + const s = sets(); + const exhausted = applyComboTargetExhaustion(target(), { + ...baseOpts, + errorText: "Forbidden", + structuredError: { message: "generic", code: "error_code: 1010" }, + result: { status: 403 }, + fallbackResult: { creditsExhausted: false }, + sets: s, + }); + assert.equal( + exhausted, + false, + "a 403 whose structuredError.code embeds error_code: 1010 must not mark auth-level" + ); + assert.equal(s.exhaustedConnections.size, 0); +}); + // #8137 regression: a SIBLING connection on the SAME provider must NOT be skipped when a // DIFFERENT connection on that provider returned 401/403 — proves the fix at the call-site // level (getExhaustedTargetSkipReason-style check), not just the raw Set contents above. @@ -544,3 +682,20 @@ test("sibling connection on the same provider is NOT skipped after a different c // The failing connection itself IS marked. assert.ok(s.exhaustedConnections.has(`${failingTarget.provider}:${failingTarget.connectionId}`)); }); + +test("401 carrying a real fingerprint signal still marks auth-level (exemption is 403-only)", () => { + // Round 4 finding: Cloudflare 1010 is a 403-only CDN signal. A 401 invalid-credential + // whose errorText carries a genuinely Cloudflare-keyed 1010 (error_code: 1010) must still + // mark auth-level exhaustion on the 401 — only a 403 earns the fingerprint exemption. + // Otherwise a 401 echoing an upstream 1010 would leave a bad credential retryable. + const s = sets(); + const exhausted = applyComboTargetExhaustion(target(), { + ...baseOpts, + errorText: "error_code: 1010, token expired", + result: { status: 401 }, + fallbackResult: { creditsExhausted: false }, + sets: s, + }); + assert.equal(exhausted, true, "a 401 with a fingerprint-looking body must still mark auth-level"); + assert.ok(s.exhaustedConnections.has("test-dedup-provider:conn-1")); +}); diff --git a/tests/unit/error-classifier.test.ts b/tests/unit/error-classifier.test.ts index afb6cd58c1..d9259f5355 100644 --- a/tests/unit/error-classifier.test.ts +++ b/tests/unit/error-classifier.test.ts @@ -1,8 +1,12 @@ import test from "node:test"; import assert from "node:assert/strict"; -const { classifyProviderError, isResourceNotFoundResponse, PROVIDER_ERROR_TYPES } = - await import("../../open-sse/services/errorClassifier.ts"); +const { + classifyProviderError, + isResourceNotFoundResponse, + isCloudflareFingerprintRejection, + PROVIDER_ERROR_TYPES, +} = await import("../../open-sse/services/errorClassifier.ts"); test("classifyProviderError: 401 + account_deactivated => ACCOUNT_DEACTIVATED", () => { const body = JSON.stringify({ @@ -183,3 +187,184 @@ test("classifyProviderError: other request-resource 404 shapes do not poison mod assert.equal(classifyProviderError(404, body, "openai"), null); } }); + +test("classifyProviderError: Cloudflare 1010 (browser signature) => FINGERPRINT_REJECTION, not FORBIDDEN/banned", () => { + // Verbatim body shape from the 2026-08-08 incident: the gateway wraps the + // upstream Cloudflare error inside error.message as a nested JSON string. + const body = JSON.stringify({ + error: { + message: + '[openai/deepseek-v4-flash-free] [403]: {"type":"https://developers.cloudflare.com/support/troubleshooting/http-status-codes/cloudflare-1xxx-errors/error-1010/","title":"Error 1010: Access denied","status":403,"detail":"The site owner has blocked access based on your browser\'s signature.","instance":"a283cb68eb52bda8","error_code":1010,"error_name":"browser_signature_banned"}', + }, + }); + const result = classifyProviderError(403, body, "opencode"); + assert.equal( + result, + PROVIDER_ERROR_TYPES.FINGERPRINT_REJECTION, + "a Cloudflare 1010 must be classified as fingerprint rejection, never FORBIDDEN" + ); + assert.notEqual( + result, + PROVIDER_ERROR_TYPES.FORBIDDEN, + "must not fall through to FORBIDDEN/banned" + ); +}); + +test("classifyProviderError: Cloudflare 1010 via error_name only (no numeric code) => FINGERPRINT_REJECTION", () => { + const body = JSON.stringify({ + error: { + message: + '[403] {"error_name":"browser_signature_banned","detail":"blocked access based on your browser\'s signature"}', + }, + }); + const result = classifyProviderError(403, body); + assert.equal(result, PROVIDER_ERROR_TYPES.FINGERPRINT_REJECTION); +}); + +test("classifyProviderError: plain 403 stays FORBIDDEN (not fingerprint rejection)", () => { + const result = classifyProviderError(403, { error: { message: "you do not have permission" } }); + assert.equal(result, PROVIDER_ERROR_TYPES.FORBIDDEN); +}); + +test("isCloudflareFingerprintRejection: error_code 10101 is NOT 1010 (boundary)", () => { + // Review finding: the code-number match must not trip on a longer numeric suffix. + assert.equal(isCloudflareFingerprintRejection('{"error_code":10101}'), false); + assert.equal(isCloudflareFingerprintRejection("error 10101: something else"), false); +}); + +test("isCloudflareFingerprintRejection: bare browser_signature_banned matches (no quotes needed)", () => { + // The error_name token may arrive bare via structuredError.code, not wrapped in + // quotes inside a JSON string. The token is unique to Cloudflare — safe to match bare. + assert.equal(isCloudflareFingerprintRejection("browser_signature_banned"), true); + assert.equal(isCloudflareFingerprintRejection('{"error_name":"browser_signature_banned"}'), true); +}); + +test("isCloudflareFingerprintRejection: Cloudflare-keyed 1010 forms are detected (round 3/4)", () => { + // The bare number 1010 is deliberately NOT a fingerprint signal (port/count/model id). + // Only explicit Cloudflare-keyed forms — error_code with = or :, the error-1010 URL path, + // the escaped-quote nested JSON form, browser_signature_banned, or the normalized type — + // qualify. + assert.equal(isCloudflareFingerprintRejection("error_code: 1010"), true, "colon code"); + assert.equal(isCloudflareFingerprintRejection("error_code = 1010"), true, "eq code"); + assert.equal( + isCloudflareFingerprintRejection('"error_code": "1010"'), + true, + "string-valued code" + ); + assert.equal( + isCloudflareFingerprintRejection("https://.../cloudflare-1xxx-errors/error-1010/"), + true, + "error-1010 URL path" + ); + // Escaped-quote form: when the upstream body is nested inside the gateway's + // error.message JSON string the quotes carry a backslash (\") — round 4 caught + // that this form was silently relying on the bare-number fallback. + assert.equal( + isCloudflareFingerprintRejection( + String.raw`{"error":{"message":"[403]: {\"error_code\":1010}"}}` + ), + true, + "escaped-quote nested error_code" + ); + assert.equal(isCloudflareFingerprintRejection("BROWSER_SIGNATURE_BANNED"), true, "case token"); + assert.equal(isCloudflareFingerprintRejection("fingerprint_rejection"), true, "normalized type"); +}); + +test("isCloudflareFingerprintRejection: a bare/unkeyed 1010 is NOT a fingerprint (round 4 FP guard)", () => { + // Round 4 finding: matching any standalone 1010 mistook ports/ids/model tokens for + // fingerprint rejections, skipping auth-level exhaustion for genuinely bad credentials. + assert.equal(isCloudflareFingerprintRejection("1010"), false, "bare numeric 1010"); + assert.equal(isCloudflareFingerprintRejection("retry after 1010 seconds"), false, "count/port"); + assert.equal( + isCloudflareFingerprintRejection("model foo-1010 is not supported"), + false, + "model id" + ); + assert.equal( + isCloudflareFingerprintRejection("project 1010 has not been used"), + false, + "project id" + ); +}); + +test("isCloudflareFingerprintRejection: larger numeric/the wrong key are NOT 1010 (boundary)", () => { + assert.equal(isCloudflareFingerprintRejection("error_code:10101"), false); + assert.equal(isCloudflareFingerprintRejection("HTTP 1019"), false); + assert.equal(isCloudflareFingerprintRejection("limit 101 tokens"), false); +}); + +test("isCloudflareFingerprintRejection: error-10101 is NOT 1010 (URL-path boundary, round 6)", () => { + // Round 6 finding: the second regex alternative (error-1010 / error_1010 URL-path form) + // lacked the (?!\d) suffix guard the error_code branch has, so a 4-digit code like 10101 + // matched as 1010. The genuine 1010 path forms still match. + assert.equal(isCloudflareFingerprintRejection("error-10101"), false); + assert.equal(isCloudflareFingerprintRejection("/error_10109/"), false); + assert.equal(isCloudflareFingerprintRejection("error-1010"), true, "hyphen path 1010"); + assert.equal( + isCloudflareFingerprintRejection("error-1010/"), + true, + "hyphen path 1010 with slash" + ); +}); + +test("isCloudflareFingerprintRejection: 1010 must not absorb an alphanumeric suffix (round 7)", () => { + // Round 7 finding: (?!\d) only refused a following DIGIT, so a letter or underscore + // suffix (errorcode1010x / error_code:1010x) sailed through as 1010. (?!\w) refuses any + // word character. The real Cloudflare body ("error_code":1010,"error_name":...) ends the + // code on a non-word boundary and still matches. + assert.equal(isCloudflareFingerprintRejection("errorcode1010x"), false, "letter suffix"); + assert.equal( + isCloudflareFingerprintRejection("error_code:1010x"), + false, + "letter suffix colon form" + ); + assert.equal(isCloudflareFingerprintRejection("error-1010_tail"), false, "underscore suffix"); + assert.equal( + isCloudflareFingerprintRejection('{"error_code":1010,"error_name":"browser_signature_banned"}'), + true, + "real Cloudflare body" + ); +}); + +test("isCloudflareFingerprintRejection: error-1010 with a real-world prefix still matches (gemini round)", () => { + // Review finding (gemini backend): the URL-path alternative required error-1010 to be at + // the string start or after a literal "/", so the real upstream phrasing "[403] error-1010" + // or "Cloudflare error-1010: Access denied" leaked through as FORBIDDEN. A word-boundary + // lookbehind (same as the error_code branch) admits any non-word prefix while still + // rejecting my_error-1010 / xerror-1010. + assert.equal(isCloudflareFingerprintRejection("[403] error-1010"), true, "bracket prefix"); + assert.equal( + isCloudflareFingerprintRejection("Cloudflare error-1010: Access denied"), + true, + "space prefix" + ); + assert.equal(isCloudflareFingerprintRejection("(error-1010)"), true, "paren prefix"); + assert.equal( + isCloudflareFingerprintRejection("my_error-1010"), + false, + "underscore prefix still FP-guarded" + ); + assert.equal( + isCloudflareFingerprintRejection("xerror-1010"), + false, + "letter prefix still FP-guarded" + ); +}); + +test("isCloudflareFingerprintRejection: no word-boundary false positives (round 5)", () => { + // error_code without a preceding word boundary matched my_error_code/twitter_error_code. + assert.equal(isCloudflareFingerprintRejection("my_error_code: 1010"), false); + assert.equal(isCloudflareFingerprintRejection("twitter_error_code 1010"), false); + // The bare phrase "error 1010" without the error_code key or URL path is too broad. + assert.equal(isCloudflareFingerprintRejection("error 1010 something else"), false); +}); + +test("isCloudflareFingerprintRejection: space-separated and URL-path forms match (round 5)", () => { + assert.equal(isCloudflareFingerprintRejection("error code: 1010"), true, "space separator"); + assert.equal(isCloudflareFingerprintRejection("error-code = 1010"), true, "hyphen key"); + assert.equal( + isCloudflareFingerprintRejection("https://.../cloudflare-1xxx-errors/error-1010/"), + true, + "URL path" + ); +});