From 20ea78c9432fb7c19c453958889d13ea05e1d020 Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza Date: Fri, 14 Aug 2026 12:42:58 -0300 Subject: [PATCH] feat(sse): restate agentrouter quota 403/400 as retryable 429 with provider-scoped error rules (#10335) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit agentrouter.org signals temporary quota exhaustion with HTTP 403/400 and a Chinese body (用户额度不足) instead of 429, so clients like Claude Code treat it as permanent and abort, and the fallback engine classified it as a generic apikey AUTH_ERROR. New registry open-sse/config/upstreamStatusRestatement.ts restates those statuses to 429 with a synthetic Retry-After at a single hook in chatCore's providerFailure block (after parseUpstreamError), so classification, combo aggregation and the client response all see a retryable error. 无权访问模型 (permanently no model access) is veto-listed and never restated. agentrouter classification rules are registered in providerErrorRules.ts and reach the real checkFallbackError path through resolveRuleMatchBody() with an exclusive FULL_TEXT_RULE_PROVIDERS allowlist — every other provider keeps its previous behavior byte-for-byte. Known limitations tracked in #10334: the rules' scope field is informational (persistence applies per-model lockout for agentrouter), the 403-only model-access rule has no production path yet, and errors embedded in 200 SSE streams are not restated. Refs #10334 --- docs/architecture/RESILIENCE_GUIDE.md | 117 +++++++++++++++ open-sse/config/providerErrorRules.ts | 94 +++++++++++- open-sse/config/upstreamStatusRestatement.ts | 129 ++++++++++++++++ open-sse/handlers/chatCore.ts | 22 +++ open-sse/services/accountFallback.ts | 16 +- stryker.conf.json | 1 + tests/unit/agentrouter-error-rules.test.ts | 141 ++++++++++++++++++ .../unit/upstream-status-restatement.test.ts | 126 ++++++++++++++++ 8 files changed, 642 insertions(+), 4 deletions(-) create mode 100644 open-sse/config/upstreamStatusRestatement.ts create mode 100644 tests/unit/agentrouter-error-rules.test.ts create mode 100644 tests/unit/upstream-status-restatement.test.ts diff --git a/docs/architecture/RESILIENCE_GUIDE.md b/docs/architecture/RESILIENCE_GUIDE.md index 2529e41b38..98629da7db 100644 --- a/docs/architecture/RESILIENCE_GUIDE.md +++ b/docs/architecture/RESILIENCE_GUIDE.md @@ -289,6 +289,123 @@ single-shot, so usage accounting and semaphore release are not duplicated. --- +## 7. Upstream Status Restatement (misstated quota errors) + +**Scope:** one upstream gateway that reports temporary quota exhaustion with the wrong HTTP status. + +**Purpose:** correct a misleading status BEFORE classification, so downstream consumers (fallback engine, combo aggregation, the client-facing response) see the true retryable nature of the failure. + +Some gateways signal TEMPORARY quota exhaustion with a non-retryable HTTP +status. `agentrouter.org` returns `403` (sometimes `400`) with a Chinese body +(`用户额度不足` / `额度不足`) instead of the standard `429`. Clients like Claude +Code treat `403` as permanent and abort the session, and without correction +the fallback engine would classify it as `AUTH_ERROR` instead of a quota +event. + +**Implementation:** + +- Registry + matcher: `open-sse/config/upstreamStatusRestatement.ts` — a + per-provider list of rules (`{id, fromStatuses, toStatus, textMarkers, +excludeMarkers, defaultRetryAfterMs}`), matched via `applyStatusRestatement()`. +- Call site: the `providerFailure:` block in `open-sse/handlers/chatCore.ts` + (around line 3654), right after `parseUpstreamError()` parses an upstream + response with an error HTTP status (`!providerResponse.ok`), and before any + classification runs, so every downstream consumer sees the corrected + status. Errors embedded inside a `200` SSE stream follow a separate, + later stream-parsing path and are **not** covered by this hook today — a + known limitation, not yet needed for agentrouter's misstatus (which + surfaces as an error HTTP status). +- Retry eligibility: `429` is in `RETRY_AFTER_ELIGIBLE_STATUSES` + (`open-sse/services/combo/unavailableRetryGate.ts`), so a restated error + carries a real retry window instead of surfacing as a dead `403`. +- The synthetic `60s` `defaultRetryAfterMs` (`upstreamStatusRestatement.ts`) + is only what the restated response tells the **client**; it is not itself + the connection's internal cooldown/lockout duration — that is governed + separately by whichever mechanism actually handles the restated error + (Connection Cooldown's escalating backoff, §2, base `3s` for API-key + providers; or Model Lockout, §3, for per-model-quota providers like + agentrouter). The router can become eligible to retry internally sooner + than the 60s window it advertises to the client — intentional headroom, + not a bug. + +Permanent errors (agentrouter's `无权访问模型` — no access to this model) are +NEVER restated: `excludeMarkers` vetoes the rule even when `textMarkers` hit, +so the error keeps its original status and nothing retries it forever. A +separate provider classification rule +(`agentrouter-model-access-denied` in `open-sse/config/providerErrorRules.ts`) +declares an `auth_error`/scope-`model` match for this text, but it does not +fire on the live production path today: the rule only matches `status === +403`, and `checkFallbackError`'s apikey-category `FORBIDDEN` branch +(`open-sse/services/accountFallback.ts`) returns early for a plain 403 +*before* the provider-rule lookup ever runs. In practice a `无权访问模型` 403 +is handled the same way as the base apikey-provider 403 path (see Connection +Cooldown, §2), not as a 6h model lockout. The rule still exists as a +declarative classification consumable by future callers of `classifyError` +with context — wiring it into the production `checkFallbackError` path is +tracked as a follow-up, not yet done. + +Restated quota errors (`额度不足`) do reach a provider rule in production +(`agentrouter-user-quota-exhausted`, scope `"connection"`), but `scope` on +`ProviderErrorRuleMatch` is currently informational — the persistence path +(`checkFallbackError` → `combo.ts`) only consumes `reason` and `cooldownMs`, +never `scope`. What actually happens for agentrouter (`passthroughModels: +true` → `hasPerModelQuota()` returns `true`) is a **per-model** lockout via +`recordModelLockoutFailure()`: the connection itself is never cooled down for +this error (`combo.ts` skips `recordProviderCooldown` for 429 when +`hasPerModelQuota` is true), so other models on the same account keep being +tried — each one burns one call and its own lockout before combo routing +moves on. Honoring `scope` end-to-end (so a `"connection"` match actually +locks the connection) is tracked as a follow-up. + +### Two-stage design: status restatement, then classification + +Status restatement (`upstreamStatusRestatement.ts`) and provider +classification rules (`open-sse/config/providerErrorRules.ts`, +`providerRuleRegistry`) are separate registries that both key on provider id +and text markers, but they run in different places and serve different +purposes: restatement rewrites the HTTP status early in `chatCore.ts`; +classification rules pick the fallback `reason` and lock `scope` +(`model` / `provider` / `connection`) inside `checkFallbackError()` +(`open-sse/services/accountFallback.ts`). + +Classification rules only see full error **text** (needed to match body +markers like `额度不足`) for providers listed in the `FULL_TEXT_RULE_PROVIDERS` +allowlist in `providerErrorRules.ts` — currently only `"agentrouter"`. For +every other provider, `checkFallbackError` hands `getProviderErrorRuleMatch` +only the structured error (`{code, type}`), which is enough for +header/status/code-based rules but blind to body-text markers. The helper +`resolveRuleMatchBody()` performs this selection: full error text for +allowlisted providers, the structured error otherwise. Adding a provider to +`FULL_TEXT_RULE_PROVIDERS` is an explicit per-provider opt-in — it exists so +that the default path for every provider not on the list stays +byte-for-byte unchanged. + +### Adding a new quota-misstating gateway + +1. Register one rule array in `statusRestatementRegistry` + (`open-sse/config/upstreamStatusRestatement.ts`). Keep `textMarkers` + provider-specific; never reuse generic English phrases that collide with + `CREDITS_EXHAUSTED_SIGNALS` (`open-sse/services/accountFallback.ts`). +2. Optionally register classification rules in + `open-sse/config/providerErrorRules.ts` (`providerRuleRegistry`) to pick + the right lock scope (`connection` for account-wide quota, `model` for + per-model errors). This step only takes effect in production for + providers whose rules need the full error text (body markers): add the + provider id to `FULL_TEXT_RULE_PROVIDERS` in the same file — otherwise + `checkFallbackError` only ever hands the rule the structured + `{code, type}` error and a body-text rule will never match live traffic. + Rules that match purely on `status`/`headers` (like Opencode's or + Minimax's) do not need this opt-in. +3. Add unit tests mirroring `tests/unit/upstream-status-restatement.test.ts` + and `tests/unit/agentrouter-error-rules.test.ts` (including the + not-permanent / not-creditsExhausted guards, and — if the provider needs + the allowlist — a test asserting `resolveRuleMatchBody()` returns the + full text only for that provider). + +No changes to `chatCore.ts`, `classifyError`, or combo are needed. + +--- + ## Other Resilience Features - **19 routing strategies** (priority, weighted, round-robin, context-relay, fill-first, p2c, random, least-used, cost-optimized, reset-aware, reset-window, headroom, strict-random, auto, lkgp, context-optimized, cache-optimized, fusion, pipeline) — see [AUTO-COMBO.md](../routing/AUTO-COMBO.md). diff --git a/open-sse/config/providerErrorRules.ts b/open-sse/config/providerErrorRules.ts index ce5c74e702..dd59e991bc 100644 --- a/open-sse/config/providerErrorRules.ts +++ b/open-sse/config/providerErrorRules.ts @@ -29,7 +29,15 @@ export type ProviderErrorRule = { export type ProviderErrorRuleMatch = { reason: ConfiguredErrorReason; - /** Default "provider" — lock the whole connection so other providers take over. */ + /** + * Intended lock scope. NOTE: this field is currently INFORMATIONAL — no + * consumer of `getProviderErrorRuleMatch` (checkFallbackError, combo.ts) + * reads `scope` today; only `reason` and `cooldownMs` are consulted. The + * actual lock scope applied at runtime is decided independently by each + * call site (e.g. `hasPerModelQuota()` deciding model- vs connection-level + * lockout). Honoring this field end-to-end is tracked as a follow-up — + * see `docs/architecture/RESILIENCE_GUIDE.md` §7. + */ scope: "model" | "provider" | "connection"; /** Optional explicit cooldown; falls back to the existing per-reason defaults. */ cooldownMs?: number; @@ -176,6 +184,61 @@ function buildOpenrouterRules(): ProviderErrorRule[] { ]; } +// ─── AgentRouter ──────────────────────────────────────────────────────────── +// agentrouter.org misstates temporary quota exhaustion as 403/400 with a +// Chinese body. upstreamStatusRestatement.ts rewrites the status to 429 +// BEFORE classification, so rules here accept both the raw 403/400 and the +// restated 429 (text is the real discriminator either way). In production, +// the raw 403 path is what actually matters here: checkFallbackError's +// apikey-category FORBIDDEN branch (~line 1699) returns EARLY for a plain +// 403, before these rules are ever consulted — these rules fire on the +// RESTATED 429 (chatCore's upstreamStatusRestatement hook runs first) via +// resolveRuleMatchBody, which is the only path in checkFallbackError that +// hands these rules the full error text instead of just {code, type}. +// - "额度不足": account-wide temporary quota → quota_exhausted, scope +// "connection" (mirror of the Opencode account-wide rationale above). +// NOTE: `scope` on ProviderErrorRuleMatch is currently informational — +// checkFallbackError/combo.ts only consume `reason` and `cooldownMs`, not +// `scope`. For agentrouter specifically (passthroughModels: true → +// hasPerModelQuota() is true), this quota_exhausted match actually +// resolves to a PER-MODEL lockout (recordModelLockoutFailure), not a +// connection-wide lock — other models on the same account keep being +// tried by combo routing (each burning one call) until they lock out +// individually. Honoring `scope` end-to-end is tracked as a follow-up. +// - "无权访问模型": declares auth_error/scope "model" (intent: lock only the +// model so the connection keeps serving the rest — Model Lockout tier). +// This rule does NOT fire on the production path today: it only matches +// `status === 403`, but checkFallbackError's apikey FORBIDDEN branch +// returns early for a plain 403 before this rule is ever consulted (see +// the note above). A live `无权访问模型` 403 is handled like the base +// apikey-provider 403 today. Wiring this rule into that path is tracked +// as a follow-up. +function buildAgentrouterRules(): ProviderErrorRule[] { + const AGENTROUTER_ERROR_STATUSES = new Set([400, 403, 429]); + return [ + { + id: "agentrouter-user-quota-exhausted", + match: ({ status, body }) => { + if (!AGENTROUTER_ERROR_STATUSES.has(status)) return null; + const text = JSON.stringify(body ?? "").toLowerCase(); + if (!text.includes("额度不足")) return null; + return { reason: "quota_exhausted", scope: "connection" }; + }, + }, + { + id: "agentrouter-model-access-denied", + match: ({ status, body }) => { + if (status !== 403) return null; + const text = JSON.stringify(body ?? "").toLowerCase(); + if (!text.includes("无权访问模型")) return null; + // 6h: effectively "until the operator fixes the key's model grants", + // without being an unrecoverable terminal state. + return { reason: "auth_error", scope: "model", cooldownMs: 6 * 60 * 60 * 1000 }; + }, + }, + ]; +} + /** * Global registry. Provider name → ordered list of rules (first match wins). * Add new providers here; the matcher in classifyError will pick them up @@ -189,8 +252,37 @@ export const providerRuleRegistry = new Map([ ["minimax-passthrough", buildMinimaxRules()], ["cloudflare-ai", buildCloudflareAiRules()], ["openrouter", buildOpenrouterRules()], + ["agentrouter", buildAgentrouterRules()], ]); +/** + * Providers whose rules match on the FULL upstream error text. + * checkFallbackError's rule lookup normally passes only the structured + * error ({code, type} — message stripped by the combo callers), which is + * enough for header/status/code rules but blind to body-text markers like + * agentrouter's "额度不足". Providers in this set get the raw error text as + * the match body instead. EXCLUSIVE allowlist by owner decision (2026-08-13): + * adding a provider here is an explicit opt-in — the default path for every + * other provider must remain byte-for-byte unchanged. + */ +const FULL_TEXT_RULE_PROVIDERS = new Set(["agentrouter"]); + +/** + * Resolve the body handed to getProviderErrorRuleMatch inside + * checkFallbackError: full error text for FULL_TEXT_RULE_PROVIDERS, + * the structured error for everyone else. + */ +export function resolveRuleMatchBody( + provider: string | null | undefined, + structuredError: unknown, + errorText: string | null | undefined +): unknown { + if (provider && FULL_TEXT_RULE_PROVIDERS.has(provider.toLowerCase()) && errorText) { + return errorText; + } + return structuredError ?? null; +} + /** * Returns the first matching rule for a provider, or null if none match. * Callers use this to (a) classify the reason and (b) decide whether to diff --git a/open-sse/config/upstreamStatusRestatement.ts b/open-sse/config/upstreamStatusRestatement.ts new file mode 100644 index 0000000000..ccadf2d704 --- /dev/null +++ b/open-sse/config/upstreamStatusRestatement.ts @@ -0,0 +1,129 @@ +/** + * Upstream status restatement — registry of gateways that MISSTATE temporary + * quota exhaustion as a non-retryable HTTP status. + * + * agentrouter.org signals "user quota exhausted" with 403 (sometimes 400) and + * a Chinese body ("用户额度不足") instead of the standard 429. Clients like + * Claude Code treat 403 as permanent and abort the whole session, and our own + * fallback engine classifies it as AUTH_ERROR instead of a quota event. + * + * applyStatusRestatement() is called from exactly ONE place — the + * `providerFailure:` block in open-sse/handlers/chatCore.ts, right after + * parseUpstreamError() parses an upstream response with an error HTTP status + * (!providerResponse.ok), and before any classification runs — so every + * downstream consumer (checkFallbackError, combo aggregation, the client + * response) sees the corrected status. Errors embedded inside a 200 SSE + * stream follow a separate, later stream-parsing path and are NOT covered by + * this hook today (known limitation; not yet needed for agentrouter's + * misstatus, which surfaces as an error HTTP status). 429 is + * Retry-After-eligible in + * open-sse/services/combo/unavailableRetryGate.ts, so the client also gets a + * retry window instead of a dead 403. + * + * Adding a future gateway with the same defect = register ONE rule array + * below (and, for cooldown-scope refinement, one entry in + * providerErrorRules.ts). No pipeline changes. + * + * Marker discipline: keep textMarkers provider-specific (the Chinese strings + * are upstream error literals, not UI copy). Generic English phrases like + * "insufficient_quota" are in CREDITS_EXHAUSTED_SIGNALS + * (accountFallback.ts) and would flip the connection into a terminal + * credits_exhausted state — never use them as markers here. + * + * Accepted trade-off: matching only on response body text means a + * legitimate 400 whose body ECHOES user-supplied content containing a + * marker (e.g. a prompt that itself contains "额度不足") would be restated to + * 429 and lose the combo's 400 stop-guard. This is treated as an acceptable + * risk because these markers are rare outside a genuine upstream error; + * keeping markers short, provider-specific, and non-generic (as above) + * minimizes false-positive restatement. + */ + +export type UpstreamStatusRestatementRule = { + id: string; + fromStatuses: ReadonlySet; + toStatus: number; + /** Lowercase markers matched against lowercased `message` + JSON(body). Any hit → restate. */ + textMarkers: readonly string[]; + /** Lowercase markers that VETO the rule even when textMarkers hit (permanent errors). */ + excludeMarkers?: readonly string[]; + /** Synthetic Retry-After used ONLY when the upstream provided none. */ + defaultRetryAfterMs?: number; +}; + +export type StatusRestatementInput = { + provider: string | null | undefined; + status: number; + message: string | null | undefined; + body?: unknown; + retryAfterMs?: number | null; +}; + +export type StatusRestatementResult = { + status: number; + retryAfterMs: number | null; + ruleId: string | null; + fromStatus: number; +}; + +// ─── agentrouter ──────────────────────────────────────────────────────────── +// Observed misstatus (ClaudeShield field reports + upstream behavior): +// 403 "用户额度不足" / "额度不足" → temporary user-quota exhaustion → 429 +// 400 variants carrying the same quota text → 429 +// 403 "无权访问模型" (no access to this model) → genuinely permanent, NEVER +// restated — it must keep flowing as 403 so nothing retries it forever. +const AGENTROUTER_RULES: UpstreamStatusRestatementRule[] = [ + { + id: "agentrouter-quota-misstatus", + fromStatuses: new Set([403, 400]), + toStatus: 429, + textMarkers: ["额度不足"], + excludeMarkers: ["无权访问"], + defaultRetryAfterMs: 60_000, + }, +]; + +/** Provider id (lowercase) → ordered rules; first match wins. */ +export const statusRestatementRegistry = new Map([ + ["agentrouter", AGENTROUTER_RULES], +]); + +function stringifyBody(body: unknown): string { + if (body === null || body === undefined) return ""; + if (typeof body === "string") return body; + try { + return JSON.stringify(body); + } catch { + return ""; + } +} + +export function applyStatusRestatement(input: StatusRestatementInput): StatusRestatementResult { + const passthrough: StatusRestatementResult = { + status: input.status, + retryAfterMs: input.retryAfterMs ?? null, + ruleId: null, + fromStatus: input.status, + }; + if (!input.provider) return passthrough; + const rules = statusRestatementRegistry.get(input.provider.toLowerCase()); + if (!rules) return passthrough; + + const haystack = `${input.message ?? ""} ${stringifyBody(input.body)}`.toLowerCase(); + if (!haystack.trim()) return passthrough; + + for (const rule of rules) { + if (!rule.fromStatuses.has(input.status)) continue; + if (!rule.textMarkers.some((marker) => haystack.includes(marker))) continue; + if (rule.excludeMarkers?.some((marker) => haystack.includes(marker))) continue; + const upstreamRetryAfterMs = + typeof input.retryAfterMs === "number" && input.retryAfterMs > 0 ? input.retryAfterMs : null; + return { + status: rule.toStatus, + retryAfterMs: upstreamRetryAfterMs ?? rule.defaultRetryAfterMs ?? null, + ruleId: rule.id, + fromStatus: input.status, + }; + } + return passthrough; +} diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 35e579a45d..8976a4272b 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -190,6 +190,7 @@ import { DEFAULT_MAX_TOKENS, STREAM_DISCONNECT_GRACE_PERIOD_MS, } from "../config/constants.ts"; +import { applyStatusRestatement } from "../config/upstreamStatusRestatement.ts"; import { createRecoverableStream, makeContinuationBody } from "../services/streamRecovery.ts"; import { resolveResilienceSettings, @@ -3667,6 +3668,27 @@ export async function handleChatCore({ upstreamErrorType = details.errorType as string | undefined; } + // Gateways like agentrouter misstate temporary quota exhaustion as 403/400, + // which downstream classification treats as AUTH_ERROR and clients like + // Claude Code treat as permanent. Restate to 429 (+ synthetic Retry-After) + // BEFORE any classification so both the fallback engine and the surfaced + // client status see a retryable error. Registry-scoped per provider. + const restatement = applyStatusRestatement({ + provider, + status: statusCode, + message, + body: upstreamErrorBody, + retryAfterMs, + }); + if (restatement.ruleId) { + statusCode = restatement.status; + retryAfterMs = restatement.retryAfterMs; + log?.info?.( + "STATUS_RESTATE", + `${provider} ${restatement.fromStatus}→${statusCode} (${restatement.ruleId})` + ); + } + const signatureRecovery = await recoverAnthropicThinkingSignature({ provider, statusCode, diff --git a/open-sse/services/accountFallback.ts b/open-sse/services/accountFallback.ts index ec8ab35cd1..5435a0e2ea 100644 --- a/open-sse/services/accountFallback.ts +++ b/open-sse/services/accountFallback.ts @@ -15,7 +15,7 @@ import { serviceSupervisorCooldown, isNimFunctionDegraded, } from "../config/errorConfig.ts"; -import { getProviderErrorRuleMatch } from "../config/providerErrorRules.ts"; +import { getProviderErrorRuleMatch, resolveRuleMatchBody } from "../config/providerErrorRules.ts"; import * as rot from "./rotationConfig.ts"; import { getPassthroughProviders, getProviderCategory } from "../config/providerRegistry.ts"; import { @@ -1743,7 +1743,12 @@ export function checkFallbackError( // specific configured reasons (e.g. 503 → SERVER_ERROR would be // shadowed by 503 → MODEL_CAPACITY). const providerMatch = provider - ? getProviderErrorRuleMatch(provider, status, headers, structuredError ?? null) + ? getProviderErrorRuleMatch( + provider, + status, + headers, + resolveRuleMatchBody(provider, structuredError ?? null, errorStr) + ) : null; const reason = providerMatch ? providerMatch.reason @@ -1776,7 +1781,12 @@ export function checkFallbackError( // generic zero-cooldown default. Mirror the backoff branch above so // provider rules win on cooldown/reason regardless of `backoff`. const providerMatch = provider - ? getProviderErrorRuleMatch(provider, status, headers, structuredError ?? null) + ? getProviderErrorRuleMatch( + provider, + status, + headers, + resolveRuleMatchBody(provider, structuredError ?? null, errorStr) + ) : null; const cooldownMs = providerMatch?.cooldownMs ?? configuredRule.cooldownMs ?? 0; return { diff --git a/stryker.conf.json b/stryker.conf.json index 7894fae4ab..af974a0ff7 100644 --- a/stryker.conf.json +++ b/stryker.conf.json @@ -63,6 +63,7 @@ "tests/unit/adaptive-admission-route-matrix.test.ts", "tests/unit/adaptive-admission-runtime.test.ts", "tests/unit/adobe-firefly.test.ts", + "tests/unit/agentrouter-error-rules.test.ts", "tests/unit/alibaba-free-tier-exhaustion.test.ts", "tests/unit/anthropic-thinking-signature-recovery.test.ts", "tests/unit/antigravity-429-quota-tdd.test.ts", diff --git a/tests/unit/agentrouter-error-rules.test.ts b/tests/unit/agentrouter-error-rules.test.ts new file mode 100644 index 0000000000..2315a3076c --- /dev/null +++ b/tests/unit/agentrouter-error-rules.test.ts @@ -0,0 +1,141 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +/** + * agentrouter.org quota model, as declared by the `providerErrorRules.ts` + * classification layer: + * - "额度不足" (quota insufficient) is ACCOUNT-wide and temporary → the rule + * declares scope "connection". + * - "无权访问模型" (no access to this model) is permanent PER MODEL → the rule + * declares scope "model". + * Status matching accepts both the raw upstream 403 AND the restated 429 + * (upstreamStatusRestatement.ts rewrites 403→429 before classification). + * + * IMPORTANT — `scope` above is what the rule DECLARES, not what production + * enforces: `ProviderErrorRuleMatch.scope` is not consumed by + * checkFallbackError/combo.ts today (only `reason`/`cooldownMs` are). For + * agentrouter (passthroughModels: true → hasPerModelQuota() true), the + * quota_exhausted match actually resolves to a PER-MODEL lockout in + * production, not a connection-wide lock — other models on the same account + * keep being tried by combo routing until they lock out individually. And + * the "无权访问模型" rule never reaches production traffic at all today: it + * only matches raw `status === 403`, but checkFallbackError's apikey + * FORBIDDEN branch returns early for a plain 403 before any provider rule is + * consulted (see A7). See `docs/architecture/RESILIENCE_GUIDE.md` §7 for the + * full writeup and the tracked follow-up to honor `scope`. + */ + +const { providerRuleRegistry, getProviderErrorRuleMatch } = await import( + "../../open-sse/config/providerErrorRules.ts" +); +const { classifyError, checkFallbackError } = await import( + "../../open-sse/services/accountFallback.ts" +); +const { RateLimitReason } = await import("../../open-sse/config/constants.ts"); + +test("A1: agentrouter is registered in providerRuleRegistry", () => { + const rules = providerRuleRegistry.get("agentrouter"); + assert.ok(rules && rules.length > 0); +}); + +test("A2: quota body → quota_exhausted scope connection (restated 429)", () => { + const match = getProviderErrorRuleMatch("agentrouter", 429, {}, { + error: { message: "用户额度不足，请充值" }, + }); + assert.ok(match, "quota body must match"); + assert.equal(match.reason, "quota_exhausted"); + assert.equal(match.scope, "connection"); +}); + +test("A3: quota body also matches the raw (pre-restatement) 403", () => { + const match = getProviderErrorRuleMatch("agentrouter", 403, {}, "用户额度不足"); + assert.ok(match); + assert.equal(match.reason, "quota_exhausted"); +}); + +test("A4: 无权访问模型 → auth_error scope model, at the RULE layer only (getProviderErrorRuleMatch directly) — this rule never receives production traffic (see A7): checkFallbackError's apikey FORBIDDEN branch returns early for a plain 403 before reaching this rule", () => { + const match = getProviderErrorRuleMatch("agentrouter", 403, {}, { + error: { message: "无权访问模型 claude-sonnet-4" }, + }); + assert.ok(match); + assert.equal(match.reason, "auth_error"); + assert.equal(match.scope, "model"); +}); + +test("A5: classifyError layer guard — quota text wins over the 403→AUTH_ERROR status fallback (classifyError itself has no production caller today; the production guard is A6/checkFallbackError)", () => { + const reason = classifyError(403, "用户额度不足", { + provider: "agentrouter", + headers: {}, + body: { error: { message: "用户额度不足" } }, + }); + assert.equal(reason, RateLimitReason.QUOTA_EXHAUSTED); +}); + +test("A6: guard — restated quota error is retryable, never terminal, and now actually classified as quota_exhausted", () => { + // Status 429 (post-restatement) reaches checkFallbackError's provider-rule + // lookup. resolveRuleMatchBody() hands agentrouter the full error text + // (instead of just the stripped {code, type} structuredError every other + // provider gets), so the "额度不足" rule actually fires here — this is the + // production path the restatement hook (Task 2) feeds into. + const result = checkFallbackError(429, "用户额度不足", 0, null, "agentrouter", null); + assert.equal(result.shouldFallback, true); + assert.equal(result.reason, "quota_exhausted"); + assert.ok(!result.permanent, "quota misstatus must never be permanent"); + assert.ok(!result.creditsExhausted, "must not trip CREDITS_EXHAUSTED_SIGNALS"); + assert.ok(result.cooldownMs > 0, "must carry a real cooldown"); +}); + +test("A7: guard — raw 403 quota (hook bypassed) is still not account-deactivation", () => { + // A raw (pre-restatement) 403 never actually reaches the agentrouter provider + // rules in production: checkFallbackError's apikey-category FORBIDDEN branch + // (status === 403 && getProviderCategory(provider) === "apikey") returns + // EARLY via resolveApiKeyForbiddenFallback before the provider-rule lookup + // is ever consulted. In the real pipeline, chatCore's upstreamStatusRestatement + // hook (Task 2) already converts 403→429 before checkFallbackError ever sees + // it, so this early-return path is what a hook-bypassed raw 403 hits — and it + // must still not be misclassified as permanent account deactivation. + const result = checkFallbackError(403, "用户额度不足", 0, null, "agentrouter", null); + assert.equal(result.shouldFallback, true); + assert.ok(!result.permanent); +}); + +test("A8: plain agentrouter 403 (no quota text) keeps the default apikey auth path", () => { + const match = getProviderErrorRuleMatch("agentrouter", 403, {}, "Invalid API key"); + assert.equal(match, null); +}); + +test("A9: resolveRuleMatchBody hands full text ONLY to allowlisted providers", async () => { + const { resolveRuleMatchBody } = await import( + "../../open-sse/config/providerErrorRules.ts" + ); + const structured = { code: "rate_limited", type: "requests" }; + assert.equal(resolveRuleMatchBody("agentrouter", structured, "用户额度不足"), "用户额度不足"); + assert.equal(resolveRuleMatchBody("opencode", structured, "monthly usage limit reached"), structured); + assert.equal(resolveRuleMatchBody("openrouter", null, "some error text"), null); + assert.equal(resolveRuleMatchBody("agentrouter", structured, ""), structured); +}); + +test("A10: other providers' checkFallbackError behavior is unchanged (exclusivity)", () => { + // opencode's body-text rule ("organization_quota_exceeded") must still NOT + // fire through checkFallbackError — the allowlist is agentrouter-only, so + // opencode keeps getting only the stripped structuredError as the match + // body (null here, since no structuredError arg is passed), same as before + // this fix. Baseline captured on the pre-fix code with this exact input: + // { shouldFallback: true, cooldownMs: 3000, baseCooldownMs: 3000, + // newBackoffLevel: 1, usedUpstreamRetryHint: false, + // reason: "rate_limit_exceeded" } + // i.e. it falls through to the generic 429 configured rule, NOT the + // opencode-quota-exhausted-body provider rule — asserting `reason` here is + // exactly what proves the allowlist didn't leak to opencode. + const result = checkFallbackError( + 429, + '{"error":{"message":"organization_quota_exceeded"}}', + 0, + null, + "opencode", + null + ); + assert.ok(result.shouldFallback); + assert.equal(result.reason, "rate_limit_exceeded"); + assert.equal(result.cooldownMs, 3000); +}); diff --git a/tests/unit/upstream-status-restatement.test.ts b/tests/unit/upstream-status-restatement.test.ts new file mode 100644 index 0000000000..7c59bc8b15 --- /dev/null +++ b/tests/unit/upstream-status-restatement.test.ts @@ -0,0 +1,126 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +/** + * Gateways like agentrouter.org misstate TEMPORARY quota exhaustion as 403/400 + * (Chinese body "用户额度不足"), which Claude Code treats as permanent and dies. + * applyStatusRestatement() rewrites such statuses to 429 (+ synthetic + * Retry-After) in ONE place, before fallback classification and before the + * status ever reaches the client. Registry-driven: future gateways with the + * same defect register one rule array — no pipeline changes. + */ + +const { applyStatusRestatement, statusRestatementRegistry } = await import( + "../../open-sse/config/upstreamStatusRestatement.ts" +); + +test("R1: agentrouter 403 + 用户额度不足 → 429 with synthetic Retry-After", () => { + const out = applyStatusRestatement({ + provider: "agentrouter", + status: 403, + message: '{"error":{"message":"用户额度不足","type":"insufficient_user_quota"}}', + retryAfterMs: null, + }); + assert.equal(out.status, 429); + assert.equal(out.fromStatus, 403); + assert.equal(out.ruleId, "agentrouter-quota-misstatus"); + assert.equal(out.retryAfterMs, 60_000); +}); + +test("R2: agentrouter 403 + 无权访问模型 (no model access) is NOT restated", () => { + const out = applyStatusRestatement({ + provider: "agentrouter", + status: 403, + message: "无权访问模型 claude-sonnet-4", + retryAfterMs: null, + }); + assert.equal(out.status, 403); + assert.equal(out.ruleId, null); +}); + +test("R3: quota marker in body (not message) still restates", () => { + const out = applyStatusRestatement({ + provider: "agentrouter", + status: 403, + message: "Forbidden", + body: { error: { message: "用户额度不足，请充值" } }, + retryAfterMs: null, + }); + assert.equal(out.status, 429); +}); + +test("R4: upstream-provided retryAfterMs wins over the synthetic default", () => { + const out = applyStatusRestatement({ + provider: "agentrouter", + status: 403, + message: "用户额度不足", + retryAfterMs: 5_000, + }); + assert.equal(out.status, 429); + assert.equal(out.retryAfterMs, 5_000); +}); + +test("R5: agentrouter 400 with quota marker also restates (gateway variant)", () => { + const out = applyStatusRestatement({ + provider: "agentrouter", + status: 400, + message: "额度不足", + retryAfterMs: null, + }); + assert.equal(out.status, 429); +}); + +test("R6: agentrouter 403 without quota markers is untouched (real auth error)", () => { + const out = applyStatusRestatement({ + provider: "agentrouter", + status: 403, + message: "Invalid API key", + retryAfterMs: null, + }); + assert.equal(out.status, 403); + assert.equal(out.ruleId, null); +}); + +test("R7: other providers never match agentrouter rules (registry-scoped)", () => { + const out = applyStatusRestatement({ + provider: "openai", + status: 403, + message: "用户额度不足", + retryAfterMs: null, + }); + assert.equal(out.status, 403); +}); + +test("R8: statuses a rule does not list pass through (already-correct 429)", () => { + const out = applyStatusRestatement({ + provider: "agentrouter", + status: 429, + message: "用户额度不足", + retryAfterMs: 1_000, + }); + assert.equal(out.status, 429); + assert.equal(out.ruleId, null); + assert.equal(out.retryAfterMs, 1_000); +}); + +test("R9: registry exposes agentrouter so future gateways copy the one-line recipe", () => { + const rules = statusRestatementRegistry.get("agentrouter"); + assert.ok(rules && rules.length > 0); +}); + +test("R10: chatCore wires applyStatusRestatement into the providerFailure block", async () => { + // chatCore is a god-file that cannot be imported standalone in unit tests + // (side-effectful DB/env wiring), so the wiring contract is asserted at the + // source level: the hook must exist, run against the parsed error, and + // reassign both statusCode and retryAfterMs BEFORE classification. + const { readFile } = await import("node:fs/promises"); + const src = await readFile( + new URL("../../open-sse/handlers/chatCore.ts", import.meta.url), + "utf8" + ); + assert.match(src, /applyStatusRestatement\(/, "chatCore must call applyStatusRestatement"); + const hookIndex = src.indexOf("applyStatusRestatement("); + const classifyIndex = src.indexOf("classifyProviderError(statusCode"); + assert.ok(hookIndex > -1 && classifyIndex > -1 && hookIndex < classifyIndex, + "restatement must run BEFORE classifyProviderError so fallback sees the corrected status"); +});