diff --git a/changelog.d/fixes/13146-opencode-400-model-lock.md b/changelog.d/fixes/13146-opencode-400-model-lock.md new file mode 100644 index 0000000000..3666287873 --- /dev/null +++ b/changelog.d/fixes/13146-opencode-400-model-lock.md @@ -0,0 +1 @@ +- **fix(providers):** lock opencode model on upstream 400 model-unavailable ([#13146](https://github.com/diegosouzapw/OmniRoute/pull/13146)) — thanks @maxmad64bis diff --git a/open-sse/config/providerErrorRules.ts b/open-sse/config/providerErrorRules.ts index 6f00d2c5f1..4c6e4e094c 100644 --- a/open-sse/config/providerErrorRules.ts +++ b/open-sse/config/providerErrorRules.ts @@ -32,7 +32,7 @@ export type ProviderErrorRuleMatch = { /** * Intended lock scope. #10334: for a BUILT-IN catalog rule, this field is * CONSUMED end-to-end only for providers in `HONORS_RULE_LOCK_SCOPE_PROVIDERS` - * (agentrouter-exclusive today, gated by `honorsRuleLockScope()`) — for those, + * (agentrouter + the opencode family, gated by `honorsRuleLockScope()`) — for * `checkFallbackError` surfaces it as `ruleScope` on its return value for the * persistence layer to honor instead of re-deriving scope from * `hasPerModelQuota()`. For every other built-in-rule provider it remains @@ -155,6 +155,19 @@ function buildOpencodeRules(): ProviderErrorRule[] { return null; }, }, + { + id: "opencode-400-model-unavailable", + match: ({ status, body }) => { + if (status !== 400) return null; + const text = JSON.stringify(body ?? "").toLowerCase(); + if (!text.includes("upstream request failed: model is unavailable.")) return null; + return { + reason: "model_capacity", + scope: "model", + cooldownMs: 3_600_000, + }; + }, + }, ]; } @@ -290,15 +303,16 @@ function buildAgentrouterRules(): ProviderErrorRule[] { ]; } +/** Providers sharing the opencode upstream envelope, hence the opencode catalog rules. */ +const OPENCODE_RULE_FAMILY = ["opencode", "opencode-zen", "opencode-go", "opencode-cli"]; + /** * Global registry. Provider name → ordered list of rules (first match wins). * Add new providers here; the matcher in classifyError will pick them up * automatically. */ export const providerRuleRegistry = new Map([ - ["opencode", buildOpencodeRules()], - ["opencode-go", buildOpencodeRules()], - ["opencode-cli", buildOpencodeRules()], + ...OPENCODE_RULE_FAMILY.map((id): [string, ProviderErrorRule[]] => [id, buildOpencodeRules()]), ["minimax", buildMinimaxRules()], ["minimax-passthrough", buildMinimaxRules()], ["cloudflare-ai", buildCloudflareAiRules()], @@ -323,7 +337,7 @@ export const providerRuleRegistry = new Map([ * mechanism (#11104) silently inert for every provider except the ones listed * below. See `hasOperatorRuleForProvider`. */ -const HONORS_RULE_LOCK_SCOPE_PROVIDERS = new Set(["agentrouter"]); +const HONORS_RULE_LOCK_SCOPE_PROVIDERS = new Set(["agentrouter", ...OPENCODE_RULE_FAMILY]); export function honorsRuleLockScope(provider: string | null | undefined): boolean { if (!provider) return false; @@ -509,3 +523,21 @@ export function parseResetCountdownMs(text: string): number | null { return null; } } + +/** + * Opencode-family "Upstream request failed: Model is unavailable." 400: the rule's + * model-scope match, or null for any other provider, status or rule. Takes the raw + * error text so it stays independent of FULL_TEXT_RULE_PROVIDERS (#10880). + */ +export function getOpencodeModelUnavailableMatch( + provider: string | null | undefined, + status: number, + headers: Headers | Record | null | undefined, + errorText: unknown +): ProviderErrorRuleMatch | null { + if (status !== 400 || !provider || !OPENCODE_RULE_FAMILY.includes(provider.toLowerCase())) { + return null; + } + const match = getProviderErrorRuleMatch(provider, status, headers, errorText); + return match?.scope === "model" && match.reason === "model_capacity" ? match : null; +} diff --git a/open-sse/services/accountFallback.ts b/open-sse/services/accountFallback.ts index 2f05ffc6e7..bf66815a5d 100644 --- a/open-sse/services/accountFallback.ts +++ b/open-sse/services/accountFallback.ts @@ -16,6 +16,7 @@ import { isNimFunctionDegraded, } from "../config/errorConfig.ts"; import { + getOpencodeModelUnavailableMatch, getProviderErrorRuleMatch, resolveRuleMatchBody, honorsRuleLockScope, @@ -1792,6 +1793,18 @@ export function checkFallbackError( return profile?.useUpstreamRetryHints ? detectRetryHint() : null; } + function ruleScopedResult(match: NonNullable>) { + const scaled = getScaledBaseCooldown(match.reason as RateLimitReasonValue, backoffLevel); + return { + shouldFallback: true, + cooldownMs: match.cooldownMs ?? scaled.cooldownMs, + baseCooldownMs: match.cooldownMs ?? scaled.baseCooldownMs, + configuredCooldownMs: match.cooldownMs, + newBackoffLevel: match.cooldownMs !== undefined ? 0 : scaled.newBackoffLevel, + reason: match.reason, + ruleScope: match.scope, + }; + } function getScaledBaseCooldown(reason: RateLimitReasonValue, level = backoffLevel) { void reason; const baseCooldownMs = @@ -2065,22 +2078,7 @@ export function checkFallbackError( headers, resolveRuleMatchBody(provider, structuredError ?? null, errorStr) ); - if (forbiddenMatch) { - const scaled = getScaledBaseCooldown( - forbiddenMatch.reason as RateLimitReasonValue, - backoffLevel - ); - const ruleCooldownMs = forbiddenMatch.cooldownMs; - return { - shouldFallback: true, - cooldownMs: ruleCooldownMs ?? scaled.cooldownMs, - baseCooldownMs: ruleCooldownMs ?? scaled.baseCooldownMs, - configuredCooldownMs: ruleCooldownMs, - newBackoffLevel: ruleCooldownMs !== undefined ? 0 : scaled.newBackoffLevel, - reason: forbiddenMatch.reason, - ruleScope: forbiddenMatch.scope, - }; - } + if (forbiddenMatch) return ruleScopedResult(forbiddenMatch); } if ( @@ -2199,6 +2197,8 @@ export function checkFallbackError( // 400 — context overflow / malformed request / model access denied if (status === HTTP_STATUS.BAD_REQUEST) { + const modelUnavailable = getOpencodeModelUnavailableMatch(provider, status, headers, errorStr); + if (modelUnavailable) return ruleScopedResult(modelUnavailable); // Check structured error codes first (more reliable, no false positives) // OpenAI: error.code === "model_not_found" // Anthropic: error.type === "not_found_error" / "permission_error" diff --git a/open-sse/services/combo/targetExhaustion.ts b/open-sse/services/combo/targetExhaustion.ts index 11e8efa654..7e636e69d0 100644 --- a/open-sse/services/combo/targetExhaustion.ts +++ b/open-sse/services/combo/targetExhaustion.ts @@ -27,9 +27,11 @@ import { import { RateLimitReason } from "../../config/constants.ts"; import { isProviderCircuitOpenResult, isRequestScopedUpstreamFailure } from "./comboPredicates.ts"; import { isCloudflareFingerprintRejection } from "../errorClassifier.ts"; -// #10334 — agentrouter-exclusive predicate shared with the persistence layer +// #10334 — connection-scope predicate shared with the persistence layer // (markAccountUnavailable) so the same-request combo skip and the persisted // connection cooldown agree on exactly which fallbackResult shapes qualify. +// Exclusive in practice to agentrouter's "额度不足" rule: no opencode-family +// rule matches 403 today, so only agentrouter reaches this predicate via 403. import { isAgentrouterConnectionQuotaScope } from "@/sse/services/auth"; import type { ComboLogger, ResolvedComboTarget } from "./types.ts"; @@ -84,9 +86,9 @@ export type ComboExhaustionSets = { export type ApplyComboTargetExhaustionOptions = { result: { status: number; headers?: Headers | null }; fallbackResult: Parameters[0] & { - /** #10334 — agentrouter-exclusive; see isAgentrouterConnectionQuotaScope + /** #10334 — agentrouter + opencode family; see isAgentrouterConnectionQuotaScope * (src/sse/services/auth.ts). Populated only for providers in - * HONORS_RULE_LOCK_SCOPE_PROVIDERS (today: agentrouter only). */ + * HONORS_RULE_LOCK_SCOPE_PROVIDERS (agentrouter + opencode family). */ ruleScope?: "model" | "provider" | "connection"; permanent?: boolean; }; @@ -115,7 +117,8 @@ export function applyComboTargetExhaustion( const { result, sets, log, tag, errorText, structuredError } = opts; const provider = target.provider; - // #10334: agentrouter-exclusive account-wide quota exhaustion ("额度不足") + // #10334: connection-scope account-wide quota exhaustion (agentrouter "额度不足"; + // exclusive in practice — no opencode-family rule matches 403 today) // must skip remaining SAME-CONNECTION targets within THIS request too, not // just via the persisted cooldown markAccountUnavailable applies for // whichever leg runs next. agentrouter is a passthroughModels provider @@ -341,7 +344,8 @@ function markAuthLevelExhaustion( } /** - * #10334: agentrouter-exclusive connection-scope account quota exhaustion. Mirrors + * #10334: connection-scope account quota exhaustion (agentrouter-exclusive in + * practice — see above). Mirrors * markAuthLevelExhaustion's connectionId-present/absent split — when the target carries a * connectionId, only that connection's account is exhausted (sibling agentrouter connections * for the same user may still have quota); fall back to whole-provider exhaustion only when no diff --git a/src/sse/services/auth.ts b/src/sse/services/auth.ts index 79fc31dfec..f9d0fad58c 100644 --- a/src/sse/services/auth.ts +++ b/src/sse/services/auth.ts @@ -2401,9 +2401,12 @@ export async function getProviderCredentialsWithQuotaPreflight( } /** - * #10334 — Guard for the agentrouter-exclusive "connection scope" quota - * cooldown branch in markAccountUnavailable. The "never terminal" invariant of - * that branch is NOT structurally guaranteed by `ruleScope === "connection"` + * #10334 — Guard for the "connection scope" quota cooldown branch in + * markAccountUnavailable (agentrouter-exclusive in practice: no opencode-family + * rule matches 403 today, so only agentrouter's "额度不足" rule reaches this + * predicate via 403 — but opencode-family 429 header-quota hits also qualify + * via the 429 path). The "never terminal" invariant of that branch is NOT + * structurally guaranteed by `ruleScope === "connection"` * alone — it also depends on the provider rule table only ever pairing scope * "connection" with a genuinely transient reason. Today * (`buildAgentrouterRules()` in providerErrorRules.ts) that is true: the only @@ -2727,8 +2730,10 @@ export async function markAccountUnavailable( const isPerModelQuotaProvider = hasPerModelQuota(provider, model, connectionPassthroughModels); - // #10334 — agentrouter EXCLUSIVE: the matched provider rule declared scope - // "connection" for account-wide quota exhaustion ("额度不足"). agentrouter is + // #10334 — connection-scope branch: the matched provider rule declared scope + // "connection" for account-wide quota exhaustion (agentrouter "额度不足"; + // exclusive in practice — no opencode-family rule matches 403 today). + // agentrouter is // a passthroughModels provider (isPerModelQuotaProvider === true), so without // this branch the next `if` would treat it like any other passthrough 429 and // lock a SINGLE model — leaving combo routing to burn one upstream call per @@ -2754,6 +2759,15 @@ export async function markAccountUnavailable( // of cooldown" ends up producing a LONGER effective block for this one rule. // Not addressed here; flagged for a future #2997 follow-up if it proves to be // a real operator complaint. + // + // HONORS note: since the opencode family joined HONORS, an opencode-family + // 429 carrying upstream quota headers (x-ratelimit-remaining-*) also lands + // here with ruleScope "connection" — before the #10880 egress branch below, + // so sibling cooling is skipped on that path. Latent today: the only + // request-path caller forwarding headers is chat.ts:2383 (chat completions), + // and opencode upstreams rarely send those headers on 429 (the observed + // envelope is the headers-less "monthly usage limit" body, which keeps + // flowing to the egress block with ruleScope undefined). if (ruleScopeIsConnection && provider && !disableCooling) { const connectionCooldownMs = fallbackResult.cooldownMs > 0 ? fallbackResult.cooldownMs : COOLDOWN_MS.rateLimit; @@ -2848,6 +2862,45 @@ export async function markAccountUnavailable( const isNvidiaModelGone = provider === "nvidia" && status === 410; const modelLockoutOptions = { maxCooldownMs: effectiveProviderProfile?.maxCooldownMs }; + // Same persisted reason the agentrouter 403 model-scope branch hard-codes + // ("forbidden"): the lock key is the getModelLockKey tuple shared with the + // combo path, and the declared 1h (same order as that combo lock) is + // operator-clamped by recordModelLockoutFailure to mlSettings.maxCooldownMs + // (~30min default) — the verbatim 1h never escapes operator control. + // Narrow scope: status === 400 only (never a 403/429 rule), adjacent to + // :2843's per-model-quota status set (which excludes 400) — malformed 400s + // carry no ruleScope and fall through unchanged. + if (model && provider && status === 400 && fallbackResult.ruleScope === "model") { + // Single source of truth: the rule's own cooldownMs (surfaced on + // fallbackResult by the 400 pre-check in checkFallbackError). The literal + // is only the fallback for a rule that declares no cooldown — editing + // the rule's cooldownMs takes effect without touching this call site. + const ruleCooldownMs = + typeof fallbackResult.cooldownMs === "number" && fallbackResult.cooldownMs > 0 + ? fallbackResult.cooldownMs + : 3_600_000; + const lockout = recordModelLockoutFailure( + provider, + connectionId, + model, + "model_capacity", + 400, + ruleCooldownMs, + effectiveProviderProfile, + { exactCooldownMs: ruleCooldownMs, maxCooldownMs: mlSettings.maxCooldownMs } + ); + updateProviderConnection(connectionId, { + lastErrorType: "model_capacity", + lastError: `Model ${model} model_capacity`, + lastErrorAt: new Date().toISOString(), + errorCode: status, + }).catch(() => {}); + log.info( + "AUTH", + `Model-only lockout for ${provider}:${model} — ${status} model_capacity ${Math.ceil(lockout.cooldownMs / 1000)}s (rule scope=model, connection stays active)` + ); + return { shouldFallback: true, cooldownMs: lockout.cooldownMs }; + } if ( isPerModelQuotaProvider && provider && diff --git a/tests/unit/agentrouter-error-rules.test.ts b/tests/unit/agentrouter-error-rules.test.ts index 272d21993b..498c31d911 100644 --- a/tests/unit/agentrouter-error-rules.test.ts +++ b/tests/unit/agentrouter-error-rules.test.ts @@ -168,10 +168,14 @@ test("A13: exclusivity — ruleScope stays undefined for other providers", () => assert.equal(openrouter.ruleScope, undefined); }); -test("A14: honorsRuleLockScope allowlist is agentrouter-only", async () => { +test("A14: honorsRuleLockScope allowlist is agentrouter + opencode family", async () => { const { honorsRuleLockScope } = await import("../../open-sse/config/providerErrorRules.ts"); assert.equal(honorsRuleLockScope("agentrouter"), true); assert.equal(honorsRuleLockScope("AgentRouter"), true); - assert.equal(honorsRuleLockScope("opencode"), false); + assert.equal(honorsRuleLockScope("opencode"), true); + assert.equal(honorsRuleLockScope("opencode-zen"), true); + assert.equal(honorsRuleLockScope("opencode-go"), true); + assert.equal(honorsRuleLockScope("opencode-cli"), true); + assert.equal(honorsRuleLockScope("openrouter"), false); assert.equal(honorsRuleLockScope(null), false); }); diff --git a/tests/unit/opencode-400-model-unavailable.test.ts b/tests/unit/opencode-400-model-unavailable.test.ts new file mode 100644 index 0000000000..278c2e35e9 --- /dev/null +++ b/tests/unit/opencode-400-model-unavailable.test.ts @@ -0,0 +1,162 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { + checkFallbackError, + recordModelLockoutFailure, + isModelLocked, + clearAllModelLockouts, +} from "../../open-sse/services/accountFallback.ts"; +import { isModelScoped400 } from "../../open-sse/services/combo/comboPredicates.ts"; +import { providerRuleRegistry } from "../../open-sse/config/providerErrorRules.ts"; + +// checkFallbackError is positional: (status, errorText, backoffLevel = 0, +// _model = null, provider = null, headers = null, profileOverride = null, +// structuredError?, …). ruleScope IS on the return type (accountFallback.ts:1686, +// #10334) but always undefined for non-allowlisted providers until the fenced +// pre-check + HONORS widening land — RED fails on values alone; the cast is +// convenience, not necessity. +const VERBATIM_BODY = `{"type":"server_error","message":"Error from provider (Console): Upstream request failed: Model is unavailable."}`; + +test("opencode 400 model-unavailable", async (t) => { + await t.test("locks the model on the pinned verbatim (opencode)", () => { + const r = checkFallbackError(400, VERBATIM_BODY, 0, null, "opencode"); + assert.equal(r.shouldFallback, true); + assert.equal((r as { ruleScope?: string }).ruleScope, "model"); + assert.equal(r.reason, "model_capacity"); + }); + + await t.test( + "locks the model on the pinned verbatim (opencode-zen, distinctly registered)", + () => { + assert.ok(providerRuleRegistry.get("opencode-zen"), "zen key registered"); + const r = checkFallbackError(400, VERBATIM_BODY, 0, null, "opencode-zen"); + assert.equal(r.shouldFallback, true); + assert.equal((r as { ruleScope?: string }).ruleScope, "model"); + } + ); + + await t.test("malformed 400 does NOT take the model lock (zero-cooldown guard preserved)", () => { + // #2101 infinite-loop guard (accountFallback.ts:2231-2237, re-pinned by + // accountfallback-ratelimit-400-4976.test.ts:38-44): a malformed 400 stays + // {shouldFallback:true, cooldownMs:0, reason:model_capacity} — "terminal" + // MEANS zero-cooldown, not shouldFallback:false. The new model-lock branch + // must not fire here: no ruleScope, no persisted lock. + const r = checkFallbackError( + 400, + `{"type":"invalid_request","message":"improperly formed request: invalid message format"}`, + 0, + null, + "opencode" + ); + assert.equal(r.shouldFallback, true); + assert.equal(r.cooldownMs, 0); + assert.equal(r.reason, "model_capacity"); + assert.equal((r as { ruleScope?: string }).ruleScope, undefined); + }); + + await t.test("model-unavailable write persists a readable model lock", () => { + // Direct round-trip on the same getModelLockKey tuple both paths share + // (exact-model key for these inputs): the auth.ts model branch calls + // recordModelLockoutFailure with the same (provider, connectionId, model, + // "model_capacity", 400) tuple, and combo routing reads it via isModelLocked. + clearAllModelLockouts(); + recordModelLockoutFailure( + "opencode", + "conn-test-400", + "deepseek-v4-flash-free", + "model_capacity", + 400, + 0, + null, + { exactCooldownMs: 3_600_000, maxCooldownMs: 1_800_000 } + ); + assert.equal(isModelLocked("opencode", "conn-test-400", "deepseek-v4-flash-free"), true); + clearAllModelLockouts(); + }); + + await t.test( + "headers-only quota rule still surfaces connection scope (pre-existing, HONORS now honors it)", + () => { + // The quota-exhausted-headers rule keys on headers alone, so it matched + // before this PR too — but ruleScope stayed undefined (opencode not in + // HONORS). Widening HONORS surfaces the rule's declared connection scope + // on header-passing paths (accountFallback 429 branch, combo executors). + // Body markers stay inert without FULL_TEXT (separate assert below). + // HONORS side effect (documented in the PR body): the pre-existing 429 + // headers rule now yields scope=connection for the whole opencode family, + // where the persistence layer previously re-derived scope via + // hasPerModelQuota(). opencode is not per-model-quota (no passthrough in + // either registry), so both derivations agree on connection — pinned here + // for all four family members plus the monthly-quota body rule, which + // keeps its exact verbatim cooldown (13 days, not the scaled default). + for (const provider of ["opencode", "opencode-zen", "opencode-go", "opencode-cli"]) { + const r = checkFallbackError(429, "rate limit reached, slow down", 0, null, provider, { + "x-ratelimit-remaining-requests": "0", + }); + assert.equal(r.reason, "quota_exhausted", provider); + assert.equal((r as { ruleScope?: string }).ruleScope, "connection", provider); + // Same body without headers: no rule fires, scope stays undefined. + const r2 = checkFallbackError( + 429, + "rate limit reached, slow down", + 0, + null, + provider, + null + ); + assert.equal((r2 as { ruleScope?: string }).ruleScope, undefined, provider); + } + // Pins parser day-granularity (parseResetCountdownMs), not this PR's code: + // relax to a range if the parser ever learns hour/minute residuals. + const monthly = checkFallbackError( + 429, + "[429] Monthly usage limit reached. Resets in 13 days.", + 0, + null, + "opencode", + null + ); + assert.equal(monthly.reason, "quota_exhausted"); + assert.ok( + monthly.cooldownMs >= 13 * 24 * 60 * 60 * 1000 && + monthly.cooldownMs < 14 * 24 * 60 * 60 * 1000 + ); + assert.equal((monthly as { ruleScope?: string }).ruleScope, undefined); + } + ); + + await t.test("quota-body markers stay inert without FULL_TEXT", () => { + // FULL_TEXT_RULE_PROVIDERS is still agentrouter-only: quota-body markers + // (organization_quota_exceeded, plan_limit_reached, account_quota_exceeded) + // must NOT surface a rule scope — the #10880 egress block stays reachable. + for (const marker of [ + "organization_quota_exceeded", + "plan_limit_reached", + "account_quota_exceeded", + ]) { + const r = checkFallbackError( + 429, + `{"error":{"message":"${marker}"}}`, + 0, + null, + "opencode", + null + ); + assert.equal(r.reason, "rate_limit_exceeded", marker); + assert.equal((r as { ruleScope?: string }).ruleScope, undefined, marker); + } + }); + + await t.test("verbatim stays terminal on non-family providers", () => { + // The new model-lock branch is fenced on OPENCODE_FAMILY: the verbatim + // under any other provider must stay shouldFallback:false (generic 400). + for (const provider of ["agentrouter", "openrouter", "minimax", "mimocode", "unknown-vendor"]) { + const r = checkFallbackError(400, VERBATIM_BODY, 0, null, provider); + assert.equal(r.shouldFallback, false, provider); + } + }); + + await t.test("combo model-scope classifier still matches (regression)", () => { + assert.equal(isModelScoped400(VERBATIM_BODY), true); + }); +});