fix(providers): lock opencode model on upstream 400 model-unavailable (#13146)

Scoping the lock to the MODEL rather than the connection is the right layer for a `400 "Model is unavailable"` — a multi-day upstream outage on one model should not darken the account. Good discipline on the two allowlist-adjacent changes: gating the `markAccountUnavailable` branch on `ruleScope === "model"` AND `status === 400` leaves every other status on its existing path, and deliberately not widening `FULL_TEXT_RULE_PROVIDERS` keeps the #10880 egress-bucketed 429 classification intact. Reading the cooldown from the rule instead of a literal at the call site is what makes it self-healing.

---

Validated in one consolidated worktree cut from `release/v3.8.51`, boarded together with the rest of this batch — zero conflicts between them.

- `typecheck:core` clean; `check:changelog-integrity` OK
- complexity 2799 / baseline 3218 and cognitive-complexity 1265 / baseline 1437 — both under baseline
- 86 focused assertions green across the batch's 10 unit test files, plus 16/16 on the v1 plugin option schema and 16/16 on the v2 option tests
- `check-file-size` rebaselined for this batch's real growth (annotation `_rebaseline_2026_09_11_mergebatch_v3851_maxmad_opencode`, landed on #13141). `open-sse/utils/stream.ts` was deliberately left frozen: it is already 3115 > 3098 on the pure tip with zero contribution from this batch.

⚠️ base-red inherited: #12732 — `Docs Gates`, `Merge integrity`, `No new ESLint warnings`, `Unit Tests fast-path` and `Fast Quality Gates` all reproduce on the pure `release/v3.8.51` tip (provider count 356 vs the 358 the modules define, SKILL.md drift, and the `stream.ts` freeze above). None of them touch these diffs.

Thanks @maxmad64bis.
This commit is contained in:
Dizzle
2026-09-11 18:51:24 +02:00
committed by GitHub
parent cc4f7ed1c4
commit a19bb2227f
7 changed files with 289 additions and 33 deletions

View File

@@ -0,0 +1 @@
- **fix(providers):** lock opencode model on upstream 400 model-unavailable ([#13146](https://github.com/diegosouzapw/OmniRoute/pull/13146)) — thanks @maxmad64bis

View File

@@ -32,7 +32,7 @@ export type ProviderErrorRuleMatch = {
/**
* Intended lock scope. #10334: for a BUILT-IN catalog rule, this field is
* CONSUMED end-to-end only for providers in `HONORS_RULE_LOCK_SCOPE_PROVIDERS`
* (agentrouter-exclusive today, gated by `honorsRuleLockScope()`) — for those,
* (agentrouter + the opencode family, gated by `honorsRuleLockScope()`) — for
* `checkFallbackError` surfaces it as `ruleScope` on its return value for the
* persistence layer to honor instead of re-deriving scope from
* `hasPerModelQuota()`. For every other built-in-rule provider it remains
@@ -155,6 +155,19 @@ function buildOpencodeRules(): ProviderErrorRule[] {
return null;
},
},
{
id: "opencode-400-model-unavailable",
match: ({ status, body }) => {
if (status !== 400) return null;
const text = JSON.stringify(body ?? "").toLowerCase();
if (!text.includes("upstream request failed: model is unavailable.")) return null;
return {
reason: "model_capacity",
scope: "model",
cooldownMs: 3_600_000,
};
},
},
];
}
@@ -290,15 +303,16 @@ function buildAgentrouterRules(): ProviderErrorRule[] {
];
}
/** Providers sharing the opencode upstream envelope, hence the opencode catalog rules. */
const OPENCODE_RULE_FAMILY = ["opencode", "opencode-zen", "opencode-go", "opencode-cli"];
/**
* Global registry. Provider name → ordered list of rules (first match wins).
* Add new providers here; the matcher in classifyError will pick them up
* automatically.
*/
export const providerRuleRegistry = new Map<string, ProviderErrorRule[]>([
["opencode", buildOpencodeRules()],
["opencode-go", buildOpencodeRules()],
["opencode-cli", buildOpencodeRules()],
...OPENCODE_RULE_FAMILY.map((id): [string, ProviderErrorRule[]] => [id, buildOpencodeRules()]),
["minimax", buildMinimaxRules()],
["minimax-passthrough", buildMinimaxRules()],
["cloudflare-ai", buildCloudflareAiRules()],
@@ -323,7 +337,7 @@ export const providerRuleRegistry = new Map<string, ProviderErrorRule[]>([
* mechanism (#11104) silently inert for every provider except the ones listed
* below. See `hasOperatorRuleForProvider`.
*/
const HONORS_RULE_LOCK_SCOPE_PROVIDERS = new Set(["agentrouter"]);
const HONORS_RULE_LOCK_SCOPE_PROVIDERS = new Set(["agentrouter", ...OPENCODE_RULE_FAMILY]);
export function honorsRuleLockScope(provider: string | null | undefined): boolean {
if (!provider) return false;
@@ -509,3 +523,21 @@ export function parseResetCountdownMs(text: string): number | null {
return null;
}
}
/**
* Opencode-family "Upstream request failed: Model is unavailable." 400: the rule's
* model-scope match, or null for any other provider, status or rule. Takes the raw
* error text so it stays independent of FULL_TEXT_RULE_PROVIDERS (#10880).
*/
export function getOpencodeModelUnavailableMatch(
provider: string | null | undefined,
status: number,
headers: Headers | Record<string, string> | null | undefined,
errorText: unknown
): ProviderErrorRuleMatch | null {
if (status !== 400 || !provider || !OPENCODE_RULE_FAMILY.includes(provider.toLowerCase())) {
return null;
}
const match = getProviderErrorRuleMatch(provider, status, headers, errorText);
return match?.scope === "model" && match.reason === "model_capacity" ? match : null;
}

View File

@@ -16,6 +16,7 @@ import {
isNimFunctionDegraded,
} from "../config/errorConfig.ts";
import {
getOpencodeModelUnavailableMatch,
getProviderErrorRuleMatch,
resolveRuleMatchBody,
honorsRuleLockScope,
@@ -1792,6 +1793,18 @@ export function checkFallbackError(
return profile?.useUpstreamRetryHints ? detectRetryHint() : null;
}
function ruleScopedResult(match: NonNullable<ReturnType<typeof getProviderErrorRuleMatch>>) {
const scaled = getScaledBaseCooldown(match.reason as RateLimitReasonValue, backoffLevel);
return {
shouldFallback: true,
cooldownMs: match.cooldownMs ?? scaled.cooldownMs,
baseCooldownMs: match.cooldownMs ?? scaled.baseCooldownMs,
configuredCooldownMs: match.cooldownMs,
newBackoffLevel: match.cooldownMs !== undefined ? 0 : scaled.newBackoffLevel,
reason: match.reason,
ruleScope: match.scope,
};
}
function getScaledBaseCooldown(reason: RateLimitReasonValue, level = backoffLevel) {
void reason;
const baseCooldownMs =
@@ -2065,22 +2078,7 @@ export function checkFallbackError(
headers,
resolveRuleMatchBody(provider, structuredError ?? null, errorStr)
);
if (forbiddenMatch) {
const scaled = getScaledBaseCooldown(
forbiddenMatch.reason as RateLimitReasonValue,
backoffLevel
);
const ruleCooldownMs = forbiddenMatch.cooldownMs;
return {
shouldFallback: true,
cooldownMs: ruleCooldownMs ?? scaled.cooldownMs,
baseCooldownMs: ruleCooldownMs ?? scaled.baseCooldownMs,
configuredCooldownMs: ruleCooldownMs,
newBackoffLevel: ruleCooldownMs !== undefined ? 0 : scaled.newBackoffLevel,
reason: forbiddenMatch.reason,
ruleScope: forbiddenMatch.scope,
};
}
if (forbiddenMatch) return ruleScopedResult(forbiddenMatch);
}
if (
@@ -2199,6 +2197,8 @@ export function checkFallbackError(
// 400 — context overflow / malformed request / model access denied
if (status === HTTP_STATUS.BAD_REQUEST) {
const modelUnavailable = getOpencodeModelUnavailableMatch(provider, status, headers, errorStr);
if (modelUnavailable) return ruleScopedResult(modelUnavailable);
// Check structured error codes first (more reliable, no false positives)
// OpenAI: error.code === "model_not_found"
// Anthropic: error.type === "not_found_error" / "permission_error"

View File

@@ -27,9 +27,11 @@ import {
import { RateLimitReason } from "../../config/constants.ts";
import { isProviderCircuitOpenResult, isRequestScopedUpstreamFailure } from "./comboPredicates.ts";
import { isCloudflareFingerprintRejection } from "../errorClassifier.ts";
// #10334 — agentrouter-exclusive predicate shared with the persistence layer
// #10334 — connection-scope predicate shared with the persistence layer
// (markAccountUnavailable) so the same-request combo skip and the persisted
// connection cooldown agree on exactly which fallbackResult shapes qualify.
// Exclusive in practice to agentrouter's "额度不足" rule: no opencode-family
// rule matches 403 today, so only agentrouter reaches this predicate via 403.
import { isAgentrouterConnectionQuotaScope } from "@/sse/services/auth";
import type { ComboLogger, ResolvedComboTarget } from "./types.ts";
@@ -84,9 +86,9 @@ export type ComboExhaustionSets = {
export type ApplyComboTargetExhaustionOptions = {
result: { status: number; headers?: Headers | null };
fallbackResult: Parameters<typeof isProviderExhaustedReason>[0] & {
/** #10334 — agentrouter-exclusive; see isAgentrouterConnectionQuotaScope
/** #10334 — agentrouter + opencode family; see isAgentrouterConnectionQuotaScope
* (src/sse/services/auth.ts). Populated only for providers in
* HONORS_RULE_LOCK_SCOPE_PROVIDERS (today: agentrouter only). */
* HONORS_RULE_LOCK_SCOPE_PROVIDERS (agentrouter + opencode family). */
ruleScope?: "model" | "provider" | "connection";
permanent?: boolean;
};
@@ -115,7 +117,8 @@ export function applyComboTargetExhaustion(
const { result, sets, log, tag, errorText, structuredError } = opts;
const provider = target.provider;
// #10334: agentrouter-exclusive account-wide quota exhaustion ("额度不足")
// #10334: connection-scope account-wide quota exhaustion (agentrouter "额度不足";
// exclusive in practice — no opencode-family rule matches 403 today)
// must skip remaining SAME-CONNECTION targets within THIS request too, not
// just via the persisted cooldown markAccountUnavailable applies for
// whichever leg runs next. agentrouter is a passthroughModels provider
@@ -341,7 +344,8 @@ function markAuthLevelExhaustion(
}
/**
* #10334: agentrouter-exclusive connection-scope account quota exhaustion. Mirrors
* #10334: connection-scope account quota exhaustion (agentrouter-exclusive in
* practice — see above). Mirrors
* markAuthLevelExhaustion's connectionId-present/absent split — when the target carries a
* connectionId, only that connection's account is exhausted (sibling agentrouter connections
* for the same user may still have quota); fall back to whole-provider exhaustion only when no

View File

@@ -2401,9 +2401,12 @@ export async function getProviderCredentialsWithQuotaPreflight(
}
/**
* #10334 — Guard for the agentrouter-exclusive "connection scope" quota
* cooldown branch in markAccountUnavailable. The "never terminal" invariant of
* that branch is NOT structurally guaranteed by `ruleScope === "connection"`
* #10334 — Guard for the "connection scope" quota cooldown branch in
* markAccountUnavailable (agentrouter-exclusive in practice: no opencode-family
* rule matches 403 today, so only agentrouter's "额度不足" rule reaches this
* predicate via 403 — but opencode-family 429 header-quota hits also qualify
* via the 429 path). The "never terminal" invariant of that branch is NOT
* structurally guaranteed by `ruleScope === "connection"`
* alone — it also depends on the provider rule table only ever pairing scope
* "connection" with a genuinely transient reason. Today
* (`buildAgentrouterRules()` in providerErrorRules.ts) that is true: the only
@@ -2727,8 +2730,10 @@ export async function markAccountUnavailable(
const isPerModelQuotaProvider = hasPerModelQuota(provider, model, connectionPassthroughModels);
// #10334 — agentrouter EXCLUSIVE: the matched provider rule declared scope
// "connection" for account-wide quota exhaustion ("额度不足"). agentrouter is
// #10334 — connection-scope branch: the matched provider rule declared scope
// "connection" for account-wide quota exhaustion (agentrouter "额度不足";
// exclusive in practice — no opencode-family rule matches 403 today).
// agentrouter is
// a passthroughModels provider (isPerModelQuotaProvider === true), so without
// this branch the next `if` would treat it like any other passthrough 429 and
// lock a SINGLE model — leaving combo routing to burn one upstream call per
@@ -2754,6 +2759,15 @@ export async function markAccountUnavailable(
// of cooldown" ends up producing a LONGER effective block for this one rule.
// Not addressed here; flagged for a future #2997 follow-up if it proves to be
// a real operator complaint.
//
// HONORS note: since the opencode family joined HONORS, an opencode-family
// 429 carrying upstream quota headers (x-ratelimit-remaining-*) also lands
// here with ruleScope "connection" — before the #10880 egress branch below,
// so sibling cooling is skipped on that path. Latent today: the only
// request-path caller forwarding headers is chat.ts:2383 (chat completions),
// and opencode upstreams rarely send those headers on 429 (the observed
// envelope is the headers-less "monthly usage limit" body, which keeps
// flowing to the egress block with ruleScope undefined).
if (ruleScopeIsConnection && provider && !disableCooling) {
const connectionCooldownMs =
fallbackResult.cooldownMs > 0 ? fallbackResult.cooldownMs : COOLDOWN_MS.rateLimit;
@@ -2848,6 +2862,45 @@ export async function markAccountUnavailable(
const isNvidiaModelGone = provider === "nvidia" && status === 410;
const modelLockoutOptions = { maxCooldownMs: effectiveProviderProfile?.maxCooldownMs };
// Same persisted reason the agentrouter 403 model-scope branch hard-codes
// ("forbidden"): the lock key is the getModelLockKey tuple shared with the
// combo path, and the declared 1h (same order as that combo lock) is
// operator-clamped by recordModelLockoutFailure to mlSettings.maxCooldownMs
// (~30min default) — the verbatim 1h never escapes operator control.
// Narrow scope: status === 400 only (never a 403/429 rule), adjacent to
// :2843's per-model-quota status set (which excludes 400) — malformed 400s
// carry no ruleScope and fall through unchanged.
if (model && provider && status === 400 && fallbackResult.ruleScope === "model") {
// Single source of truth: the rule's own cooldownMs (surfaced on
// fallbackResult by the 400 pre-check in checkFallbackError). The literal
// is only the fallback for a rule that declares no cooldown — editing
// the rule's cooldownMs takes effect without touching this call site.
const ruleCooldownMs =
typeof fallbackResult.cooldownMs === "number" && fallbackResult.cooldownMs > 0
? fallbackResult.cooldownMs
: 3_600_000;
const lockout = recordModelLockoutFailure(
provider,
connectionId,
model,
"model_capacity",
400,
ruleCooldownMs,
effectiveProviderProfile,
{ exactCooldownMs: ruleCooldownMs, maxCooldownMs: mlSettings.maxCooldownMs }
);
updateProviderConnection(connectionId, {
lastErrorType: "model_capacity",
lastError: `Model ${model} model_capacity`,
lastErrorAt: new Date().toISOString(),
errorCode: status,
}).catch(() => {});
log.info(
"AUTH",
`Model-only lockout for ${provider}:${model}${status} model_capacity ${Math.ceil(lockout.cooldownMs / 1000)}s (rule scope=model, connection stays active)`
);
return { shouldFallback: true, cooldownMs: lockout.cooldownMs };
}
if (
isPerModelQuotaProvider &&
provider &&

View File

@@ -168,10 +168,14 @@ test("A13: exclusivity — ruleScope stays undefined for other providers", () =>
assert.equal(openrouter.ruleScope, undefined);
});
test("A14: honorsRuleLockScope allowlist is agentrouter-only", async () => {
test("A14: honorsRuleLockScope allowlist is agentrouter + opencode family", async () => {
const { honorsRuleLockScope } = await import("../../open-sse/config/providerErrorRules.ts");
assert.equal(honorsRuleLockScope("agentrouter"), true);
assert.equal(honorsRuleLockScope("AgentRouter"), true);
assert.equal(honorsRuleLockScope("opencode"), false);
assert.equal(honorsRuleLockScope("opencode"), true);
assert.equal(honorsRuleLockScope("opencode-zen"), true);
assert.equal(honorsRuleLockScope("opencode-go"), true);
assert.equal(honorsRuleLockScope("opencode-cli"), true);
assert.equal(honorsRuleLockScope("openrouter"), false);
assert.equal(honorsRuleLockScope(null), false);
});

View File

@@ -0,0 +1,162 @@
import test from "node:test";
import assert from "node:assert/strict";
import {
checkFallbackError,
recordModelLockoutFailure,
isModelLocked,
clearAllModelLockouts,
} from "../../open-sse/services/accountFallback.ts";
import { isModelScoped400 } from "../../open-sse/services/combo/comboPredicates.ts";
import { providerRuleRegistry } from "../../open-sse/config/providerErrorRules.ts";
// checkFallbackError is positional: (status, errorText, backoffLevel = 0,
// _model = null, provider = null, headers = null, profileOverride = null,
// structuredError?, …). ruleScope IS on the return type (accountFallback.ts:1686,
// #10334) but always undefined for non-allowlisted providers until the fenced
// pre-check + HONORS widening land — RED fails on values alone; the cast is
// convenience, not necessity.
const VERBATIM_BODY = `{"type":"server_error","message":"Error from provider (Console): Upstream request failed: Model is unavailable."}`;
test("opencode 400 model-unavailable", async (t) => {
await t.test("locks the model on the pinned verbatim (opencode)", () => {
const r = checkFallbackError(400, VERBATIM_BODY, 0, null, "opencode");
assert.equal(r.shouldFallback, true);
assert.equal((r as { ruleScope?: string }).ruleScope, "model");
assert.equal(r.reason, "model_capacity");
});
await t.test(
"locks the model on the pinned verbatim (opencode-zen, distinctly registered)",
() => {
assert.ok(providerRuleRegistry.get("opencode-zen"), "zen key registered");
const r = checkFallbackError(400, VERBATIM_BODY, 0, null, "opencode-zen");
assert.equal(r.shouldFallback, true);
assert.equal((r as { ruleScope?: string }).ruleScope, "model");
}
);
await t.test("malformed 400 does NOT take the model lock (zero-cooldown guard preserved)", () => {
// #2101 infinite-loop guard (accountFallback.ts:2231-2237, re-pinned by
// accountfallback-ratelimit-400-4976.test.ts:38-44): a malformed 400 stays
// {shouldFallback:true, cooldownMs:0, reason:model_capacity} — "terminal"
// MEANS zero-cooldown, not shouldFallback:false. The new model-lock branch
// must not fire here: no ruleScope, no persisted lock.
const r = checkFallbackError(
400,
`{"type":"invalid_request","message":"improperly formed request: invalid message format"}`,
0,
null,
"opencode"
);
assert.equal(r.shouldFallback, true);
assert.equal(r.cooldownMs, 0);
assert.equal(r.reason, "model_capacity");
assert.equal((r as { ruleScope?: string }).ruleScope, undefined);
});
await t.test("model-unavailable write persists a readable model lock", () => {
// Direct round-trip on the same getModelLockKey tuple both paths share
// (exact-model key for these inputs): the auth.ts model branch calls
// recordModelLockoutFailure with the same (provider, connectionId, model,
// "model_capacity", 400) tuple, and combo routing reads it via isModelLocked.
clearAllModelLockouts();
recordModelLockoutFailure(
"opencode",
"conn-test-400",
"deepseek-v4-flash-free",
"model_capacity",
400,
0,
null,
{ exactCooldownMs: 3_600_000, maxCooldownMs: 1_800_000 }
);
assert.equal(isModelLocked("opencode", "conn-test-400", "deepseek-v4-flash-free"), true);
clearAllModelLockouts();
});
await t.test(
"headers-only quota rule still surfaces connection scope (pre-existing, HONORS now honors it)",
() => {
// The quota-exhausted-headers rule keys on headers alone, so it matched
// before this PR too — but ruleScope stayed undefined (opencode not in
// HONORS). Widening HONORS surfaces the rule's declared connection scope
// on header-passing paths (accountFallback 429 branch, combo executors).
// Body markers stay inert without FULL_TEXT (separate assert below).
// HONORS side effect (documented in the PR body): the pre-existing 429
// headers rule now yields scope=connection for the whole opencode family,
// where the persistence layer previously re-derived scope via
// hasPerModelQuota(). opencode is not per-model-quota (no passthrough in
// either registry), so both derivations agree on connection — pinned here
// for all four family members plus the monthly-quota body rule, which
// keeps its exact verbatim cooldown (13 days, not the scaled default).
for (const provider of ["opencode", "opencode-zen", "opencode-go", "opencode-cli"]) {
const r = checkFallbackError(429, "rate limit reached, slow down", 0, null, provider, {
"x-ratelimit-remaining-requests": "0",
});
assert.equal(r.reason, "quota_exhausted", provider);
assert.equal((r as { ruleScope?: string }).ruleScope, "connection", provider);
// Same body without headers: no rule fires, scope stays undefined.
const r2 = checkFallbackError(
429,
"rate limit reached, slow down",
0,
null,
provider,
null
);
assert.equal((r2 as { ruleScope?: string }).ruleScope, undefined, provider);
}
// Pins parser day-granularity (parseResetCountdownMs), not this PR's code:
// relax to a range if the parser ever learns hour/minute residuals.
const monthly = checkFallbackError(
429,
"[429] Monthly usage limit reached. Resets in 13 days.",
0,
null,
"opencode",
null
);
assert.equal(monthly.reason, "quota_exhausted");
assert.ok(
monthly.cooldownMs >= 13 * 24 * 60 * 60 * 1000 &&
monthly.cooldownMs < 14 * 24 * 60 * 60 * 1000
);
assert.equal((monthly as { ruleScope?: string }).ruleScope, undefined);
}
);
await t.test("quota-body markers stay inert without FULL_TEXT", () => {
// FULL_TEXT_RULE_PROVIDERS is still agentrouter-only: quota-body markers
// (organization_quota_exceeded, plan_limit_reached, account_quota_exceeded)
// must NOT surface a rule scope — the #10880 egress block stays reachable.
for (const marker of [
"organization_quota_exceeded",
"plan_limit_reached",
"account_quota_exceeded",
]) {
const r = checkFallbackError(
429,
`{"error":{"message":"${marker}"}}`,
0,
null,
"opencode",
null
);
assert.equal(r.reason, "rate_limit_exceeded", marker);
assert.equal((r as { ruleScope?: string }).ruleScope, undefined, marker);
}
});
await t.test("verbatim stays terminal on non-family providers", () => {
// The new model-lock branch is fenced on OPENCODE_FAMILY: the verbatim
// under any other provider must stay shouldFallback:false (generic 400).
for (const provider of ["agentrouter", "openrouter", "minimax", "mimocode", "unknown-vendor"]) {
const r = checkFallbackError(400, VERBATIM_BODY, 0, null, provider);
assert.equal(r.shouldFallback, false, provider);
}
});
await t.test("combo model-scope classifier still matches (regression)", () => {
assert.equal(isModelScoped400(VERBATIM_BODY), true);
});
});