From e6aee3d26cbcffb800440cc13b811ab1c9b5a05e Mon Sep 17 00:00:00 2001 From: InkshadeWoods <144514307+InkshadeWoods@users.noreply.github.com> Date: Wed, 13 May 2026 06:57:26 +0800 Subject: [PATCH] feat: add ModelScope provider-specific 429 handling and retry logic (#2202) Integrated into release/v3.8.0 after syncing the contributor branch, removing unrelated workflow/docker/package-lock changes, tightening ModelScope 429 classification, and validating policy coverage locally. --- open-sse/handlers/chatCore.ts | 59 ++++++++++++++++-- open-sse/services/modelscopePolicy.ts | 88 +++++++++++++++++++++++++++ tests/unit/modelscope-policy.test.ts | 64 +++++++++++++++++++ 3 files changed, 207 insertions(+), 4 deletions(-) create mode 100644 open-sse/services/modelscopePolicy.ts create mode 100644 tests/unit/modelscope-policy.test.ts diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 3d903eb939..bd8b4dd33b 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -128,7 +128,7 @@ import { buildAccountSemaphoreKey, markBlocked as markAccountSemaphoreBlocked, } from "../services/accountSemaphore.ts"; -import { lockModelIfPerModelQuota } from "../services/accountFallback.ts"; +import { lockModel, lockModelIfPerModelQuota } from "../services/accountFallback.ts"; import { generateSignature, getCachedResponse, @@ -185,6 +185,11 @@ import { import { setGeminiThoughtSignatureMode } from "../services/geminiThoughtSignatureStore.ts"; import { fetchLiveProviderLimits } from "@/lib/usage/providerLimits"; import { isClaudeExtraUsageBlockEnabled } from "@/lib/providers/claudeExtraUsage"; +import { + classifyModelScope429, + getModelScopeRetryDelayMs, + isModelScopeProvider, +} from "../services/modelscopePolicy.ts"; const MEMORY_EXTRACTION_TEXT_LIMIT = 64 * 1024; @@ -1160,6 +1165,7 @@ export async function handleChatCore({ let { provider, model, extendedContext } = modelInfo; const requestedModel = typeof body?.model === "string" && body.model.trim().length > 0 ? body.model : model; + const isModelScope = () => isModelScopeProvider(provider, credentials?.providerSpecificData); const startTime = Date.now(); // Per-request trace id + checkpoint helper. Lets us see exactly which await // a hung request was sitting on in `[STAGE_TRACE]` log lines. @@ -2970,7 +2976,14 @@ export async function handleChatCore({ async () => { trace("inside_rate_limit"); let attempts = 0; - const maxAttempts = provider === "qwen" ? 3 : provider === "codex" ? 3 : 1; + const isModelScopeForRequest = isModelScope(); + const maxAttempts = isModelScopeForRequest + ? 3 + : provider === "qwen" + ? 3 + : provider === "codex" + ? 3 + : 1; // ── Codex 429 account-rotation state ───────────────────────────────── // Track excluded connection IDs for codex failover across attempts. @@ -3016,6 +3029,28 @@ export async function handleChatCore({ } } + if ( + isModelScope() && + res.response.status === 429 && + attempts < maxAttempts - 1 + ) { + const bodyPeek = await res.response + .clone() + .text() + .catch(() => ""); + const decision = classifyModelScope429(bodyPeek, res.response.headers); + if (decision.retryable) { + const delay = getModelScopeRetryDelayMs(res.response.headers, attempts); + log?.warn?.( + "MODELSCOPE_RETRY", + `429 ${decision.kind}; retrying in ${delay}ms (model remaining: ${decision.snapshot.modelRemaining ?? "unknown"})` + ); + await new Promise((r) => setTimeout(r, delay)); + attempts++; + continue; + } + } + // Codex 429 account-rotation failover (disabled for context-relay so combo.ts can inject handoff) if ( provider === "codex" && @@ -3422,7 +3457,18 @@ export async function handleChatCore({ } // T06/T10/T36: classify provider errors and persist terminal account states. - const errorType = classifyProviderError(statusCode, message, provider); + let errorType = classifyProviderError(statusCode, message, provider); + if (statusCode === 429 && isModelScope()) { + const decision = classifyModelScope429(message, providerResponse.headers); + errorType = + decision.kind === "quota_exhausted" + ? PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED + : PROVIDER_ERROR_TYPES.RATE_LIMITED; + log?.warn?.( + "MODELSCOPE_429", + `${decision.kind} (model remaining: ${decision.snapshot.modelRemaining ?? "unknown"}, total remaining: ${decision.snapshot.totalRemaining ?? "unknown"})` + ); + } if (connectionId && errorType) { try { if (errorType === PROVIDER_ERROR_TYPES.FORBIDDEN) { @@ -3459,7 +3505,12 @@ export async function handleChatCore({ if (accountSemaphoreKey) { markAccountSemaphoreBlocked(accountSemaphoreKey, quotaCooldownMs); } - if ( + if (isModelScope() && connectionId) { + lockModel(provider, connectionId, model, "quota_exhausted", quotaCooldownMs); + console.warn( + `[provider] Node ${connectionId} ModelScope model quota exhausted (${statusCode}) for ${model} - ${Math.ceil(quotaCooldownMs / 1000)}s (connection stays active)` + ); + } else if ( lockModelIfPerModelQuota( provider, connectionId, diff --git a/open-sse/services/modelscopePolicy.ts b/open-sse/services/modelscopePolicy.ts new file mode 100644 index 0000000000..af284aa347 --- /dev/null +++ b/open-sse/services/modelscopePolicy.ts @@ -0,0 +1,88 @@ +type ModelScopeRateLimitSnapshot = { + modelRemaining: number | null; + modelLimit: number | null; + totalRemaining: number | null; + totalLimit: number | null; +}; + +type ModelScope429Decision = + | { kind: "quota_exhausted"; retryable: false; snapshot: ModelScopeRateLimitSnapshot } + | { kind: "rate_limited"; retryable: true; snapshot: ModelScopeRateLimitSnapshot }; + +const MODELSCOPE_HOST_MARKERS = ["modelscope.cn", "modelscope.aliyuncs.com"]; +const MODELSCOPE_QUOTA_EXHAUSTED_SIGNALS = ["free allocated quota exceeded"]; +const MODELSCOPE_THROTTLE_SIGNALS = [ + "throttling", + "throttled", + "rate limit", + "too many requests", + "batch requests", + "allocated quota exceeded", + "exceeded your current quota", +]; + +function parseHeaderInteger(value: string | null): number | null { + if (value === null || value.trim() === "") return null; + const parsed = Number.parseInt(value, 10); + return Number.isFinite(parsed) ? parsed : null; +} + +function getProviderBaseUrl(providerSpecificData?: unknown): string { + if (!providerSpecificData || typeof providerSpecificData !== "object") return ""; + const data = providerSpecificData as Record; + const value = data.baseUrl ?? data.baseURL ?? data.url ?? data.endpoint; + return typeof value === "string" ? value.toLowerCase() : ""; +} + +export function isModelScopeProvider( + provider: string | null | undefined, + providerSpecificData?: unknown +): boolean { + if ( + String(provider || "") + .trim() + .toLowerCase() === "modelscope" + ) + return true; + const baseUrl = getProviderBaseUrl(providerSpecificData); + return MODELSCOPE_HOST_MARKERS.some((marker) => baseUrl.includes(marker)); +} + +export function parseModelScopeRateLimitHeaders(headers: Headers): ModelScopeRateLimitSnapshot { + return { + modelRemaining: parseHeaderInteger( + headers.get("modelscope-ratelimit-model-requests-remaining") + ), + modelLimit: parseHeaderInteger(headers.get("modelscope-ratelimit-model-requests-limit")), + totalRemaining: parseHeaderInteger(headers.get("modelscope-ratelimit-requests-remaining")), + totalLimit: parseHeaderInteger(headers.get("modelscope-ratelimit-requests-limit")), + }; +} + +export function classifyModelScope429(errorText: string, headers: Headers): ModelScope429Decision { + const snapshot = parseModelScopeRateLimitHeaders(headers); + const lower = String(errorText || "").toLowerCase(); + + if (MODELSCOPE_QUOTA_EXHAUSTED_SIGNALS.some((signal) => lower.includes(signal))) { + return { kind: "quota_exhausted", retryable: false, snapshot }; + } + + if (snapshot.modelRemaining !== null || snapshot.totalRemaining !== null) { + return { kind: "rate_limited", retryable: true, snapshot }; + } + + if (MODELSCOPE_THROTTLE_SIGNALS.some((signal) => lower.includes(signal))) { + return { kind: "rate_limited", retryable: true, snapshot }; + } + + return { kind: "rate_limited", retryable: true, snapshot }; +} + +export function getModelScopeRetryDelayMs(headers: Headers, attempt: number): number { + const retryAfter = headers.get("retry-after"); + if (retryAfter) { + const parsed = Number.parseFloat(retryAfter); + if (Number.isFinite(parsed) && parsed > 0) return parsed * 1000; + } + return 3000 * (attempt + 1); +} diff --git a/tests/unit/modelscope-policy.test.ts b/tests/unit/modelscope-policy.test.ts new file mode 100644 index 0000000000..39ea704f85 --- /dev/null +++ b/tests/unit/modelscope-policy.test.ts @@ -0,0 +1,64 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +const { + classifyModelScope429, + getModelScopeRetryDelayMs, + isModelScopeProvider, + parseModelScopeRateLimitHeaders, +} = await import("../../open-sse/services/modelscopePolicy.ts"); + +test("ModelScope policy detects provider ids and ModelScope host markers", () => { + assert.equal(isModelScopeProvider("modelscope"), true); + assert.equal( + isModelScopeProvider("openai-compatible-custom", { + baseUrl: "https://api-inference.modelscope.cn/v1", + }), + true + ); + assert.equal(isModelScopeProvider("openai", { baseUrl: "https://api.openai.com/v1" }), false); +}); + +test("ModelScope policy parses per-model and total rate-limit headers", () => { + const snapshot = parseModelScopeRateLimitHeaders( + new Headers({ + "modelscope-ratelimit-model-requests-remaining": "0", + "modelscope-ratelimit-model-requests-limit": "10", + "modelscope-ratelimit-requests-remaining": "17", + "modelscope-ratelimit-requests-limit": "20", + }) + ); + + assert.deepEqual(snapshot, { + modelRemaining: 0, + modelLimit: 10, + totalRemaining: 17, + totalLimit: 20, + }); +}); + +test("ModelScope policy keeps temporary 429 headers retryable", () => { + const decision = classifyModelScope429( + "Throttling: current batch requests reached the limit", + new Headers({ + "modelscope-ratelimit-model-requests-remaining": "0", + "modelscope-ratelimit-model-requests-limit": "10", + }) + ); + + assert.equal(decision.kind, "rate_limited"); + assert.equal(decision.retryable, true); + assert.equal(decision.snapshot.modelRemaining, 0); +}); + +test("ModelScope policy treats explicit free quota exhaustion as terminal", () => { + const decision = classifyModelScope429("Free allocated quota exceeded", new Headers()); + + assert.equal(decision.kind, "quota_exhausted"); + assert.equal(decision.retryable, false); +}); + +test("ModelScope retry delay respects Retry-After seconds before backoff fallback", () => { + assert.equal(getModelScopeRetryDelayMs(new Headers({ "retry-after": "2.5" }), 0), 2500); + assert.equal(getModelScopeRetryDelayMs(new Headers(), 1), 6000); +});