feat: add ModelScope provider-specific 429 handling and retry logic (#2202)

Integrated into release/v3.8.0 after syncing the contributor branch, removing unrelated workflow/docker/package-lock changes, tightening ModelScope 429 classification, and validating policy coverage locally.

open-sse/handlers/chatCore.ts

@@ -128,7 +128,7 @@ import {
   buildAccountSemaphoreKey,
   markBlocked as markAccountSemaphoreBlocked,
 } from "../services/accountSemaphore.ts";
-import { lockModelIfPerModelQuota } from "../services/accountFallback.ts";
+import { lockModel, lockModelIfPerModelQuota } from "../services/accountFallback.ts";
 import {
   generateSignature,
   getCachedResponse,
@@ -185,6 +185,11 @@ import {
 import { setGeminiThoughtSignatureMode } from "../services/geminiThoughtSignatureStore.ts";
 import { fetchLiveProviderLimits } from "@/lib/usage/providerLimits";
 import { isClaudeExtraUsageBlockEnabled } from "@/lib/providers/claudeExtraUsage";
+import {
+  classifyModelScope429,
+  getModelScopeRetryDelayMs,
+  isModelScopeProvider,
+} from "../services/modelscopePolicy.ts";
 
 const MEMORY_EXTRACTION_TEXT_LIMIT = 64 * 1024;
 
@@ -1160,6 +1165,7 @@ export async function handleChatCore({
   let { provider, model, extendedContext } = modelInfo;
   const requestedModel =
     typeof body?.model === "string" && body.model.trim().length > 0 ? body.model : model;
+  const isModelScope = () => isModelScopeProvider(provider, credentials?.providerSpecificData);
   const startTime = Date.now();
   // Per-request trace id + checkpoint helper. Lets us see exactly which await
   // a hung request was sitting on in `[STAGE_TRACE]` log lines.
@@ -2970,7 +2976,14 @@ export async function handleChatCore({
           async () => {
             trace("inside_rate_limit");
             let attempts = 0;
-            const maxAttempts = provider === "qwen" ? 3 : provider === "codex" ? 3 : 1;
+            const isModelScopeForRequest = isModelScope();
+            const maxAttempts = isModelScopeForRequest
+              ? 3
+              : provider === "qwen"
+                ? 3
+                : provider === "codex"
+                  ? 3
+                  : 1;
 
             // ── Codex 429 account-rotation state ─────────────────────────────────
             // Track excluded connection IDs for codex failover across attempts.
@@ -3016,6 +3029,28 @@ export async function handleChatCore({
                 }
               }
 
+              if (
+                isModelScope() &&
+                res.response.status === 429 &&
+                attempts < maxAttempts - 1
+              ) {
+                const bodyPeek = await res.response
+                  .clone()
+                  .text()
+                  .catch(() => "");
+                const decision = classifyModelScope429(bodyPeek, res.response.headers);
+                if (decision.retryable) {
+                  const delay = getModelScopeRetryDelayMs(res.response.headers, attempts);
+                  log?.warn?.(
+                    "MODELSCOPE_RETRY",
+                    `429 ${decision.kind}; retrying in ${delay}ms (model remaining: ${decision.snapshot.modelRemaining ?? "unknown"})`
+                  );
+                  await new Promise((r) => setTimeout(r, delay));
+                  attempts++;
+                  continue;
+                }
+              }
+
               // Codex 429 account-rotation failover (disabled for context-relay so combo.ts can inject handoff)
               if (
                 provider === "codex" &&
@@ -3422,7 +3457,18 @@ export async function handleChatCore({
     }
 
     // T06/T10/T36: classify provider errors and persist terminal account states.
-    const errorType = classifyProviderError(statusCode, message, provider);
+    let errorType = classifyProviderError(statusCode, message, provider);
+    if (statusCode === 429 && isModelScope()) {
+      const decision = classifyModelScope429(message, providerResponse.headers);
+      errorType =
+        decision.kind === "quota_exhausted"
+          ? PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED
+          : PROVIDER_ERROR_TYPES.RATE_LIMITED;
+      log?.warn?.(
+        "MODELSCOPE_429",
+        `${decision.kind} (model remaining: ${decision.snapshot.modelRemaining ?? "unknown"}, total remaining: ${decision.snapshot.totalRemaining ?? "unknown"})`
+      );
+    }
     if (connectionId && errorType) {
       try {
         if (errorType === PROVIDER_ERROR_TYPES.FORBIDDEN) {
@@ -3459,7 +3505,12 @@ export async function handleChatCore({
           if (accountSemaphoreKey) {
             markAccountSemaphoreBlocked(accountSemaphoreKey, quotaCooldownMs);
           }
-          if (
+          if (isModelScope() && connectionId) {
+            lockModel(provider, connectionId, model, "quota_exhausted", quotaCooldownMs);
+            console.warn(
+              `[provider] Node ${connectionId} ModelScope model quota exhausted (${statusCode}) for ${model} - ${Math.ceil(quotaCooldownMs / 1000)}s (connection stays active)`
+            );
+          } else if (
             lockModelIfPerModelQuota(
               provider,
               connectionId,

open-sse/services/modelscopePolicy.ts

@@ -0,0 +1,88 @@
+type ModelScopeRateLimitSnapshot = {
+  modelRemaining: number | null;
+  modelLimit: number | null;
+  totalRemaining: number | null;
+  totalLimit: number | null;
+};
+
+type ModelScope429Decision =
+  | { kind: "quota_exhausted"; retryable: false; snapshot: ModelScopeRateLimitSnapshot }
+  | { kind: "rate_limited"; retryable: true; snapshot: ModelScopeRateLimitSnapshot };
+
+const MODELSCOPE_HOST_MARKERS = ["modelscope.cn", "modelscope.aliyuncs.com"];
+const MODELSCOPE_QUOTA_EXHAUSTED_SIGNALS = ["free allocated quota exceeded"];
+const MODELSCOPE_THROTTLE_SIGNALS = [
+  "throttling",
+  "throttled",
+  "rate limit",
+  "too many requests",
+  "batch requests",
+  "allocated quota exceeded",
+  "exceeded your current quota",
+];
+
+function parseHeaderInteger(value: string | null): number | null {
+  if (value === null || value.trim() === "") return null;
+  const parsed = Number.parseInt(value, 10);
+  return Number.isFinite(parsed) ? parsed : null;
+}
+
+function getProviderBaseUrl(providerSpecificData?: unknown): string {
+  if (!providerSpecificData || typeof providerSpecificData !== "object") return "";
+  const data = providerSpecificData as Record<string, unknown>;
+  const value = data.baseUrl ?? data.baseURL ?? data.url ?? data.endpoint;
+  return typeof value === "string" ? value.toLowerCase() : "";
+}
+
+export function isModelScopeProvider(
+  provider: string | null | undefined,
+  providerSpecificData?: unknown
+): boolean {
+  if (
+    String(provider || "")
+      .trim()
+      .toLowerCase() === "modelscope"
+  )
+    return true;
+  const baseUrl = getProviderBaseUrl(providerSpecificData);
+  return MODELSCOPE_HOST_MARKERS.some((marker) => baseUrl.includes(marker));
+}
+
+export function parseModelScopeRateLimitHeaders(headers: Headers): ModelScopeRateLimitSnapshot {
+  return {
+    modelRemaining: parseHeaderInteger(
+      headers.get("modelscope-ratelimit-model-requests-remaining")
+    ),
+    modelLimit: parseHeaderInteger(headers.get("modelscope-ratelimit-model-requests-limit")),
+    totalRemaining: parseHeaderInteger(headers.get("modelscope-ratelimit-requests-remaining")),
+    totalLimit: parseHeaderInteger(headers.get("modelscope-ratelimit-requests-limit")),
+  };
+}
+
+export function classifyModelScope429(errorText: string, headers: Headers): ModelScope429Decision {
+  const snapshot = parseModelScopeRateLimitHeaders(headers);
+  const lower = String(errorText || "").toLowerCase();
+
+  if (MODELSCOPE_QUOTA_EXHAUSTED_SIGNALS.some((signal) => lower.includes(signal))) {
+    return { kind: "quota_exhausted", retryable: false, snapshot };
+  }
+
+  if (snapshot.modelRemaining !== null || snapshot.totalRemaining !== null) {
+    return { kind: "rate_limited", retryable: true, snapshot };
+  }
+
+  if (MODELSCOPE_THROTTLE_SIGNALS.some((signal) => lower.includes(signal))) {
+    return { kind: "rate_limited", retryable: true, snapshot };
+  }
+
+  return { kind: "rate_limited", retryable: true, snapshot };
+}
+
+export function getModelScopeRetryDelayMs(headers: Headers, attempt: number): number {
+  const retryAfter = headers.get("retry-after");
+  if (retryAfter) {
+    const parsed = Number.parseFloat(retryAfter);
+    if (Number.isFinite(parsed) && parsed > 0) return parsed * 1000;
+  }
+  return 3000 * (attempt + 1);
+}

tests/unit/modelscope-policy.test.ts

@@ -0,0 +1,64 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+
+const {
+  classifyModelScope429,
+  getModelScopeRetryDelayMs,
+  isModelScopeProvider,
+  parseModelScopeRateLimitHeaders,
+} = await import("../../open-sse/services/modelscopePolicy.ts");
+
+test("ModelScope policy detects provider ids and ModelScope host markers", () => {
+  assert.equal(isModelScopeProvider("modelscope"), true);
+  assert.equal(
+    isModelScopeProvider("openai-compatible-custom", {
+      baseUrl: "https://api-inference.modelscope.cn/v1",
+    }),
+    true
+  );
+  assert.equal(isModelScopeProvider("openai", { baseUrl: "https://api.openai.com/v1" }), false);
+});
+
+test("ModelScope policy parses per-model and total rate-limit headers", () => {
+  const snapshot = parseModelScopeRateLimitHeaders(
+    new Headers({
+      "modelscope-ratelimit-model-requests-remaining": "0",
+      "modelscope-ratelimit-model-requests-limit": "10",
+      "modelscope-ratelimit-requests-remaining": "17",
+      "modelscope-ratelimit-requests-limit": "20",
+    })
+  );
+
+  assert.deepEqual(snapshot, {
+    modelRemaining: 0,
+    modelLimit: 10,
+    totalRemaining: 17,
+    totalLimit: 20,
+  });
+});
+
+test("ModelScope policy keeps temporary 429 headers retryable", () => {
+  const decision = classifyModelScope429(
+    "Throttling: current batch requests reached the limit",
+    new Headers({
+      "modelscope-ratelimit-model-requests-remaining": "0",
+      "modelscope-ratelimit-model-requests-limit": "10",
+    })
+  );
+
+  assert.equal(decision.kind, "rate_limited");
+  assert.equal(decision.retryable, true);
+  assert.equal(decision.snapshot.modelRemaining, 0);
+});
+
+test("ModelScope policy treats explicit free quota exhaustion as terminal", () => {
+  const decision = classifyModelScope429("Free allocated quota exceeded", new Headers());
+
+  assert.equal(decision.kind, "quota_exhausted");
+  assert.equal(decision.retryable, false);
+});
+
+test("ModelScope retry delay respects Retry-After seconds before backoff fallback", () => {
+  assert.equal(getModelScopeRetryDelayMs(new Headers({ "retry-after": "2.5" }), 0), 2500);
+  assert.equal(getModelScopeRetryDelayMs(new Headers(), 1), 6000);
+});