diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index fbcde7c28c..2cf6762d37 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -602,14 +602,39 @@ function toFiniteNumberOrNull(value: unknown): number | null { return null; } -function isSemaphoreTimeoutError(error: unknown): error is Error & { code: string } { +function isSemaphoreCapacityError(error: unknown): error is Error & { code: string } { return ( !!error && typeof error === "object" && - (error as { code?: unknown }).code === "SEMAPHORE_TIMEOUT" + ((error as { code?: unknown }).code === "SEMAPHORE_TIMEOUT" || + (error as { code?: unknown }).code === "SEMAPHORE_QUEUE_FULL") ); } +function createStreamingErrorResult(statusCode: number, message: string, code?: string) { + const errorBody = buildErrorBody(statusCode, message); + if (code) { + errorBody.error.code = code; + } + + const body = `data: ${JSON.stringify(errorBody)}\n\ndata: [DONE]\n\n`; + + return { + success: false as const, + status: statusCode, + error: message, + response: new Response(body, { + status: statusCode, + headers: { + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache, no-transform", + Connection: "keep-alive", + "X-Accel-Buffering": "no", + }, + }), + }; +} + function wrapReadableStreamWithFinalize( readable: ReadableStream, finalize: () => void @@ -3051,16 +3076,13 @@ export async function handleChatCore({ ); } catch (error) { trackPendingRequest(model, provider, connectionId, false); - if (isSemaphoreTimeoutError(error)) { + if (isSemaphoreCapacityError(error)) { appendRequestLog({ model, provider, connectionId, status: `FAILED ${error.code}`, }).catch(() => {}); - if (isCombo) { - throw error; - } const failureMessage = error.message || "Semaphore timeout"; persistAttemptLogs({ status: HTTP_STATUS.RATE_LIMITED, @@ -3071,7 +3093,14 @@ export async function handleChatCore({ cacheSource: "upstream", }); persistFailureUsage(HTTP_STATUS.RATE_LIMITED, error.code); - return createErrorResult(HTTP_STATUS.RATE_LIMITED, failureMessage); + const result = stream + ? createStreamingErrorResult(HTTP_STATUS.RATE_LIMITED, failureMessage, error.code) + : createErrorResult(HTTP_STATUS.RATE_LIMITED, failureMessage); + return { + ...result, + errorType: "account_semaphore_capacity", + errorCode: error.code, + }; } const failureStatus = error.name === "AbortError" diff --git a/open-sse/services/usage.ts b/open-sse/services/usage.ts index 8424b89af2..b55959ddd6 100644 --- a/open-sse/services/usage.ts +++ b/open-sse/services/usage.ts @@ -156,6 +156,12 @@ function getGlmTokenQuotaName( return existingQuotas.session ? "weekly" : "session"; } +function getGlmQuotaDisplayName(quotaName: string): string { + if (quotaName === "session") return "5 Hours Quota"; + if (quotaName === "weekly") return "Weekly Quota"; + return quotaName; +} + function getFieldValue(source: unknown, snakeKey: string, camelKey: string): unknown { const obj = toRecord(source); return obj[snakeKey] ?? obj[camelKey] ?? null; @@ -635,6 +641,13 @@ async function getGlmUsage(apiKey: string, providerSpecificData?: Record ({ + name: String(m.model || ""), + used: toNumber(m.percentage, 0), + })) + : [], unlimited: false, }; continue; @@ -876,6 +889,7 @@ export async function getUsageForProvider(connection, options: { forceRefresh?: return await getQoderUsage(accessToken); case "glm": case "glm-cn": + case "zai": case "glmt": return await getGlmUsage(apiKey, { ...(providerSpecificData || {}), diff --git a/src/app/(dashboard)/dashboard/usage/components/ProviderLimits/utils.tsx b/src/app/(dashboard)/dashboard/usage/components/ProviderLimits/utils.tsx index 615dd51c61..6851c621e3 100644 --- a/src/app/(dashboard)/dashboard/usage/components/ProviderLimits/utils.tsx +++ b/src/app/(dashboard)/dashboard/usage/components/ProviderLimits/utils.tsx @@ -26,6 +26,11 @@ const QUOTA_LABEL_MAP: Record = { "search-prime": "Web Search", "web-reader": "Web Reader", zread: "Zread", + "5 Hours Quota": "5 Hours", + "Weekly Quota": "Weekly", + "Monthly Tools": "Monthly Tools", + tokens: "Tokens", + time_limit: "Time Limit", }; const GLM_QUOTA_ORDER: Record = { diff --git a/src/app/api/usage/analytics/route.ts b/src/app/api/usage/analytics/route.ts index e3fada7b03..fc76fbce7b 100644 --- a/src/app/api/usage/analytics/route.ts +++ b/src/app/api/usage/analytics/route.ts @@ -555,6 +555,7 @@ export async function GET(request: Request) { COALESCE(NULLIF(service_tier, ''), 'standard') as serviceTier, LOWER(provider) as provider, LOWER(model) as model, + COALESCE(NULLIF(service_tier, ''), 'standard') as serviceTier, COUNT(*) as requests, COALESCE(SUM(tokens_input), 0) as promptTokens, COALESCE(SUM(tokens_output), 0) as completionTokens, @@ -563,6 +564,7 @@ export async function GET(request: Request) { COALESCE(SUM(tokens_reasoning), 0) as reasoningTokens, COALESCE(SUM(tokens_input + tokens_output), 0) as totalTokens FROM usage_history + ${whereClause} GROUP BY serviceTier, LOWER(provider), LOWER(model) ` diff --git a/src/lib/apiBridgeServer.ts b/src/lib/apiBridgeServer.ts index d289d6f752..4109dfd6ac 100644 --- a/src/lib/apiBridgeServer.ts +++ b/src/lib/apiBridgeServer.ts @@ -22,7 +22,22 @@ function isOpenAiCompatiblePath(pathname: string): boolean { return OPENAI_COMPAT_PATHS.some((pattern) => pattern.test(pathname)); } +function requestWantsStreaming(req: IncomingMessage): boolean { + const accept = String(req.headers.accept || "").toLowerCase(); + if (accept.includes("text/event-stream")) return true; + + const pathname = (req.url || "/").split("?")[0] || "/"; + return /^\/(?:v1\/)?(?:responses|chat\/completions)(?:\/|$)/.test(pathname); +} + +function getProxyTimeoutMs(req: IncomingMessage): number { + if (!requestWantsStreaming(req)) return API_BRIDGE_TIMEOUTS.proxyTimeoutMs; + + return Math.max(API_BRIDGE_TIMEOUTS.proxyTimeoutMs, API_BRIDGE_TIMEOUTS.serverRequestTimeoutMs); +} + function proxyRequest(req: IncomingMessage, res: ServerResponse, dashboardPort: number): void { + const proxyTimeoutMs = getProxyTimeoutMs(req); const targetReq = http.request( { hostname: "127.0.0.1", @@ -33,9 +48,14 @@ function proxyRequest(req: IncomingMessage, res: ServerResponse, dashboardPort: ...req.headers, host: `127.0.0.1:${dashboardPort}`, }, - timeout: API_BRIDGE_TIMEOUTS.proxyTimeoutMs, + timeout: proxyTimeoutMs, }, (targetRes) => { + const contentType = String(targetRes.headers["content-type"] || "").toLowerCase(); + if (contentType.includes("text/event-stream")) { + targetReq.setTimeout(0); + } + res.writeHead(targetRes.statusCode || 502, targetRes.headers); targetRes.pipe(res); } @@ -48,7 +68,7 @@ function proxyRequest(req: IncomingMessage, res: ServerResponse, dashboardPort: res.end( JSON.stringify({ error: "api_bridge_timeout", - detail: `Proxy request timed out after ${API_BRIDGE_TIMEOUTS.proxyTimeoutMs}ms`, + detail: `Proxy request timed out after ${proxyTimeoutMs}ms`, }) ); }); diff --git a/src/lib/usage/providerLimits.ts b/src/lib/usage/providerLimits.ts index df189ceb5a..11081b6332 100644 --- a/src/lib/usage/providerLimits.ts +++ b/src/lib/usage/providerLimits.ts @@ -47,6 +47,7 @@ interface ProviderConnectionLike { const PROVIDER_LIMITS_APIKEY_PROVIDERS = new Set([ "glm", "glm-cn", + "zai", "glmt", "minimax", "minimax-cn", diff --git a/src/shared/constants/providers.ts b/src/shared/constants/providers.ts index 72064f737b..ef53efc4b1 100644 --- a/src/shared/constants/providers.ts +++ b/src/shared/constants/providers.ts @@ -1982,6 +1982,7 @@ export const USAGE_SUPPORTED_PROVIDERS = [ "kimi-coding", "glm", "glm-cn", + "zai", "glmt", "minimax", "minimax-cn", diff --git a/src/shared/utils/runtimeTimeouts.ts b/src/shared/utils/runtimeTimeouts.ts index 672466a1c2..14e0e67afb 100644 --- a/src/shared/utils/runtimeTimeouts.ts +++ b/src/shared/utils/runtimeTimeouts.ts @@ -11,7 +11,7 @@ export const DEFAULT_STREAM_IDLE_TIMEOUT_MS = 600_000; export const DEFAULT_STREAM_READINESS_TIMEOUT_MS = 30_000; export const DEFAULT_FETCH_CONNECT_TIMEOUT_MS = 30_000; export const DEFAULT_FETCH_KEEPALIVE_TIMEOUT_MS = 4_000; -export const DEFAULT_API_BRIDGE_PROXY_TIMEOUT_MS = 30_000; +export const DEFAULT_API_BRIDGE_PROXY_TIMEOUT_MS = 600_000; export const DEFAULT_API_BRIDGE_SERVER_REQUEST_TIMEOUT_MS = 300_000; export const DEFAULT_API_BRIDGE_SERVER_HEADERS_TIMEOUT_MS = 60_000; export const DEFAULT_API_BRIDGE_SERVER_KEEPALIVE_TIMEOUT_MS = 5_000; diff --git a/src/sse/handlers/chat.ts b/src/sse/handlers/chat.ts index 8d57cfe438..62915eb998 100644 --- a/src/sse/handlers/chat.ts +++ b/src/sse/handlers/chat.ts @@ -874,6 +874,25 @@ async function handleSingleModelChat( return result.response; } + if (result.errorType === "account_semaphore_capacity") { + // Local concurrency pressure is not an upstream quota failure. Prefer another + // account when possible; pinned combo steps fall through to combo orchestration. + if (hasForcedConnection) { + return result.response; + } + + log.warn( + "AUTH", + `Account ${accountId}... at local concurrency cap, trying fallback account` + ); + excludedConnectionIds.add(credentials.connectionId); + lastError = result.error; + lastStatus = result.status; + requestRetryLastError = result.error; + requestRetryLastStatus = result.status; + continue; + } + // Emergency fallback for budget exhaustion (402 / billing / quota keywords): // reroute to a free model (default provider/model: nvidia + openai/gpt-oss-120b) exactly once. if (!runtimeOptions.emergencyFallbackTried) { diff --git a/tests/unit/runtime-timeouts.test.ts b/tests/unit/runtime-timeouts.test.ts index 3ac71b5de6..2a588a63b6 100644 --- a/tests/unit/runtime-timeouts.test.ts +++ b/tests/unit/runtime-timeouts.test.ts @@ -94,3 +94,10 @@ test("API bridge timeouts align request timeout with long proxy timeout by defau serverSocketTimeoutMs: 0, }); }); + +test("API bridge proxy timeout defaults to the long upstream request window", () => { + const config = runtimeTimeouts.getApiBridgeTimeoutConfig({}); + + assert.equal(config.proxyTimeoutMs, 600000); + assert.equal(config.serverRequestTimeoutMs, 600000); +});