diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 480e803ab2..fe312660ec 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -2347,6 +2347,10 @@ components: type: string isActive: type: boolean + maxConcurrent: + type: integer + nullable: true + minimum: 0 priority: type: integer testStatus: @@ -2372,6 +2376,10 @@ components: isActive: type: boolean default: true + maxConcurrent: + type: integer + nullable: true + minimum: 0 ApiKey: type: object diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 264e14a79c..15cf6abd51 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -90,6 +90,11 @@ import { updateFromResponseBody, initializeRateLimits, } from "../services/rateLimitManager.ts"; +import { + acquire as acquireAccountSemaphore, + buildAccountSemaphoreKey, + markBlocked as markAccountSemaphoreBlocked, +} from "../services/accountSemaphore.ts"; import { generateSignature, getCachedResponse, @@ -465,6 +470,72 @@ function getHeaderValueCaseInsensitive( return null; } +function toFiniteNumberOrNull(value: unknown): number | null { + if (typeof value === "number" && Number.isFinite(value)) { + return value; + } + if (typeof value === "string" && value.trim().length > 0) { + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : null; + } + return null; +} + +function isSemaphoreTimeoutError(error: unknown): error is Error & { code: string } { + return ( + !!error && + typeof error === "object" && + (error as { code?: unknown }).code === "SEMAPHORE_TIMEOUT" + ); +} + +function resolveAccountSemaphoreAccountKey( + connectionId: string | null | undefined, + credentials: Record | null | undefined +): string | null { + if (typeof connectionId === "string" && connectionId.trim().length > 0) { + return connectionId; + } + + const candidateKeys = [ + credentials?.connectionId, + credentials?.id, + credentials?.email, + credentials?.name, + credentials?.displayName, + ]; + + for (const candidate of candidateKeys) { + if (typeof candidate === "string" && candidate.trim().length > 0) { + return candidate.trim(); + } + } + + return null; +} + +function resolveAccountSemaphoreMaxConcurrency( + credentials: Record | null | undefined +): number | null { + return toFiniteNumberOrNull(credentials?.maxConcurrent); +} + +function resolveAccountSemaphoreKey({ + provider, + model, + connectionId, + credentials, +}: { + provider: string | null | undefined; + model: string; + connectionId: string | null | undefined; + credentials: Record | null | undefined; +}): string | null { + const accountKey = resolveAccountSemaphoreAccountKey(connectionId, credentials); + if (!accountKey || !provider) return null; + return buildAccountSemaphoreKey({ provider, accountKey }); +} + function buildClaudePromptCacheLogMeta( targetFormat: string, finalBody: Record | null | undefined, @@ -1783,6 +1854,15 @@ export async function handleChatCore({ const executeProviderRequest = async (modelToCall = effectiveModel, allowDedup = false) => { const execute = async () => { + const executionCredentials = getExecutionCredentials(); + const accountSemaphoreMaxConcurrency = + resolveAccountSemaphoreMaxConcurrency(executionCredentials); + const accountSemaphoreKey = resolveAccountSemaphoreKey({ + provider, + model: modelToCall, + connectionId, + credentials: executionCredentials, + }); let bodyToSend = translatedBody.model === modelToCall ? translatedBody @@ -1850,6 +1930,13 @@ export async function handleChatCore({ } } + const acquireAccountSemaphoreRelease = + accountSemaphoreKey && accountSemaphoreMaxConcurrency != null + ? await acquireAccountSemaphore(accountSemaphoreKey, { + maxConcurrency: accountSemaphoreMaxConcurrency, + }) + : () => {}; + const rawResult = await withRateLimit(provider, connectionId, modelToCall, async () => { let attempts = 0; const maxAttempts = provider === "qwen" ? 3 : 1; @@ -1859,7 +1946,7 @@ export async function handleChatCore({ model: modelToCall, body: bodyToSend, stream: upstreamStream, - credentials: getExecutionCredentials(), + credentials: executionCredentials, signal: streamController.signal, log, extendedContext, @@ -1882,17 +1969,39 @@ export async function handleChatCore({ continue; } } + + // For streaming: wrap response body to release semaphore only when stream is fully consumed + if (stream) { + const originalBody = res.response.body; + const wrappedBody = originalBody + ? originalBody.pipeThrough( + new TransformStream({ + flush: () => { + acquireAccountSemaphoreRelease(); + }, + }) + ) + : null; + return { + ...res, + response: new Response(wrappedBody, { + status: res.response.status, + statusText: res.response.statusText, + headers: res.response.headers, + }), + }; + } + return res; } }); - if (stream) return rawResult; - - // Non-stream responses need cloning for shared dedup consumers. + // Non-stream: release semaphore immediately after reading full response body const status = rawResult.response.status; const statusText = rawResult.response.statusText; const headers = Array.from(rawResult.response.headers.entries()) as [string, string][]; const payload = await rawResult.response.text(); + acquireAccountSemaphoreRelease(); return { ...rawResult, @@ -1967,6 +2076,28 @@ export async function handleChatCore({ ); } catch (error) { trackPendingRequest(model, provider, connectionId, false); + if (isSemaphoreTimeoutError(error)) { + appendRequestLog({ + model, + provider, + connectionId, + status: `FAILED ${error.code}`, + }).catch(() => {}); + if (isCombo) { + throw error; + } + const failureMessage = error.message || "Semaphore timeout"; + persistAttemptLogs({ + status: HTTP_STATUS.RATE_LIMITED, + error: failureMessage, + providerRequest: finalBody || translatedBody, + clientResponse: buildErrorBody(HTTP_STATUS.RATE_LIMITED, failureMessage), + claudeCacheMeta: claudePromptCacheLogMeta, + cacheSource: "upstream", + }); + persistFailureUsage(HTTP_STATUS.RATE_LIMITED, error.code); + return createErrorResult(HTTP_STATUS.RATE_LIMITED, failureMessage); + } const failureStatus = error.name === "AbortError" ? 499 @@ -2138,6 +2269,80 @@ export async function handleChatCore({ console.warn( `[provider] Node ${connectionId} account deactivated (${statusCode}) — disabling permanently` ); + } else if (errorType === PROVIDER_ERROR_TYPES.RATE_LIMITED) { + // For providers with per-model quotas (passthrough providers, Gemini), + // each model has independent quota. A 429 on one model must NOT lock out + // the entire connection — other models may still have quota available. + const rateLimitCooldownMs = retryAfterMs || COOLDOWN_MS.rateLimit; + const accountSemaphoreKey = resolveAccountSemaphoreKey({ + provider, + model: currentModel, + connectionId, + credentials, + }); + if (accountSemaphoreKey) { + markAccountSemaphoreBlocked(accountSemaphoreKey, rateLimitCooldownMs); + } + if ( + lockModelIfPerModelQuota( + provider, + connectionId, + model, + "rate_limited", + rateLimitCooldownMs + ) + ) { + console.warn( + `[provider] Node ${connectionId} model-only rate limited (${statusCode}) for ${model} - ${Math.ceil(rateLimitCooldownMs / 1000)}s (connection stays active)` + ); + } else { + const rateLimitedUntil = new Date(Date.now() + rateLimitCooldownMs).toISOString(); + await updateProviderConnection(connectionId, { + rateLimitedUntil: rateLimitedUntil, + testStatus: "unavailable", + lastErrorType: errorType, + lastError: message, + errorCode: statusCode, + healthCheckInterval: null, + lastHealthCheckAt: null, + }); + console.warn( + `[provider] Node ${connectionId} rate limited (${statusCode}) - Next available at ${rateLimitedUntil}` + ); + } + } else if (errorType === PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED) { + // Providers with per-model quotas — lock the model only, not the connection + const quotaCooldownMs = retryAfterMs || COOLDOWN_MS.rateLimit; + const accountSemaphoreKey = resolveAccountSemaphoreKey({ + provider, + model: currentModel, + connectionId, + credentials, + }); + if (accountSemaphoreKey) { + markAccountSemaphoreBlocked(accountSemaphoreKey, quotaCooldownMs); + } + if ( + lockModelIfPerModelQuota( + provider, + connectionId, + model, + "quota_exhausted", + quotaCooldownMs + ) + ) { + console.warn( + `[provider] Node ${connectionId} model-only quota exhausted (${statusCode}) for ${model} - ${Math.ceil(quotaCooldownMs / 1000)}s (connection stays active)` + ); + } else { + await updateProviderConnection(connectionId, { + testStatus: "credits_exhausted", + lastErrorType: errorType, + lastError: message, + errorCode: statusCode, + }); + console.warn(`[provider] Node ${connectionId} exhausted quota (${statusCode})`); + } } else if (errorType === PROVIDER_ERROR_TYPES.ACCOUNT_DEACTIVATED) { await updateProviderConnection(connectionId, { isActive: false, diff --git a/open-sse/services/accountSemaphore.ts b/open-sse/services/accountSemaphore.ts new file mode 100644 index 0000000000..0f96dec799 --- /dev/null +++ b/open-sse/services/accountSemaphore.ts @@ -0,0 +1,288 @@ +/** + * Account Semaphore + * + * In-memory provider/account concurrency limiter keyed by provider and account. + * Requests beyond the configured concurrency cap wait in a FIFO queue until a slot opens, + * the gate is unblocked, or the queue timeout expires. + */ + +export interface AccountSemaphoreKeyParts { + provider: string; + accountKey: string; +} + +interface QueuedAcquire { + resolve: (release: () => void) => void; + reject: (error: Error) => void; + timer: ReturnType; +} + +interface AccountGate { + running: number; + maxConcurrency: number; + queue: QueuedAcquire[]; + blockedUntil: number | null; + cleanupTimer: ReturnType | null; +} + +export interface AcquireAccountSemaphoreOptions { + maxConcurrency?: number | null; + timeoutMs?: number; +} + +export interface AccountSemaphoreStatsEntry { + running: number; + queued: number; + maxConcurrency: number; + blockedUntil: string | null; +} + +const DEFAULT_TIMEOUT_MS = 30_000; + +const gates = new Map(); + +/** + * Build the canonical account semaphore key. + */ +export function buildAccountSemaphoreKey({ + provider, + accountKey, +}: AccountSemaphoreKeyParts): string { + return `${String(provider)}:${String(accountKey)}`; +} + +function isBypassed(maxConcurrency?: number | null): boolean { + return maxConcurrency == null || maxConcurrency <= 0; +} + +function createNoopReleaseFn(): () => void { + let released = false; + + return () => { + if (released) return; + released = true; + }; +} + +function ensureGate(semaphoreKey: string, maxConcurrency: number): AccountGate { + const existing = gates.get(semaphoreKey); + if (existing) { + existing.maxConcurrency = maxConcurrency; + return existing; + } + + const created: AccountGate = { + running: 0, + maxConcurrency, + queue: [], + blockedUntil: null, + cleanupTimer: null, + }; + gates.set(semaphoreKey, created); + return created; +} + +function isBlocked(gate: AccountGate): boolean { + if (!gate.blockedUntil) return false; + if (Date.now() >= gate.blockedUntil) { + gate.blockedUntil = null; + return false; + } + return true; +} + +function clearCleanupTimer(gate: AccountGate): void { + if (!gate.cleanupTimer) return; + clearTimeout(gate.cleanupTimer); + gate.cleanupTimer = null; +} + +function cleanupGateIfIdle(semaphoreKey: string): void { + const gate = gates.get(semaphoreKey); + if (!gate) return; + if (gate.running > 0 || gate.queue.length > 0 || isBlocked(gate)) return; + clearCleanupTimer(gate); + gates.delete(semaphoreKey); +} + +function scheduleCleanup(semaphoreKey: string): void { + const gate = gates.get(semaphoreKey); + if (!gate) return; + clearCleanupTimer(gate); + + gate.cleanupTimer = setTimeout(() => { + gate.cleanupTimer = null; + cleanupGateIfIdle(semaphoreKey); + }, 0); + + gate.cleanupTimer.unref?.(); +} + +function drainQueue(semaphoreKey: string): void { + const gate = gates.get(semaphoreKey); + if (!gate) return; + + while (gate.queue.length > 0 && gate.running < gate.maxConcurrency && !isBlocked(gate)) { + const next = gate.queue.shift(); + if (!next) break; + clearTimeout(next.timer); + gate.running++; + next.resolve(createReleaseFn(semaphoreKey)); + } + + if (gate.running === 0 && gate.queue.length === 0) { + scheduleCleanup(semaphoreKey); + } +} + +function createReleaseFn(semaphoreKey: string): () => void { + let released = false; + + return () => { + if (released) return; + released = true; + + const gate = gates.get(semaphoreKey); + if (!gate) return; + if (gate.running > 0) { + gate.running--; + } + + if (gate.queue.length > 0) { + drainQueue(semaphoreKey); + return; + } + + scheduleCleanup(semaphoreKey); + }; +} + +function createSemaphoreTimeoutError( + semaphoreKey: string, + timeoutMs: number +): Error & { code: string } { + const error = new Error(`Semaphore timeout after ${timeoutMs}ms for ${semaphoreKey}`) as Error & { + code: string; + }; + error.code = "SEMAPHORE_TIMEOUT"; + return error; +} + +/** + * Acquire a slot for a provider/model/account tuple. + * Returns an idempotent release function that is safe to call in finally blocks. + */ +export function acquire( + semaphoreKey: string, + { maxConcurrency = null, timeoutMs = DEFAULT_TIMEOUT_MS }: AcquireAccountSemaphoreOptions = {} +): Promise<() => void> { + if (isBypassed(maxConcurrency)) { + return Promise.resolve(createNoopReleaseFn()); + } + + const gate = ensureGate(semaphoreKey, maxConcurrency); + clearCleanupTimer(gate); + + if (gate.running < gate.maxConcurrency && !isBlocked(gate)) { + gate.running++; + return Promise.resolve(createReleaseFn(semaphoreKey)); + } + + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + const nextGate = gates.get(semaphoreKey); + if (!nextGate) { + reject(createSemaphoreTimeoutError(semaphoreKey, timeoutMs)); + return; + } + + const queueIndex = nextGate.queue.findIndex((item) => item.timer === timer); + if (queueIndex !== -1) { + nextGate.queue.splice(queueIndex, 1); + } + + if (nextGate.running === 0 && nextGate.queue.length === 0) { + scheduleCleanup(semaphoreKey); + } + + reject(createSemaphoreTimeoutError(semaphoreKey, timeoutMs)); + }, timeoutMs); + + timer.unref?.(); + gate.queue.push({ resolve, reject, timer }); + }); +} + +/** + * Temporarily block new acquisitions for a key while allowing in-flight requests to finish. + */ +export function markBlocked(semaphoreKey: string, cooldownMs: number): void { + const safeCooldownMs = Number.isFinite(cooldownMs) && cooldownMs > 0 ? cooldownMs : 0; + if (safeCooldownMs <= 0) { + const gate = gates.get(semaphoreKey); + if (!gate) return; + gate.blockedUntil = null; + drainQueue(semaphoreKey); + return; + } + + const gate = gates.get(semaphoreKey) ?? ensureGate(semaphoreKey, 1); + clearCleanupTimer(gate); + gate.blockedUntil = Date.now() + safeCooldownMs; + + const timer = setTimeout(() => { + const nextGate = gates.get(semaphoreKey); + if (!nextGate) return; + if (nextGate.blockedUntil && Date.now() >= nextGate.blockedUntil) { + nextGate.blockedUntil = null; + drainQueue(semaphoreKey); + if (nextGate.running === 0 && nextGate.queue.length === 0) { + scheduleCleanup(semaphoreKey); + } + } + }, safeCooldownMs + 50); + + timer.unref?.(); +} + +/** + * Return the current in-memory semaphore snapshot. + */ +export function getStats(): Record { + const stats: Record = {}; + + for (const [key, gate] of gates) { + stats[key] = { + running: gate.running, + queued: gate.queue.length, + maxConcurrency: gate.maxConcurrency, + blockedUntil: gate.blockedUntil ? new Date(gate.blockedUntil).toISOString() : null, + }; + } + + return stats; +} + +/** + * Reset a single key and reject queued waiters. + */ +export function reset(semaphoreKey: string): void { + const gate = gates.get(semaphoreKey); + if (!gate) return; + + clearCleanupTimer(gate); + for (const entry of gate.queue) { + clearTimeout(entry.timer); + entry.reject(new Error("Semaphore reset")); + } + gates.delete(semaphoreKey); +} + +/** + * Reset all keys and reject queued waiters. + */ +export function resetAll(): void { + for (const key of gates.keys()) { + reset(key); + } +} diff --git a/src/app/(dashboard)/dashboard/combos/page.tsx b/src/app/(dashboard)/dashboard/combos/page.tsx index 6406158e7b..376f037b49 100644 --- a/src/app/(dashboard)/dashboard/combos/page.tsx +++ b/src/app/(dashboard)/dashboard/combos/page.tsx @@ -140,8 +140,10 @@ const STRATEGY_GUIDANCE_FALLBACK = { const ADVANCED_FIELD_HELP_FALLBACK = { maxRetries: "How many retries are attempted before failing the request.", retryDelay: "Initial delay between retries. Higher values reduce burst pressure.", - concurrencyPerModel: "Max simultaneous requests sent to each model in round-robin.", - queueTimeout: "How long a request can wait in queue before timeout in round-robin.", + concurrencyPerModel: + "Round-robin combo/model limit: max simultaneous requests sent to each model target. This is separate from any provider account-only cap.", + queueTimeout: + "How long a request can wait for a round-robin model slot before timing out. This queue is separate from any account-only concurrency cap.", }; const LEGACY_COMBO_RESILIENCE_KEYS = new Set([ diff --git a/src/app/(dashboard)/dashboard/providers/[id]/page.tsx b/src/app/(dashboard)/dashboard/providers/[id]/page.tsx index 11370b1d42..b8d45573a9 100644 --- a/src/app/(dashboard)/dashboard/providers/[id]/page.tsx +++ b/src/app/(dashboard)/dashboard/providers/[id]/page.tsx @@ -543,6 +543,7 @@ interface EditConnectionModalConnection { name?: string; email?: string; priority?: number; + maxConcurrent?: number | null; authType?: string; provider?: string; providerSpecificData?: Record; @@ -5910,6 +5911,7 @@ function EditConnectionModal({ isOpen, connection, onSave, onClose }: EditConnec const [formData, setFormData] = useState({ name: "", priority: 1, + maxConcurrent: "", apiKey: "", healthCheckInterval: 60, baseUrl: "", @@ -5980,6 +5982,10 @@ function EditConnectionModal({ isOpen, connection, onSave, onClose }: EditConnec setFormData({ name: connection.name || "", priority: connection.priority || 1, + maxConcurrent: + connection.maxConcurrent === null || connection.maxConcurrent === undefined + ? "" + : String(connection.maxConcurrent), apiKey: "", healthCheckInterval: connection.healthCheckInterval ?? 60, baseUrl: existingBaseUrl || defaultBaseUrl, @@ -6077,9 +6083,21 @@ function EditConnectionModal({ isOpen, connection, onSave, onClose }: EditConnec setSaving(true); setSaveError(null); try { + const trimmedMaxConcurrent = formData.maxConcurrent.trim(); + let parsedMaxConcurrent: number | null = null; + if (trimmedMaxConcurrent) { + const numericMaxConcurrent = Number(trimmedMaxConcurrent); + if (!Number.isInteger(numericMaxConcurrent) || numericMaxConcurrent < 0) { + setSaveError("Max concurrent must be a whole number greater than or equal to 0."); + return; + } + parsedMaxConcurrent = numericMaxConcurrent; + } + const updates: any = { name: formData.name, priority: formData.priority, + maxConcurrent: parsedMaxConcurrent, healthCheckInterval: formData.healthCheckInterval, }; @@ -6340,6 +6358,30 @@ function EditConnectionModal({ isOpen, connection, onSave, onClose }: EditConnec setFormData({ ...formData, priority: Number.parseInt(e.target.value) || 1 }) } /> + { + const nextValue = e.target.value; + setFormData({ ...formData, maxConcurrent: nextValue }); + if (saveError && nextValue.trim()) { + const numericValue = Number(nextValue); + if (Number.isInteger(numericValue) && numericValue >= 0) { + setSaveError(null); + } + } + }} + placeholder="0" + hint={t("accountConcurrencyCapHint")} + /> + {saveError && ( +
+ {saveError} +
+ )} {!isOAuth && ( <>
@@ -6385,11 +6427,6 @@ function EditConnectionModal({ isOpen, connection, onSave, onClose }: EditConnec {validationResult === "success" ? t("valid") : t("invalid")} )} - {saveError && ( -
- {saveError} -
- )}