/** * Shared combo (model combo) handling with fallback support * Supports: priority, weighted, round-robin, random, least-used, and cost-optimized strategies */ import { checkFallbackError, formatRetryAfter, getProviderProfile } from "./accountFallback.ts"; import { unavailableResponse } from "../utils/error.ts"; import { recordComboIntent, recordComboRequest, getComboMetrics } from "./comboMetrics.ts"; import { resolveComboConfig, getDefaultComboConfig } from "./comboConfig.ts"; import * as semaphore from "./rateLimitSemaphore.ts"; import { getCircuitBreaker } from "../../src/shared/utils/circuitBreaker"; import { fisherYatesShuffle, getNextFromDeck } from "../../src/shared/utils/shuffleDeck"; import { parseModel } from "./model.ts"; import { applyComboAgentMiddleware, injectModelTag } from "./comboAgentMiddleware.ts"; import { classifyWithConfig, DEFAULT_INTENT_CONFIG } from "./intentClassifier.ts"; import { selectProvider as selectAutoProvider } from "./autoCombo/engine.ts"; import { selectWithStrategy } from "./autoCombo/routerStrategy.ts"; import { DEFAULT_WEIGHTS, scorePool } from "./autoCombo/scoring.ts"; import { supportsToolCalling } from "./modelCapabilities.ts"; // Status codes that should mark semaphore + record circuit breaker failures const TRANSIENT_FOR_BREAKER = [429, 502, 503, 504]; const MAX_COMBO_DEPTH = 3; // Bootstrap defaults from ClawRouter benchmark (used when no local latency history exists yet) const DEFAULT_MODEL_P95_MS = { "grok-4-fast-non-reasoning": 1143, "grok-4-1-fast-non-reasoning": 1244, "gemini-2.5-flash": 1238, "kimi-k2.5": 1646, "gpt-4o-mini": 2764, "claude-sonnet-4.6": 4000, "claude-opus-4.6": 6000, "deepseek-chat": 2000, }; const MIN_HISTORY_SAMPLES = 10; // In-memory atomic counter per combo for round-robin distribution // Resets on server restart (by design — no stale state) const rrCounters = new Map(); /** * Normalize a model entry to { model, weight } * Supports both legacy string format and new object format */ function normalizeModelEntry(entry) { if (typeof entry === "string") return { model: entry, weight: 0 }; return { model: entry.model, weight: entry.weight || 0 }; } /** * Get combo models from combos data (for open-sse standalone use) * @param {string} modelStr - Model string to check * @param {Array|Object} combosData - Array of combos or object with combos * @returns {Object|null} Full combo object or null if not a combo */ export function getComboFromData(modelStr, combosData) { const combos = Array.isArray(combosData) ? combosData : combosData?.combos || []; const combo = combos.find((c) => c.name === modelStr); if (combo && combo.models && combo.models.length > 0) { return combo; } return null; } /** * Legacy: Get combo models as string array (backward compat) */ export function getComboModelsFromData(modelStr, combosData) { const combo = getComboFromData(modelStr, combosData); if (!combo) return null; return combo.models.map((m) => normalizeModelEntry(m).model); } /** * Validate combo DAG — detect circular references and enforce max depth * @param {string} comboName - Name of the combo to validate * @param {Array} allCombos - All combos in the system * @param {Set} [visited] - Set of already visited combo names (for cycle detection) * @param {number} [depth] - Current depth level * @throws {Error} If circular reference or max depth exceeded */ export function validateComboDAG(comboName, allCombos, visited = new Set(), depth = 0) { if (depth > MAX_COMBO_DEPTH) { throw new Error(`Max combo nesting depth (${MAX_COMBO_DEPTH}) exceeded at "${comboName}"`); } if (visited.has(comboName)) { throw new Error(`Circular combo reference detected: ${comboName}`); } visited.add(comboName); const combos = Array.isArray(allCombos) ? allCombos : allCombos?.combos || []; const combo = combos.find((c) => c.name === comboName); if (!combo || !combo.models) return; for (const entry of combo.models) { const modelName = normalizeModelEntry(entry).model; // Check if this model name is itself a combo (not a provider/model pattern) const nestedCombo = combos.find((c) => c.name === modelName); if (nestedCombo) { validateComboDAG(modelName, combos, new Set(visited), depth + 1); } } } /** * Resolve nested combos by expanding inline to a flat model list * Respects max depth and detects cycles * @param {Object} combo - The combo object * @param {Array} allCombos - All combos in the system * @param {Set} [visited] - For cycle detection * @param {number} [depth] - Current depth * @returns {Array} Flat array of model strings */ export function resolveNestedComboModels(combo, allCombos, visited = new Set(), depth = 0) { if (depth > MAX_COMBO_DEPTH) return combo.models.map((m) => normalizeModelEntry(m).model); if (visited.has(combo.name)) return []; // cycle safety visited.add(combo.name); const combos = Array.isArray(allCombos) ? allCombos : allCombos?.combos || []; const resolved = []; for (const entry of combo.models || []) { const modelName = normalizeModelEntry(entry).model; const nestedCombo = combos.find((c) => c.name === modelName); if (nestedCombo) { // Recursively expand the nested combo const nested = resolveNestedComboModels(nestedCombo, combos, new Set(visited), depth + 1); resolved.push(...nested); } else { resolved.push(modelName); } } return resolved; } /** * Select a model using weighted random distribution * @param {Array} models - Array of { model, weight } entries * @returns {string} Selected model string */ function selectWeightedModel(models) { const entries = models.map((m) => normalizeModelEntry(m)); const totalWeight = entries.reduce((sum, m) => sum + m.weight, 0); if (totalWeight <= 0) { // All weights are 0 → uniform random return entries[Math.floor(Math.random() * entries.length)].model; } let random = Math.random() * totalWeight; for (const entry of entries) { random -= entry.weight; if (random <= 0) return entry.model; } return entries[entries.length - 1].model; // safety fallback } /** * Order models for weighted fallback (selected first, then by descending weight) */ function orderModelsForWeightedFallback(models, selectedModel) { const entries = models.map((m) => normalizeModelEntry(m)); const selected = entries.find((e) => e.model === selectedModel); const rest = entries.filter((e) => e.model !== selectedModel).sort((a, b) => b.weight - a.weight); // highest weight first for fallback return [selected, ...rest].filter(Boolean).map((e) => e.model); } // shuffleArray and getNextModelFromDeck moved to src/shared/utils/shuffleDeck.ts // combo.ts now uses the shared, mutex-protected getNextFromDeck with "combo:" namespace. /** * Sort models by pricing (cheapest first) for cost-optimized strategy * @param {Array} models - Model strings in "provider/model" format * @returns {Promise>} Sorted model strings */ async function sortModelsByCost(models) { try { const { getPricingForModel } = await import("../../src/lib/localDb"); const withCost = await Promise.all( models.map(async (modelStr) => { const parsed = parseModel(modelStr); const provider = parsed.provider || parsed.providerAlias || "unknown"; const model = parsed.model || modelStr; try { const pricing = await getPricingForModel(provider, model); return { modelStr, cost: pricing?.input ?? Infinity }; } catch { return { modelStr, cost: Infinity }; } }) ); withCost.sort((a, b) => a.cost - b.cost); return withCost.map((e) => e.modelStr); } catch { // If pricing lookup fails entirely, return original order return models; } } /** * Sort models by usage count (least-used first) for least-used strategy * @param {Array} models - Model strings * @param {string} comboName - Combo name for metrics lookup * @returns {Array} Sorted model strings */ function sortModelsByUsage(models, comboName) { const metrics = getComboMetrics(comboName); if (!metrics || !metrics.byModel) return models; const withUsage = models.map((modelStr) => ({ modelStr, requests: metrics.byModel[modelStr]?.requests ?? 0, })); withUsage.sort((a, b) => a.requests - b.requests); return withUsage.map((e) => e.modelStr); } function toTextContent(content) { if (typeof content === "string") return content; if (!Array.isArray(content)) return ""; return content .map((part) => { if (!part || typeof part !== "object") return ""; if (typeof part.text === "string") return part.text; return ""; }) .join("\n"); } function extractPromptForIntent(body) { if (!body || typeof body !== "object") return ""; const fromMessages = Array.isArray(body.messages) ? [...body.messages].reverse().find((m) => m && typeof m === "object" && m.role === "user") : null; if (fromMessages) return toTextContent(fromMessages.content); if (typeof body.input === "string") return body.input; if (Array.isArray(body.input)) { const text = body.input .map((item) => { if (!item || typeof item !== "object") return ""; if (typeof item.content === "string") return item.content; if (typeof item.text === "string") return item.text; return ""; }) .filter(Boolean) .join("\n"); if (text) return text; } if (typeof body.prompt === "string") return body.prompt; return ""; } function mapIntentToTaskType(intent) { switch (intent) { case "code": return "coding"; case "reasoning": return "analysis"; case "simple": return "default"; case "medium": default: return "default"; } } function toStringArray(input) { if (Array.isArray(input)) { return input.map((v) => (typeof v === "string" ? v.trim() : "")).filter(Boolean); } if (typeof input === "string") { return input .split(",") .map((v) => v.trim()) .filter(Boolean); } return []; } function getIntentConfig(settings, combo) { const comboIntentConfig = combo?.autoConfig?.intentConfig || combo?.config?.auto?.intentConfig || combo?.config?.intentConfig || {}; return { ...DEFAULT_INTENT_CONFIG, ...comboIntentConfig, ...(typeof settings?.intentDetectionEnabled === "boolean" ? { enabled: settings.intentDetectionEnabled } : {}), ...(Number.isFinite(Number(settings?.intentSimpleMaxWords)) ? { simpleMaxWords: Number(settings.intentSimpleMaxWords) } : {}), ...(toStringArray(settings?.intentExtraCodeKeywords).length > 0 ? { extraCodeKeywords: toStringArray(settings.intentExtraCodeKeywords) } : {}), ...(toStringArray(settings?.intentExtraReasoningKeywords).length > 0 ? { extraReasoningKeywords: toStringArray(settings.intentExtraReasoningKeywords) } : {}), ...(toStringArray(settings?.intentExtraSimpleKeywords).length > 0 ? { extraSimpleKeywords: toStringArray(settings.intentExtraSimpleKeywords) } : {}), }; } function getBootstrapLatencyMs(modelId) { const normalized = String(modelId || "").toLowerCase(); return DEFAULT_MODEL_P95_MS[normalized] ?? 1500; } async function buildAutoCandidates(modelStrings, comboName) { const metrics = getComboMetrics(comboName); const { getPricingForModel } = await import("../../src/lib/localDb"); let historicalLatencyStats = {}; try { const { getModelLatencyStats } = await import("../../src/lib/usageDb"); historicalLatencyStats = await getModelLatencyStats({ windowHours: 24, minSamples: 3, maxRows: 10000, }); } catch { // keep empty stats — auto-combo will use runtime + bootstrap signals } const candidates = await Promise.all( modelStrings.map(async (modelStr) => { const parsed = parseModel(modelStr); const provider = parsed.provider || parsed.providerAlias || "unknown"; const model = parsed.model || modelStr; const historicalKey = `${provider}/${model}`; const historicalModelMetric = historicalLatencyStats[historicalKey] || null; const historicalTotal = Number(historicalModelMetric?.totalRequests); const hasHistoricalSignal = Number.isFinite(historicalTotal) && historicalTotal >= MIN_HISTORY_SAMPLES; let costPer1MTokens = 1; try { const pricing = await getPricingForModel(provider, model); const inputPrice = Number(pricing?.input); if (Number.isFinite(inputPrice) && inputPrice >= 0) { costPer1MTokens = inputPrice; } } catch { // keep default cost } const modelMetric = metrics?.byModel?.[modelStr] || null; const avgLatency = Number(modelMetric?.avgLatencyMs); const successRate = Number(modelMetric?.successRate); const historicalP95Latency = Number(historicalModelMetric?.p95LatencyMs); const historicalStdDev = Number(historicalModelMetric?.latencyStdDev); const historicalSuccessRate = Number(historicalModelMetric?.successRate); // 0..1 const p95LatencyMs = hasHistoricalSignal ? Number.isFinite(historicalP95Latency) && historicalP95Latency > 0 ? historicalP95Latency : getBootstrapLatencyMs(model) : Number.isFinite(avgLatency) && avgLatency > 0 ? avgLatency : getBootstrapLatencyMs(model); const errorRate = hasHistoricalSignal ? Number.isFinite(historicalSuccessRate) && historicalSuccessRate >= 0 && historicalSuccessRate <= 1 ? 1 - historicalSuccessRate : 0.05 : Number.isFinite(successRate) && successRate >= 0 && successRate <= 100 ? 1 - successRate / 100 : 0.05; const latencyStdDev = hasHistoricalSignal && Number.isFinite(historicalStdDev) && historicalStdDev > 0 ? Math.max(10, historicalStdDev) : Math.max(10, p95LatencyMs * 0.1); const breakerStateRaw = getCircuitBreaker(`combo:${modelStr}`)?.getStatus?.()?.state; const circuitBreakerState = breakerStateRaw === "OPEN" || breakerStateRaw === "HALF_OPEN" ? breakerStateRaw : "CLOSED"; return { provider, model, quotaRemaining: 100, quotaTotal: 100, circuitBreakerState, costPer1MTokens, p95LatencyMs, latencyStdDev, errorRate, accountTier: "standard", quotaResetIntervalSecs: 86400, }; }) ); return candidates; } /** * Handle combo chat with fallback * Supports all 6 strategies: priority, weighted, round-robin, random, least-used, cost-optimized * @param {Object} options * @param {Object} options.body - Request body * @param {Object} options.combo - Full combo object { name, models, strategy, config } * @param {Function} options.handleSingleModel - Function: (body, modelStr) => Promise * @param {Function} [options.isModelAvailable] - Optional pre-check: (modelStr) => Promise * @param {Object} options.log - Logger object * @returns {Promise} */ /** @param {object} options */ export async function handleComboChat({ body, combo, handleSingleModel, isModelAvailable, log, settings, allCombos, }) { const strategy = combo.strategy || "priority"; const models = combo.models || []; // ── Combo Agent Middleware (#399 + #401) ──────────────────────────────── // Apply system_message override, tool_filter_regex, and extract pinned model // from context caching tag. These are all opt-in per combo config. const { body: agentBody, pinnedModel } = applyComboAgentMiddleware( body, combo, "" // provider/model not yet known — resolved per-model in loop ); body = agentBody; if (pinnedModel) { log.info("COMBO", `[#401] Context caching: pinned model=${pinnedModel}`); } // Wrap handleSingleModel to inject context caching tag on response (#401) const handleSingleModelWrapped = combo.context_cache_protection ? async (b, modelStr) => { const res = await handleSingleModel(b, modelStr); if (!res.ok) return res; // Non-streaming: inject tag into JSON response (existing logic) if (!b.stream) { try { const json = await res.clone().json(); const msgs = Array.isArray(json?.messages) ? json.messages : []; if (msgs.length > 0) { const tagged = injectModelTag(msgs, modelStr); return new Response(JSON.stringify({ ...json, messages: tagged }), { status: res.status, headers: res.headers, }); } } catch { /* non-JSON — skip tagging */ } return res; } // Streaming (Fix #490 + #511): prepend omniModel tag into the first // non-empty content chunk so it arrives BEFORE finish_reason:stop. // SDKs close the connection on finish_reason, so anything sent after // that marker is silently dropped. if (!res.body) return res; const tagContent = `\\n${modelStr}\\n`; const encoder = new TextEncoder(); const decoder = new TextDecoder(); let tagInjected = false; const transform = new TransformStream({ transform(chunk, controller) { if (tagInjected) { // Already injected — passthrough controller.enqueue(chunk); return; } const text = decoder.decode(chunk, { stream: true }); // Look for the first SSE data line with non-empty content // Pattern: "content":"" — we inject tag at the start const contentMatch = text.match(/"content":"([^"]+)/); if (contentMatch) { // Inject tag at the beginning of the first content value const injected = text.replace( /"content":"([^"]+)/, `"content":"${tagContent.replace(/"/g, '\\"')}$1` ); tagInjected = true; controller.enqueue(encoder.encode(injected)); return; } // No content yet — passthrough controller.enqueue(chunk); }, flush(controller) { // If stream ends without ever finding content (edge case), // inject tag as a standalone chunk before the stream closes if (!tagInjected) { const tagChunk = `data: ${JSON.stringify({ choices: [ { delta: { content: tagContent }, index: 0, finish_reason: null, }, ], })}\n\n`; controller.enqueue(encoder.encode(tagChunk)); } }, }); // FIX #585: Sanitize outbound stream — strip tags from // visible content so they don't leak to the user. The tag is still // present in the full response for round-trip context pinning, but // we clean it from each SSE chunk's content field before delivery. // // IMPORTANT: Use a SEPARATE TextDecoder from the transform stream above. // The transform stream's decoder accumulates UTF-8 state; reusing it here // would corrupt multi-byte characters split across chunk boundaries. const sanitizeDecoder = new TextDecoder(); const sanitize = new TransformStream({ transform(chunk, controller) { const text = sanitizeDecoder.decode(chunk, { stream: true }); if (text) { if (text.includes("")) { const cleaned = text.replace(/\n?[^<]+<\/omniModel>\n?/g, ""); if (cleaned) controller.enqueue(encoder.encode(cleaned)); } else { controller.enqueue(encoder.encode(text)); } } }, flush(controller) { const tail = sanitizeDecoder.decode(); if (tail) { if (tail.includes("")) { const cleaned = tail.replace(/\n?[^<]+<\/omniModel>\n?/g, ""); if (cleaned) controller.enqueue(encoder.encode(cleaned)); } else { controller.enqueue(encoder.encode(tail)); } } }, }); const transformedStream = res.body.pipeThrough(transform).pipeThrough(sanitize); // Add model info as response header for clients that support it const headers = new Headers(res.headers); headers.set("X-OmniRoute-Model", modelStr); return new Response(transformedStream, { status: res.status, headers, }); } : handleSingleModel; // ───────────────────────────────────────────────────────────────────────── // Route to pinned model if context caching specifies one (Fix #679) if (pinnedModel) { log.info( "COMBO", `Bypassing strategy — routing directly to pinned context model: ${pinnedModel}` ); return handleSingleModelWrapped(body, pinnedModel); } // Route to round-robin handler if strategy matches if (strategy === "round-robin") { return handleRoundRobinCombo({ body, combo, handleSingleModel: handleSingleModelWrapped, isModelAvailable, log, settings, allCombos, }); } // Use config cascade if settings provided const config = settings ? resolveComboConfig(combo, settings) : { ...getDefaultComboConfig(), ...(combo.config || {}) }; const maxRetries = config.maxRetries ?? 1; const retryDelayMs = config.retryDelayMs ?? 2000; let orderedModels; // Resolve nested combos if allCombos provided if (allCombos) { const flatModels = resolveNestedComboModels(combo, allCombos); if (strategy === "weighted") { // For weighted + nested, select from original models then fallback sequentially const selected = selectWeightedModel(models); orderedModels = orderModelsForWeightedFallback(models, selected); // If entries were nested, they are already resolved to flat orderedModels = orderedModels.flatMap((m) => { const combos = Array.isArray(allCombos) ? allCombos : allCombos?.combos || []; const nested = combos.find((c) => c.name === m); if (nested) return resolveNestedComboModels(nested, allCombos); return [m]; }); log.info( "COMBO", `Weighted selection with nested resolution: ${orderedModels.length} total models` ); } else { orderedModels = flatModels; log.info("COMBO", `${strategy} with nested resolution: ${orderedModels.length} total models`); } } else if (strategy === "weighted") { const selected = selectWeightedModel(models); orderedModels = orderModelsForWeightedFallback(models, selected); log.info("COMBO", `Weighted selection: ${selected} (from ${models.length} models)`); } else { orderedModels = models.map((m) => normalizeModelEntry(m).model); } // Apply strategy-specific ordering if (strategy === "auto") { const requestHasTools = Array.isArray(body?.tools) && body.tools.length > 0; let eligibleModels = [...orderedModels]; if (requestHasTools) { const filtered = eligibleModels.filter((m) => supportsToolCalling(m)); if (filtered.length > 0) { eligibleModels = filtered; } else { log.warn( "COMBO", "Auto strategy: all candidates filtered by tool-calling policy, falling back to full pool" ); } } const prompt = extractPromptForIntent(body); const systemPrompt = typeof combo?.system_message === "string" ? combo.system_message : undefined; const intentConfig = getIntentConfig(settings, combo); const intent = classifyWithConfig(prompt, intentConfig, systemPrompt); recordComboIntent(combo.name, intent); const taskType = mapIntentToTaskType(intent); const autoConfigSource = combo?.autoConfig || combo?.config?.auto || combo?.config || {}; const routingStrategy = typeof autoConfigSource.routingStrategy === "string" ? autoConfigSource.routingStrategy : typeof autoConfigSource.strategyName === "string" ? autoConfigSource.strategyName : "rules"; const candidatePool = Array.isArray(autoConfigSource.candidatePool) ? autoConfigSource.candidatePool : [ ...new Set( eligibleModels.map((m) => { const parsed = parseModel(m); return parsed.provider || parsed.providerAlias || "unknown"; }) ), ]; const weights = autoConfigSource.weights && typeof autoConfigSource.weights === "object" ? autoConfigSource.weights : DEFAULT_WEIGHTS; const explorationRate = Number.isFinite(Number(autoConfigSource.explorationRate)) ? Number(autoConfigSource.explorationRate) : 0.05; const budgetCap = Number.isFinite(Number(autoConfigSource.budgetCap)) ? Number(autoConfigSource.budgetCap) : undefined; const modePack = typeof autoConfigSource.modePack === "string" ? autoConfigSource.modePack : undefined; const candidates = await buildAutoCandidates(eligibleModels, combo.name); if (candidates.length > 0) { let selectedProvider = null; let selectedModel = null; let selectionReason = ""; if (routingStrategy !== "rules") { try { const decision = selectWithStrategy( candidates, { taskType, requestHasTools }, routingStrategy ); selectedProvider = decision.provider; selectedModel = decision.model; selectionReason = decision.reason; } catch (err) { log.warn( "COMBO", `Auto strategy '${routingStrategy}' failed (${err?.message || "unknown"}), falling back to rules` ); } } if (!selectedProvider || !selectedModel) { const selection = selectAutoProvider( { id: combo.id || combo.name, name: combo.name, type: "auto", candidatePool, weights, modePack, budgetCap, explorationRate, }, candidates, taskType ); selectedProvider = selection.provider; selectedModel = selection.model; selectionReason = `score=${selection.score.toFixed(3)}${selection.isExploration ? " (exploration)" : ""}`; } const modelLookup = new Map(); for (const modelStr of eligibleModels) { const parsed = parseModel(modelStr); const provider = parsed.provider || parsed.providerAlias || "unknown"; const modelId = parsed.model || modelStr; modelLookup.set(`${provider}/${modelId}`, modelStr); } const ranked = scorePool(candidates, taskType, weights) .map((r) => modelLookup.get(`${r.provider}/${r.model}`) || `${r.provider}/${r.model}`) .filter(Boolean); const selectedModelStr = modelLookup.get(`${selectedProvider}/${selectedModel}`) || `${selectedProvider}/${selectedModel}`; orderedModels = [...new Set([selectedModelStr, ...ranked, ...eligibleModels])]; log.info( "COMBO", `Auto selection: ${selectedModelStr} | intent=${intent} task=${taskType} | strategy=${routingStrategy} | ${selectionReason}` ); } else { log.warn("COMBO", "Auto strategy has no candidates, keeping default ordering"); } } else if (strategy === "strict-random") { const selectedId = await getNextFromDeck(`combo:${combo.name}`, orderedModels); // Put selected model first so the fallback loop tries it first const rest = orderedModels.filter((m) => m !== selectedId); orderedModels = [selectedId, ...rest]; log.info( "COMBO", `Strict-random deck: ${selectedId} selected (${orderedModels.length} models)` ); } else if (strategy === "random") { orderedModels = fisherYatesShuffle([...orderedModels]); log.info("COMBO", `Random shuffle: ${orderedModels.length} models`); } else if (strategy === "least-used") { orderedModels = sortModelsByUsage(orderedModels, combo.name); log.info("COMBO", `Least-used ordering: ${orderedModels[0]} has fewest requests`); } else if (strategy === "cost-optimized") { orderedModels = await sortModelsByCost(orderedModels); log.info("COMBO", `Cost-optimized ordering: cheapest first (${orderedModels[0]})`); } let lastError = null; let earliestRetryAfter = null; let lastStatus = null; const startTime = Date.now(); let resolvedByModel = null; let fallbackCount = 0; for (let i = 0; i < orderedModels.length; i++) { const modelStr = orderedModels[i]; const parsed = parseModel(modelStr); const provider = parsed.provider || parsed.providerAlias || "unknown"; const profile = getProviderProfile(provider); const breakerKey = `combo:${modelStr}`; const breaker = getCircuitBreaker(breakerKey, { failureThreshold: profile.circuitBreakerThreshold, resetTimeout: profile.circuitBreakerReset, }); // Skip model if circuit breaker is OPEN if (!breaker.canExecute()) { log.info("COMBO", `Skipping ${modelStr}: circuit breaker OPEN for ${provider}`); if (i > 0) fallbackCount++; continue; } // Pre-check: skip models where all accounts are in cooldown if (isModelAvailable) { const available = await isModelAvailable(modelStr); if (!available) { log.info("COMBO", `Skipping ${modelStr} (all accounts in cooldown)`); if (i > 0) fallbackCount++; continue; } } // Retry loop for transient errors for (let retry = 0; retry <= maxRetries; retry++) { if (retry > 0) { log.info( "COMBO", `Retrying ${modelStr} in ${retryDelayMs}ms (attempt ${retry + 1}/${maxRetries + 1})` ); await new Promise((r) => setTimeout(r, retryDelayMs)); } log.info( "COMBO", `Trying model ${i + 1}/${orderedModels.length}: ${modelStr}${retry > 0 ? ` (retry ${retry})` : ""}` ); const result = await handleSingleModelWrapped(body, modelStr); // Success — return response if (result.ok) { resolvedByModel = modelStr; const latencyMs = Date.now() - startTime; log.info( "COMBO", `Model ${modelStr} succeeded (${latencyMs}ms, ${fallbackCount} fallbacks)` ); recordComboRequest(combo.name, modelStr, { success: true, latencyMs, fallbackCount, strategy, }); return result; } // Extract error info from response let errorText = result.statusText || ""; let retryAfter = null; try { const cloned = result.clone(); try { const errorBody = await cloned.json(); errorText = errorBody?.error?.message || errorBody?.error || errorBody?.message || errorText; retryAfter = errorBody?.retryAfter || null; } catch { try { const text = await result.text(); if (text) errorText = text.substring(0, 500); } catch { /* Body consumed */ } } } catch { /* Clone failed */ } // Track earliest retryAfter if ( retryAfter && (!earliestRetryAfter || new Date(retryAfter) < new Date(earliestRetryAfter)) ) { earliestRetryAfter = retryAfter; } // Normalize error text if (typeof errorText !== "string") { try { errorText = JSON.stringify(errorText); } catch { errorText = String(errorText); } } const { shouldFallback, cooldownMs } = checkFallbackError( result.status, errorText, 0, null, provider, result.headers ); // Record failure in circuit breaker for transient errors if (TRANSIENT_FOR_BREAKER.includes(result.status)) { breaker._onFailure(); } if (!shouldFallback) { log.warn("COMBO", `Model ${modelStr} failed (no fallback)`, { status: result.status }); return result; } // Check if this is a transient error worth retrying on same model const isTransient = [408, 429, 500, 502, 503, 504].includes(result.status); if (retry < maxRetries && isTransient) { continue; // Retry same model } // Done retrying this model lastError = errorText || String(result.status); if (!lastStatus) lastStatus = result.status; if (i > 0) fallbackCount++; log.warn("COMBO", `Model ${modelStr} failed, trying next`, { status: result.status }); if ([502, 503, 504].includes(result.status) && cooldownMs > 0 && cooldownMs <= 5000) { log.info("COMBO", `Waiting ${cooldownMs}ms before fallback to next model`); await new Promise((r) => setTimeout(r, cooldownMs)); } break; // Move to next model } } // Early exit: check if all models have breaker OPEN const allBreakersOpen = orderedModels.every((m) => { return !getCircuitBreaker(`combo:${m}`).canExecute(); }); // All models failed const latencyMs = Date.now() - startTime; recordComboRequest(combo.name, null, { success: false, latencyMs, fallbackCount, strategy }); if (allBreakersOpen) { log.warn("COMBO", "All models have circuit breaker OPEN — aborting"); return unavailableResponse( 503, "All providers temporarily unavailable (circuit breakers open)" ); } if (!lastStatus) { return new Response( JSON.stringify({ error: { message: "Service temporarily unavailable: all upstream accounts are inactive", type: "service_unavailable", code: "ALL_ACCOUNTS_INACTIVE", }, }), { status: 503, headers: { "Content-Type": "application/json" } } ); } const status = lastStatus; const msg = lastError || "All combo models unavailable"; if (earliestRetryAfter) { const retryHuman = formatRetryAfter(earliestRetryAfter); log.warn("COMBO", `All models failed | ${msg} (${retryHuman})`); return unavailableResponse(status, msg, earliestRetryAfter, retryHuman); } log.warn("COMBO", `All models failed | ${msg}`); return new Response(JSON.stringify({ error: { message: msg } }), { status, headers: { "Content-Type": "application/json" }, }); } /** * Handle round-robin combo: each request goes to the next model in circular order. * Uses semaphore-based concurrency control with queue + rate-limit awareness. * * Flow: * 1. Pick target model via atomic counter (counter % models.length) * 2. Acquire semaphore slot (may queue if at max concurrency) * 3. Send request to target model * 4. On 429 → mark model rate-limited, try next model in rotation * 5. On semaphore timeout → fallback to next available model */ async function handleRoundRobinCombo({ body, combo, handleSingleModel, isModelAvailable, log, settings, allCombos, }) { const models = combo.models || []; const config = settings ? resolveComboConfig(combo, settings) : { ...getDefaultComboConfig(), ...(combo.config || {}) }; const concurrency = config.concurrencyPerModel ?? 3; const queueTimeout = config.queueTimeoutMs ?? 30000; const maxRetries = config.maxRetries ?? 1; const retryDelayMs = config.retryDelayMs ?? 2000; // Resolve models (support nested combos) let orderedModels; if (allCombos) { orderedModels = resolveNestedComboModels(combo, allCombos); } else { orderedModels = models.map((m) => normalizeModelEntry(m).model); } const modelCount = orderedModels.length; if (modelCount === 0) { return unavailableResponse(503, "Round-robin combo has no models"); } // Get and increment atomic counter const counter = rrCounters.get(combo.name) || 0; rrCounters.set(combo.name, counter + 1); const startIndex = counter % modelCount; const startTime = Date.now(); let lastError = null; let lastStatus = null; let earliestRetryAfter = null; let fallbackCount = 0; // Try each model starting from the round-robin target for (let offset = 0; offset < modelCount; offset++) { const modelIndex = (startIndex + offset) % modelCount; const modelStr = orderedModels[modelIndex]; const parsed = parseModel(modelStr); const provider = parsed.provider || parsed.providerAlias || "unknown"; const profile = getProviderProfile(provider); const breakerKey = `combo:${modelStr}`; const breaker = getCircuitBreaker(breakerKey, { failureThreshold: profile.circuitBreakerThreshold, resetTimeout: profile.circuitBreakerReset, }); // Skip model if circuit breaker is OPEN if (!breaker.canExecute()) { log.info("COMBO-RR", `Skipping ${modelStr}: circuit breaker OPEN for ${provider}`); if (offset > 0) fallbackCount++; continue; } // Pre-check availability if (isModelAvailable) { const available = await isModelAvailable(modelStr); if (!available) { log.info("COMBO-RR", `Skipping ${modelStr} (all accounts in cooldown)`); if (offset > 0) fallbackCount++; continue; } } // Acquire semaphore slot (may wait in queue) let release; try { release = await semaphore.acquire(modelStr, { maxConcurrency: concurrency, timeoutMs: queueTimeout, }); } catch (err) { if (err.code === "SEMAPHORE_TIMEOUT") { log.warn("COMBO-RR", `Semaphore timeout for ${modelStr}, trying next model`); if (offset > 0) fallbackCount++; continue; } throw err; } // Retry loop within this model try { for (let retry = 0; retry <= maxRetries; retry++) { if (retry > 0) { log.info( "COMBO-RR", `Retrying ${modelStr} in ${retryDelayMs}ms (attempt ${retry + 1}/${maxRetries + 1})` ); await new Promise((r) => setTimeout(r, retryDelayMs)); } log.info( "COMBO-RR", `[RR #${counter}] → ${modelStr}${offset > 0 ? ` (fallback +${offset})` : ""}${retry > 0 ? ` (retry ${retry})` : ""}` ); const result = await handleSingleModel(body, modelStr); // Success if (result.ok) { const latencyMs = Date.now() - startTime; log.info( "COMBO-RR", `${modelStr} succeeded (${latencyMs}ms, ${fallbackCount} fallbacks)` ); recordComboRequest(combo.name, modelStr, { success: true, latencyMs, fallbackCount, strategy: "round-robin", }); return result; } // Extract error info let errorText = result.statusText || ""; let retryAfter = null; try { const cloned = result.clone(); try { const errorBody = await cloned.json(); errorText = errorBody?.error?.message || errorBody?.error || errorBody?.message || errorText; retryAfter = errorBody?.retryAfter || null; } catch { try { const text = await result.text(); if (text) errorText = text.substring(0, 500); } catch { /* Body consumed */ } } } catch { /* Clone failed */ } if ( retryAfter && (!earliestRetryAfter || new Date(retryAfter) < new Date(earliestRetryAfter)) ) { earliestRetryAfter = retryAfter; } if (typeof errorText !== "string") { try { errorText = JSON.stringify(errorText); } catch { errorText = String(errorText); } } const { shouldFallback, cooldownMs } = checkFallbackError( result.status, errorText, 0, null, provider, result.headers ); // Transient errors → mark in semaphore AND record circuit breaker failure if (TRANSIENT_FOR_BREAKER.includes(result.status) && cooldownMs > 0) { semaphore.markRateLimited(modelStr, cooldownMs); breaker._onFailure(); log.warn( "COMBO-RR", `${modelStr} error ${result.status}, cooldown ${cooldownMs}ms (breaker: ${breaker.getStatus().failureCount}/${profile.circuitBreakerThreshold})` ); } if (!shouldFallback) { log.warn("COMBO-RR", `${modelStr} failed (no fallback)`, { status: result.status }); return result; } // Transient error → retry same model const isTransient = [408, 429, 500, 502, 503, 504].includes(result.status); if (retry < maxRetries && isTransient) { continue; } // Done with this model lastError = errorText || String(result.status); if (!lastStatus) lastStatus = result.status; if (offset > 0) fallbackCount++; log.warn("COMBO-RR", `${modelStr} failed, trying next model`, { status: result.status }); if ([502, 503, 504].includes(result.status) && cooldownMs > 0 && cooldownMs <= 5000) { log.info("COMBO-RR", `Waiting ${cooldownMs}ms before fallback to next model`); await new Promise((r) => setTimeout(r, cooldownMs)); } break; } } finally { // ALWAYS release semaphore slot release(); } } // All models exhausted const latencyMs = Date.now() - startTime; recordComboRequest(combo.name, null, { success: false, latencyMs, fallbackCount, strategy: "round-robin", }); // Early exit: check if all models have breaker OPEN const allBreakersOpen = orderedModels.every((m) => { return !getCircuitBreaker(`combo:${m}`).canExecute(); }); if (allBreakersOpen) { log.warn("COMBO-RR", "All models have circuit breaker OPEN — aborting"); return unavailableResponse( 503, "All providers temporarily unavailable (circuit breakers open)" ); } if (!lastStatus) { return new Response( JSON.stringify({ error: { message: "Service temporarily unavailable: all upstream accounts are inactive", type: "service_unavailable", code: "ALL_ACCOUNTS_INACTIVE", }, }), { status: 503, headers: { "Content-Type": "application/json" } } ); } const status = lastStatus; const msg = lastError || "All round-robin combo models unavailable"; if (earliestRetryAfter) { const retryHuman = formatRetryAfter(earliestRetryAfter); log.warn("COMBO-RR", `All models failed | ${msg} (${retryHuman})`); return unavailableResponse(status, msg, earliestRetryAfter, retryHuman); } log.warn("COMBO-RR", `All models failed | ${msg}`); return new Response(JSON.stringify({ error: { message: msg } }), { status, headers: { "Content-Type": "application/json" }, }); }