From 798fce727c1524f10f045e6f27e4909cd97c88cc Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza <8016841+diegosouzapw@users.noreply.github.com> Date: Thu, 18 Jun 2026 12:52:06 -0300 Subject: [PATCH] =?UTF-8?q?refactor(combo):=20god-file=20split=20part=202?= =?UTF-8?q?=20=E2=80=94=20shadow=20+=20sorters=20+=20structure=20(QG=20v2?= =?UTF-8?q?=20Fase=209=20T5=20D4-D6)=20(#4175)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Integrated into release/v3.8.29 — combo.ts god-file split part 2 (QG v2 Fase 9 T5 D4-D6): shadow routing, target sorters, and combo structure resolution extracted byte-identically to combo/{shadowRouting,targetSorters,comboStructure,comboData}.ts; combo.ts 4740->3819, public surface re-exported (callers unchanged). Validated on the rebased+merged tree: typecheck:core, file-size, test-discovery green; 356/356 full combo suite + 51/51 smoke pass. Resynced with current release (baseline union: combo-split entries + #4176 free-models entry; combo.ts=3819, rateLimitManager=1035). --- config/quality/file-size-baseline.json | 5 +- open-sse/services/combo.ts | 971 +--------------------- open-sse/services/combo/comboData.ts | 24 + open-sse/services/combo/comboStructure.ts | 637 ++++++++++++++ open-sse/services/combo/shadowRouting.ts | 178 ++++ open-sse/services/combo/targetSorters.ts | 173 ++++ 6 files changed, 1041 insertions(+), 947 deletions(-) create mode 100644 open-sse/services/combo/comboData.ts create mode 100644 open-sse/services/combo/comboStructure.ts create mode 100644 open-sse/services/combo/shadowRouting.ts create mode 100644 open-sse/services/combo/targetSorters.ts diff --git a/config/quality/file-size-baseline.json b/config/quality/file-size-baseline.json index 561a3fe937..58fcf5ed79 100644 --- a/config/quality/file-size-baseline.json +++ b/config/quality/file-size-baseline.json @@ -3,6 +3,9 @@ "_rebaseline_2026_06_18_4176_free_models": "PR #4176 own growth: AddApiKeyModal.tsx 845->866 (+21) and EditConnectionModal.tsx 1174->1204 (+30) = the 'import only free models' connection option — a free-models Toggle gated by providerHasFreeModels() plus its form-state wiring (importFreeModelsOnly field + providerSpecificData persistence, explicit-false on edit so the PUT merge doesn't keep a stale true) added to both connection modals, mirroring the prior per-modal toggle bumps #3879 (redact-thinking) and #2997 (disable-cooling). Detection lives in the new shared src/shared/utils/freeModels.ts (112 LOC, 2531 (+19 = one PROVIDER_MODELS_CONFIG entry for `qwen-web` + a 4-line comment). qwen-web was missing from the config map so its model-discovery page returned nothing (the OAuth fallback only fires for provider===qwen). Pure additive config entry pointing at the public chat.qwen.ai/api/v2/models endpoint; standard per-provider addition, not extractable.", "_rebaseline_2026_06_18_4165_queue_timeout_msg": "Issue #4165 own growth: rateLimitManager.ts 1022->1035 (+13 at the existing withRateLimit catch chokepoint). Bottleneck's raw `This job timed out after ms.` is rewritten into a clear OmniRoute-owned error (names resilienceSettings.requestQueue.maxWaitMs, disclaims upstream, keeps the original as `cause`, tags code=RATE_LIMIT_QUEUE_TIMEOUT) so queue-saturation 502s stop masquerading as provider outages. The branch already existed (it only logged); this adds the error construction at the same point. Not extractable — closes over provider/model/maxWaitMs locals of the single catch.", + "_rebaseline_2026_06_18_qg9_combo_split_d4": "QG v2 Fase 9 T5 D4: combo.ts 4740->4589 — shadow routing extracted byte-identically to the new open-sse/services/combo/shadowRouting.ts (4430 — target sorters extracted byte-identically to the new open-sse/services/combo/targetSorters.ts (3819 — combo structure resolution extracted byte-identically to the new open-sse/services/combo/comboStructure.ts (638, 1022 (+5 = one import + one `await awaitProviderDefaultSlot(...)` call + a 2-line comment at the existing withRateLimit chokepoint). All sliding-window logic was extracted to the new open-sse/services/providerDefaultRateLimit.ts + open-sse/services/slidingWindowLimiter.ts (both 1440 (+5 = appendNoThinkingVariants(finalModels) call + comment at the existing finalModels chokepoint) and chat.ts 1458->1471 (+13 = applyNoThinkingAlias(body) call + comment right after body.model is read, before model resolution). All real logic lives in the new open-sse/utils/noThinkingAlias.ts (6009 (+29 at the existing streaming-return chokepoint = capture streamRecovery.continueMidStream alongside .enabled; refactor the early-retry reopen thunk into a shared runUpstreamStream(body) helper — net DRY — and add the gated continueStream(assistantSoFar) thunk that re-runs the upstream with makeContinuationBody(bodyToSend, …), plus the onContinue log). All continuation logic (scanOpenAiSseText, makeContinuationBody, trimContinuationOverlap, the createRecoverableStream continuation path) lives in open-sse/services/streamRecovery.ts ( = { @@ -271,36 +276,6 @@ type QuotaFetchCacheConfig = { }; type ResetWindowConfig = ReturnType; -function isRecord(value: unknown): value is Record { - return !!value && typeof value === "object" && !Array.isArray(value); -} - -function toTrimmedString(value: unknown): string | null { - return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; -} - -function toComboLike(combo: ComboInput): ComboLike { - return { - ...combo, - id: toTrimmedString(combo.id) || undefined, - name: toTrimmedString(combo.name) || "", - models: Array.isArray(combo.models) ? combo.models : [], - config: isRecord(combo.config) ? combo.config : null, - autoConfig: isRecord(combo.autoConfig) ? combo.autoConfig : null, - context_cache_protection: - typeof combo.context_cache_protection === "boolean" || - typeof combo.context_cache_protection === "number" - ? combo.context_cache_protection - : undefined, - system_message: typeof combo.system_message === "string" ? combo.system_message : null, - }; -} - -function getCombosArray(allCombos: ComboCollectionLike): ComboLike[] { - const combos = Array.isArray(allCombos) ? allCombos : allCombos?.combos || []; - return combos.map((combo) => toComboLike(combo)); -} - // In-memory atomic counter per combo for round-robin distribution // Resets on server restart (by design — no stale state) // Eviction limits to prevent unbounded memory growth @@ -319,17 +294,6 @@ const resetAwareQuotaCache = new Map< { fetchedAt: number; quota: unknown; refreshPromise: Promise | null } >(); -/** - * Normalize a model entry to { model, weight } - * Supports both legacy string format and new object format - */ -function normalizeModelEntry(entry: unknown): { model: string; weight: number } { - return { - model: getComboStepTarget(entry) || "", - weight: getComboStepWeight(entry), - }; -} - function clampStickyRoundRobinTargetLimit(value: unknown): number { const numericValue = Number(value); if (!Number.isFinite(numericValue)) return 1; @@ -374,836 +338,6 @@ function recordStickyRoundRobinSuccess( rrStickyTargets.set(comboName, { executionKey: target.executionKey, successCount }); } -function normalizeShadowRoutingConfig(config: Record): ShadowRoutingConfig { - const raw = isRecord(config.shadowRouting) ? config.shadowRouting : {}; - const sampleRate = Number(raw.sampleRate ?? 1); - const maxTargets = Number(raw.maxTargets ?? 2); - const timeoutMs = Number(raw.timeoutMs ?? 30000); - return { - enabled: raw.enabled === true, - targets: Array.isArray(raw.targets) ? raw.targets : [], - sampleRate: Number.isFinite(sampleRate) ? Math.max(0, Math.min(1, sampleRate)) : 1, - maxTargets: Number.isFinite(maxTargets) ? Math.max(1, Math.min(10, Math.floor(maxTargets))) : 2, - timeoutMs: Number.isFinite(timeoutMs) - ? Math.max(1000, Math.min(120000, Math.floor(timeoutMs))) - : 30000, - }; -} - -function resolveShadowTargets( - combo: ComboLike, - config: Record, - allCombos: ComboCollectionLike -): ResolvedComboTarget[] { - const shadowConfig = normalizeShadowRoutingConfig(config); - if (!shadowConfig.enabled || shadowConfig.targets.length === 0) return []; - if (shadowConfig.sampleRate <= 0 || Math.random() > shadowConfig.sampleRate) return []; - - const shadowCombo: ComboLike = { - ...combo, - name: `${combo.name}:shadow`, - models: shadowConfig.targets, - }; - return resolveNestedComboTargets(shadowCombo, allCombos, new Set([combo.name]), 0, ["shadow"]) - .slice(0, shadowConfig.maxTargets) - .map((target) => ({ - ...target, - trafficType: "shadow" as const, - })); -} - -async function drainShadowResponse(response: Response): Promise { - try { - if (!response.body) return; - await response.arrayBuffer(); - } catch { - // Shadow draining is best-effort and must never affect the production response. - } -} - -function withTimeout(promise: Promise, timeoutMs: number): Promise { - return new Promise((resolve, reject) => { - const timer = setTimeout(() => reject(new Error("Shadow route timed out")), timeoutMs); - promise.then( - (value) => { - clearTimeout(timer); - resolve(value); - }, - (error) => { - clearTimeout(timer); - reject(error); - } - ); - }); -} - -function cloneRequestBodyForShadowRouting(body: Record): Record { - if (typeof structuredClone === "function") { - return structuredClone(body) as Record; - } - - return JSON.parse(JSON.stringify(body)) as Record; -} - -function scheduleShadowRouting( - combo: ComboLike, - config: Record, - body: Record, - targets: ResolvedComboTarget[], - handleSingleModel: HandleSingleModel, - isModelAvailable: IsModelAvailable | undefined, - strategy: string, - log: ComboLogger -): void { - if (targets.length === 0) return; - const shadowConfig = normalizeShadowRoutingConfig(config); - let shadowBaseBody: Record; - try { - shadowBaseBody = cloneRequestBodyForShadowRouting(body); - } catch (error) { - log.warn("COMBO", "Shadow routing skipped: failed to clone request body", { - error: error instanceof Error ? error.message : String(error), - }); - return; - } - const run = async () => { - await Promise.all( - targets.map(async (target) => { - const startedAt = Date.now(); - try { - const shadowBody = { - ...cloneRequestBodyForShadowRouting(shadowBaseBody), - model: target.modelStr, - stream: false, - }; - if (isModelAvailable) { - const available = await isModelAvailable(target.modelStr, target); - if (!available) { - recordComboShadowRequest(combo.name, target.modelStr, { - success: false, - latencyMs: Date.now() - startedAt, - target: toRecordedTarget(target), - }); - log.info("COMBO", `Shadow target skipped (unavailable): ${target.modelStr}`); - return; - } - } - - const response = await withTimeout( - handleSingleModel(shadowBody, target.modelStr, { - ...target, - failoverBeforeRetry: true, - trafficType: "shadow", - }), - shadowConfig.timeoutMs - ); - await drainShadowResponse(response.clone()); - recordComboShadowRequest(combo.name, target.modelStr, { - success: response.ok, - latencyMs: Date.now() - startedAt, - target: toRecordedTarget(target), - }); - log.info( - "COMBO", - `Shadow target ${target.modelStr} completed with status ${response.status} (${strategy})` - ); - } catch (error) { - recordComboShadowRequest(combo.name, target.modelStr, { - success: false, - latencyMs: Date.now() - startedAt, - target: toRecordedTarget(target), - }); - log.warn("COMBO", `Shadow target ${target.modelStr} failed`, { - error: error instanceof Error ? error.message : String(error), - }); - } - }) - ); - }; - - setTimeout(() => void run(), 0); -} - -function buildExecutionKey(path: string[], stepId: string): string { - return [...path, stepId].join(">"); -} - -function normalizeRuntimeStep( - entry: unknown, - comboName: string, - index: number, - allCombos: ComboCollectionLike, - path: string[] = [] -): ComboRuntimeStep | null { - const step = normalizeComboStep(entry, { - comboName, - index, - allCombos, - }); - if (!step) return null; - - const executionKey = buildExecutionKey(path, step.id); - const label = typeof step.label === "string" ? step.label : null; - const weight = step.weight || 0; - - if (step.kind === "combo-ref") { - return { - kind: "combo-ref", - stepId: step.id, - executionKey, - comboName: step.comboName, - weight, - label, - }; - } - - const modelStr = getComboModelString(step); - if (!modelStr) return null; - - return { - kind: "model", - stepId: step.id, - executionKey, - modelStr, - provider: getTargetProvider(modelStr, step.providerId), - providerId: step.providerId || null, - connectionId: step.connectionId || null, - weight, - label, - } satisfies ResolvedComboTarget; -} - -function getDirectComboTargets(combo: ComboLike): ResolvedComboTarget[] { - return getOrderedTopLevelRuntimeSteps(combo, null).filter( - (entry): entry is ResolvedComboTarget => entry?.kind === "model" - ); -} - -function getTopLevelRuntimeSteps( - combo: ComboLike, - allCombos: ComboCollectionLike, - path: string[] = [] -): ComboRuntimeStep[] { - return (combo.models || []) - .map((entry, index) => normalizeRuntimeStep(entry, combo.name, index, allCombos, path)) - .filter((entry): entry is ComboRuntimeStep => entry !== null); -} - -function getCompositeTierStepOrder(combo: ComboLike): string[] { - const compositeTiers = isRecord(combo?.config) ? combo.config.compositeTiers : null; - if (!isRecord(compositeTiers)) return []; - - const defaultTier = toTrimmedString(compositeTiers.defaultTier); - const tiers = isRecord(compositeTiers.tiers) ? compositeTiers.tiers : null; - if (!defaultTier || !tiers) return []; - - const orderedStepIds: string[] = []; - const visitedTiers = new Set(); - const seenStepIds = new Set(); - type CompositeTierEntry = readonly [ - string, - { readonly stepId: string; readonly fallbackTier: string | null }, - ]; - const tierEntries = new Map( - Object.entries(tiers) - .map(([tierName, rawTier]) => { - if (!isRecord(rawTier)) return null; - const normalizedTierName = toTrimmedString(tierName); - const stepId = toTrimmedString(rawTier.stepId); - const fallbackTier = toTrimmedString(rawTier.fallbackTier); - if (!normalizedTierName || !stepId) return null; - return [normalizedTierName, { stepId, fallbackTier }] as const; - }) - .filter((entry): entry is CompositeTierEntry => entry !== null) - ); - - let currentTier: string | null = defaultTier; - while (currentTier && tierEntries.has(currentTier) && !visitedTiers.has(currentTier)) { - visitedTiers.add(currentTier); - const entry = tierEntries.get(currentTier); - if (!entry) break; - if (!seenStepIds.has(entry.stepId)) { - orderedStepIds.push(entry.stepId); - seenStepIds.add(entry.stepId); - } - currentTier = entry.fallbackTier; - } - - for (const entry of tierEntries.values()) { - if (!seenStepIds.has(entry.stepId)) { - orderedStepIds.push(entry.stepId); - seenStepIds.add(entry.stepId); - } - } - - return orderedStepIds; -} - -function hasCompositeTierRuntimeOrder(combo: ComboLike): boolean { - return getCompositeTierStepOrder(combo).length > 0; -} - -function orderRuntimeStepsByCompositeTiers( - steps: ComboRuntimeStep[], - combo: ComboLike -): ComboRuntimeStep[] { - const orderedStepIds = getCompositeTierStepOrder(combo); - if (orderedStepIds.length === 0) return steps; - - const byStepId = new Map(steps.map((step) => [step.stepId, step])); - const seen = new Set(); - const ordered: ComboRuntimeStep[] = []; - - for (const stepId of orderedStepIds) { - const step = byStepId.get(stepId); - if (!step || seen.has(step.stepId)) continue; - ordered.push(step); - seen.add(step.stepId); - } - - for (const step of steps) { - if (seen.has(step.stepId)) continue; - ordered.push(step); - seen.add(step.stepId); - } - - return ordered; -} - -function getOrderedTopLevelRuntimeSteps( - combo: ComboLike, - allCombos: ComboCollectionLike, - path: string[] = [] -): ComboRuntimeStep[] { - return orderRuntimeStepsByCompositeTiers(getTopLevelRuntimeSteps(combo, allCombos, path), combo); -} - -function expandRuntimeStep( - step: ComboRuntimeStep, - allCombos: ComboCollectionLike, - visited = new Set(), - depth = 0, - path: string[] = [], - maxDepth: number = MAX_COMBO_DEPTH -): ResolvedComboTarget[] { - if (step.kind === "model") return [step]; - if (depth > maxDepth) return []; - - const combos = getCombosArray(allCombos); - const nestedCombo = combos.find((combo) => combo.name === step.comboName); - if (!nestedCombo || visited.has(step.comboName)) return []; - - return resolveNestedComboTargets( - nestedCombo, - combos, - new Set(visited), - depth + 1, - [...path, step.stepId], - maxDepth - ); -} - -export function resolveNestedComboTargets( - combo: ComboLike, - allCombos: ComboCollectionLike, - visited = new Set(), - depth = 0, - path: string[] = [], - maxDepth: number = MAX_COMBO_DEPTH -): ResolvedComboTarget[] { - const directTargets = (combo.models || []) - .map((entry, index) => normalizeRuntimeStep(entry, combo.name, index, null, path)) - .filter((entry): entry is ResolvedComboTarget => entry?.kind === "model"); - - if (depth > maxDepth) return directTargets; - if (visited.has(combo.name)) return []; - visited.add(combo.name); - - const runtimeSteps = getOrderedTopLevelRuntimeSteps(combo, allCombos, path); - const resolved: ResolvedComboTarget[] = []; - - for (const step of runtimeSteps) { - if (step.kind === "combo-ref") { - resolved.push(...expandRuntimeStep(step, allCombos, new Set(visited), depth, path, maxDepth)); - continue; - } - resolved.push(step); - } - - return resolved; -} - -/** - * Get combo models from combos data (for open-sse standalone use) - * @param {string} modelStr - Model string to check - * @param {Array|Object} combosData - Array of combos or object with combos - * @returns {Object|null} Full combo object or null if not a combo - */ -export function getComboFromData( - modelStr: string, - combosData: ComboCollectionLike -): ComboLike | null { - const combos = getCombosArray(combosData); - const combo = combos.find((c) => c.name === modelStr); - if (combo?.models && combo.models.length > 0) { - return combo; - } - return null; -} - -/** - * Legacy: Get combo models as string array (backward compat) - */ -export function getComboModelsFromData( - modelStr: string, - combosData: ComboCollectionLike -): string[] | null { - const combo = getComboFromData(modelStr, combosData); - if (!combo) return null; - return combo.models.map((m) => normalizeModelEntry(m).model); -} - -/** - * Validate combo DAG — detect circular references and enforce max depth - * @param {string} comboName - Name of the combo to validate - * @param {Array} allCombos - All combos in the system - * @param {Set} [visited] - Set of already visited combo names (for cycle detection) - * @param {number} [depth] - Current depth level - * @throws {Error} If circular reference or max depth exceeded - */ -export function validateComboDAG( - comboName: string, - allCombos: ComboCollectionLike, - visited = new Set(), - depth = 0, - maxDepth: number = MAX_COMBO_DEPTH -): void { - if (depth > maxDepth) { - throw new Error(`Max combo nesting depth (${maxDepth}) exceeded at "${comboName}"`); - } - if (visited.has(comboName)) { - throw new Error(`Circular combo reference detected: ${comboName}`); - } - visited.add(comboName); - - const combos = getCombosArray(allCombos); - const combo = combos.find((c) => c.name === comboName); - if (!combo?.models) return; - - for (const entry of combo.models) { - const modelName = normalizeModelEntry(entry).model; - // Check if this model name is itself a combo (not a provider/model pattern) - const nestedCombo = combos.find((c) => c.name === modelName); - if (nestedCombo) { - validateComboDAG(modelName, combos, new Set(visited), depth + 1, maxDepth); - } - } -} - -/** - * Resolve nested combos by expanding inline to a flat model list - * Respects max depth and detects cycles - * @param {Object} combo - The combo object - * @param {Array} allCombos - All combos in the system - * @param {Set} [visited] - For cycle detection - * @param {number} [depth] - Current depth - * @returns {Array} Flat array of model strings - */ -export function resolveNestedComboModels( - combo: ComboLike, - allCombos: ComboCollectionLike, - visited = new Set(), - depth = 0, - maxDepth: number = MAX_COMBO_DEPTH -): string[] { - if (depth > maxDepth) return combo.models.map((m) => normalizeModelEntry(m).model); - if (visited.has(combo.name)) return []; // cycle safety - visited.add(combo.name); - - const combos = getCombosArray(allCombos); - const resolved: string[] = []; - - for (const entry of combo.models || []) { - const modelName = normalizeModelEntry(entry).model; - const nestedCombo = combos.find((c) => c.name === modelName); - - if (nestedCombo) { - // Recursively expand the nested combo - const nested = resolveNestedComboModels( - nestedCombo, - combos, - new Set(visited), - depth + 1, - maxDepth - ); - resolved.push(...nested); - } else { - resolved.push(modelName); - } - } - - return resolved; -} - -function selectWeightedTarget(targets: T[]) { - if (targets.length === 0) return null; - - const totalWeight = targets.reduce((sum, target) => sum + (target.weight || 0), 0); - if (totalWeight <= 0) { - return targets[Math.floor(Math.random() * targets.length)]; - } - - let random = Math.random() * totalWeight; - for (const target of targets) { - random -= target.weight || 0; - if (random <= 0) return target; - } - - return targets.at(-1); -} - -function orderTargetsForWeightedFallback( - targets: T[], - selectedExecutionKey: string, - preserveExistingOrder = false -): T[] { - const selected = targets.find((target) => target.executionKey === selectedExecutionKey); - const rest = targets.filter((target) => target.executionKey !== selectedExecutionKey); - if (!preserveExistingOrder) { - rest.sort((a, b) => b.weight - a.weight); - } - return selected ? [selected, ...rest] : rest; -} - -// shuffleArray and getNextModelFromDeck moved to src/shared/utils/shuffleDeck.ts -// combo.ts now uses the shared, mutex-protected getNextFromDeck with "combo:" namespace. - -/** - * Sort models by pricing (cheapest first) for cost-optimized strategy - * @param {Array} models - Model strings in "provider/model" format - * @returns {Promise>} Sorted model strings - */ -async function sortModelsByCost(models: string[]): Promise { - try { - const { getPricingForModel } = await import("../../src/lib/localDb"); - const withCost = await Promise.all( - models.map(async (modelStr) => { - const parsed = parseModel(modelStr); - const provider = parsed.provider || parsed.providerAlias || "unknown"; - const model = parsed.model || modelStr; - try { - const pricing = await getPricingForModel(provider, model); - const cost = Number(pricing?.input); - return { modelStr, cost: Number.isFinite(cost) ? cost : Infinity }; - } catch { - return { modelStr, cost: Infinity }; - } - }) - ); - withCost.sort((a, b) => a.cost - b.cost); - return withCost.map((e) => e.modelStr); - } catch { - // If pricing lookup fails entirely, return original order - return models; - } -} - -async function sortTargetsByCost(targets: ResolvedComboTarget[]) { - const orderedModels = await sortModelsByCost(targets.map((target) => target.modelStr)); - const byModel = new Map(); - for (const target of targets) { - const queue = byModel.get(target.modelStr) || []; - queue.push(target); - byModel.set(target.modelStr, queue); - } - return orderedModels - .map((modelStr) => { - const queue = byModel.get(modelStr); - return queue?.shift() || null; - }) - .filter((target): target is ResolvedComboTarget => target !== null); -} - -/** - * Sort models by usage count (least-used first) for least-used strategy - * @param {Array} models - Model strings - * @param {string} comboName - Combo name for metrics lookup - * @returns {Array} Sorted model strings - */ -function sortModelsByUsage(models: string[], comboName: string): string[] { - const metrics = getComboMetrics(comboName); - if (!metrics?.byModel) return models; - - const withUsage = models.map((modelStr) => ({ - modelStr, - requests: metrics.byModel[modelStr]?.requests ?? 0, - })); - withUsage.sort((a, b) => a.requests - b.requests); - return withUsage.map((e) => e.modelStr); -} - -function sortTargetsByUsage(targets: ResolvedComboTarget[], comboName: string) { - const orderedModels = sortModelsByUsage( - targets.map((target) => target.modelStr), - comboName - ); - const byModel = new Map(); - for (const target of targets) { - const queue = byModel.get(target.modelStr) || []; - queue.push(target); - byModel.set(target.modelStr, queue); - } - return orderedModels - .map((modelStr) => { - const queue = byModel.get(modelStr); - return queue?.shift() || null; - }) - .filter((target): target is ResolvedComboTarget => target !== null); -} - -/** - * Sort models by context window size (largest first) for context-optimized strategy. - * Uses models.dev synced capabilities to get context limits. - * @param {Array} models - Model strings in "provider/model" format - * @returns {Array} Sorted model strings (largest context first) - */ -function sortModelsByContextSize(models: string[]): string[] { - const withContext = models.map((modelStr) => { - return { modelStr, context: getModelContextLimitForModelString(modelStr) ?? 0 }; - }); - withContext.sort((a, b) => b.context - a.context); - return withContext.map((e) => e.modelStr); -} - -function getModelContextLimitForModelString(modelStr: string) { - const parsed = parseModel(modelStr); - const provider = parsed.provider || parsed.providerAlias || "unknown"; - const model = parsed.model || modelStr; - return getModelContextLimit(provider, model); -} - -type RequestCompatibilityRequirements = { - requiresTools: boolean; - requiresVision: boolean; - requiresStructuredOutput: boolean; - estimatedInputTokens: number; - requestedOutputTokens: number; - requiredContextTokens: number; -}; - -function getPositiveTokenCount(value: unknown): number { - const count = Number(value); - return Number.isFinite(count) && count > 0 ? Math.ceil(count) : 0; -} - -function requestRequiresTools(body: Record): boolean { - if (Array.isArray(body.tools) && body.tools.length > 0) return true; - if (Array.isArray(body.functions) && body.functions.length > 0) return true; - return false; -} - -function requestRequiresStructuredOutput(body: Record): boolean { - const responseFormat = isRecord(body.response_format) ? body.response_format : null; - const type = typeof responseFormat?.type === "string" ? responseFormat.type : null; - return type === "json_object" || type === "json_schema"; -} - -function estimateRequestInputTokens(body: Record): number { - const estimatePayload: Record = {}; - for (const key of ["messages", "input", "tools", "functions", "response_format"]) { - if (body[key] !== undefined) estimatePayload[key] = body[key]; - } - return Object.keys(estimatePayload).length > 0 ? estimateTokens(estimatePayload) : 0; -} - -function valueContainsImagePart(value: unknown, depth = 0): boolean { - if (depth > 8 || value === null || value === undefined) return false; - if (typeof value === "string") return value.startsWith("data:image/"); - if (Array.isArray(value)) return value.some((entry) => valueContainsImagePart(entry, depth + 1)); - if (!isRecord(value)) return false; - - const type = typeof value.type === "string" ? value.type.toLowerCase() : null; - if (type === "image" || type === "image_url" || type === "input_image") return true; - if ("image_url" in value || "input_image" in value) return true; - - const source = isRecord(value.source) ? value.source : null; - const mediaType = typeof source?.media_type === "string" ? source.media_type.toLowerCase() : ""; - if (mediaType.startsWith("image/")) return true; - - return Object.values(value).some((entry) => valueContainsImagePart(entry, depth + 1)); -} - -function deriveRequestCompatibilityRequirements( - body: Record -): RequestCompatibilityRequirements { - const estimatedInputTokens = estimateRequestInputTokens(body); - const requestedOutputTokens = Math.max( - getPositiveTokenCount(body.max_tokens), - getPositiveTokenCount(body.max_completion_tokens) - ); - return { - requiresTools: requestRequiresTools(body), - requiresVision: valueContainsImagePart(body.messages) || valueContainsImagePart(body.input), - requiresStructuredOutput: requestRequiresStructuredOutput(body), - estimatedInputTokens, - requestedOutputTokens, - requiredContextTokens: estimatedInputTokens + requestedOutputTokens, - }; -} - -function getTargetCompatibilityFailures( - target: ResolvedComboTarget, - requirements: RequestCompatibilityRequirements -): string[] { - const capabilities = getResolvedModelCapabilities(target.modelStr); - const failures: string[] = []; - - if ( - requirements.requiresTools && - (capabilities.supportsTools === false || !capabilities.toolCalling) - ) { - failures.push("tools"); - } - - // For a request that carries an image, only route to a target whose vision - // support is *confirmed* (`=== true`). Treat `false` AND `null` (unknown) as - // incompatible: an unknown-capability model receiving the image is exactly how - // a text-only model (e.g. ministral) ended up answering "image not provided". - // The caller keeps all targets when none qualify, so combos with no - // confirmed-vision member still behave as before. - if (requirements.requiresVision && capabilities.supportsVision !== true) { - failures.push("vision"); - } - - if (requirements.requiresStructuredOutput && capabilities.structuredOutput === false) { - failures.push("structured_output"); - } - - if ( - requirements.requestedOutputTokens > 0 && - Number.isFinite(capabilities.maxOutputTokens) && - capabilities.maxOutputTokens < requirements.requestedOutputTokens - ) { - failures.push("output_tokens"); - } - - const contextLimit = capabilities.maxInputTokens ?? capabilities.contextWindow ?? null; - if ( - requirements.requiredContextTokens > 0 && - contextLimit !== null && - contextLimit !== undefined && - contextLimit < requirements.requiredContextTokens - ) { - failures.push("context_window"); - } - - return failures; -} - -export function filterTargetsByRequestCompatibility( - targets: ResolvedComboTarget[], - body: Record, - log: ComboLogger, - label = "Context-aware fallback" -): ResolvedComboTarget[] { - if (targets.length === 0) return targets; - const requirements = deriveRequestCompatibilityRequirements(body); - const needsFiltering = - requirements.requiresTools || - requirements.requiresVision || - requirements.requiresStructuredOutput || - requirements.requiredContextTokens > 0; - if (!needsFiltering) return targets; - - const rejected: Array<{ target: ResolvedComboTarget; reasons: string[] }> = []; - const compatible = targets.filter((target) => { - const reasons = getTargetCompatibilityFailures(target, requirements); - if (reasons.length === 0) return true; - rejected.push({ target, reasons }); - return false; - }); - - if (compatible.length === targets.length) return targets; - if (compatible.length === 0) { - log.warn( - "COMBO", - `${label}: all ${targets.length} targets were filtered by request requirements; preserving strategy order` - ); - log.debug?.( - "COMBO", - `${label}: rejected targets ${rejected - .map((entry) => `${entry.target.modelStr}(${entry.reasons.join("+")})`) - .join(", ")}` - ); - return targets; - } - - log.info( - "COMBO", - `${label}: kept ${compatible.length}/${targets.length} targets for request requirements` - ); - log.debug?.( - "COMBO", - `${label}: rejected targets ${rejected - .map((entry) => `${entry.target.modelStr}(${entry.reasons.join("+")})`) - .join(", ")}` - ); - return compatible; -} - -function sortTargetsByContextSize(targets: ResolvedComboTarget[]) { - const hasKnownContext = targets.some( - (target) => getModelContextLimitForModelString(target.modelStr) != null - ); - if (!hasKnownContext) return targets; - - const orderedModels = sortModelsByContextSize(targets.map((target) => target.modelStr)); - const byModel = new Map(); - for (const target of targets) { - const queue = byModel.get(target.modelStr) || []; - queue.push(target); - byModel.set(target.modelStr, queue); - } - return orderedModels - .map((modelStr) => { - const queue = byModel.get(modelStr); - return queue?.shift() || null; - }) - .filter((target): target is ResolvedComboTarget => target !== null); -} - -function getP2CTargetScore( - target: ResolvedComboTarget, - metrics: ReturnType -): number { - const breakerState = getCircuitBreaker(target.provider)?.getStatus?.()?.state; - if (breakerState === "OPEN") return -Infinity; - const modelMetric = metrics?.byModel?.[target.modelStr] || null; - const successRate = Number(modelMetric?.successRate); - const avgLatency = Number(modelMetric?.avgLatencyMs); - const successScore = Number.isFinite(successRate) ? successRate / 100 : 0.5; - const latencyScore = - Number.isFinite(avgLatency) && avgLatency > 0 ? 1 / Math.log10(avgLatency + 10) : 0.25; - const breakerPenalty = breakerState === "HALF_OPEN" ? 0.25 : 0; - return successScore + latencyScore - breakerPenalty; -} - -function orderTargetsByPowerOfTwoChoices(targets: ResolvedComboTarget[], comboName: string) { - if (targets.length <= 1) return targets; - const metrics = getComboMetrics(comboName); - const firstIndex = Math.floor(Math.random() * targets.length); - let secondIndex = Math.floor(Math.random() * (targets.length - 1)); - if (secondIndex >= firstIndex) secondIndex++; - - const first = targets[firstIndex]; - const second = targets[secondIndex]; - const selectedIndex = - getP2CTargetScore(second, metrics) > getP2CTargetScore(first, metrics) - ? secondIndex - : firstIndex; - return [targets[selectedIndex], ...targets.filter((_, index) => index !== selectedIndex)]; -} - function finiteNumberOrNull(value: unknown): number | null { const numericValue = Number(value); return Number.isFinite(numericValue) ? numericValue : null; @@ -2281,15 +1415,6 @@ async function buildAutoCandidates( return candidates; } -function dedupeTargetsByExecutionKey(targets: ResolvedComboTarget[]) { - const seen = new Set(); - return targets.filter((target) => { - if (seen.has(target.executionKey)) return false; - seen.add(target.executionKey); - return true; - }); -} - async function applyRequestTagRouting( targets: ResolvedComboTarget[], body: Record | null | undefined, @@ -2382,52 +1507,6 @@ async function applyRequestTagRouting( return filteredTargets; } -export function resolveComboTargets( - combo: ComboLike, - allCombos: ComboCollectionLike, - maxDepth: number = MAX_COMBO_DEPTH -): ResolvedComboTarget[] { - return allCombos - ? resolveNestedComboTargets(combo, allCombos, new Set(), 0, [], maxDepth) - : getDirectComboTargets(combo); -} - -function resolveWeightedTargets( - combo: ComboLike, - allCombos: ComboCollectionLike -): { - orderedTargets: ResolvedComboTarget[]; - selectedStep: ComboRuntimeStep | null; -} { - const topLevelSteps = getOrderedTopLevelRuntimeSteps(combo, allCombos); - if (topLevelSteps.length === 0) { - return { orderedTargets: [], selectedStep: null }; - } - - const selectedStep = selectWeightedTarget(topLevelSteps); - if (!selectedStep) { - return { orderedTargets: [], selectedStep: null }; - } - - const orderedSteps = orderTargetsForWeightedFallback( - topLevelSteps, - selectedStep.executionKey, - hasCompositeTierRuntimeOrder(combo) - ); - const expandedTargets = orderedSteps.flatMap((step) => { - if (!step) return []; - if (!allCombos) { - return step.kind === "model" ? [step] : []; - } - return expandRuntimeStep(step, allCombos, new Set([combo.name])); - }); - - return { - orderedTargets: dedupeTargetsByExecutionKey(expandedTargets), - selectedStep, - }; -} - export function scoreAutoTargets( targets: ResolvedComboTarget[], candidates: AutoProviderCandidate[], diff --git a/open-sse/services/combo/comboData.ts b/open-sse/services/combo/comboData.ts new file mode 100644 index 0000000000..6fb42a435f --- /dev/null +++ b/open-sse/services/combo/comboData.ts @@ -0,0 +1,24 @@ +/** + * Shared combo data-normalization helpers extracted from combo.ts. + * + * Tiny side-effect-free guards/normalizers that several combo submodules depend + * on (shadow routing, combo structure resolution) plus combo.ts itself. Moving + * them to this leaf module out of the combo.ts god-file (Quality Gate v2 / + * Fase 9) lets the submodules import them without reaching back into the barrel + * (no cycles). Logic unchanged; combo.ts imports them back for compatibility. + */ + +import type { ResolvedComboTarget } from "./types.ts"; + +export function isRecord(value: unknown): value is Record { + return !!value && typeof value === "object" && !Array.isArray(value); +} + +export function dedupeTargetsByExecutionKey(targets: ResolvedComboTarget[]) { + const seen = new Set(); + return targets.filter((target) => { + if (seen.has(target.executionKey)) return false; + seen.add(target.executionKey); + return true; + }); +} diff --git a/open-sse/services/combo/comboStructure.ts b/open-sse/services/combo/comboStructure.ts new file mode 100644 index 0000000000..69850e76e2 --- /dev/null +++ b/open-sse/services/combo/comboStructure.ts @@ -0,0 +1,637 @@ +/** + * Combo structure resolution extracted from combo.ts. + * + * Runtime-step normalization, nested-combo / DAG expansion, weighted/direct + * target resolution, and request-compatibility filtering moved out of the + * combo.ts god-file (Quality Gate v2 / Fase 9). Logic unchanged; the public + * entry points (resolveComboTargets, resolveNestedComboTargets, getComboFromData, + * getComboModelsFromData, validateComboDAG, resolveNestedComboModels, + * filterTargetsByRequestCompatibility) are re-exported from combo.ts for the + * ~20 external consumers (chatCore.ts, the /api/combos routes, embeddings, etc.). + * No barrel import — depends only on sibling leaves. + */ + +import { getModelContextLimit } from "../../../src/lib/modelCapabilities"; +import { getComboModelString, normalizeComboStep } from "../../../src/lib/combos/steps.ts"; +import { estimateTokens } from "../contextManager.ts"; +import { getResolvedModelCapabilities } from "../modelCapabilities.ts"; +import { parseModel } from "../model.ts"; +import { dedupeTargetsByExecutionKey, isRecord } from "./comboData.ts"; +import { getTargetProvider, MAX_COMBO_DEPTH } from "./comboPredicates.ts"; +import { + normalizeModelEntry, + orderTargetsForWeightedFallback, + selectWeightedTarget, +} from "./targetSorters.ts"; +import type { + ComboCollectionLike, + ComboInput, + ComboLike, + ComboLogger, + ComboRuntimeStep, + ResolvedComboTarget, +} from "./types.ts"; + +function toTrimmedString(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; +} + +function toComboLike(combo: ComboInput): ComboLike { + return { + ...combo, + id: toTrimmedString(combo.id) || undefined, + name: toTrimmedString(combo.name) || "", + models: Array.isArray(combo.models) ? combo.models : [], + config: isRecord(combo.config) ? combo.config : null, + autoConfig: isRecord(combo.autoConfig) ? combo.autoConfig : null, + context_cache_protection: + typeof combo.context_cache_protection === "boolean" || + typeof combo.context_cache_protection === "number" + ? combo.context_cache_protection + : undefined, + system_message: typeof combo.system_message === "string" ? combo.system_message : null, + }; +} + +function getCombosArray(allCombos: ComboCollectionLike): ComboLike[] { + const combos = Array.isArray(allCombos) ? allCombos : allCombos?.combos || []; + return combos.map((combo) => toComboLike(combo)); +} + +function buildExecutionKey(path: string[], stepId: string): string { + return [...path, stepId].join(">"); +} + +function normalizeRuntimeStep( + entry: unknown, + comboName: string, + index: number, + allCombos: ComboCollectionLike, + path: string[] = [] +): ComboRuntimeStep | null { + const step = normalizeComboStep(entry, { + comboName, + index, + allCombos, + }); + if (!step) return null; + + const executionKey = buildExecutionKey(path, step.id); + const label = typeof step.label === "string" ? step.label : null; + const weight = step.weight || 0; + + if (step.kind === "combo-ref") { + return { + kind: "combo-ref", + stepId: step.id, + executionKey, + comboName: step.comboName, + weight, + label, + }; + } + + const modelStr = getComboModelString(step); + if (!modelStr) return null; + + return { + kind: "model", + stepId: step.id, + executionKey, + modelStr, + provider: getTargetProvider(modelStr, step.providerId), + providerId: step.providerId || null, + connectionId: step.connectionId || null, + weight, + label, + } satisfies ResolvedComboTarget; +} + +function getDirectComboTargets(combo: ComboLike): ResolvedComboTarget[] { + return getOrderedTopLevelRuntimeSteps(combo, null).filter( + (entry): entry is ResolvedComboTarget => entry?.kind === "model" + ); +} + +function getTopLevelRuntimeSteps( + combo: ComboLike, + allCombos: ComboCollectionLike, + path: string[] = [] +): ComboRuntimeStep[] { + return (combo.models || []) + .map((entry, index) => normalizeRuntimeStep(entry, combo.name, index, allCombos, path)) + .filter((entry): entry is ComboRuntimeStep => entry !== null); +} + +function getCompositeTierStepOrder(combo: ComboLike): string[] { + const compositeTiers = isRecord(combo?.config) ? combo.config.compositeTiers : null; + if (!isRecord(compositeTiers)) return []; + + const defaultTier = toTrimmedString(compositeTiers.defaultTier); + const tiers = isRecord(compositeTiers.tiers) ? compositeTiers.tiers : null; + if (!defaultTier || !tiers) return []; + + const orderedStepIds: string[] = []; + const visitedTiers = new Set(); + const seenStepIds = new Set(); + type CompositeTierEntry = readonly [ + string, + { readonly stepId: string; readonly fallbackTier: string | null }, + ]; + const tierEntries = new Map( + Object.entries(tiers) + .map(([tierName, rawTier]) => { + if (!isRecord(rawTier)) return null; + const normalizedTierName = toTrimmedString(tierName); + const stepId = toTrimmedString(rawTier.stepId); + const fallbackTier = toTrimmedString(rawTier.fallbackTier); + if (!normalizedTierName || !stepId) return null; + return [normalizedTierName, { stepId, fallbackTier }] as const; + }) + .filter((entry): entry is CompositeTierEntry => entry !== null) + ); + + let currentTier: string | null = defaultTier; + while (currentTier && tierEntries.has(currentTier) && !visitedTiers.has(currentTier)) { + visitedTiers.add(currentTier); + const entry = tierEntries.get(currentTier); + if (!entry) break; + if (!seenStepIds.has(entry.stepId)) { + orderedStepIds.push(entry.stepId); + seenStepIds.add(entry.stepId); + } + currentTier = entry.fallbackTier; + } + + for (const entry of tierEntries.values()) { + if (!seenStepIds.has(entry.stepId)) { + orderedStepIds.push(entry.stepId); + seenStepIds.add(entry.stepId); + } + } + + return orderedStepIds; +} + +function hasCompositeTierRuntimeOrder(combo: ComboLike): boolean { + return getCompositeTierStepOrder(combo).length > 0; +} + +function orderRuntimeStepsByCompositeTiers( + steps: ComboRuntimeStep[], + combo: ComboLike +): ComboRuntimeStep[] { + const orderedStepIds = getCompositeTierStepOrder(combo); + if (orderedStepIds.length === 0) return steps; + + const byStepId = new Map(steps.map((step) => [step.stepId, step])); + const seen = new Set(); + const ordered: ComboRuntimeStep[] = []; + + for (const stepId of orderedStepIds) { + const step = byStepId.get(stepId); + if (!step || seen.has(step.stepId)) continue; + ordered.push(step); + seen.add(step.stepId); + } + + for (const step of steps) { + if (seen.has(step.stepId)) continue; + ordered.push(step); + seen.add(step.stepId); + } + + return ordered; +} + +function getOrderedTopLevelRuntimeSteps( + combo: ComboLike, + allCombos: ComboCollectionLike, + path: string[] = [] +): ComboRuntimeStep[] { + return orderRuntimeStepsByCompositeTiers(getTopLevelRuntimeSteps(combo, allCombos, path), combo); +} + +function expandRuntimeStep( + step: ComboRuntimeStep, + allCombos: ComboCollectionLike, + visited = new Set(), + depth = 0, + path: string[] = [], + maxDepth: number = MAX_COMBO_DEPTH +): ResolvedComboTarget[] { + if (step.kind === "model") return [step]; + if (depth > maxDepth) return []; + + const combos = getCombosArray(allCombos); + const nestedCombo = combos.find((combo) => combo.name === step.comboName); + if (!nestedCombo || visited.has(step.comboName)) return []; + + return resolveNestedComboTargets( + nestedCombo, + combos, + new Set(visited), + depth + 1, + [...path, step.stepId], + maxDepth + ); +} + +export function resolveNestedComboTargets( + combo: ComboLike, + allCombos: ComboCollectionLike, + visited = new Set(), + depth = 0, + path: string[] = [], + maxDepth: number = MAX_COMBO_DEPTH +): ResolvedComboTarget[] { + const directTargets = (combo.models || []) + .map((entry, index) => normalizeRuntimeStep(entry, combo.name, index, null, path)) + .filter((entry): entry is ResolvedComboTarget => entry?.kind === "model"); + + if (depth > maxDepth) return directTargets; + if (visited.has(combo.name)) return []; + visited.add(combo.name); + + const runtimeSteps = getOrderedTopLevelRuntimeSteps(combo, allCombos, path); + const resolved: ResolvedComboTarget[] = []; + + for (const step of runtimeSteps) { + if (step.kind === "combo-ref") { + resolved.push(...expandRuntimeStep(step, allCombos, new Set(visited), depth, path, maxDepth)); + continue; + } + resolved.push(step); + } + + return resolved; +} + +/** + * Get combo models from combos data (for open-sse standalone use) + * @param {string} modelStr - Model string to check + * @param {Array|Object} combosData - Array of combos or object with combos + * @returns {Object|null} Full combo object or null if not a combo + */ +export function getComboFromData( + modelStr: string, + combosData: ComboCollectionLike +): ComboLike | null { + const combos = getCombosArray(combosData); + const combo = combos.find((c) => c.name === modelStr); + if (combo?.models && combo.models.length > 0) { + return combo; + } + return null; +} + +/** + * Legacy: Get combo models as string array (backward compat) + */ +export function getComboModelsFromData( + modelStr: string, + combosData: ComboCollectionLike +): string[] | null { + const combo = getComboFromData(modelStr, combosData); + if (!combo) return null; + return combo.models.map((m) => normalizeModelEntry(m).model); +} + +/** + * Validate combo DAG — detect circular references and enforce max depth + * @param {string} comboName - Name of the combo to validate + * @param {Array} allCombos - All combos in the system + * @param {Set} [visited] - Set of already visited combo names (for cycle detection) + * @param {number} [depth] - Current depth level + * @throws {Error} If circular reference or max depth exceeded + */ +export function validateComboDAG( + comboName: string, + allCombos: ComboCollectionLike, + visited = new Set(), + depth = 0, + maxDepth: number = MAX_COMBO_DEPTH +): void { + if (depth > maxDepth) { + throw new Error(`Max combo nesting depth (${maxDepth}) exceeded at "${comboName}"`); + } + if (visited.has(comboName)) { + throw new Error(`Circular combo reference detected: ${comboName}`); + } + visited.add(comboName); + + const combos = getCombosArray(allCombos); + const combo = combos.find((c) => c.name === comboName); + if (!combo?.models) return; + + for (const entry of combo.models) { + const modelName = normalizeModelEntry(entry).model; + // Check if this model name is itself a combo (not a provider/model pattern) + const nestedCombo = combos.find((c) => c.name === modelName); + if (nestedCombo) { + validateComboDAG(modelName, combos, new Set(visited), depth + 1, maxDepth); + } + } +} + +/** + * Resolve nested combos by expanding inline to a flat model list + * Respects max depth and detects cycles + * @param {Object} combo - The combo object + * @param {Array} allCombos - All combos in the system + * @param {Set} [visited] - For cycle detection + * @param {number} [depth] - Current depth + * @returns {Array} Flat array of model strings + */ +export function resolveNestedComboModels( + combo: ComboLike, + allCombos: ComboCollectionLike, + visited = new Set(), + depth = 0, + maxDepth: number = MAX_COMBO_DEPTH +): string[] { + if (depth > maxDepth) return combo.models.map((m) => normalizeModelEntry(m).model); + if (visited.has(combo.name)) return []; // cycle safety + visited.add(combo.name); + + const combos = getCombosArray(allCombos); + const resolved: string[] = []; + + for (const entry of combo.models || []) { + const modelName = normalizeModelEntry(entry).model; + const nestedCombo = combos.find((c) => c.name === modelName); + + if (nestedCombo) { + // Recursively expand the nested combo + const nested = resolveNestedComboModels( + nestedCombo, + combos, + new Set(visited), + depth + 1, + maxDepth + ); + resolved.push(...nested); + } else { + resolved.push(modelName); + } + } + + return resolved; +} + +/** + * Sort models by context window size (largest first) for context-optimized strategy. + * Uses models.dev synced capabilities to get context limits. + * @param {Array} models - Model strings in "provider/model" format + * @returns {Array} Sorted model strings (largest context first) + */ +function sortModelsByContextSize(models: string[]): string[] { + const withContext = models.map((modelStr) => { + return { modelStr, context: getModelContextLimitForModelString(modelStr) ?? 0 }; + }); + withContext.sort((a, b) => b.context - a.context); + return withContext.map((e) => e.modelStr); +} + +export function getModelContextLimitForModelString(modelStr: string) { + const parsed = parseModel(modelStr); + const provider = parsed.provider || parsed.providerAlias || "unknown"; + const model = parsed.model || modelStr; + return getModelContextLimit(provider, model); +} + +type RequestCompatibilityRequirements = { + requiresTools: boolean; + requiresVision: boolean; + requiresStructuredOutput: boolean; + estimatedInputTokens: number; + requestedOutputTokens: number; + requiredContextTokens: number; +}; + +function getPositiveTokenCount(value: unknown): number { + const count = Number(value); + return Number.isFinite(count) && count > 0 ? Math.ceil(count) : 0; +} + +function requestRequiresTools(body: Record): boolean { + if (Array.isArray(body.tools) && body.tools.length > 0) return true; + if (Array.isArray(body.functions) && body.functions.length > 0) return true; + return false; +} + +function requestRequiresStructuredOutput(body: Record): boolean { + const responseFormat = isRecord(body.response_format) ? body.response_format : null; + const type = typeof responseFormat?.type === "string" ? responseFormat.type : null; + return type === "json_object" || type === "json_schema"; +} + +function estimateRequestInputTokens(body: Record): number { + const estimatePayload: Record = {}; + for (const key of ["messages", "input", "tools", "functions", "response_format"]) { + if (body[key] !== undefined) estimatePayload[key] = body[key]; + } + return Object.keys(estimatePayload).length > 0 ? estimateTokens(estimatePayload) : 0; +} + +function valueContainsImagePart(value: unknown, depth = 0): boolean { + if (depth > 8 || value === null || value === undefined) return false; + if (typeof value === "string") return value.startsWith("data:image/"); + if (Array.isArray(value)) return value.some((entry) => valueContainsImagePart(entry, depth + 1)); + if (!isRecord(value)) return false; + + const type = typeof value.type === "string" ? value.type.toLowerCase() : null; + if (type === "image" || type === "image_url" || type === "input_image") return true; + if ("image_url" in value || "input_image" in value) return true; + + const source = isRecord(value.source) ? value.source : null; + const mediaType = typeof source?.media_type === "string" ? source.media_type.toLowerCase() : ""; + if (mediaType.startsWith("image/")) return true; + + return Object.values(value).some((entry) => valueContainsImagePart(entry, depth + 1)); +} + +function deriveRequestCompatibilityRequirements( + body: Record +): RequestCompatibilityRequirements { + const estimatedInputTokens = estimateRequestInputTokens(body); + const requestedOutputTokens = Math.max( + getPositiveTokenCount(body.max_tokens), + getPositiveTokenCount(body.max_completion_tokens) + ); + return { + requiresTools: requestRequiresTools(body), + requiresVision: valueContainsImagePart(body.messages) || valueContainsImagePart(body.input), + requiresStructuredOutput: requestRequiresStructuredOutput(body), + estimatedInputTokens, + requestedOutputTokens, + requiredContextTokens: estimatedInputTokens + requestedOutputTokens, + }; +} + +function getTargetCompatibilityFailures( + target: ResolvedComboTarget, + requirements: RequestCompatibilityRequirements +): string[] { + const capabilities = getResolvedModelCapabilities(target.modelStr); + const failures: string[] = []; + + if ( + requirements.requiresTools && + (capabilities.supportsTools === false || !capabilities.toolCalling) + ) { + failures.push("tools"); + } + + // For a request that carries an image, only route to a target whose vision + // support is *confirmed* (`=== true`). Treat `false` AND `null` (unknown) as + // incompatible: an unknown-capability model receiving the image is exactly how + // a text-only model (e.g. ministral) ended up answering "image not provided". + // The caller keeps all targets when none qualify, so combos with no + // confirmed-vision member still behave as before. + if (requirements.requiresVision && capabilities.supportsVision !== true) { + failures.push("vision"); + } + + if (requirements.requiresStructuredOutput && capabilities.structuredOutput === false) { + failures.push("structured_output"); + } + + if ( + requirements.requestedOutputTokens > 0 && + Number.isFinite(capabilities.maxOutputTokens) && + capabilities.maxOutputTokens < requirements.requestedOutputTokens + ) { + failures.push("output_tokens"); + } + + const contextLimit = capabilities.maxInputTokens ?? capabilities.contextWindow ?? null; + if ( + requirements.requiredContextTokens > 0 && + contextLimit !== null && + contextLimit !== undefined && + contextLimit < requirements.requiredContextTokens + ) { + failures.push("context_window"); + } + + return failures; +} + +export function filterTargetsByRequestCompatibility( + targets: ResolvedComboTarget[], + body: Record, + log: ComboLogger, + label = "Context-aware fallback" +): ResolvedComboTarget[] { + if (targets.length === 0) return targets; + const requirements = deriveRequestCompatibilityRequirements(body); + const needsFiltering = + requirements.requiresTools || + requirements.requiresVision || + requirements.requiresStructuredOutput || + requirements.requiredContextTokens > 0; + if (!needsFiltering) return targets; + + const rejected: Array<{ target: ResolvedComboTarget; reasons: string[] }> = []; + const compatible = targets.filter((target) => { + const reasons = getTargetCompatibilityFailures(target, requirements); + if (reasons.length === 0) return true; + rejected.push({ target, reasons }); + return false; + }); + + if (compatible.length === targets.length) return targets; + if (compatible.length === 0) { + log.warn( + "COMBO", + `${label}: all ${targets.length} targets were filtered by request requirements; preserving strategy order` + ); + log.debug?.( + "COMBO", + `${label}: rejected targets ${rejected + .map((entry) => `${entry.target.modelStr}(${entry.reasons.join("+")})`) + .join(", ")}` + ); + return targets; + } + + log.info( + "COMBO", + `${label}: kept ${compatible.length}/${targets.length} targets for request requirements` + ); + log.debug?.( + "COMBO", + `${label}: rejected targets ${rejected + .map((entry) => `${entry.target.modelStr}(${entry.reasons.join("+")})`) + .join(", ")}` + ); + return compatible; +} + +export function sortTargetsByContextSize(targets: ResolvedComboTarget[]) { + const hasKnownContext = targets.some( + (target) => getModelContextLimitForModelString(target.modelStr) != null + ); + if (!hasKnownContext) return targets; + + const orderedModels = sortModelsByContextSize(targets.map((target) => target.modelStr)); + const byModel = new Map(); + for (const target of targets) { + const queue = byModel.get(target.modelStr) || []; + queue.push(target); + byModel.set(target.modelStr, queue); + } + return orderedModels + .map((modelStr) => { + const queue = byModel.get(modelStr); + return queue?.shift() || null; + }) + .filter((target): target is ResolvedComboTarget => target !== null); +} + +export function resolveComboTargets( + combo: ComboLike, + allCombos: ComboCollectionLike, + maxDepth: number = MAX_COMBO_DEPTH +): ResolvedComboTarget[] { + return allCombos + ? resolveNestedComboTargets(combo, allCombos, new Set(), 0, [], maxDepth) + : getDirectComboTargets(combo); +} + +export function resolveWeightedTargets( + combo: ComboLike, + allCombos: ComboCollectionLike +): { + orderedTargets: ResolvedComboTarget[]; + selectedStep: ComboRuntimeStep | null; +} { + const topLevelSteps = getOrderedTopLevelRuntimeSteps(combo, allCombos); + if (topLevelSteps.length === 0) { + return { orderedTargets: [], selectedStep: null }; + } + + const selectedStep = selectWeightedTarget(topLevelSteps); + if (!selectedStep) { + return { orderedTargets: [], selectedStep: null }; + } + + const orderedSteps = orderTargetsForWeightedFallback( + topLevelSteps, + selectedStep.executionKey, + hasCompositeTierRuntimeOrder(combo) + ); + const expandedTargets = orderedSteps.flatMap((step) => { + if (!step) return []; + if (!allCombos) { + return step.kind === "model" ? [step] : []; + } + return expandRuntimeStep(step, allCombos, new Set([combo.name])); + }); + + return { + orderedTargets: dedupeTargetsByExecutionKey(expandedTargets), + selectedStep, + }; +} diff --git a/open-sse/services/combo/shadowRouting.ts b/open-sse/services/combo/shadowRouting.ts new file mode 100644 index 0000000000..2fe214df77 --- /dev/null +++ b/open-sse/services/combo/shadowRouting.ts @@ -0,0 +1,178 @@ +/** + * Combo shadow-routing helpers extracted from combo.ts. + * + * Shadow routing mirrors a sampled fraction of production traffic to extra + * targets (fire-and-forget) to compare behavior without affecting the real + * response. Moved out of the combo.ts god-file (Quality Gate v2 / Fase 9) — + * logic unchanged; `resolveShadowTargets` and `scheduleShadowRouting` are + * re-exported from combo.ts for backward compatibility and used by + * handleComboChat / handleRoundRobinCombo (which stay in combo.ts). + * + * NOTE: `resolveNestedComboTargets` lives in combo/comboStructure.ts (extracted + * in the same Fase 9 split). It is only invoked at request time inside + * `resolveShadowTargets`, never during module init. + */ + +import { recordComboShadowRequest } from "../comboMetrics.ts"; +import { isRecord } from "./comboData.ts"; +import { resolveNestedComboTargets } from "./comboStructure.ts"; +import { toRecordedTarget } from "./comboPredicates.ts"; +import type { + ComboLike, + ComboCollectionLike, + ComboLogger, + HandleSingleModel, + IsModelAvailable, + ResolvedComboTarget, + ShadowRoutingConfig, +} from "./types.ts"; + +function normalizeShadowRoutingConfig(config: Record): ShadowRoutingConfig { + const raw = isRecord(config.shadowRouting) ? config.shadowRouting : {}; + const sampleRate = Number(raw.sampleRate ?? 1); + const maxTargets = Number(raw.maxTargets ?? 2); + const timeoutMs = Number(raw.timeoutMs ?? 30000); + return { + enabled: raw.enabled === true, + targets: Array.isArray(raw.targets) ? raw.targets : [], + sampleRate: Number.isFinite(sampleRate) ? Math.max(0, Math.min(1, sampleRate)) : 1, + maxTargets: Number.isFinite(maxTargets) ? Math.max(1, Math.min(10, Math.floor(maxTargets))) : 2, + timeoutMs: Number.isFinite(timeoutMs) + ? Math.max(1000, Math.min(120000, Math.floor(timeoutMs))) + : 30000, + }; +} + +export function resolveShadowTargets( + combo: ComboLike, + config: Record, + allCombos: ComboCollectionLike +): ResolvedComboTarget[] { + const shadowConfig = normalizeShadowRoutingConfig(config); + if (!shadowConfig.enabled || shadowConfig.targets.length === 0) return []; + if (shadowConfig.sampleRate <= 0 || Math.random() > shadowConfig.sampleRate) return []; + + const shadowCombo: ComboLike = { + ...combo, + name: `${combo.name}:shadow`, + models: shadowConfig.targets, + }; + return resolveNestedComboTargets(shadowCombo, allCombos, new Set([combo.name]), 0, ["shadow"]) + .slice(0, shadowConfig.maxTargets) + .map((target) => ({ + ...target, + trafficType: "shadow" as const, + })); +} + +async function drainShadowResponse(response: Response): Promise { + try { + if (!response.body) return; + await response.arrayBuffer(); + } catch { + // Shadow draining is best-effort and must never affect the production response. + } +} + +function withTimeout(promise: Promise, timeoutMs: number): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error("Shadow route timed out")), timeoutMs); + promise.then( + (value) => { + clearTimeout(timer); + resolve(value); + }, + (error) => { + clearTimeout(timer); + reject(error); + } + ); + }); +} + +function cloneRequestBodyForShadowRouting(body: Record): Record { + if (typeof structuredClone === "function") { + return structuredClone(body) as Record; + } + + return JSON.parse(JSON.stringify(body)) as Record; +} + +export function scheduleShadowRouting( + combo: ComboLike, + config: Record, + body: Record, + targets: ResolvedComboTarget[], + handleSingleModel: HandleSingleModel, + isModelAvailable: IsModelAvailable | undefined, + strategy: string, + log: ComboLogger +): void { + if (targets.length === 0) return; + const shadowConfig = normalizeShadowRoutingConfig(config); + let shadowBaseBody: Record; + try { + shadowBaseBody = cloneRequestBodyForShadowRouting(body); + } catch (error) { + log.warn("COMBO", "Shadow routing skipped: failed to clone request body", { + error: error instanceof Error ? error.message : String(error), + }); + return; + } + const run = async () => { + await Promise.all( + targets.map(async (target) => { + const startedAt = Date.now(); + try { + const shadowBody = { + ...cloneRequestBodyForShadowRouting(shadowBaseBody), + model: target.modelStr, + stream: false, + }; + if (isModelAvailable) { + const available = await isModelAvailable(target.modelStr, target); + if (!available) { + recordComboShadowRequest(combo.name, target.modelStr, { + success: false, + latencyMs: Date.now() - startedAt, + target: toRecordedTarget(target), + }); + log.info("COMBO", `Shadow target skipped (unavailable): ${target.modelStr}`); + return; + } + } + + const response = await withTimeout( + handleSingleModel(shadowBody, target.modelStr, { + ...target, + failoverBeforeRetry: true, + trafficType: "shadow", + }), + shadowConfig.timeoutMs + ); + await drainShadowResponse(response.clone()); + recordComboShadowRequest(combo.name, target.modelStr, { + success: response.ok, + latencyMs: Date.now() - startedAt, + target: toRecordedTarget(target), + }); + log.info( + "COMBO", + `Shadow target ${target.modelStr} completed with status ${response.status} (${strategy})` + ); + } catch (error) { + recordComboShadowRequest(combo.name, target.modelStr, { + success: false, + latencyMs: Date.now() - startedAt, + target: toRecordedTarget(target), + }); + log.warn("COMBO", `Shadow target ${target.modelStr} failed`, { + error: error instanceof Error ? error.message : String(error), + }); + } + }) + ); + }; + + setTimeout(() => void run(), 0); +} diff --git a/open-sse/services/combo/targetSorters.ts b/open-sse/services/combo/targetSorters.ts new file mode 100644 index 0000000000..7829e7512b --- /dev/null +++ b/open-sse/services/combo/targetSorters.ts @@ -0,0 +1,173 @@ +/** + * Combo target-ordering / sorting helpers extracted from combo.ts. + * + * Weighted selection, cost / usage / power-of-two-choices ordering, and the + * model-entry normalizer moved out of the combo.ts god-file (Quality Gate v2 / + * Fase 9). Logic unchanged; the helpers used by the combo handlers (which stay + * in combo.ts) are imported back from this module. No barrel import — pure leaf. + */ + +import { getCircuitBreaker } from "../../../src/shared/utils/circuitBreaker"; +import { getComboStepTarget, getComboStepWeight } from "../../../src/lib/combos/steps.ts"; +import { getComboMetrics } from "../comboMetrics.ts"; +import { parseModel } from "../model.ts"; +import type { ResolvedComboTarget } from "./types.ts"; + +/** + * Normalize a model entry to { model, weight } + * Supports both legacy string format and new object format + */ +export function normalizeModelEntry(entry: unknown): { model: string; weight: number } { + return { + model: getComboStepTarget(entry) || "", + weight: getComboStepWeight(entry), + }; +} + +export function selectWeightedTarget(targets: T[]) { + if (targets.length === 0) return null; + + const totalWeight = targets.reduce((sum, target) => sum + (target.weight || 0), 0); + if (totalWeight <= 0) { + return targets[Math.floor(Math.random() * targets.length)]; + } + + let random = Math.random() * totalWeight; + for (const target of targets) { + random -= target.weight || 0; + if (random <= 0) return target; + } + + return targets.at(-1); +} + +export function orderTargetsForWeightedFallback( + targets: T[], + selectedExecutionKey: string, + preserveExistingOrder = false +): T[] { + const selected = targets.find((target) => target.executionKey === selectedExecutionKey); + const rest = targets.filter((target) => target.executionKey !== selectedExecutionKey); + if (!preserveExistingOrder) { + rest.sort((a, b) => b.weight - a.weight); + } + return selected ? [selected, ...rest] : rest; +} + +// shuffleArray and getNextModelFromDeck moved to src/shared/utils/shuffleDeck.ts +// combo.ts now uses the shared, mutex-protected getNextFromDeck with "combo:" namespace. + +/** + * Sort models by pricing (cheapest first) for cost-optimized strategy + * @param {Array} models - Model strings in "provider/model" format + * @returns {Promise>} Sorted model strings + */ +export async function sortModelsByCost(models: string[]): Promise { + try { + const { getPricingForModel } = await import("../../../src/lib/localDb"); + const withCost = await Promise.all( + models.map(async (modelStr) => { + const parsed = parseModel(modelStr); + const provider = parsed.provider || parsed.providerAlias || "unknown"; + const model = parsed.model || modelStr; + try { + const pricing = await getPricingForModel(provider, model); + const cost = Number(pricing?.input); + return { modelStr, cost: Number.isFinite(cost) ? cost : Infinity }; + } catch { + return { modelStr, cost: Infinity }; + } + }) + ); + withCost.sort((a, b) => a.cost - b.cost); + return withCost.map((e) => e.modelStr); + } catch { + // If pricing lookup fails entirely, return original order + return models; + } +} + +export async function sortTargetsByCost(targets: ResolvedComboTarget[]) { + const orderedModels = await sortModelsByCost(targets.map((target) => target.modelStr)); + const byModel = new Map(); + for (const target of targets) { + const queue = byModel.get(target.modelStr) || []; + queue.push(target); + byModel.set(target.modelStr, queue); + } + return orderedModels + .map((modelStr) => { + const queue = byModel.get(modelStr); + return queue?.shift() || null; + }) + .filter((target): target is ResolvedComboTarget => target !== null); +} + +/** + * Sort models by usage count (least-used first) for least-used strategy + * @param {Array} models - Model strings + * @param {string} comboName - Combo name for metrics lookup + * @returns {Array} Sorted model strings + */ +export function sortModelsByUsage(models: string[], comboName: string): string[] { + const metrics = getComboMetrics(comboName); + if (!metrics?.byModel) return models; + + const withUsage = models.map((modelStr) => ({ + modelStr, + requests: metrics.byModel[modelStr]?.requests ?? 0, + })); + withUsage.sort((a, b) => a.requests - b.requests); + return withUsage.map((e) => e.modelStr); +} + +export function sortTargetsByUsage(targets: ResolvedComboTarget[], comboName: string) { + const orderedModels = sortModelsByUsage( + targets.map((target) => target.modelStr), + comboName + ); + const byModel = new Map(); + for (const target of targets) { + const queue = byModel.get(target.modelStr) || []; + queue.push(target); + byModel.set(target.modelStr, queue); + } + return orderedModels + .map((modelStr) => { + const queue = byModel.get(modelStr); + return queue?.shift() || null; + }) + .filter((target): target is ResolvedComboTarget => target !== null); +} + +function getP2CTargetScore( + target: ResolvedComboTarget, + metrics: ReturnType +): number { + const breakerState = getCircuitBreaker(target.provider)?.getStatus?.()?.state; + if (breakerState === "OPEN") return -Infinity; + const modelMetric = metrics?.byModel?.[target.modelStr] || null; + const successRate = Number(modelMetric?.successRate); + const avgLatency = Number(modelMetric?.avgLatencyMs); + const successScore = Number.isFinite(successRate) ? successRate / 100 : 0.5; + const latencyScore = + Number.isFinite(avgLatency) && avgLatency > 0 ? 1 / Math.log10(avgLatency + 10) : 0.25; + const breakerPenalty = breakerState === "HALF_OPEN" ? 0.25 : 0; + return successScore + latencyScore - breakerPenalty; +} + +export function orderTargetsByPowerOfTwoChoices(targets: ResolvedComboTarget[], comboName: string) { + if (targets.length <= 1) return targets; + const metrics = getComboMetrics(comboName); + const firstIndex = Math.floor(Math.random() * targets.length); + let secondIndex = Math.floor(Math.random() * (targets.length - 1)); + if (secondIndex >= firstIndex) secondIndex++; + + const first = targets[firstIndex]; + const second = targets[secondIndex]; + const selectedIndex = + getP2CTargetScore(second, metrics) > getP2CTargetScore(first, metrics) + ? secondIndex + : firstIndex; + return [targets[selectedIndex], ...targets.filter((_, index) => index !== selectedIndex)]; +}