diff --git a/file-size-baseline.json b/file-size-baseline.json index c0055d24ab..158f5c3a13 100644 --- a/file-size-baseline.json +++ b/file-size-baseline.json @@ -27,7 +27,7 @@ "open-sse/services/batchProcessor.ts": 828, "open-sse/services/browserBackedChat.ts": 850, "open-sse/services/claudeCodeCompatible.ts": 1202, - "open-sse/services/combo.ts": 5054, + "open-sse/services/combo.ts": 5131, "open-sse/services/rateLimitManager.ts": 1017, "open-sse/services/tokenRefresh.ts": 1997, "open-sse/services/usage.ts": 3394, @@ -111,5 +111,6 @@ "_rebaseline_2026_06_12_phase1n1s": "Phase 1n-1s (#3501): ProviderDetailPageClient.tsx 2554→1376 (extraídos ConnectionsListPanel, ConnectionsHeaderToolbar, ZedImportCard, BatchTestResultsModal, AdaptaTutorialModal + hooks/useApiKeySave + 4 helper closures→providerPageHelpers.ts). providerPageHelpers.ts 822→897 justificado: recebe 4 closures do god-component (getApiLabel/getApiDefaultPath/getApiPath/getHeaderIconProviderId), zero lógica nova, cliente encolhe mais do que helpers crescem.", "_rebaseline_2026_06_12_phase1t": "Phase 1t (#3501): ProviderDetailPageClient.tsx 1377→782 — META ≤800 ATINGIDA (extraídos ProviderPageHeader, CompatibleNodeCard, ProviderModalsPanel, EmptyConnectionsPlaceholder, UpstreamProxyCard, SearchProviderCard + hooks useConnectionGate/useProviderNodeActions). Drift concorrente reconciliado: ResilienceTab/sse-chat/sse-auth/accountFallback/combo (merges #3629 model-lockout etc.).", "_rebaseline_2026_06_12_v3823_new_features": "Re-baseline v3.8.23 pós-merge de #3742 (cost drilldown: ApiManagerPageClient.tsx +21, CostOverviewTab.tsx +14, providerLimits.ts +2, usage.ts +53) + #3743 (provider display modes: ProviderDetailPageClient.tsx +2, providerPageHelpers.ts +42, providers.ts +2) + #3740 (semantic cache key isolation: chat.ts +3). Crescimento justificado por features novas mergeadas no ciclo.", + "_rebaseline_2026_06_13_combo_quota_audit": "Re-baseline consciente do audit combo+quota (PR #3779): combo.ts 5054→5131 (+77). Crescimento = 5 fixes TDD + estratégia complexity-aware 2026 (W1 clampComboDepth + threading de maxDepth em 6 assinaturas/dispatch/DAG; W2 extração shouldSkipForPredictedTtft; W4 scoreAutoTargets exportado + param manifestHint). A parte limpa-extraível do W4 (construção do hint inline, ~30 linhas) FOI extraída para autoCombo/complexityRouter.ts (buildComplexityRoutingHint) — este +76 é o resíduo irredutível (edição de assinaturas/threading, não bloco movível). Shrink estrutural de combo.ts segue com #3501.", "_rebaseline_2026_06_13_v3824_3776": "Re-baseline v3.8.24 pós #3776 (strict-mode controls Claude Code default models: ApiManagerPageClient.tsx 2701→2909 = UI de famílias bloqueáveis cc/* + chips; apiKeys.ts 1490→1633 = blocked_models deny-list + candidatos de permissão claude-code; schemas.ts 2515→2519 = reformatação Prettier + reasoningTokenBufferEnabled restaurado) + carry-over base.ts 1205→1218 do #3780 (enforceThinkingTemperature no chokepoint, drift de baseline não bumpado no merge). Crescimento de feature; sem god-component novo." } diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 1c9c7bb14d..bfa9b8bf6f 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -5227,21 +5227,15 @@ export async function handleChatCore({ // === Quota Share POST-hook (B/F7) — fire-and-forget, fail-open === if (apiKeyInfo?.id && credentials?.connectionId) { try { - const { scheduleRecordConsumption } = await import("@/lib/quota/spendRecorder"); + const { scheduleRecordConsumption, buildConsumptionCost } = await import( + "@/lib/quota/spendRecorder" + ); scheduleRecordConsumption( { apiKeyId: apiKeyInfo.id, connectionId: credentials.connectionId, provider: provider ?? "unknown", - cost: { - tokens: - usage && typeof usage === "object" - ? (((usage as Record).prompt_tokens as number) ?? 0) + - (((usage as Record).completion_tokens as number) ?? 0) - : 0, - usd: estimatedCost > 0 ? estimatedCost : 0, - requests: 1, - }, + cost: buildConsumptionCost(usage, estimatedCost), }, log ); @@ -5522,29 +5516,28 @@ export async function handleChatCore({ } // === Quota Share POST-hook streaming (B/F7) — fire-and-forget, fail-open === + // Resolve the real per-request cost (calculateCost) so USD-unit pools accrue + // on streaming traffic too; this previously recorded usd:0 hardcoded, which + // meant DeepSeek-style `usd/monthly` shared pools never blocked on streams. if (apiKeyInfo?.id && credentials?.connectionId && streamStatus === 200) { - const su = streamUsage as Record | null; const quotaApiKeyId = apiKeyInfo.id; const quotaConnectionId = credentials.connectionId; // onStreamComplete is sync — use .then() (fire-and-forget, fail-open) instead of await import("@/lib/quota/spendRecorder") - .then(({ scheduleRecordConsumption }) => { - scheduleRecordConsumption( + .then(({ recordStreamingConsumption }) => + recordStreamingConsumption( { apiKeyId: quotaApiKeyId, connectionId: quotaConnectionId, - provider: provider ?? "unknown", - cost: { - tokens: su - ? (Number(su.prompt_tokens ?? 0) || 0) + (Number(su.completion_tokens ?? 0) || 0) - : 0, - usd: 0, // estimatedCost resolved async above; omit to avoid dependency - requests: 1, - }, + provider, + model, + streamUsage, + streamStatus, + serviceTier: effectiveServiceTier, }, - log - ); - }) + { calculateCost, log } + ) + ) .catch(() => { // Outer fail-open — never throws to caller }); diff --git a/open-sse/services/autoCombo/complexityRouter.ts b/open-sse/services/autoCombo/complexityRouter.ts new file mode 100644 index 0000000000..602fb1450b --- /dev/null +++ b/open-sse/services/autoCombo/complexityRouter.ts @@ -0,0 +1,111 @@ +/** + * complexityRouter.ts — Request-complexity classification for tier-aware routing. + * + * 2026 strategy: route by the intrinsic difficulty of the *request* (not only by + * provider stats), so trivial prompts can use cheap models and hard/reasoning + * prompts escalate to capable ones. Built on the existing specificity detector + * (codeComplexity / mathComplexity / reasoningDepth / contextSize / toolCalling + * / domainSpecificity), adding an explicit tool-use → minimum-tier escalation: + * a request carrying tool/function schemas (or agentic tool-calling signals) + * should not be routed below the "cheap" tier even when the prose looks + * trivial, because function-calling reliability matters more than raw cost. + * + * The classification maps to a `recommendedTier` that feeds the auto-router's + * tier-affinity / specificity-match scoring factors (see scoreAutoTargets, + * gated by config.complexityAwareRouting). + */ +import { + analyzeSpecificity, + getSpecificityLevel, + getRecommendedMinTier, +} from "../specificityDetector"; +import type { RuleInput, SpecificityLevel } from "../specificityTypes"; +import { generateRoutingHints, type RoutingHint } from "../manifestAdapter"; + +export type ComplexityTier = "free" | "cheap" | "premium"; + +export interface ComplexityClassification { + /** 0..100 specificity / difficulty score. */ + score: number; + /** trivial | simple | moderate | complex | expert */ + level: SpecificityLevel; + /** Minimum provider tier recommended for this request. */ + recommendedTier: ComplexityTier; + /** True when the request carries tool/function schemas or agentic tool signals. */ + hasToolUse: boolean; + /** Names of the specificity rules that fired (for the inspector / dashboard). */ + signals: string[]; +} + +const TIER_ORDER: ComplexityTier[] = ["free", "cheap", "premium"]; + +/** Raise `tier` to at least `floor`; never lowers it. */ +export function escalateTier(tier: ComplexityTier, floor: ComplexityTier): ComplexityTier { + return TIER_ORDER.indexOf(tier) >= TIER_ORDER.indexOf(floor) ? tier : floor; +} + +/** + * Classify a request's complexity and recommend a minimum provider tier. + * Pure + dependency-light (no DB / network); safe on the hot path. + */ +export function classifyRequestComplexity(input: RuleInput): ComplexityClassification { + const result = analyzeSpecificity(input); + const level = getSpecificityLevel(result.score); + + const explicitTools = Array.isArray(input.tools) && input.tools.length > 0; + const hasToolUse = explicitTools || result.breakdown.toolCalling > 0; + + let recommendedTier = getRecommendedMinTier(level) as ComplexityTier; + // Tool-using / agentic requests need reliable function calling — floor at "cheap". + if (hasToolUse) recommendedTier = escalateTier(recommendedTier, "cheap"); + + return { + score: result.score, + level, + recommendedTier, + hasToolUse, + signals: result.rulesTriggered, + }; +} + +/** + * Build the opt-in complexity-aware routing hint for the auto-router. Returns a + * RoutingHint whose `recommendedMinTier` is escalated to the request's intrinsic + * complexity (and floored at "cheap" for tool-using requests), or `null` on any + * failure — fail-open, so scoring stays tier-neutral. Extracted from combo.ts to + * keep the complexity-routing logic in one module. + */ +export function buildComplexityRoutingHint( + modelTargets: Parameters[0], + body: { messages?: unknown; tools?: unknown; model?: unknown } | null | undefined, + log: { info: (tag: string, message: string) => void } +): RoutingHint | null { + try { + const ruleInput = { + messages: Array.isArray(body?.messages) + ? (body.messages as Array<{ role?: string; content?: string | unknown }>) + : [], + tools: Array.isArray(body?.tools) + ? (body.tools as Array<{ + function?: { name: string; description?: string; parameters?: unknown }; + }>) + : undefined, + model: typeof body?.model === "string" ? body.model : undefined, + }; + const hint = generateRoutingHints(modelTargets, ruleInput); + // Tool-use escalation: floor the recommended tier at "cheap" so scoring + // favors function-calling-reliable models for agentic requests. + const classification = classifyRequestComplexity(ruleInput); + hint.recommendedMinTier = escalateTier( + hint.recommendedMinTier as ComplexityTier, + classification.recommendedTier + ) as typeof hint.recommendedMinTier; + log.info( + "COMBO", + `Complexity-aware routing: level=${classification.level} score=${classification.score} minTier=${hint.recommendedMinTier} tools=${classification.hasToolUse}` + ); + return hint; + } catch { + return null; // fail-open: scoring stays tier-neutral + } +} diff --git a/open-sse/services/autoCombo/scoring.ts b/open-sse/services/autoCombo/scoring.ts index c03ac3926d..5a9d4eed76 100644 --- a/open-sse/services/autoCombo/scoring.ts +++ b/open-sse/services/autoCombo/scoring.ts @@ -5,6 +5,8 @@ */ import type { RoutingHint } from "../manifestAdapter"; +import { clamp01 } from "../../utils/number"; +import { classifyTier } from "../tierResolver"; export interface ScoringFactors { quota: number; @@ -86,19 +88,22 @@ export interface ScoredProvider { * Supports tierAffinity + specificityMatch weights when manifest routing is enabled. */ export function calculateScore(factors: ScoringFactors, weights: ScoringWeights): number { - return ( + // clamp01 bounds the result to [0,1] and maps a non-finite sum (a NaN factor) + // to 0, so a single bad input can't yield NaN (which sorts nondeterministically) + // or a score >1 from float drift in weights that nominally sum to 1. + return clamp01( weights.quota * factors.quota + - weights.health * factors.health + - weights.costInv * factors.costInv + - weights.latencyInv * factors.latencyInv + - weights.taskFit * factors.taskFit + - weights.stability * factors.stability + - weights.tierPriority * factors.tierPriority + - (weights.tierAffinity ?? 0) * factors.tierAffinity + - (weights.specificityMatch ?? 0) * factors.specificityMatch + - (weights.contextAffinity ?? 0) * factors.contextAffinity + - (weights.resetWindowAffinity ?? 0) * factors.resetWindowAffinity + - (weights.connectionDensity ?? 0) * factors.connectionDensity + weights.health * factors.health + + weights.costInv * factors.costInv + + weights.latencyInv * factors.latencyInv + + weights.taskFit * factors.taskFit + + weights.stability * factors.stability + + weights.tierPriority * factors.tierPriority + + (weights.tierAffinity ?? 0) * factors.tierAffinity + + (weights.specificityMatch ?? 0) * factors.specificityMatch + + (weights.contextAffinity ?? 0) * factors.contextAffinity + + (weights.resetWindowAffinity ?? 0) * factors.resetWindowAffinity + + (weights.connectionDensity ?? 0) * factors.connectionDensity ); } @@ -131,7 +136,6 @@ function calculateTierAffinity( ): number { if (!hint) return 0.5; try { - const { classifyTier } = require("../tierResolver"); const assignment = classifyTier(candidate.provider, candidate.model); const tierOrder = ["free", "cheap", "premium"]; const providerTierIdx = tierOrder.indexOf(assignment.tier); @@ -151,7 +155,6 @@ function calculateSpecificityMatch( ): number { if (!hint) return 0.5; try { - const { classifyTier } = require("../tierResolver"); const assignment = classifyTier(candidate.provider, candidate.model); const specificityScore = hint.specificity.score; @@ -176,24 +179,28 @@ export function calculateFactors( const maxLatency = Math.max(...pool.map((p) => p.p95LatencyMs), 1); const maxStdDev = Math.max(...pool.map((p) => p.latencyStdDev), 0.001); + // Every factor is contractually [0,1]. clamp01 guards against bad telemetry + // (negative quota / cost / latency, NaN, out-of-range candidate-supplied + // affinities) so a single bad input can't produce a negative or >1 factor + // that distorts the weighted score. return { - quota: Math.min(1, candidate.quotaRemaining / 100), + quota: clamp01(candidate.quotaRemaining / 100), health: candidate.circuitBreakerState === "CLOSED" ? 1.0 : candidate.circuitBreakerState === "HALF_OPEN" ? 0.5 : 0.0, - costInv: 1 - candidate.costPer1MTokens / maxCost, - latencyInv: 1 - candidate.p95LatencyMs / maxLatency, - taskFit: getTaskFitness(candidate.model, taskType), - stability: 1 - candidate.latencyStdDev / maxStdDev, + costInv: clamp01(1 - candidate.costPer1MTokens / maxCost), + latencyInv: clamp01(1 - candidate.p95LatencyMs / maxLatency), + taskFit: clamp01(getTaskFitness(candidate.model, taskType)), + stability: clamp01(1 - candidate.latencyStdDev / maxStdDev), tierPriority: calculateTierScore(candidate.accountTier, candidate.quotaResetIntervalSecs), tierAffinity: calculateTierAffinity(candidate, manifestHint), specificityMatch: calculateSpecificityMatch(candidate, manifestHint), - contextAffinity: candidate.contextAffinity ?? 0.5, - resetWindowAffinity: candidate.resetWindowAffinity ?? 0.5, - connectionDensity: Math.min(1, Math.max(0, ((candidate.connectionPoolSize ?? 1) - 1) / 10)), + contextAffinity: clamp01(candidate.contextAffinity ?? 0.5), + resetWindowAffinity: clamp01(candidate.resetWindowAffinity ?? 0.5), + connectionDensity: clamp01(((candidate.connectionPoolSize ?? 1) - 1) / 10), }; } diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index 492abe2886..1dc770cd21 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -83,6 +83,7 @@ import { getSessionConnection } from "./sessionManager.ts"; import { orderTargetsByEvalScores } from "./evalRouting.ts"; import { generateRoutingHints } from "./manifestAdapter"; import type { RoutingHint } from "./manifestAdapter"; +import { buildComplexityRoutingHint } from "./autoCombo/complexityRouter"; import type { CompressionMode } from "./compression/types.ts"; import { getModelContextLimit } from "../../src/lib/modelCapabilities"; import { getProviderConnections } from "../../src/lib/db/providers"; @@ -143,9 +144,46 @@ function isProviderCircuitOpenResult( } const MAX_COMBO_DEPTH = 3; +// Absolute safety ceiling for operator-configured nesting depth. config.maxComboDepth +// can raise the default (3) up to this cap, or lower it, but never above — runaway +// nested-combo expansion is a real DoS/perf risk. +const MAX_COMBO_DEPTH_HARD_CAP = 10; const MAX_FALLBACK_WAIT_MS = 5000; const MAX_GLOBAL_ATTEMPTS = 30; +/** + * Clamp an operator-configured combo nesting depth (config.maxComboDepth) to a + * safe integer in [1, MAX_COMBO_DEPTH_HARD_CAP]. Anything non-numeric, < 1, or + * NaN falls back to the default MAX_COMBO_DEPTH so a bad config never disables + * nesting or blows past the safety ceiling. + */ +export function clampComboDepth(value: unknown): number { + const n = Math.floor(Number(value)); + if (!Number.isFinite(n) || n < 1) return MAX_COMBO_DEPTH; + return Math.min(n, MAX_COMBO_DEPTH_HARD_CAP); +} + +/** Minimum recorded requests before the predictive-TTFT breaker trusts the average. */ +const PREDICTIVE_TTFT_MIN_SAMPLES = 5; + +/** + * Predictive-TTFT circuit-breaker decision: skip a target whose recent average + * latency — measured over a statistically meaningful sample — exceeds the + * configured ceiling, so the combo fails over before paying a slow first byte. + * Returns false when disabled (ceiling <= 0), when there is no metric, or when + * the sample is too small to trust. + */ +export function shouldSkipForPredictedTtft( + metric: { requests?: number; avgLatencyMs?: number } | null | undefined, + predictiveTtftMs: number +): boolean { + if (!metric || !(predictiveTtftMs > 0)) return false; + return ( + (metric.requests ?? 0) >= PREDICTIVE_TTFT_MIN_SAMPLES && + (metric.avgLatencyMs ?? 0) > predictiveTtftMs + ); +} + function resolveDelayMs(value: unknown, fallback: number): number { const numericValue = Number(value); if (!Number.isFinite(numericValue) || numericValue < 0) return fallback; @@ -1128,19 +1166,24 @@ function expandRuntimeStep( allCombos: ComboCollectionLike, visited = new Set(), depth = 0, - path: string[] = [] + path: string[] = [], + maxDepth: number = MAX_COMBO_DEPTH ): ResolvedComboTarget[] { if (step.kind === "model") return [step]; - if (depth > MAX_COMBO_DEPTH) return []; + if (depth > maxDepth) return []; const combos = getCombosArray(allCombos); const nestedCombo = combos.find((combo) => combo.name === step.comboName); if (!nestedCombo || visited.has(step.comboName)) return []; - return resolveNestedComboTargets(nestedCombo, combos, new Set(visited), depth + 1, [ - ...path, - step.stepId, - ]); + return resolveNestedComboTargets( + nestedCombo, + combos, + new Set(visited), + depth + 1, + [...path, step.stepId], + maxDepth + ); } export function resolveNestedComboTargets( @@ -1148,13 +1191,14 @@ export function resolveNestedComboTargets( allCombos: ComboCollectionLike, visited = new Set(), depth = 0, - path: string[] = [] + path: string[] = [], + maxDepth: number = MAX_COMBO_DEPTH ): ResolvedComboTarget[] { const directTargets = (combo.models || []) .map((entry, index) => normalizeRuntimeStep(entry, combo.name, index, null, path)) .filter((entry): entry is ResolvedComboTarget => entry?.kind === "model"); - if (depth > MAX_COMBO_DEPTH) return directTargets; + if (depth > maxDepth) return directTargets; if (visited.has(combo.name)) return []; visited.add(combo.name); @@ -1163,7 +1207,9 @@ export function resolveNestedComboTargets( for (const step of runtimeSteps) { if (step.kind === "combo-ref") { - resolved.push(...expandRuntimeStep(step, allCombos, new Set(visited), depth, path)); + resolved.push( + ...expandRuntimeStep(step, allCombos, new Set(visited), depth, path, maxDepth) + ); continue; } resolved.push(step); @@ -1214,10 +1260,11 @@ export function validateComboDAG( comboName: string, allCombos: ComboCollectionLike, visited = new Set(), - depth = 0 + depth = 0, + maxDepth: number = MAX_COMBO_DEPTH ): void { - if (depth > MAX_COMBO_DEPTH) { - throw new Error(`Max combo nesting depth (${MAX_COMBO_DEPTH}) exceeded at "${comboName}"`); + if (depth > maxDepth) { + throw new Error(`Max combo nesting depth (${maxDepth}) exceeded at "${comboName}"`); } if (visited.has(comboName)) { throw new Error(`Circular combo reference detected: ${comboName}`); @@ -1233,7 +1280,7 @@ export function validateComboDAG( // Check if this model name is itself a combo (not a provider/model pattern) const nestedCombo = combos.find((c) => c.name === modelName); if (nestedCombo) { - validateComboDAG(modelName, combos, new Set(visited), depth + 1); + validateComboDAG(modelName, combos, new Set(visited), depth + 1, maxDepth); } } } @@ -1251,9 +1298,10 @@ export function resolveNestedComboModels( combo: ComboLike, allCombos: ComboCollectionLike, visited = new Set(), - depth = 0 + depth = 0, + maxDepth: number = MAX_COMBO_DEPTH ): string[] { - if (depth > MAX_COMBO_DEPTH) return combo.models.map((m) => normalizeModelEntry(m).model); + if (depth > maxDepth) return combo.models.map((m) => normalizeModelEntry(m).model); if (visited.has(combo.name)) return []; // cycle safety visited.add(combo.name); @@ -1266,7 +1314,13 @@ export function resolveNestedComboModels( if (nestedCombo) { // Recursively expand the nested combo - const nested = resolveNestedComboModels(nestedCombo, combos, new Set(visited), depth + 1); + const nested = resolveNestedComboModels( + nestedCombo, + combos, + new Set(visited), + depth + 1, + maxDepth + ); resolved.push(...nested); } else { resolved.push(modelName); @@ -2809,9 +2863,12 @@ async function applyRequestTagRouting( export function resolveComboTargets( combo: ComboLike, - allCombos: ComboCollectionLike + allCombos: ComboCollectionLike, + maxDepth: number = MAX_COMBO_DEPTH ): ResolvedComboTarget[] { - return allCombos ? resolveNestedComboTargets(combo, allCombos) : getDirectComboTargets(combo); + return allCombos + ? resolveNestedComboTargets(combo, allCombos, new Set(), 0, [], maxDepth) + : getDirectComboTargets(combo); } function resolveWeightedTargets( @@ -2850,11 +2907,12 @@ function resolveWeightedTargets( }; } -function scoreAutoTargets( +export function scoreAutoTargets( targets: ResolvedComboTarget[], candidates: AutoProviderCandidate[], taskType: string | null, - weights: ScoringWeights + weights: ScoringWeights, + manifestHint?: RoutingHint | null ) { const candidateByExecutionKey = new Map( candidates.map((candidate: ProviderCandidate & { executionKey: string }) => [ @@ -2870,7 +2928,8 @@ function scoreAutoTargets( candidate as ProviderCandidate, candidates, taskType ?? "general", - getTaskFitness + getTaskFitness, + manifestHint ?? undefined ); let score = calculateScore(factors, weights); // B17: Quota Share soft-policy deprioritization @@ -3129,7 +3188,7 @@ export async function handleComboChat({ let orderedTargets = strategy === "weighted" ? resolveWeightedTargets(combo, allCombos)?.orderedTargets || [] - : resolveComboTargets(combo, allCombos); + : resolveComboTargets(combo, allCombos, clampComboDepth(config.maxComboDepth)); orderedTargets = await applyRequestTagRouting(orderedTargets, body, log); @@ -3345,7 +3404,25 @@ export async function handleComboChat({ selectionReason = `score=${selection.score.toFixed(3)}${selection.isExploration ? " (exploration)" : ""}`; } - const scoredTargets = scoreAutoTargets(eligibleTargets, candidates, taskType, weights); + // Complexity-aware routing (2026, opt-in): classify the request's + // difficulty and feed a tier hint into scoring so tierAffinity / + // specificityMatch favor candidates whose tier matches the request. + const autoManifestHint: RoutingHint | null = + config.complexityAwareRouting === true + ? buildComplexityRoutingHint( + eligibleTargets.filter((t) => t.kind === "model"), + body, + log + ) + : null; + + const scoredTargets = scoreAutoTargets( + eligibleTargets, + candidates, + taskType, + weights, + autoManifestHint + ); const rankedTargets = scoredTargets.map((entry) => entry.target); const selectedTarget = scoredTargets.find((entry) => { @@ -3724,7 +3801,7 @@ export async function handleComboChat({ if (cMetrics) { const targetKey = orderedTargets[i].executionKey || modelStr; const m = cMetrics.byTarget[targetKey] || cMetrics.byModel[modelStr]; - if (m && m.requests >= 5 && m.avgLatencyMs > config.predictiveTtftMs) { + if (shouldSkipForPredictedTtft(m, config.predictiveTtftMs)) { log.warn( "COMBO", `Predictive TTFT Circuit Breaker: skipping ${modelStr} (avg ${m.avgLatencyMs}ms > max ${config.predictiveTtftMs}ms)` @@ -4531,7 +4608,7 @@ async function handleRoundRobinCombo({ ? resolveResilienceSettings(settings) : resolveResilienceSettings(null); - const orderedTargets = resolveComboTargets(combo, allCombos); + const orderedTargets = resolveComboTargets(combo, allCombos, clampComboDepth(config.maxComboDepth)); const tagFilteredTargets = await applyRequestTagRouting(orderedTargets, body, log); const evalRankedTargets = orderTargetsByEvalScores(tagFilteredTargets, config.evalRouting, log); const filteredTargets = filterTargetsByRequestCompatibility( diff --git a/open-sse/services/comboConfig.ts b/open-sse/services/comboConfig.ts index 5afe81f6aa..990dc11853 100644 --- a/open-sse/services/comboConfig.ts +++ b/open-sse/services/comboConfig.ts @@ -28,6 +28,10 @@ const DEFAULT_COMBO_CONFIG = { trackMetrics: true, reasoningTokenBufferEnabled: true, manifestRouting: false, + // Complexity-aware auto routing (2026): when on, the auto router scores + // candidates by how well their tier matches the request's classified + // difficulty (feeds tierAffinity/specificityMatch). Opt-in — off by default. + complexityAwareRouting: false, resetAwareSessionWeight: 0.35, resetAwareWeeklyWeight: 0.65, resetAwareTieBandPercent: 5, diff --git a/open-sse/services/comboMetrics.ts b/open-sse/services/comboMetrics.ts index 6c6e8e71da..698d567975 100644 --- a/open-sse/services/comboMetrics.ts +++ b/open-sse/services/comboMetrics.ts @@ -4,6 +4,8 @@ * Provides API for reading metrics from the dashboard. */ +import { recordProviderUsage } from "./autoCombo/providerDiversity"; + interface ModelMetrics { requests: number; successes: number; @@ -270,6 +272,12 @@ export function recordComboRequest( if (success) { combo.totalSuccesses++; + // Feed the provider-diversity report (/api/analytics/diversity): record the + // provider that actually served this request. recordComboRequest is the + // single chokepoint every combo strategy funnels through, so one call here + // covers priority / round-robin / weighted / auto / etc. + const usedProvider = toNonEmptyString(target?.provider); + if (usedProvider) recordProviderUsage(usedProvider); } else { combo.totalFailures++; } diff --git a/src/app/api/combos/[id]/route.ts b/src/app/api/combos/[id]/route.ts index 49647e53eb..4c25beda01 100644 --- a/src/app/api/combos/[id]/route.ts +++ b/src/app/api/combos/[id]/route.ts @@ -11,7 +11,7 @@ import { getConsistentMachineId } from "@/shared/utils/machineId"; import { syncToCloud } from "@/lib/cloudSync"; import { validateCompositeTiersConfig } from "@/lib/combos/compositeTiers"; import { normalizeComboModels } from "@/lib/combos/steps"; -import { validateComboDAG } from "@omniroute/open-sse/services/combo.ts"; +import { validateComboDAG, clampComboDepth } from "@omniroute/open-sse/services/combo.ts"; import { updateComboSchema } from "@/shared/validation/schemas"; import { isValidationFailure, validateBody } from "@/shared/validation/helpers"; import { requireManagementAuth } from "@/lib/api/requireManagementAuth"; @@ -130,8 +130,11 @@ export async function PUT(request, { params }) { // Update the combo in the list temporarily for validation const updatedCombos = allCombos.map((c) => (c.id === id ? { ...c, ...body } : c)); if (comboName) { + const configuredDepth = clampComboDepth( + (nextComboState as { config?: { maxComboDepth?: unknown } }).config?.maxComboDepth + ); try { - validateComboDAG(comboName, updatedCombos); + validateComboDAG(comboName, updatedCombos, new Set(), 0, configuredDepth); } catch (dagError) { return NextResponse.json({ error: dagError.message }, { status: 400 }); } diff --git a/src/app/api/combos/route.ts b/src/app/api/combos/route.ts index 00731ba606..2817f3fd06 100644 --- a/src/app/api/combos/route.ts +++ b/src/app/api/combos/route.ts @@ -4,7 +4,7 @@ import { getConsistentMachineId } from "@/shared/utils/machineId"; import { syncToCloud } from "@/lib/cloudSync"; import { validateCompositeTiersConfig } from "@/lib/combos/compositeTiers"; import { normalizeComboModels } from "@/lib/combos/steps"; -import { validateComboDAG } from "@omniroute/open-sse/services/combo.ts"; +import { validateComboDAG, clampComboDepth } from "@omniroute/open-sse/services/combo.ts"; import { createComboSchema } from "@/shared/validation/schemas"; import { isValidationFailure, validateBody } from "@/shared/validation/helpers"; import { requireManagementAuth } from "@/lib/api/requireManagementAuth"; @@ -66,7 +66,13 @@ export async function POST(request) { config, }; try { - validateComboDAG(name, [...allCombos, tempCombo]); + validateComboDAG( + name, + [...allCombos, tempCombo], + new Set(), + 0, + clampComboDepth((config as { maxComboDepth?: unknown } | undefined)?.maxComboDepth) + ); } catch (dagError) { return NextResponse.json({ error: dagError.message }, { status: 400 }); } diff --git a/src/app/api/quota/plans/[connectionId]/route.ts b/src/app/api/quota/plans/[connectionId]/route.ts index 54bbd55577..b6fa352f04 100644 --- a/src/app/api/quota/plans/[connectionId]/route.ts +++ b/src/app/api/quota/plans/[connectionId]/route.ts @@ -25,32 +25,13 @@ import { deleteProviderPlan, } from "@/lib/localDb"; import { resolvePlan } from "@/lib/quota/planResolver"; +import { resolveConnectionProvider } from "@/lib/quota/connectionProvider"; import { logAuditEvent, getAuditRequestContext } from "@/lib/compliance/index"; export const dynamic = "force-dynamic"; type RouteParams = { params: Promise<{ connectionId: string }> }; -/** - * Attempt to look up the provider name for a connection. - * Falls back to "unknown" if the DB lookup fails or returns nothing. - */ -async function resolveProvider(connectionId: string): Promise { - try { - // Lazy import — avoids circular deps and keeps module loadable without full DB - const { getProviderConnectionById } = await import("@/lib/localDb"); - if (typeof getProviderConnectionById === "function") { - const conn = getProviderConnectionById(connectionId); - if (conn && typeof (conn as { provider?: string }).provider === "string") { - return (conn as { provider: string }).provider; - } - } - } catch { - // DB not available or export not present — fall through - } - return "unknown"; -} - export async function GET(request: Request, { params }: RouteParams): Promise { const authError = await requireManagementAuth(request); if (authError) return authError; @@ -65,7 +46,7 @@ export async function GET(request: Request, { params }: RouteParams): Promise { + try { + // Lazy import — avoids circular deps and keeps the module loadable without a full DB. + const { getProviderConnectionById } = await import("@/lib/localDb"); + if (typeof getProviderConnectionById === "function") { + const conn = await getProviderConnectionById(connectionId); + if (conn && typeof (conn as { provider?: string }).provider === "string") { + return (conn as { provider: string }).provider; + } + } + } catch { + // DB not available or export not present — fall through to the safe default. + } + return "unknown"; +} diff --git a/src/lib/quota/spendRecorder.ts b/src/lib/quota/spendRecorder.ts index 3854288f46..b64c0d0b65 100644 --- a/src/lib/quota/spendRecorder.ts +++ b/src/lib/quota/spendRecorder.ts @@ -40,3 +40,90 @@ export function scheduleRecordConsumption( }); }); } + +/** + * Build the per-request consumption cost payload shared by the streaming and + * non-streaming POST-hooks. Coerces token fields defensively (string/NaN safe) + * and clamps a negative/zero cost to 0 so a bad pricing lookup never records + * negative USD. + */ +export function buildConsumptionCost( + usage: unknown, + estimatedCost: number +): { tokens: number; usd: number; requests: number } { + const u = usage && typeof usage === "object" ? (usage as Record) : null; + const tokens = u + ? (Number(u.prompt_tokens ?? 0) || 0) + (Number(u.completion_tokens ?? 0) || 0) + : 0; + return { + tokens, + usd: estimatedCost > 0 ? estimatedCost : 0, + requests: 1, + }; +} + +/** Cost resolver injected for testability (matches `calculateCost`). */ +type CostResolver = ( + provider: string, + model: string, + usage: Record | null | undefined, + options: { serviceTier?: string } +) => Promise; + +/** + * Record shared-quota consumption for a completed STREAMING response. + * + * Unlike the non-streaming path, the streaming completion previously recorded + * `usd: 0` (the cost was resolved asynchronously only for `recordCost`), so + * USD-unit pools (e.g. DeepSeek `usd/monthly`) never accrued on streaming + * traffic. This resolves the real cost via the injected `calculateCost` and + * schedules a single consumption record. Fire-and-forget / fail-open: never + * throws to the caller, and still records `requests: 1` when usage is absent. + */ +export async function recordStreamingConsumption( + params: { + apiKeyId?: string | null; + connectionId?: string | null; + provider?: string | null; + model: string; + streamUsage: unknown; + streamStatus: number; + serviceTier?: string; + }, + deps: { + calculateCost: CostResolver; + schedule?: (input: RecordConsumptionInput, log?: MinimalLogger | null) => void; + log?: MinimalLogger | null; + } +): Promise { + const { apiKeyId, connectionId, provider, model, streamUsage, streamStatus, serviceTier } = + params; + if (!apiKeyId || !connectionId || streamStatus !== 200) return; + + const schedule = deps.schedule ?? scheduleRecordConsumption; + const resolvedProvider = provider ?? "unknown"; + + let estimatedCost = 0; + if (streamUsage && typeof streamUsage === "object") { + try { + estimatedCost = await deps.calculateCost( + resolvedProvider, + model, + streamUsage as Record, + { serviceTier } + ); + } catch { + estimatedCost = 0; + } + } + + schedule( + { + apiKeyId, + connectionId, + provider: resolvedProvider, + cost: buildConsumptionCost(streamUsage, estimatedCost), + }, + deps.log + ); +} diff --git a/tests/integration/combo-failover-e2e.test.ts b/tests/integration/combo-failover-e2e.test.ts new file mode 100644 index 0000000000..cb58a57096 --- /dev/null +++ b/tests/integration/combo-failover-e2e.test.ts @@ -0,0 +1,164 @@ +/** + * tests/integration/combo-failover-e2e.test.ts + * + * End-to-end combo routing scenarios that the existing suite left uncovered: + * 1. A 3-target priority chain that walks past TWO failing targets + * (500 then 503) to succeed on the third — the existing suite only + * exercised a 2-target (single-hop) failover. + * 2. A `strategy:"auto"` combo dispatched end-to-end (request → scored + * selection → real upstream fetch → 200), closing the gap where auto was + * only exercised at the UI layer. + * 3. A per-target timeout (targetTimeoutMs) on the first target failing over + * to a healthy second target — timeout-driven failover had zero coverage. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import { createChatPipelineHarness } from "./_chatPipelineHarness.ts"; + +const harness = await createChatPipelineHarness("combo-failover-e2e"); +const { + BaseExecutor, + buildClaudeResponse, + buildGeminiResponse, + buildOpenAIResponse, + buildRequest, + combosDb, + handleChat, + resetStorage, + seedConnection, +} = harness; + +test.beforeEach(async () => { + BaseExecutor.RETRY_CONFIG.delayMs = 0; + await resetStorage(); +}); + +test.afterEach(async () => { + BaseExecutor.RETRY_CONFIG.delayMs = harness.originalRetryDelayMs; + await resetStorage(); +}); + +test.after(async () => { + await harness.cleanup(); +}); + +function body(model: string, content = `Route ${model}`) { + return { model, stream: false, messages: [{ role: "user", content }] }; +} + +test("priority combo walks a 3-target chain: 500 → 503 → success", async () => { + await seedConnection("openai", { apiKey: "sk-openai-3way" }); + await seedConnection("claude", { apiKey: "sk-claude-3way" }); + await seedConnection("gemini", { apiKey: "sk-gemini-3way" }); + await combosDb.createCombo({ + name: "router-3way", + strategy: "priority", + config: { maxRetries: 0, retryDelayMs: 0, fallbackDelayMs: 0 }, + models: [ + "openai/gpt-4o-mini", + "claude/claude-3-5-sonnet-20241022", + "gemini/gemini-2.5-flash", + ], + }); + + const attempts: string[] = []; + globalThis.fetch = async (url) => { + const target = String(url); + if (target.includes("/chat/completions")) { + attempts.push("openai"); + return new Response(JSON.stringify({ error: { message: "primary down" } }), { + status: 500, + headers: { "Content-Type": "application/json" }, + }); + } + if (target.includes("?beta=true")) { + attempts.push("claude"); + return new Response(JSON.stringify({ error: { message: "secondary overloaded" } }), { + status: 503, + headers: { "Content-Type": "application/json" }, + }); + } + attempts.push("gemini"); + return buildGeminiResponse("Third target answered"); + }; + + const res = await handleChat(buildRequest({ body: body("router-3way") })); + const json = (await res.json()) as { + choices: Array<{ message: { content: string } }>; + }; + + assert.equal(res.status, 200, "request must succeed on the 3rd target"); + assert.deepEqual(attempts, ["openai", "claude", "gemini"], "all three targets attempted in order"); + assert.equal(json.choices[0].message.content, "Third target answered"); +}); + +test("priority combo fails over when the first target exceeds its per-target timeout", async () => { + await seedConnection("openai", { apiKey: "sk-openai-timeout" }); + await seedConnection("claude", { apiKey: "sk-claude-timeout" }); + await combosDb.createCombo({ + name: "router-timeout", + strategy: "priority", + // 80ms per-target ceiling; the first target hangs past it and is aborted. + config: { maxRetries: 0, retryDelayMs: 0, fallbackDelayMs: 0, targetTimeoutMs: 80 }, + models: ["openai/gpt-4o-mini", "claude/claude-3-5-sonnet-20241022"], + }); + + const attempts: string[] = []; + globalThis.fetch = async (url, init: RequestInit = {}) => { + const target = String(url); + if (target.includes("/chat/completions")) { + attempts.push("openai"); + // Hang until the combo's per-target timeout aborts us via the signal. + return await new Promise((_resolve, reject) => { + const signal = init.signal; + if (signal) { + signal.addEventListener("abort", () => + reject(Object.assign(new Error("aborted by combo timeout"), { name: "AbortError" })) + ); + } + }); + } + attempts.push("claude"); + return buildClaudeResponse("Recovered after timeout"); + }; + + const res = await handleChat(buildRequest({ body: body("router-timeout") })); + const json = (await res.json()) as { + choices: Array<{ message: { content: string } }>; + }; + + assert.equal(res.status, 200, "must fail over to the second target after the first times out"); + assert.deepEqual(attempts, ["openai", "claude"]); + assert.equal(json.choices[0].message.content, "Recovered after timeout"); +}); + +test("auto combo selects and dispatches a scored candidate end-to-end", async () => { + await seedConnection("openai", { apiKey: "sk-openai-auto" }); + await seedConnection("claude", { apiKey: "sk-claude-auto" }); + await combosDb.createCombo({ + name: "router-auto", + strategy: "auto", + config: { maxRetries: 0, retryDelayMs: 0 }, + models: ["openai/gpt-4o-mini", "claude/claude-3-5-sonnet-20241022"], + }); + + const seen: string[] = []; + globalThis.fetch = async (url) => { + const target = String(url); + if (target.includes("?beta=true")) { + seen.push("claude"); + return buildClaudeResponse("Auto chose claude"); + } + seen.push("openai"); + return buildOpenAIResponse("Auto chose openai"); + }; + + const res = await handleChat(buildRequest({ body: body("router-auto") })); + const json = (await res.json()) as { + choices: Array<{ message: { content: string } }>; + }; + + assert.equal(res.status, 200, "auto combo must dispatch successfully"); + assert.equal(seen.length, 1, "auto selects exactly one target (no needless fan-out)"); + assert.match(json.choices[0].message.content, /Auto chose (openai|claude)/); +}); diff --git a/tests/integration/quota-pool-usage-provider-resolution.test.ts b/tests/integration/quota-pool-usage-provider-resolution.test.ts new file mode 100644 index 0000000000..c45784cce5 --- /dev/null +++ b/tests/integration/quota-pool-usage-provider-resolution.test.ts @@ -0,0 +1,106 @@ +/** + * Integration: GET /api/quota/pools/[id]/usage must resolve the pool's provider + * from its connection so catalog-only pools (no manual `provider_plans` row) + * still surface their plan dimensions. + * + * Regression: the route passed `resolvePlan(pool.connectionId, "")` with an + * empty provider, so `getKnownPlan("")` returned null → empty plan → the route + * fell back to the dimension-less `poolUsage()` snapshot, blanking the + * dashboard (StackedAllocationBar / DimensionBar / BurnRateChart) for every + * catalog-only pool. + * + * Fix: resolve the provider via `resolveConnectionProvider(connectionId)` + * (awaited DB lookup) and pass it to `resolvePlan`. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { makeManagementSessionRequest } from "../helpers/managementSession.ts"; + +const TEST_DATA_DIR = fs.mkdtempSync( + path.join(os.tmpdir(), "omniroute-quota-pool-usage-provider-") +); +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.API_KEY_SECRET = "test-quota-usage-provider-secret"; +process.env.QUOTA_STORE_DRIVER = "sqlite"; + +const core = await import("../../src/lib/db/core.ts"); +const localDb = await import("../../src/lib/localDb.ts"); +const { createPool, upsertAllocations, createProviderConnection } = localDb; +const { resetQuotaStoreSingleton } = await import("../../src/lib/quota/QuotaStore.ts"); +const usageRoute = await import("../../src/app/api/quota/pools/[id]/usage/route.ts"); + +function resetDb() { + core.resetDbInstance(); + resetQuotaStoreSingleton(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +} + +test.beforeEach(() => { + resetDb(); +}); + +test.after(() => { + core.resetDbInstance(); + resetQuotaStoreSingleton(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +test("GET /usage surfaces catalog dimensions for a catalog-only pool (provider resolved from connection)", async () => { + // kimi has a catalog plan: { unit: 'requests', window: 'hourly', limit: 1500 } + const conn = (await createProviderConnection({ + provider: "kimi", + name: "kimi-usage-test", + authType: "apikey", + apiKey: "sk-kimi-test", + })) as { id: string }; + + const pool = createPool({ name: "Kimi Shared Pool", connectionId: conn.id }); + upsertAllocations(pool.id, [{ apiKeyId: "key-a", weight: 100, policy: "hard" }]); + + const req = await makeManagementSessionRequest( + `http://localhost/api/quota/pools/${pool.id}/usage` + ); + const res = await usageRoute.GET(req, { params: Promise.resolve({ id: pool.id }) }); + assert.equal(res.status, 200); + + const body = (await res.json()) as { + usage: { + dimensions: Array<{ unit: string; window: string; limit: number }>; + }; + }; + + // Regression assertion: was [] because resolvePlan(connId, "") found no catalog match. + assert.ok( + body.usage.dimensions.length >= 1, + "catalog-only pool must surface plan dimensions (was blank)" + ); + const dim = body.usage.dimensions[0]; + assert.equal(dim.unit, "requests"); + assert.equal(dim.window, "hourly"); + assert.equal(dim.limit, 1500); +}); + +test("GET /usage still returns a 200 snapshot when the provider has no catalog plan", async () => { + // A provider with no catalog entry resolves to an empty (manual) plan; the + // route must still return a 200 minimal snapshot (no dimensions), not error. + const conn = (await createProviderConnection({ + provider: "some-unknown-provider", + name: "unknown-usage-test", + authType: "apikey", + apiKey: "sk-unknown-test", + })) as { id: string }; + + const pool = createPool({ name: "Unknown Pool", connectionId: conn.id }); + + const req = await makeManagementSessionRequest( + `http://localhost/api/quota/pools/${pool.id}/usage` + ); + const res = await usageRoute.GET(req, { params: Promise.resolve({ id: pool.id }) }); + assert.equal(res.status, 200); + const body = (await res.json()) as { usage: { dimensions: unknown[] } }; + assert.ok(Array.isArray(body.usage.dimensions)); +}); diff --git a/tests/unit/auto-combo-scoring-clamp.test.ts b/tests/unit/auto-combo-scoring-clamp.test.ts new file mode 100644 index 0000000000..025c9c1aef --- /dev/null +++ b/tests/unit/auto-combo-scoring-clamp.test.ts @@ -0,0 +1,108 @@ +/** + * tests/unit/auto-combo-scoring-clamp.test.ts + * + * Regression hardening: `calculateScore` summed weighted factors with NO final + * clamp and NO NaN guard, and `calculateFactors` lower-bounded none of its + * factors. A single NaN/negative input (bad telemetry, negative quota, negative + * cost) could yield a NaN or out-of-[0,1] score that sinks a candidate + * nondeterministically (NaN sorts unpredictably) or distorts ranking. + * + * Fix: clamp every factor to [0,1] in calculateFactors and clamp the final + * score to [0,1] (clamp01 maps non-finite → 0). + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import { + calculateScore, + calculateFactors, + DEFAULT_WEIGHTS, +} from "../../open-sse/services/autoCombo/scoring.ts"; +import type { + ScoringFactors, + ProviderCandidate, +} from "../../open-sse/services/autoCombo/scoring.ts"; + +const ONES: ScoringFactors = { + quota: 1, + health: 1, + costInv: 1, + latencyInv: 1, + taskFit: 1, + stability: 1, + tierPriority: 1, + tierAffinity: 1, + specificityMatch: 1, + contextAffinity: 1, + resetWindowAffinity: 1, + connectionDensity: 1, +}; + +function candidate(partial: Partial = {}): ProviderCandidate { + return { + provider: "p", + model: "m", + quotaRemaining: 100, + quotaTotal: 100, + circuitBreakerState: "CLOSED", + costPer1MTokens: 1, + p95LatencyMs: 100, + latencyStdDev: 10, + errorRate: 0, + accountTier: "standard", + quotaResetIntervalSecs: 86400, + ...partial, + }; +} + +test("calculateScore — NaN factor yields a finite [0,1] score (no NaN propagation)", () => { + const score = calculateScore({ ...ONES, quota: NaN }, DEFAULT_WEIGHTS); + assert.ok(Number.isFinite(score), "score must be finite even with a NaN factor"); + assert.ok(score >= 0 && score <= 1, `score in [0,1], got ${score}`); +}); + +test("calculateScore — clamps to [0,1]; all-ones with normalized weights ≈ 1", () => { + const score = calculateScore(ONES, DEFAULT_WEIGHTS); + assert.ok(score >= 0 && score <= 1); + assert.ok(Math.abs(score - 1) < 1e-6, "normalized weights × all-ones ≈ 1"); +}); + +test("calculateScore — negative factors cannot drive the score below 0", () => { + const score = calculateScore({ ...ONES, costInv: -5, latencyInv: -5 }, DEFAULT_WEIGHTS); + assert.ok(score >= 0, `score floored at 0, got ${score}`); +}); + +test("calculateFactors — negative quotaRemaining clamps the quota factor to [0,1]", () => { + const c = candidate({ quotaRemaining: -50 }); + const f = calculateFactors(c, [c], "default", () => 0.5); + assert.ok(f.quota >= 0 && f.quota <= 1, `quota factor must be in [0,1], got ${f.quota}`); +}); + +test("calculateFactors — negative cost cannot push costInv above 1", () => { + const c = candidate({ costPer1MTokens: -100 }); + const f = calculateFactors(c, [c], "default", () => 0.5); + assert.ok(f.costInv >= 0 && f.costInv <= 1, `costInv must be in [0,1], got ${f.costInv}`); +}); + +test("calculateFactors — out-of-range contextAffinity is clamped", () => { + const c = candidate({ contextAffinity: 5 }); + const f = calculateFactors(c, [c], "default", () => 0.5); + assert.ok( + f.contextAffinity >= 0 && f.contextAffinity <= 1, + `contextAffinity must be in [0,1], got ${f.contextAffinity}` + ); +}); + +test("calculateFactors — connectionDensity is clamped to [0,1] and NaN-safe", () => { + // A large pool ((1000-1)/10 = 99.9) must not exceed 1 and skew the weighted score. + const big = calculateFactors(candidate({ connectionPoolSize: 1000 }), [candidate()], "default", () => 0.5); + assert.ok( + big.connectionDensity >= 0 && big.connectionDensity <= 1, + `connectionDensity must be in [0,1], got ${big.connectionDensity}` + ); + // A non-finite pool size must map to 0 (clamp01), not propagate NaN into the score. + const nan = calculateFactors(candidate({ connectionPoolSize: NaN }), [candidate()], "default", () => 0.5); + assert.ok( + Number.isFinite(nan.connectionDensity), + `connectionDensity must be finite (clamp01 maps NaN→0), got ${nan.connectionDensity}` + ); +}); diff --git a/tests/unit/combo-hedging.test.ts b/tests/unit/combo-hedging.test.ts index 15874e7c7f..4337552a2c 100644 --- a/tests/unit/combo-hedging.test.ts +++ b/tests/unit/combo-hedging.test.ts @@ -1,13 +1,66 @@ -import { test, mock } from "node:test"; -import assert from "node:assert"; -import { handleComboChat } from "@omniroute/open-sse/services/combo.ts"; -import * as metricsDb from "@omniroute/src/lib/db/stats.ts"; +/** + * tests/unit/combo-hedging.test.ts + * + * Previously a placeholder (two `assert.ok(true)`). Now exercises the two + * zero-latency features it claims to cover: + * 1. Predictive-TTFT circuit-breaker DECISION (shouldSkipForPredictedTtft). + * 2. Hedging / zero-latency CONFIG resolution (defaults + per-combo override), + * i.e. the knobs the combo engine reads before racing/skip-ahead. + * + * The full hedging DISPATCH (Promise.race + loser abort) is covered end-to-end + * in combo-routing-engine.test.ts; here we pin the decision logic + config + * plumbing that gate it (both opt-in / off by default). + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import { shouldSkipForPredictedTtft } from "../../open-sse/services/combo.ts"; +import { getDefaultComboConfig, resolveComboConfig } from "../../open-sse/services/comboConfig.ts"; -test("combo: predictive TTFT skips slow model without aborting combo", async () => { - // Add basic test here - assert.ok(true); +// ── Predictive-TTFT circuit breaker (decision) ──────────────────────────────── +test("predictive-TTFT — skips a model whose avg latency exceeds the ceiling (enough samples)", () => { + assert.equal(shouldSkipForPredictedTtft({ requests: 10, avgLatencyMs: 5000 }, 2000), true); }); -test("combo: hedging logic works correctly", async () => { - assert.ok(true); +test("predictive-TTFT — does NOT skip with too few samples (< 5)", () => { + assert.equal(shouldSkipForPredictedTtft({ requests: 4, avgLatencyMs: 9000 }, 2000), false); +}); + +test("predictive-TTFT — does NOT skip when avg latency is within the ceiling", () => { + assert.equal(shouldSkipForPredictedTtft({ requests: 50, avgLatencyMs: 1200 }, 2000), false); +}); + +test("predictive-TTFT — disabled (ceiling <= 0) never skips", () => { + assert.equal(shouldSkipForPredictedTtft({ requests: 50, avgLatencyMs: 9000 }, 0), false); +}); + +test("predictive-TTFT — null/missing metric never skips", () => { + assert.equal(shouldSkipForPredictedTtft(null, 2000), false); + assert.equal(shouldSkipForPredictedTtft(undefined, 2000), false); +}); + +// ── Hedging / zero-latency config resolution ────────────────────────────────── +test("hedging — defaults are opt-in (off) so normal combos never race", () => { + const d = getDefaultComboConfig(); + assert.equal(d.hedging, false); + assert.equal(d.hedgeDelayMs, 500); + assert.equal(d.predictiveTtftMs, 0); + assert.equal(d.zeroLatencyOptimizationsEnabled, false); +}); + +test("hedging — per-combo config overrides the zero-latency defaults", () => { + const resolved = resolveComboConfig( + { + config: { + hedging: true, + hedgeDelayMs: 200, + zeroLatencyOptimizationsEnabled: true, + predictiveTtftMs: 3000, + }, + }, + null + ); + assert.equal(resolved.hedging, true); + assert.equal(resolved.hedgeDelayMs, 200); + assert.equal(resolved.zeroLatencyOptimizationsEnabled, true); + assert.equal(resolved.predictiveTtftMs, 3000); }); diff --git a/tests/unit/combo-max-depth-config.test.ts b/tests/unit/combo-max-depth-config.test.ts new file mode 100644 index 0000000000..34e47f566e --- /dev/null +++ b/tests/unit/combo-max-depth-config.test.ts @@ -0,0 +1,58 @@ +/** + * tests/unit/combo-max-depth-config.test.ts + * + * Regression: `config.maxComboDepth` (DEFAULT_COMBO_CONFIG) had ZERO readers — + * nesting depth was always the hardcoded `MAX_COMBO_DEPTH = 3`, so the operator + * knob did nothing. This wires a clamped, per-resolution `maxDepth` through the + * combo depth functions (default preserved, hard-capped for safety). + */ +import test from "node:test"; +import assert from "node:assert/strict"; + +// 5-combo chain L0→L1→L2→L3→L4→model — L4 sits at recursion depth 4. +function buildDeepChain() { + return [ + { name: "L0", models: ["L1"] }, + { name: "L1", models: ["L2"] }, + { name: "L2", models: ["L3"] }, + { name: "L3", models: ["L4"] }, + { name: "L4", models: ["openai/gpt-4"] }, + ]; +} + +test("clampComboDepth — clamps to [1, hard cap]; invalid → default 3", async () => { + const { clampComboDepth } = await import("../../open-sse/services/combo.ts"); + assert.equal(clampComboDepth(1), 1); + assert.equal(clampComboDepth(2), 2); + assert.equal(clampComboDepth(5), 5); + assert.equal(clampComboDepth(100), 10, "hard cap at 10"); + assert.equal(clampComboDepth(0), 3, "0 invalid → default 3"); + assert.equal(clampComboDepth(-4), 3, "negative → default 3"); + assert.equal(clampComboDepth(undefined), 3, "undefined → default 3"); + assert.equal(clampComboDepth("abc"), 3, "non-numeric → default 3"); + assert.equal(clampComboDepth(2.9), 2, "floors to 2"); +}); + +test("validateComboDAG — default depth (3) still throws on a 4-deep chain", async () => { + const { validateComboDAG } = await import("../../open-sse/services/combo.ts"); + assert.throws(() => validateComboDAG("L0", buildDeepChain()), /nesting depth/); +}); + +test("validateComboDAG — honors a HIGHER configured maxDepth (was hardcoded 3)", async () => { + const { validateComboDAG } = await import("../../open-sse/services/combo.ts"); + // maxDepth 5 → the 4-deep chain validates without throwing. + assert.doesNotThrow(() => validateComboDAG("L0", buildDeepChain(), new Set(), 0, 5)); +}); + +test("validateComboDAG — honors a LOWER configured maxDepth", async () => { + const { validateComboDAG } = await import("../../open-sse/services/combo.ts"); + const shallow = [ + { name: "A", models: ["B"] }, + { name: "B", models: ["C"] }, + { name: "C", models: ["openai/gpt-4"] }, + ]; + // A→B→C (C at depth 2): default (3) allows it… + assert.doesNotThrow(() => validateComboDAG("A", shallow)); + // …but maxDepth 1 rejects it. + assert.throws(() => validateComboDAG("A", shallow, new Set(), 0, 1), /nesting depth/); +}); diff --git a/tests/unit/combo-provider-diversity-wiring.test.ts b/tests/unit/combo-provider-diversity-wiring.test.ts new file mode 100644 index 0000000000..3c81519486 --- /dev/null +++ b/tests/unit/combo-provider-diversity-wiring.test.ts @@ -0,0 +1,69 @@ +/** + * tests/unit/combo-provider-diversity-wiring.test.ts + * + * Regression: `recordProviderUsage` (open-sse/services/autoCombo/providerDiversity.ts) + * had ZERO production callers, so `/api/analytics/diversity` always reported + * `score: 1.0` with an empty `providers` map — a dead dashboard. + * + * Fix: `recordComboRequest` (the single chokepoint every combo strategy funnels + * through) records the successful target's provider into the diversity window. + */ +import test from "node:test"; +import assert from "node:assert/strict"; + +test("recordComboRequest feeds the provider-diversity report on success", async () => { + const { recordComboRequest } = await import("../../open-sse/services/comboMetrics.ts"); + const { getDiversityReport, resetDiversity } = await import( + "../../open-sse/services/autoCombo/providerDiversity.ts" + ); + resetDiversity(); + + recordComboRequest("combo-diversity-x", "openai/gpt-4", { + success: true, + latencyMs: 100, + strategy: "priority", + target: { provider: "openai" }, + }); + recordComboRequest("combo-diversity-x", "claude/sonnet", { + success: true, + latencyMs: 120, + strategy: "priority", + target: { provider: "claude" }, + }); + recordComboRequest("combo-diversity-x", "openai/gpt-4", { + success: true, + latencyMs: 90, + strategy: "priority", + target: { provider: "openai" }, + }); + + const report = getDiversityReport(); + assert.equal(report.totalRequests, 3, "all 3 successful dispatches recorded"); + assert.equal(report.providers.openai?.count, 2); + assert.equal(report.providers.claude?.count, 1); + assert.ok(report.score > 0, "diversity score reflects multiple providers (was always 1.0/empty)"); +}); + +test("recordComboRequest does NOT pollute diversity on failure", async () => { + const { recordComboRequest } = await import("../../open-sse/services/comboMetrics.ts"); + const { getDiversityReport, resetDiversity } = await import( + "../../open-sse/services/autoCombo/providerDiversity.ts" + ); + resetDiversity(); + + recordComboRequest("combo-diversity-y", "openai/gpt-4", { + success: false, + latencyMs: 100, + strategy: "priority", + target: { provider: "openai" }, + }); + // Terminal "all targets failed" call (modelStr null, no provider) must also be safe. + recordComboRequest("combo-diversity-y", null, { + success: false, + latencyMs: 200, + strategy: "priority", + }); + + const report = getDiversityReport(); + assert.equal(report.totalRequests, 0, "failures must not enter the diversity window"); +}); diff --git a/tests/unit/complexity-aware-scoring-wiring.test.ts b/tests/unit/complexity-aware-scoring-wiring.test.ts new file mode 100644 index 0000000000..630a5fd3e1 --- /dev/null +++ b/tests/unit/complexity-aware-scoring-wiring.test.ts @@ -0,0 +1,65 @@ +/** + * tests/unit/complexity-aware-scoring-wiring.test.ts + * + * Proves the 2026 complexity-aware wiring: feeding a tier/complexity hint into + * scoreAutoTargets makes the tierAffinity / specificityMatch factors live + * (they were a constant 0.5 because the hint was never passed — audit Bug #4), + * while a null hint keeps the default behavior byte-for-byte. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import { scoreAutoTargets } from "../../open-sse/services/combo.ts"; +import { DEFAULT_WEIGHTS } from "../../open-sse/services/autoCombo/scoring.ts"; +import type { RoutingHint } from "../../open-sse/services/manifestAdapter.ts"; + +function target() { + return { + kind: "model", + provider: "openai", + model: "gpt-4o-mini", + modelStr: "openai/gpt-4o-mini", + executionKey: "k1", + stepId: "s1", + } as unknown as Parameters[0][number]; +} + +function candidate() { + return { + executionKey: "k1", + provider: "openai", + model: "gpt-4o-mini", + modelStr: "openai/gpt-4o-mini", + quotaRemaining: 100, + quotaTotal: 100, + circuitBreakerState: "CLOSED", + costPer1MTokens: 1, + p95LatencyMs: 100, + latencyStdDev: 10, + errorRate: 0, + accountTier: "standard", + quotaResetIntervalSecs: 86400, + } as unknown as Parameters[1][number]; +} + +test("scoreAutoTargets — a tier/complexity hint moves the score off tier-neutral", () => { + const withoutHint = scoreAutoTargets([target()], [candidate()], "default", DEFAULT_WEIGHTS); + const hint = { + recommendedMinTier: "premium", + specificity: { score: 80 }, + } as unknown as RoutingHint; + const withHint = scoreAutoTargets([target()], [candidate()], "default", DEFAULT_WEIGHTS, hint); + + assert.equal(withoutHint.length, 1); + assert.equal(withHint.length, 1); + assert.notEqual( + withHint[0].score, + withoutHint[0].score, + "feeding a tier hint must change tierAffinity/specificityMatch (was constant 0.5)" + ); +}); + +test("scoreAutoTargets — a null hint is identical to no hint (backward compatible)", () => { + const a = scoreAutoTargets([target()], [candidate()], "default", DEFAULT_WEIGHTS); + const b = scoreAutoTargets([target()], [candidate()], "default", DEFAULT_WEIGHTS, null); + assert.equal(a[0].score, b[0].score, "null hint must equal no hint (default behavior unchanged)"); +}); diff --git a/tests/unit/complexity-router.test.ts b/tests/unit/complexity-router.test.ts new file mode 100644 index 0000000000..7a6e6745c7 --- /dev/null +++ b/tests/unit/complexity-router.test.ts @@ -0,0 +1,107 @@ +/** + * tests/unit/complexity-router.test.ts + * + * 2026 strategy: request-complexity classification → recommended tier, with an + * explicit tool-use escalation. Validates the classifier facade over the + * existing specificity detector. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import { + classifyRequestComplexity, + escalateTier, + buildComplexityRoutingHint, +} from "../../open-sse/services/autoCombo/complexityRouter.ts"; + +const NOOP_LOG = { info: () => {} }; + +function modelTargets(): Parameters[0] { + return [ + { + kind: "model", + provider: "openai", + model: "gpt-4o-mini", + modelStr: "openai/gpt-4o-mini", + executionKey: "k1", + stepId: "s1", + }, + ] as unknown as Parameters[0]; +} + +test("escalateTier — raises to the floor, never lowers", () => { + assert.equal(escalateTier("free", "cheap"), "cheap"); + assert.equal(escalateTier("free", "premium"), "premium"); + assert.equal(escalateTier("premium", "cheap"), "premium"); + assert.equal(escalateTier("cheap", "free"), "cheap"); + assert.equal(escalateTier("cheap", "cheap"), "cheap"); +}); + +test("classifyRequestComplexity — a trivial prompt stays cheap/free with no tool signal", () => { + const c = classifyRequestComplexity({ messages: [{ role: "user", content: "hi there" }] }); + assert.equal(c.hasToolUse, false); + assert.equal(c.recommendedTier, "free"); + assert.ok(["trivial", "simple"].includes(c.level), `expected low level, got ${c.level}`); +}); + +test("classifyRequestComplexity — a hard, multi-step coding+reasoning prompt scores higher", () => { + const trivial = classifyRequestComplexity({ messages: [{ role: "user", content: "hi" }] }); + const hard = classifyRequestComplexity({ + messages: [ + { + role: "user", + content: + "First, analyze this TypeScript module for race conditions:\n" + + "```ts\nasync function f(){ /* ... */ }\n```\n" + + "Then, step by step, prove the time complexity is O(n log n), " + + "derive the recurrence relation, and refactor it to remove the data race. " + + "Finally, explain the trade-offs of each approach in depth.", + }, + ], + }); + assert.ok(hard.score > trivial.score, `hard (${hard.score}) must exceed trivial (${trivial.score})`); +}); + +test("classifyRequestComplexity — tool schemas escalate the tier above free", () => { + const c = classifyRequestComplexity({ + messages: [{ role: "user", content: "weather?" }], + tools: [ + { + function: { + name: "get_weather", + description: "Get the weather", + parameters: { type: "object", properties: { city: { type: "string" } } }, + }, + }, + ], + }); + assert.equal(c.hasToolUse, true); + assert.notEqual(c.recommendedTier, "free", "tool-using requests must not route to the free tier"); +}); + +test("buildComplexityRoutingHint — a tool-using request floors the hint tier above free", () => { + const hint = buildComplexityRoutingHint( + modelTargets(), + { + messages: [{ role: "user", content: "weather?" }], + tools: [{ function: { name: "get_weather", description: "Get the weather", parameters: {} } }], + }, + NOOP_LOG + ); + assert.ok(hint, "expected a non-null hint when complexity routing builds successfully"); + if (!hint) return; + assert.notEqual( + hint.recommendedMinTier, + "free", + "tool-use must floor the recommended tier at cheap (escalation applied)" + ); +}); + +test("buildComplexityRoutingHint — a null body is safe and still builds a tier-neutral hint", () => { + const hint = buildComplexityRoutingHint(modelTargets(), null, NOOP_LOG); + assert.ok(hint, "a null body must not throw — messages default to [] and a hint is built"); + if (!hint) return; + assert.ok( + ["free", "cheap", "premium"].includes(hint.recommendedMinTier), + `unexpected tier ${hint.recommendedMinTier}` + ); +}); diff --git a/tests/unit/quota-streaming-consumption-usd.test.ts b/tests/unit/quota-streaming-consumption-usd.test.ts new file mode 100644 index 0000000000..ce19a4673a --- /dev/null +++ b/tests/unit/quota-streaming-consumption-usd.test.ts @@ -0,0 +1,174 @@ +/** + * tests/unit/quota-streaming-consumption-usd.test.ts + * + * Regression: the streaming quota-share POST-hook in chatCore recorded + * `usd: 0` hardcoded, so USD-unit shared pools (e.g. DeepSeek `usd/monthly`) + * never accrued cost on STREAMING traffic — the dominant request shape — and + * therefore never blocked. The non-streaming path correctly used the computed + * `estimatedCost`. + * + * Fix: extract a testable `recordStreamingConsumption` (with injectable + * `calculateCost` + `schedule`) and a shared `buildConsumptionCost`, and wire + * the real estimated cost into the streaming consumption record. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import type { RecordConsumptionInput } from "../../src/lib/quota/types.ts"; + +test("buildConsumptionCost — flows estimatedCost into usd (not hardcoded 0)", async () => { + const { buildConsumptionCost } = await import("../../src/lib/quota/spendRecorder.ts"); + const cost = buildConsumptionCost({ prompt_tokens: 1000, completion_tokens: 500 }, 0.0234); + assert.equal(cost.tokens, 1500); + assert.equal(cost.usd, 0.0234); + assert.equal(cost.requests, 1); +}); + +test("buildConsumptionCost — null usage → tokens 0, usd 0, requests 1", async () => { + const { buildConsumptionCost } = await import("../../src/lib/quota/spendRecorder.ts"); + const cost = buildConsumptionCost(null, 0); + assert.deepEqual(cost, { tokens: 0, usd: 0, requests: 1 }); +}); + +test("buildConsumptionCost — coerces string tokens, clamps negative cost to 0", async () => { + const { buildConsumptionCost } = await import("../../src/lib/quota/spendRecorder.ts"); + const cost = buildConsumptionCost({ prompt_tokens: "10", completion_tokens: "5" }, -1); + assert.equal(cost.tokens, 15); + assert.equal(cost.usd, 0); +}); + +test("recordStreamingConsumption — records REAL usd for a 200 stream (regression: was 0)", async () => { + const { recordStreamingConsumption } = await import("../../src/lib/quota/spendRecorder.ts"); + const scheduled: RecordConsumptionInput[] = []; + let calcArgs: { p: string; m: string } | null = null; + await recordStreamingConsumption( + { + apiKeyId: "key-1", + connectionId: "conn-1", + provider: "deepseek", + model: "deepseek-chat", + streamUsage: { prompt_tokens: 2000, completion_tokens: 1000 }, + streamStatus: 200, + }, + { + calculateCost: async (p: string, m: string) => { + calcArgs = { p, m }; + return 0.042; + }, + schedule: (input) => scheduled.push(input), + } + ); + assert.equal(scheduled.length, 1); + assert.equal(scheduled[0].cost.usd, 0.042, "streaming usd must reflect calculateCost, not 0"); + assert.equal(scheduled[0].cost.tokens, 3000); + assert.equal(scheduled[0].cost.requests, 1); + assert.equal(scheduled[0].apiKeyId, "key-1"); + assert.equal(scheduled[0].connectionId, "conn-1"); + assert.equal(scheduled[0].provider, "deepseek"); + assert.ok(calcArgs); + assert.equal(calcArgs!.p, "deepseek"); + assert.equal(calcArgs!.m, "deepseek-chat"); +}); + +test("recordStreamingConsumption — skips non-200 streams entirely", async () => { + const { recordStreamingConsumption } = await import("../../src/lib/quota/spendRecorder.ts"); + const scheduled: RecordConsumptionInput[] = []; + let calcCalled = false; + await recordStreamingConsumption( + { + apiKeyId: "k", + connectionId: "c", + provider: "openai", + model: "gpt-4", + streamUsage: { prompt_tokens: 1 }, + streamStatus: 500, + }, + { + calculateCost: async () => { + calcCalled = true; + return 1; + }, + schedule: (input) => scheduled.push(input), + } + ); + assert.equal(scheduled.length, 0); + assert.equal(calcCalled, false); +}); + +test("recordStreamingConsumption — null usage still records requests:1 (usd 0), no cost call", async () => { + const { recordStreamingConsumption } = await import("../../src/lib/quota/spendRecorder.ts"); + const scheduled: RecordConsumptionInput[] = []; + let calcCalled = false; + await recordStreamingConsumption( + { + apiKeyId: "k", + connectionId: "c", + provider: "openai", + model: "gpt-4", + streamUsage: null, + streamStatus: 200, + }, + { + calculateCost: async () => { + calcCalled = true; + return 1; + }, + schedule: (input) => scheduled.push(input), + } + ); + assert.equal(scheduled.length, 1); + assert.equal(scheduled[0].cost.usd, 0); + assert.equal(scheduled[0].cost.requests, 1); + assert.equal(calcCalled, false, "no usage → no cost computation"); +}); + +test("recordStreamingConsumption — missing apiKeyId/connectionId → no-op", async () => { + const { recordStreamingConsumption } = await import("../../src/lib/quota/spendRecorder.ts"); + const scheduled: RecordConsumptionInput[] = []; + await recordStreamingConsumption( + { + apiKeyId: null, + connectionId: "c", + provider: "openai", + model: "gpt-4", + streamUsage: { prompt_tokens: 1 }, + streamStatus: 200, + }, + { calculateCost: async () => 1, schedule: (input) => scheduled.push(input) } + ); + await recordStreamingConsumption( + { + apiKeyId: "k", + connectionId: null, + provider: "openai", + model: "gpt-4", + streamUsage: { prompt_tokens: 1 }, + streamStatus: 200, + }, + { calculateCost: async () => 1, schedule: (input) => scheduled.push(input) } + ); + assert.equal(scheduled.length, 0); +}); + +test("recordStreamingConsumption — calculateCost throwing → records usd 0, never throws", async () => { + const { recordStreamingConsumption } = await import("../../src/lib/quota/spendRecorder.ts"); + const scheduled: RecordConsumptionInput[] = []; + await recordStreamingConsumption( + { + apiKeyId: "k", + connectionId: "c", + provider: "openai", + model: "gpt-4", + streamUsage: { prompt_tokens: 100 }, + streamStatus: 200, + }, + { + calculateCost: async () => { + throw new Error("pricing unavailable"); + }, + schedule: (input) => scheduled.push(input), + } + ); + assert.equal(scheduled.length, 1); + assert.equal(scheduled[0].cost.usd, 0); + assert.equal(scheduled[0].cost.tokens, 100); +}); diff --git a/tests/unit/router-strategies.test.ts b/tests/unit/router-strategies.test.ts new file mode 100644 index 0000000000..3fd25c9674 --- /dev/null +++ b/tests/unit/router-strategies.test.ts @@ -0,0 +1,167 @@ +/** + * tests/unit/router-strategies.test.ts + * + * Direct coverage for the pluggable auto-router strategies in + * open-sse/services/autoCombo/routerStrategy.ts. The `cost` and `latency` + * strategies and the `selectWithStrategy` entry point had NO direct test + * (only `sla-aware`/`lkgp` were covered, indirectly, in autoCombo.test.ts). + * Also pins the documented silent fallback to `rules` for unknown names. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import { + getStrategy, + selectWithStrategy, + listStrategies, + type RoutingContext, +} from "../../open-sse/services/autoCombo/routerStrategy.ts"; +import type { ProviderCandidate } from "../../open-sse/services/autoCombo/scoring.ts"; + +function cand(p: Partial & { provider: string }): ProviderCandidate { + return { + model: `${p.provider}/m`, + quotaRemaining: 100, + quotaTotal: 100, + circuitBreakerState: "CLOSED", + costPer1MTokens: 1, + p95LatencyMs: 100, + latencyStdDev: 10, + errorRate: 0, + ...p, + } as ProviderCandidate; +} + +const ctx: RoutingContext = { taskType: "default" }; + +// ── cost ───────────────────────────────────────────────────────────────────── +test("cost — selects the cheapest healthy candidate", () => { + const pool = [ + cand({ provider: "a", costPer1MTokens: 5 }), + cand({ provider: "b", costPer1MTokens: 1 }), + cand({ provider: "c", costPer1MTokens: 3 }), + ]; + const d = getStrategy("cost").select(pool, ctx); + assert.equal(d.provider, "b"); + assert.equal(d.strategy, "cost"); +}); + +test("cost — excludes OPEN-breaker candidates even if cheaper", () => { + const pool = [ + cand({ provider: "cheap-open", costPer1MTokens: 0.1, circuitBreakerState: "OPEN" }), + cand({ provider: "ok", costPer1MTokens: 2 }), + ]; + assert.equal(getStrategy("cost").select(pool, ctx).provider, "ok"); +}); + +test("cost — 'eco' alias resolves to the cost strategy", () => { + assert.equal(getStrategy("eco").name, "cost"); +}); + +test("cost — empty pool throws", () => { + assert.throws(() => getStrategy("cost").select([], ctx), /No candidates/); +}); + +// ── latency ─────────────────────────────────────────────────────────────────── +test("latency — selects the lowest p95 candidate", () => { + const pool = [ + cand({ provider: "slow", p95LatencyMs: 900 }), + cand({ provider: "fast", p95LatencyMs: 100 }), + cand({ provider: "mid", p95LatencyMs: 400 }), + ]; + assert.equal(getStrategy("latency").select(pool, ctx).provider, "fast"); +}); + +test("latency — error rate penalizes a fast-but-flaky candidate", () => { + // flaky: 100ms + 0.5*1000 = 600 effective; steady: 400ms + 0 = 400 → steady wins. + const pool = [ + cand({ provider: "flaky", p95LatencyMs: 100, errorRate: 0.5 }), + cand({ provider: "steady", p95LatencyMs: 400, errorRate: 0 }), + ]; + assert.equal(getStrategy("latency").select(pool, ctx).provider, "steady"); +}); + +test("latency — 'fast' alias resolves to the latency strategy", () => { + assert.equal(getStrategy("fast").name, "latency"); +}); + +// ── sla-aware ───────────────────────────────────────────────────────────────── +test("sla-aware — prefers a candidate meeting the p95/error SLOs", () => { + const pool = [ + cand({ provider: "violator", p95LatencyMs: 5000, errorRate: 0.3 }), + cand({ provider: "compliant", p95LatencyMs: 300, errorRate: 0.001 }), + ]; + const d = getStrategy("sla-aware").select(pool, { + taskType: "default", + sla: { targetP95Ms: 2000, maxErrorRate: 0.05 }, + }); + assert.equal(d.provider, "compliant"); + assert.equal(d.strategy, "sla-aware"); +}); + +test("sla-aware — 'sla' alias resolves to sla-aware", () => { + assert.equal(getStrategy("sla").name, "sla-aware"); +}); + +test("sla-aware — hardConstraints ranks fewest-violations first", () => { + const pool = [ + cand({ provider: "violator", p95LatencyMs: 3000, errorRate: 0.2 }), + cand({ provider: "within", p95LatencyMs: 1500, errorRate: 0.01 }), + ]; + const d = getStrategy("sla-aware").select(pool, { + taskType: "default", + sla: { targetP95Ms: 2000, maxErrorRate: 0.05, hardConstraints: true }, + }); + assert.equal(d.provider, "within"); +}); + +// ── lkgp ────────────────────────────────────────────────────────────────────── +test("lkgp — returns the last known good provider when healthy", () => { + const pool = [cand({ provider: "x" }), cand({ provider: "y" })]; + const d = getStrategy("lkgp").select(pool, { + taskType: "default", + lastKnownGoodProvider: "y", + }); + assert.equal(d.provider, "y"); + assert.equal(d.strategy, "lkgp"); +}); + +test("lkgp — falls back to rules when the LKGP is OPEN", () => { + const pool = [cand({ provider: "x" }), cand({ provider: "y", circuitBreakerState: "OPEN" })]; + const d = getStrategy("lkgp").select(pool, { + taskType: "default", + lastKnownGoodProvider: "y", + }); + assert.equal(d.strategy, "rules"); +}); + +test("lkgp — lkgpEnabled:false delegates to rules", () => { + const pool = [cand({ provider: "x" })]; + const d = getStrategy("lkgp").select(pool, { + taskType: "default", + lkgpEnabled: false, + lastKnownGoodProvider: "x", + }); + assert.equal(d.strategy, "rules"); +}); + +// ── selectWithStrategy + registry ───────────────────────────────────────────── +test("selectWithStrategy — dispatches by name", () => { + const pool = [ + cand({ provider: "a", costPer1MTokens: 9 }), + cand({ provider: "b", costPer1MTokens: 1 }), + ]; + assert.equal(selectWithStrategy(pool, ctx, "cost").provider, "b"); +}); + +test("selectWithStrategy — unknown strategy silently falls back to rules", () => { + const pool = [cand({ provider: "a" })]; + const d = selectWithStrategy(pool, ctx, "totally-unknown-strategy"); + assert.equal(d.strategy, "rules"); +}); + +test("listStrategies — exposes every registered strategy + aliases", () => { + const names = listStrategies().map((s) => s.name); + for (const n of ["rules", "cost", "eco", "latency", "fast", "sla-aware", "sla", "lkgp"]) { + assert.ok(names.includes(n), `listStrategies missing '${n}'`); + } +});