From 43b13928761a664c8100f8b0ffa70159ea56eaf9 Mon Sep 17 00:00:00 2001 From: Paijo <14921983+oyi77@users.noreply.github.com> Date: Wed, 3 Jun 2026 16:36:47 +0700 Subject: [PATCH] fix(autoCombo): rotate across all connections, never waste provider capacity (#3078) Integrated into release/v3.8.9. Clean merge, all 146 vitest tests pass. --- open-sse/services/autoCombo/engine.ts | 164 ++++++++++++-- open-sse/services/autoCombo/modePacks.ts | 12 +- open-sse/services/autoCombo/scoring.ts | 17 +- open-sse/services/combo.ts | 49 ++++- tests/unit/autoCombo/tieredRotation.test.ts | 230 ++++++++++++++++++++ vitest.mcp.config.ts | 1 + 6 files changed, 450 insertions(+), 23 deletions(-) create mode 100644 tests/unit/autoCombo/tieredRotation.test.ts diff --git a/open-sse/services/autoCombo/engine.ts b/open-sse/services/autoCombo/engine.ts index 74c85b896a..870b7c3dc6 100644 --- a/open-sse/services/autoCombo/engine.ts +++ b/open-sse/services/autoCombo/engine.ts @@ -42,6 +42,136 @@ export interface SelectionResult { isExploration: boolean; factors: Record; excluded: string[]; + connectionId?: string; +} + +type TierName = "top" | "mid" | "rest"; + +const TIER_PREFERENCES: Record> = { + smart: { top: 0.5, mid: 0.3, rest: 0.2 }, + fast: { top: 0.3, mid: 0.5, rest: 0.2 }, + cheap: { top: 0.2, mid: 0.3, rest: 0.5 }, + coding: { top: 0.6, mid: 0.25, rest: 0.15 }, + default: { top: 0.45, mid: 0.35, rest: 0.2 }, +}; + +function tierPreferencesForName(name: string): Record { + const key = name.toLowerCase(); + if (TIER_PREFERENCES[key]) return TIER_PREFERENCES[key]; + for (const prefix of Object.keys(TIER_PREFERENCES)) { + if (key.startsWith(`${prefix}-`) || key.includes(prefix)) return TIER_PREFERENCES[prefix]; + } + return TIER_PREFERENCES.default; +} + +const SCORE_EPSILON = 1e-4; +const CLEAR_WINNER_THRESHOLD = 0.1; + +class ScoreTierRotator { + private readonly tierCounters = new Map(); + private rrCounter = 0; + + constructor(private readonly comboName: string) {} + + pick(candidates: ScoredProvider[]): ScoredProvider { + if (candidates.length === 0) { + throw new Error(`ScoreTierRotator: no candidates to pick from for combo=${this.comboName}`); + } + if (candidates.length === 1) return candidates[0]; + + const tiers = groupIntoTiers(candidates); + const best = candidates[0].score; + const worst = candidates[candidates.length - 1].score; + if (tiers.top.length > 0 && (best - worst) >= CLEAR_WINNER_THRESHOLD) { + return this.pickFromPool(tiers.top); + } + const prefs = tierPreferencesForName(this.comboName); + const chosen = chooseTierWeighted(tiers, prefs, (pool) => this.pickFromPool(pool), () => + this.advance(tiers, prefs, candidates) + ); + return chosen; + } + + private advance( + tiers: Record, + prefs: Record, + candidates: ScoredProvider[] + ): ScoredProvider { + const order: TierName[] = ["top", "mid", "rest"]; + for (const tier of order) { + if (tiers[tier].length > 0 && prefs[tier] > 0) { + const idx = this.tierCounters.get(tier) ?? 0; + const picked = tiers[tier][idx % tiers[tier].length]; + this.tierCounters.set(tier, idx + 1); + return picked; + } + } + return tiers.top[0] ?? tiers.mid[0] ?? tiers.rest[0] ?? candidates[0]; + } + + private pickFromPool(pool: ScoredProvider[]): ScoredProvider { + if (pool.length === 0) throw new Error("pickFromPool: empty pool"); + if (pool.length === 1) return pool[0]; + const picked = pool[this.rrCounter % pool.length]; + this.rrCounter = (this.rrCounter + 1) % pool.length; + return picked; + } +} + +function groupIntoTiers(candidates: ScoredProvider[]): Record { + if (candidates.length === 0) return { top: [], mid: [], rest: [] }; + const best = candidates[0].score; + const worst = candidates[candidates.length - 1].score; + const range = best - worst; + + const top: ScoredProvider[] = []; + const mid: ScoredProvider[] = []; + const rest: ScoredProvider[] = []; + + for (const c of candidates) { + const delta = best - c.score; + if (delta <= SCORE_EPSILON) top.push(c); + else if (range <= SCORE_EPSILON || delta <= range * 0.3) mid.push(c); + else rest.push(c); + } + + if (mid.length === 0 && rest.length > 0) { + const half = Math.ceil(rest.length / 2); + mid.push(...rest.splice(0, half)); + } + + return { top, mid, rest }; +} + +function chooseTierWeighted( + tiers: Record, + prefs: Record, + pickFromPool: (pool: ScoredProvider[]) => ScoredProvider, + fallback: () => ScoredProvider +): ScoredProvider { + const active = { + top: tiers.top.length > 0 ? prefs.top : 0, + mid: tiers.mid.length > 0 ? prefs.mid : 0, + rest: tiers.rest.length > 0 ? prefs.rest : 0, + }; + const total = active.top + active.mid + active.rest; + if (total <= 0) return fallback(); + const r = Math.random() * total; + let acc = 0; + if (active.top > 0 && (acc += active.top) >= r) return pickFromPool(tiers.top); + if (active.mid > 0 && (acc += active.mid) >= r) return pickFromPool(tiers.mid); + if (active.rest > 0) return pickFromPool(tiers.rest); + return fallback(); +} + +const comboRotators = new Map(); +function getRotator(comboName: string): ScoreTierRotator { + let r = comboRotators.get(comboName); + if (!r) { + r = new ScoreTierRotator(comboName); + comboRotators.set(comboName, r); + } + return r; } /** @@ -137,27 +267,32 @@ export function selectProvider( const isExploration = Math.random() < effectiveExplorationRate && candidates_.length > 1; if (isExploration) { - // Random selection (bandit exploration) const idx = Math.floor(Math.random() * candidates_.length); selected = candidates_[idx]; } else { - // Greedy: highest score - selected = candidates_[0]; + const rotator = getRotator(config.name); + selected = rotator.pick(candidates_); } // Budget cap enforcement if (config.budgetCap) { - const candidate = candidates.find((c) => c.provider === selected.provider); - if (candidate) { - const estimatedCost = (candidate.costPer1MTokens / 1_000_000) * 1000; // approx for 1K tokens - if (estimatedCost > config.budgetCap) { - // Degrade to cheapest - const cheapest = candidates_ - .map((s) => ({ - ...s, - cost: candidates.find((c) => c.provider === s.provider)?.costPer1MTokens || 0, - })) - .sort((a, b) => a.cost - b.cost)[0]; + const costMap = new Map(); + for (const c of candidates) { + costMap.set(`${c.provider}\0${c.model}`, c.costPer1MTokens); + } + const estimatedCostFor = (s: ScoredProvider) => { + const cost = costMap.get(`${s.provider}\0${s.model}`) ?? 0; + return (cost / 1_000_000) * 1000; + }; + if (estimatedCostFor(selected) > config.budgetCap) { + const budgetOk = candidates_.filter((s) => estimatedCostFor(s) <= config.budgetCap!); + if (budgetOk.length > 0) { + const rotator = getRotator(`${config.name}#budget`); + selected = rotator.pick(budgetOk); + } else { + const cheapest = [...candidates_].sort( + (a, b) => estimatedCostFor(a) - estimatedCostFor(b) + )[0]; if (cheapest) selected = cheapest; } } @@ -170,6 +305,7 @@ export function selectProvider( isExploration, factors: selected.factors as unknown as Record, excluded, + connectionId: selected.connectionId, }; } diff --git a/open-sse/services/autoCombo/modePacks.ts b/open-sse/services/autoCombo/modePacks.ts index 6278deb33d..a912fdc389 100644 --- a/open-sse/services/autoCombo/modePacks.ts +++ b/open-sse/services/autoCombo/modePacks.ts @@ -23,8 +23,9 @@ export const MODE_PACKS: Record = { tierPriority: 0.05, tierAffinity: 0, specificityMatch: 0, - contextAffinity: 0.06, + contextAffinity: 0.01, resetWindowAffinity: 0, + connectionDensity: 0.05, }, // Prioritize cost. tierPriority replaces 0.05 from stability. "cost-saver": { @@ -37,8 +38,9 @@ export const MODE_PACKS: Record = { tierPriority: 0.05, tierAffinity: 0, specificityMatch: 0, - contextAffinity: 0.05, + contextAffinity: 0.0, resetWindowAffinity: 0, + connectionDensity: 0.05, }, // Prioritize task fitness. tierPriority replaces 0.05 from latencyInv. "quality-first": { @@ -51,8 +53,9 @@ export const MODE_PACKS: Record = { tierPriority: 0.05, tierAffinity: 0, specificityMatch: 0, - contextAffinity: 0.05, + contextAffinity: 0.0, resetWindowAffinity: 0, + connectionDensity: 0.05, }, // Prioritize quota availability. tierPriority replaces 0.05 from taskFit. "offline-friendly": { @@ -65,8 +68,9 @@ export const MODE_PACKS: Record = { tierPriority: 0.05, tierAffinity: 0, specificityMatch: 0, - contextAffinity: 0.05, + contextAffinity: 0.0, resetWindowAffinity: 0, + connectionDensity: 0.05, }, }; diff --git a/open-sse/services/autoCombo/scoring.ts b/open-sse/services/autoCombo/scoring.ts index f140d52c7e..c03ac3926d 100644 --- a/open-sse/services/autoCombo/scoring.ts +++ b/open-sse/services/autoCombo/scoring.ts @@ -18,6 +18,7 @@ export interface ScoringFactors { specificityMatch: number; contextAffinity: number; resetWindowAffinity: number; + connectionDensity: number; } export interface ScoringWeights { @@ -32,20 +33,22 @@ export interface ScoringWeights { specificityMatch: number; contextAffinity: number; resetWindowAffinity: number; + connectionDensity: number; } export const DEFAULT_WEIGHTS: ScoringWeights = { - quota: 0.16, + quota: 0.15, health: 0.2, - costInv: 0.16, + costInv: 0.15, latencyInv: 0.12, taskFit: 0.08, stability: 0.05, tierPriority: 0.05, tierAffinity: 0.05, specificityMatch: 0.05, - contextAffinity: 0.08, + contextAffinity: 0.05, resetWindowAffinity: 0, + connectionDensity: 0.05, }; export interface ProviderCandidate { @@ -66,6 +69,8 @@ export interface ProviderCandidate { contextAffinity?: number; /** Score [0..1] for quota reset-window preference; sooner selected reset windows score higher. */ resetWindowAffinity?: number; + connectionPoolSize?: number; + connectionId?: string; } export interface ScoredProvider { @@ -73,6 +78,7 @@ export interface ScoredProvider { model: string; score: number; factors: ScoringFactors; + connectionId?: string; } /** @@ -91,7 +97,8 @@ export function calculateScore(factors: ScoringFactors, weights: ScoringWeights) (weights.tierAffinity ?? 0) * factors.tierAffinity + (weights.specificityMatch ?? 0) * factors.specificityMatch + (weights.contextAffinity ?? 0) * factors.contextAffinity + - (weights.resetWindowAffinity ?? 0) * factors.resetWindowAffinity + (weights.resetWindowAffinity ?? 0) * factors.resetWindowAffinity + + (weights.connectionDensity ?? 0) * factors.connectionDensity ); } @@ -186,6 +193,7 @@ export function calculateFactors( specificityMatch: calculateSpecificityMatch(candidate, manifestHint), contextAffinity: candidate.contextAffinity ?? 0.5, resetWindowAffinity: candidate.resetWindowAffinity ?? 0.5, + connectionDensity: Math.min(1, Math.max(0, ((candidate.connectionPoolSize ?? 1) - 1) / 10)), }; } @@ -204,6 +212,7 @@ export function scorePool( model: candidate.model, score: calculateScore(factors, weights), factors, + connectionId: candidate.connectionId, }; }) .sort((a, b) => b.score - a.score); diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index aafd147238..5f6dfa2ee6 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -2243,8 +2243,53 @@ async function buildAutoCandidates( // keep empty stats — auto-combo will use runtime + bootstrap signals } + const uniqueProviders = Array.from( + new Set( + targets.map((target) => target.provider || parseModel(target.modelStr).provider || "unknown") + ) + ); + const connectionPoolCounts = new Map(); + const connectionsByProvider = new Map>>(); + await Promise.all( + uniqueProviders.map(async (provider) => { + try { + const connections = await getProviderConnections({ provider, isActive: true }); + const active = Array.isArray(connections) ? connections : []; + connectionPoolCounts.set(provider, active.length); + connectionsByProvider.set(provider, active); + } catch { + connectionPoolCounts.set(provider, 0); + connectionsByProvider.set(provider, []); + } + }) + ); + + const expandedTargets: ResolvedComboTarget[] = []; + for (const target of targets) { + const provider = target.provider || parseModel(target.modelStr).provider || "unknown"; + const providerConnections = connectionsByProvider.get(provider) || []; + if (target.connectionId) { + expandedTargets.push(target); + continue; + } + const connectionIds = providerConnections + .map((c) => (c && typeof c === "object" && typeof c.id === "string" ? c.id : null)) + .filter((id): id is string => id !== null); + if (connectionIds.length === 0) { + expandedTargets.push(target); + continue; + } + for (const connectionId of connectionIds) { + expandedTargets.push({ + ...target, + connectionId, + executionKey: `${target.executionKey}@${connectionId}`, + }); + } + } + const candidates = await Promise.all( - targets.map(async (target) => { + expandedTargets.map(async (target) => { const modelStr = target.modelStr; const parsed = parseModel(modelStr); const provider = target.provider || parsed.provider || parsed.providerAlias || "unknown"; @@ -2343,6 +2388,8 @@ async function buildAutoCandidates( quotaResetIntervalSecs: 86400, contextAffinity, resetWindowAffinity, + connectionPoolSize: connectionPoolCounts.get(provider) ?? 1, + connectionId: target.connectionId ?? undefined, }; }) ); diff --git a/tests/unit/autoCombo/tieredRotation.test.ts b/tests/unit/autoCombo/tieredRotation.test.ts new file mode 100644 index 0000000000..21c74ed098 --- /dev/null +++ b/tests/unit/autoCombo/tieredRotation.test.ts @@ -0,0 +1,230 @@ +/** + * Tests for ScoreTierRotator and connectionDensity factor. + * Verifies that multi-connection providers surface in ranked candidates + * and that tiered rotation distributes traffic fairly. + */ + +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { selectProvider, type AutoComboConfig } from "../../../open-sse/services/autoCombo/engine"; +import { + calculateFactors, + calculateScore, + DEFAULT_WEIGHTS, + scorePool, + type ProviderCandidate, + type ScoredProvider, +} from "../../../open-sse/services/autoCombo/scoring"; +import { getTaskFitness } from "../../../open-sse/services/autoCombo/taskFitness"; +import { resetDiversity } from "../../../open-sse/services/autoCombo/providerDiversity"; + +function makeCandidate(overrides: Partial): ProviderCandidate { + return { + provider: "unknown", + model: "unknown-model", + quotaRemaining: 100, + quotaTotal: 100, + circuitBreakerState: "CLOSED", + costPer1MTokens: 1, + p95LatencyMs: 1000, + latencyStdDev: 100, + errorRate: 0.01, + ...overrides, + }; +} + +function makeConfig(name: string): AutoComboConfig { + return { + id: `test-${name}`, + name, + type: "auto", + candidatePool: [], + weights: { ...DEFAULT_WEIGHTS }, + explorationRate: 0, + routerStrategy: "rules", + }; +} + +describe("Connection Density Factor", () => { + const baseCandidate = makeCandidate({ provider: "cerebras", model: "llama-70b" }); + + it("multi-connection provider scores higher than single-connection at same quality", () => { + const multiConn = makeCandidate({ provider: "cerebras", model: "llama-70b", connectionPoolSize: 43 }); + const singleConn = makeCandidate({ provider: "anthropic", model: "claude-sonnet", connectionPoolSize: 1 }); + const pool = [multiConn, singleConn]; + + const multiFactors = calculateFactors(multiConn, pool, "coding", getTaskFitness); + const singleFactors = calculateFactors(singleConn, pool, "coding", getTaskFitness); + const multiScore = calculateScore(multiFactors, DEFAULT_WEIGHTS); + const singleScore = calculateScore(singleFactors, DEFAULT_WEIGHTS); + + expect(multiFactors.connectionDensity).toBe(1.0); + expect(singleFactors.connectionDensity).toBe(0.0); + expect(multiScore).toBeGreaterThan(singleScore); + }); + + it("density scales linearly from 0 to 10 connections, caps at 10+", () => { + const make = (size: number) => makeCandidate({ connectionPoolSize: size }); + const sizes = [1, 2, 5, 10, 20, 43]; + const densities = sizes.map((s) => { + const c = make(s); + const pool = [c]; + return calculateFactors(c, pool, "coding", getTaskFitness).connectionDensity; + }); + + expect(densities[0]).toBeCloseTo(0.0, 5); + expect(densities[1]).toBeCloseTo(0.1, 5); + expect(densities[2]).toBeCloseTo(0.4, 5); + expect(densities[3]).toBeCloseTo(0.9, 5); + expect(densities[4]).toBe(1.0); + expect(densities[5]).toBe(1.0); + }); + + it("missing connectionPoolSize defaults to 1 (backward compat)", () => { + const candidate = makeCandidate({ provider: "x" }); + const pool = [candidate]; + const factors = calculateFactors(candidate, pool, "coding", getTaskFitness); + expect(factors.connectionDensity).toBe(0.0); + }); + + it("DEFAULT_WEIGHTS still sum to 1.0 after adding density", () => { + const sum = Object.values(DEFAULT_WEIGHTS).reduce((a, b) => a + b, 0); + expect(Math.abs(sum - 1.0)).toBeLessThan(0.01); + }); +}); + +describe("Tiered Rotation in selectProvider", () => { + beforeEach(() => { + resetDiversity(); + }); + + it("smart combo rotates within top tier across many requests", () => { + const topA = makeCandidate({ provider: "openai", model: "gpt-4o", quotaRemaining: 95 }); + const topB = makeCandidate({ provider: "anthropic", model: "claude-opus", quotaRemaining: 90 }); + const topC = makeCandidate({ provider: "google", model: "gemini-ultra", quotaRemaining: 88 }); + const mid = makeCandidate({ provider: "mistral", model: "mistral-large", quotaRemaining: 70 }); + const pool = [topA, topB, topC, mid]; + + const config = makeConfig("smart"); + const seen = new Set(); + for (let i = 0; i < 50; i++) { + const result = selectProvider(config, pool, "coding"); + seen.add(`${result.provider}/${result.model}`); + } + expect(seen.size).toBeGreaterThanOrEqual(2); + expect(seen.has("openai/gpt-4o") || seen.has("anthropic/claude-opus")).toBe(true); + }); + + it("cheap combo pulls from rest tier (lower scores) more often than smart", () => { + const top = makeCandidate({ provider: "openai", model: "gpt-4o", quotaRemaining: 100 }); + const rest = makeCandidate({ + provider: "cheap-provider", + model: "cheap-model", + quotaRemaining: 100, + costPer1MTokens: 0, + p95LatencyMs: 5000, + }); + const pool = [top, rest]; + + const config = makeConfig("cheap"); + const counts: Record = {}; + for (let i = 0; i < 200; i++) { + const result = selectProvider(config, pool, "coding"); + counts[result.provider] = (counts[result.provider] ?? 0) + 1; + } + expect(counts["cheap-provider"]).toBeGreaterThan(0); + }); + + it("single-candidate pool always returns the same candidate", () => { + const only = makeCandidate({ provider: "only", model: "only-model" }); + const config = makeConfig("smart"); + for (let i = 0; i < 10; i++) { + const result = selectProvider(config, [only], "coding"); + expect(result.provider).toBe("only"); + expect(result.model).toBe("only-model"); + } + }); +}); + +describe("scorePool with connectionDensity", () => { + it("Cerebras with 43 keys ranks above single-connection providers of similar quality", () => { + const cerebras = makeCandidate({ + provider: "cerebras", + model: "llama-3.1-70b", + connectionPoolSize: 43, + quotaRemaining: 100, + }); + const anthropic = makeCandidate({ + provider: "anthropic", + model: "claude-sonnet", + connectionPoolSize: 1, + quotaRemaining: 100, + }); + const pool = [cerebras, anthropic]; + const scored = scorePool(pool, "coding", DEFAULT_WEIGHTS, getTaskFitness); + expect(scored[0].provider).toBe("cerebras"); + }); +}); + +describe("Per-Connection Rotation", () => { + it("rotates across all 43 Cerebras connection IDs, not just one", () => { + const cerebrasCandidates: ProviderCandidate[] = Array.from({ length: 43 }, (_, i) => + makeCandidate({ + provider: "cerebras", + model: "llama-3.1-70b", + connectionId: `cerebras-conn-${i + 1}`, + }) + ); + const config = makeConfig("smart"); + + const seenConnections = new Set(); + for (let i = 0; i < 200; i++) { + const result = selectProvider(config, cerebrasCandidates, "coding"); + if (result.connectionId) seenConnections.add(result.connectionId); + } + expect(seenConnections.size).toBeGreaterThanOrEqual(10); + }); + + it("different combos maintain independent round-robin state", () => { + const candidates: ProviderCandidate[] = Array.from({ length: 5 }, (_, i) => + makeCandidate({ provider: "p", model: "m", connectionId: `c-${i}` }) + ); + const smartConfig = makeConfig("smart-A"); + const fastConfig = makeConfig("fast-B"); + + for (let i = 0; i < 5; i++) { + selectProvider(smartConfig, candidates, "coding"); + } + const smartResults: string[] = []; + for (let i = 0; i < 5; i++) { + const r = selectProvider(smartConfig, candidates, "coding"); + if (r.connectionId) smartResults.push(r.connectionId); + } + + for (let i = 0; i < 5; i++) { + selectProvider(fastConfig, candidates, "coding"); + } + const fastResults: string[] = []; + for (let i = 0; i < 5; i++) { + const r = selectProvider(fastConfig, candidates, "coding"); + if (r.connectionId) fastResults.push(r.connectionId); + } + + expect(smartResults.length).toBe(5); + expect(fastResults.length).toBe(5); + expect(new Set(smartResults).size).toBeGreaterThan(1); + expect(new Set(fastResults).size).toBeGreaterThan(1); + }); + + it("tied-score candidates from same provider+model are all reachable", () => { + const candidates: ProviderCandidate[] = Array.from({ length: 5 }, (_, i) => + makeCandidate({ provider: "free", model: "free-model", connectionId: `key-${i}` }) + ); + const config = makeConfig("smart"); + const visited = new Set(); + for (let i = 0; i < 20; i++) { + const result = selectProvider(config, candidates, "coding"); + if (result.connectionId) visited.add(result.connectionId); + } + expect(visited.size).toBeGreaterThan(1); + }); +}); diff --git a/vitest.mcp.config.ts b/vitest.mcp.config.ts index 732db920b5..a393b1fff3 100644 --- a/vitest.mcp.config.ts +++ b/vitest.mcp.config.ts @@ -12,6 +12,7 @@ export default defineConfig({ include: [ "open-sse/mcp-server/__tests__/**/*.test.ts", "open-sse/services/autoCombo/__tests__/**/*.test.ts", + "tests/unit/autoCombo/**/*.test.ts", "tests/unit/encryption.spec.ts", "src/shared/components/**/*.test.tsx", "src/shared/hooks/__tests__/**/*.test.tsx",