fix(autoCombo): rotate across all connections, never waste provider capacity (#3078)

Integrated into release/v3.8.9. Clean merge, all 146 vitest tests pass.
This commit is contained in:
Paijo
2026-06-03 16:36:47 +07:00
committed by GitHub
parent 9141e98458
commit 43b1392876
6 changed files with 450 additions and 23 deletions

View File

@@ -42,6 +42,136 @@ export interface SelectionResult {
isExploration: boolean;
factors: Record<string, number>;
excluded: string[];
connectionId?: string;
}
type TierName = "top" | "mid" | "rest";
const TIER_PREFERENCES: Record<string, Record<TierName, number>> = {
smart: { top: 0.5, mid: 0.3, rest: 0.2 },
fast: { top: 0.3, mid: 0.5, rest: 0.2 },
cheap: { top: 0.2, mid: 0.3, rest: 0.5 },
coding: { top: 0.6, mid: 0.25, rest: 0.15 },
default: { top: 0.45, mid: 0.35, rest: 0.2 },
};
function tierPreferencesForName(name: string): Record<TierName, number> {
const key = name.toLowerCase();
if (TIER_PREFERENCES[key]) return TIER_PREFERENCES[key];
for (const prefix of Object.keys(TIER_PREFERENCES)) {
if (key.startsWith(`${prefix}-`) || key.includes(prefix)) return TIER_PREFERENCES[prefix];
}
return TIER_PREFERENCES.default;
}
const SCORE_EPSILON = 1e-4;
const CLEAR_WINNER_THRESHOLD = 0.1;
class ScoreTierRotator {
private readonly tierCounters = new Map<TierName, number>();
private rrCounter = 0;
constructor(private readonly comboName: string) {}
pick(candidates: ScoredProvider[]): ScoredProvider {
if (candidates.length === 0) {
throw new Error(`ScoreTierRotator: no candidates to pick from for combo=${this.comboName}`);
}
if (candidates.length === 1) return candidates[0];
const tiers = groupIntoTiers(candidates);
const best = candidates[0].score;
const worst = candidates[candidates.length - 1].score;
if (tiers.top.length > 0 && (best - worst) >= CLEAR_WINNER_THRESHOLD) {
return this.pickFromPool(tiers.top);
}
const prefs = tierPreferencesForName(this.comboName);
const chosen = chooseTierWeighted(tiers, prefs, (pool) => this.pickFromPool(pool), () =>
this.advance(tiers, prefs, candidates)
);
return chosen;
}
private advance(
tiers: Record<TierName, ScoredProvider[]>,
prefs: Record<TierName, number>,
candidates: ScoredProvider[]
): ScoredProvider {
const order: TierName[] = ["top", "mid", "rest"];
for (const tier of order) {
if (tiers[tier].length > 0 && prefs[tier] > 0) {
const idx = this.tierCounters.get(tier) ?? 0;
const picked = tiers[tier][idx % tiers[tier].length];
this.tierCounters.set(tier, idx + 1);
return picked;
}
}
return tiers.top[0] ?? tiers.mid[0] ?? tiers.rest[0] ?? candidates[0];
}
private pickFromPool(pool: ScoredProvider[]): ScoredProvider {
if (pool.length === 0) throw new Error("pickFromPool: empty pool");
if (pool.length === 1) return pool[0];
const picked = pool[this.rrCounter % pool.length];
this.rrCounter = (this.rrCounter + 1) % pool.length;
return picked;
}
}
function groupIntoTiers(candidates: ScoredProvider[]): Record<TierName, ScoredProvider[]> {
if (candidates.length === 0) return { top: [], mid: [], rest: [] };
const best = candidates[0].score;
const worst = candidates[candidates.length - 1].score;
const range = best - worst;
const top: ScoredProvider[] = [];
const mid: ScoredProvider[] = [];
const rest: ScoredProvider[] = [];
for (const c of candidates) {
const delta = best - c.score;
if (delta <= SCORE_EPSILON) top.push(c);
else if (range <= SCORE_EPSILON || delta <= range * 0.3) mid.push(c);
else rest.push(c);
}
if (mid.length === 0 && rest.length > 0) {
const half = Math.ceil(rest.length / 2);
mid.push(...rest.splice(0, half));
}
return { top, mid, rest };
}
function chooseTierWeighted(
tiers: Record<TierName, ScoredProvider[]>,
prefs: Record<TierName, number>,
pickFromPool: (pool: ScoredProvider[]) => ScoredProvider,
fallback: () => ScoredProvider
): ScoredProvider {
const active = {
top: tiers.top.length > 0 ? prefs.top : 0,
mid: tiers.mid.length > 0 ? prefs.mid : 0,
rest: tiers.rest.length > 0 ? prefs.rest : 0,
};
const total = active.top + active.mid + active.rest;
if (total <= 0) return fallback();
const r = Math.random() * total;
let acc = 0;
if (active.top > 0 && (acc += active.top) >= r) return pickFromPool(tiers.top);
if (active.mid > 0 && (acc += active.mid) >= r) return pickFromPool(tiers.mid);
if (active.rest > 0) return pickFromPool(tiers.rest);
return fallback();
}
const comboRotators = new Map<string, ScoreTierRotator>();
function getRotator(comboName: string): ScoreTierRotator {
let r = comboRotators.get(comboName);
if (!r) {
r = new ScoreTierRotator(comboName);
comboRotators.set(comboName, r);
}
return r;
}
/**
@@ -137,27 +267,32 @@ export function selectProvider(
const isExploration = Math.random() < effectiveExplorationRate && candidates_.length > 1;
if (isExploration) {
// Random selection (bandit exploration)
const idx = Math.floor(Math.random() * candidates_.length);
selected = candidates_[idx];
} else {
// Greedy: highest score
selected = candidates_[0];
const rotator = getRotator(config.name);
selected = rotator.pick(candidates_);
}
// Budget cap enforcement
if (config.budgetCap) {
const candidate = candidates.find((c) => c.provider === selected.provider);
if (candidate) {
const estimatedCost = (candidate.costPer1MTokens / 1_000_000) * 1000; // approx for 1K tokens
if (estimatedCost > config.budgetCap) {
// Degrade to cheapest
const cheapest = candidates_
.map((s) => ({
...s,
cost: candidates.find((c) => c.provider === s.provider)?.costPer1MTokens || 0,
}))
.sort((a, b) => a.cost - b.cost)[0];
const costMap = new Map<string, number>();
for (const c of candidates) {
costMap.set(`${c.provider}\0${c.model}`, c.costPer1MTokens);
}
const estimatedCostFor = (s: ScoredProvider) => {
const cost = costMap.get(`${s.provider}\0${s.model}`) ?? 0;
return (cost / 1_000_000) * 1000;
};
if (estimatedCostFor(selected) > config.budgetCap) {
const budgetOk = candidates_.filter((s) => estimatedCostFor(s) <= config.budgetCap!);
if (budgetOk.length > 0) {
const rotator = getRotator(`${config.name}#budget`);
selected = rotator.pick(budgetOk);
} else {
const cheapest = [...candidates_].sort(
(a, b) => estimatedCostFor(a) - estimatedCostFor(b)
)[0];
if (cheapest) selected = cheapest;
}
}
@@ -170,6 +305,7 @@ export function selectProvider(
isExploration,
factors: selected.factors as unknown as Record<string, number>,
excluded,
connectionId: selected.connectionId,
};
}

View File

@@ -23,8 +23,9 @@ export const MODE_PACKS: Record<string, ScoringWeights> = {
tierPriority: 0.05,
tierAffinity: 0,
specificityMatch: 0,
contextAffinity: 0.06,
contextAffinity: 0.01,
resetWindowAffinity: 0,
connectionDensity: 0.05,
},
// Prioritize cost. tierPriority replaces 0.05 from stability.
"cost-saver": {
@@ -37,8 +38,9 @@ export const MODE_PACKS: Record<string, ScoringWeights> = {
tierPriority: 0.05,
tierAffinity: 0,
specificityMatch: 0,
contextAffinity: 0.05,
contextAffinity: 0.0,
resetWindowAffinity: 0,
connectionDensity: 0.05,
},
// Prioritize task fitness. tierPriority replaces 0.05 from latencyInv.
"quality-first": {
@@ -51,8 +53,9 @@ export const MODE_PACKS: Record<string, ScoringWeights> = {
tierPriority: 0.05,
tierAffinity: 0,
specificityMatch: 0,
contextAffinity: 0.05,
contextAffinity: 0.0,
resetWindowAffinity: 0,
connectionDensity: 0.05,
},
// Prioritize quota availability. tierPriority replaces 0.05 from taskFit.
"offline-friendly": {
@@ -65,8 +68,9 @@ export const MODE_PACKS: Record<string, ScoringWeights> = {
tierPriority: 0.05,
tierAffinity: 0,
specificityMatch: 0,
contextAffinity: 0.05,
contextAffinity: 0.0,
resetWindowAffinity: 0,
connectionDensity: 0.05,
},
};

View File

@@ -18,6 +18,7 @@ export interface ScoringFactors {
specificityMatch: number;
contextAffinity: number;
resetWindowAffinity: number;
connectionDensity: number;
}
export interface ScoringWeights {
@@ -32,20 +33,22 @@ export interface ScoringWeights {
specificityMatch: number;
contextAffinity: number;
resetWindowAffinity: number;
connectionDensity: number;
}
export const DEFAULT_WEIGHTS: ScoringWeights = {
quota: 0.16,
quota: 0.15,
health: 0.2,
costInv: 0.16,
costInv: 0.15,
latencyInv: 0.12,
taskFit: 0.08,
stability: 0.05,
tierPriority: 0.05,
tierAffinity: 0.05,
specificityMatch: 0.05,
contextAffinity: 0.08,
contextAffinity: 0.05,
resetWindowAffinity: 0,
connectionDensity: 0.05,
};
export interface ProviderCandidate {
@@ -66,6 +69,8 @@ export interface ProviderCandidate {
contextAffinity?: number;
/** Score [0..1] for quota reset-window preference; sooner selected reset windows score higher. */
resetWindowAffinity?: number;
connectionPoolSize?: number;
connectionId?: string;
}
export interface ScoredProvider {
@@ -73,6 +78,7 @@ export interface ScoredProvider {
model: string;
score: number;
factors: ScoringFactors;
connectionId?: string;
}
/**
@@ -91,7 +97,8 @@ export function calculateScore(factors: ScoringFactors, weights: ScoringWeights)
(weights.tierAffinity ?? 0) * factors.tierAffinity +
(weights.specificityMatch ?? 0) * factors.specificityMatch +
(weights.contextAffinity ?? 0) * factors.contextAffinity +
(weights.resetWindowAffinity ?? 0) * factors.resetWindowAffinity
(weights.resetWindowAffinity ?? 0) * factors.resetWindowAffinity +
(weights.connectionDensity ?? 0) * factors.connectionDensity
);
}
@@ -186,6 +193,7 @@ export function calculateFactors(
specificityMatch: calculateSpecificityMatch(candidate, manifestHint),
contextAffinity: candidate.contextAffinity ?? 0.5,
resetWindowAffinity: candidate.resetWindowAffinity ?? 0.5,
connectionDensity: Math.min(1, Math.max(0, ((candidate.connectionPoolSize ?? 1) - 1) / 10)),
};
}
@@ -204,6 +212,7 @@ export function scorePool(
model: candidate.model,
score: calculateScore(factors, weights),
factors,
connectionId: candidate.connectionId,
};
})
.sort((a, b) => b.score - a.score);

View File

@@ -2243,8 +2243,53 @@ async function buildAutoCandidates(
// keep empty stats — auto-combo will use runtime + bootstrap signals
}
const uniqueProviders = Array.from(
new Set(
targets.map((target) => target.provider || parseModel(target.modelStr).provider || "unknown")
)
);
const connectionPoolCounts = new Map<string, number>();
const connectionsByProvider = new Map<string, Array<Record<string, unknown>>>();
await Promise.all(
uniqueProviders.map(async (provider) => {
try {
const connections = await getProviderConnections({ provider, isActive: true });
const active = Array.isArray(connections) ? connections : [];
connectionPoolCounts.set(provider, active.length);
connectionsByProvider.set(provider, active);
} catch {
connectionPoolCounts.set(provider, 0);
connectionsByProvider.set(provider, []);
}
})
);
const expandedTargets: ResolvedComboTarget[] = [];
for (const target of targets) {
const provider = target.provider || parseModel(target.modelStr).provider || "unknown";
const providerConnections = connectionsByProvider.get(provider) || [];
if (target.connectionId) {
expandedTargets.push(target);
continue;
}
const connectionIds = providerConnections
.map((c) => (c && typeof c === "object" && typeof c.id === "string" ? c.id : null))
.filter((id): id is string => id !== null);
if (connectionIds.length === 0) {
expandedTargets.push(target);
continue;
}
for (const connectionId of connectionIds) {
expandedTargets.push({
...target,
connectionId,
executionKey: `${target.executionKey}@${connectionId}`,
});
}
}
const candidates = await Promise.all(
targets.map(async (target) => {
expandedTargets.map(async (target) => {
const modelStr = target.modelStr;
const parsed = parseModel(modelStr);
const provider = target.provider || parsed.provider || parsed.providerAlias || "unknown";
@@ -2343,6 +2388,8 @@ async function buildAutoCandidates(
quotaResetIntervalSecs: 86400,
contextAffinity,
resetWindowAffinity,
connectionPoolSize: connectionPoolCounts.get(provider) ?? 1,
connectionId: target.connectionId ?? undefined,
};
})
);

View File

@@ -0,0 +1,230 @@
/**
* Tests for ScoreTierRotator and connectionDensity factor.
* Verifies that multi-connection providers surface in ranked candidates
* and that tiered rotation distributes traffic fairly.
*/
import { describe, it, expect, beforeEach, afterEach } from "vitest";
import { selectProvider, type AutoComboConfig } from "../../../open-sse/services/autoCombo/engine";
import {
calculateFactors,
calculateScore,
DEFAULT_WEIGHTS,
scorePool,
type ProviderCandidate,
type ScoredProvider,
} from "../../../open-sse/services/autoCombo/scoring";
import { getTaskFitness } from "../../../open-sse/services/autoCombo/taskFitness";
import { resetDiversity } from "../../../open-sse/services/autoCombo/providerDiversity";
function makeCandidate(overrides: Partial<ProviderCandidate>): ProviderCandidate {
return {
provider: "unknown",
model: "unknown-model",
quotaRemaining: 100,
quotaTotal: 100,
circuitBreakerState: "CLOSED",
costPer1MTokens: 1,
p95LatencyMs: 1000,
latencyStdDev: 100,
errorRate: 0.01,
...overrides,
};
}
function makeConfig(name: string): AutoComboConfig {
return {
id: `test-${name}`,
name,
type: "auto",
candidatePool: [],
weights: { ...DEFAULT_WEIGHTS },
explorationRate: 0,
routerStrategy: "rules",
};
}
describe("Connection Density Factor", () => {
const baseCandidate = makeCandidate({ provider: "cerebras", model: "llama-70b" });
it("multi-connection provider scores higher than single-connection at same quality", () => {
const multiConn = makeCandidate({ provider: "cerebras", model: "llama-70b", connectionPoolSize: 43 });
const singleConn = makeCandidate({ provider: "anthropic", model: "claude-sonnet", connectionPoolSize: 1 });
const pool = [multiConn, singleConn];
const multiFactors = calculateFactors(multiConn, pool, "coding", getTaskFitness);
const singleFactors = calculateFactors(singleConn, pool, "coding", getTaskFitness);
const multiScore = calculateScore(multiFactors, DEFAULT_WEIGHTS);
const singleScore = calculateScore(singleFactors, DEFAULT_WEIGHTS);
expect(multiFactors.connectionDensity).toBe(1.0);
expect(singleFactors.connectionDensity).toBe(0.0);
expect(multiScore).toBeGreaterThan(singleScore);
});
it("density scales linearly from 0 to 10 connections, caps at 10+", () => {
const make = (size: number) => makeCandidate({ connectionPoolSize: size });
const sizes = [1, 2, 5, 10, 20, 43];
const densities = sizes.map((s) => {
const c = make(s);
const pool = [c];
return calculateFactors(c, pool, "coding", getTaskFitness).connectionDensity;
});
expect(densities[0]).toBeCloseTo(0.0, 5);
expect(densities[1]).toBeCloseTo(0.1, 5);
expect(densities[2]).toBeCloseTo(0.4, 5);
expect(densities[3]).toBeCloseTo(0.9, 5);
expect(densities[4]).toBe(1.0);
expect(densities[5]).toBe(1.0);
});
it("missing connectionPoolSize defaults to 1 (backward compat)", () => {
const candidate = makeCandidate({ provider: "x" });
const pool = [candidate];
const factors = calculateFactors(candidate, pool, "coding", getTaskFitness);
expect(factors.connectionDensity).toBe(0.0);
});
it("DEFAULT_WEIGHTS still sum to 1.0 after adding density", () => {
const sum = Object.values(DEFAULT_WEIGHTS).reduce((a, b) => a + b, 0);
expect(Math.abs(sum - 1.0)).toBeLessThan(0.01);
});
});
describe("Tiered Rotation in selectProvider", () => {
beforeEach(() => {
resetDiversity();
});
it("smart combo rotates within top tier across many requests", () => {
const topA = makeCandidate({ provider: "openai", model: "gpt-4o", quotaRemaining: 95 });
const topB = makeCandidate({ provider: "anthropic", model: "claude-opus", quotaRemaining: 90 });
const topC = makeCandidate({ provider: "google", model: "gemini-ultra", quotaRemaining: 88 });
const mid = makeCandidate({ provider: "mistral", model: "mistral-large", quotaRemaining: 70 });
const pool = [topA, topB, topC, mid];
const config = makeConfig("smart");
const seen = new Set<string>();
for (let i = 0; i < 50; i++) {
const result = selectProvider(config, pool, "coding");
seen.add(`${result.provider}/${result.model}`);
}
expect(seen.size).toBeGreaterThanOrEqual(2);
expect(seen.has("openai/gpt-4o") || seen.has("anthropic/claude-opus")).toBe(true);
});
it("cheap combo pulls from rest tier (lower scores) more often than smart", () => {
const top = makeCandidate({ provider: "openai", model: "gpt-4o", quotaRemaining: 100 });
const rest = makeCandidate({
provider: "cheap-provider",
model: "cheap-model",
quotaRemaining: 100,
costPer1MTokens: 0,
p95LatencyMs: 5000,
});
const pool = [top, rest];
const config = makeConfig("cheap");
const counts: Record<string, number> = {};
for (let i = 0; i < 200; i++) {
const result = selectProvider(config, pool, "coding");
counts[result.provider] = (counts[result.provider] ?? 0) + 1;
}
expect(counts["cheap-provider"]).toBeGreaterThan(0);
});
it("single-candidate pool always returns the same candidate", () => {
const only = makeCandidate({ provider: "only", model: "only-model" });
const config = makeConfig("smart");
for (let i = 0; i < 10; i++) {
const result = selectProvider(config, [only], "coding");
expect(result.provider).toBe("only");
expect(result.model).toBe("only-model");
}
});
});
describe("scorePool with connectionDensity", () => {
it("Cerebras with 43 keys ranks above single-connection providers of similar quality", () => {
const cerebras = makeCandidate({
provider: "cerebras",
model: "llama-3.1-70b",
connectionPoolSize: 43,
quotaRemaining: 100,
});
const anthropic = makeCandidate({
provider: "anthropic",
model: "claude-sonnet",
connectionPoolSize: 1,
quotaRemaining: 100,
});
const pool = [cerebras, anthropic];
const scored = scorePool(pool, "coding", DEFAULT_WEIGHTS, getTaskFitness);
expect(scored[0].provider).toBe("cerebras");
});
});
describe("Per-Connection Rotation", () => {
it("rotates across all 43 Cerebras connection IDs, not just one", () => {
const cerebrasCandidates: ProviderCandidate[] = Array.from({ length: 43 }, (_, i) =>
makeCandidate({
provider: "cerebras",
model: "llama-3.1-70b",
connectionId: `cerebras-conn-${i + 1}`,
})
);
const config = makeConfig("smart");
const seenConnections = new Set<string>();
for (let i = 0; i < 200; i++) {
const result = selectProvider(config, cerebrasCandidates, "coding");
if (result.connectionId) seenConnections.add(result.connectionId);
}
expect(seenConnections.size).toBeGreaterThanOrEqual(10);
});
it("different combos maintain independent round-robin state", () => {
const candidates: ProviderCandidate[] = Array.from({ length: 5 }, (_, i) =>
makeCandidate({ provider: "p", model: "m", connectionId: `c-${i}` })
);
const smartConfig = makeConfig("smart-A");
const fastConfig = makeConfig("fast-B");
for (let i = 0; i < 5; i++) {
selectProvider(smartConfig, candidates, "coding");
}
const smartResults: string[] = [];
for (let i = 0; i < 5; i++) {
const r = selectProvider(smartConfig, candidates, "coding");
if (r.connectionId) smartResults.push(r.connectionId);
}
for (let i = 0; i < 5; i++) {
selectProvider(fastConfig, candidates, "coding");
}
const fastResults: string[] = [];
for (let i = 0; i < 5; i++) {
const r = selectProvider(fastConfig, candidates, "coding");
if (r.connectionId) fastResults.push(r.connectionId);
}
expect(smartResults.length).toBe(5);
expect(fastResults.length).toBe(5);
expect(new Set(smartResults).size).toBeGreaterThan(1);
expect(new Set(fastResults).size).toBeGreaterThan(1);
});
it("tied-score candidates from same provider+model are all reachable", () => {
const candidates: ProviderCandidate[] = Array.from({ length: 5 }, (_, i) =>
makeCandidate({ provider: "free", model: "free-model", connectionId: `key-${i}` })
);
const config = makeConfig("smart");
const visited = new Set<string>();
for (let i = 0; i < 20; i++) {
const result = selectProvider(config, candidates, "coding");
if (result.connectionId) visited.add(result.connectionId);
}
expect(visited.size).toBeGreaterThan(1);
});
});

View File

@@ -12,6 +12,7 @@ export default defineConfig({
include: [
"open-sse/mcp-server/__tests__/**/*.test.ts",
"open-sse/services/autoCombo/__tests__/**/*.test.ts",
"tests/unit/autoCombo/**/*.test.ts",
"tests/unit/encryption.spec.ts",
"src/shared/components/**/*.test.tsx",
"src/shared/hooks/__tests__/**/*.test.tsx",