diff --git a/open-sse/services/autoCombo/engine.ts b/open-sse/services/autoCombo/engine.ts index 6d238fc98f..9e91552f36 100644 --- a/open-sse/services/autoCombo/engine.ts +++ b/open-sse/services/autoCombo/engine.ts @@ -11,8 +11,8 @@ import { scorePool, - validateWeights, DEFAULT_WEIGHTS, + normalizeScoringWeights, type ScoringWeights, type ProviderCandidate, type ScoredProvider, @@ -110,12 +110,15 @@ class ScoreTierRotator { const tiers = groupIntoTiers(candidates); const best = candidates[0].score; const worst = candidates[candidates.length - 1].score; - if (tiers.top.length > 0 && (best - worst) >= CLEAR_WINNER_THRESHOLD) { + if (tiers.top.length > 0 && best - worst >= CLEAR_WINNER_THRESHOLD) { return this.pickFromPool(tiers.top); } const prefs = tierPreferencesForName(this.comboName); - const chosen = chooseTierWeighted(tiers, prefs, (pool) => this.pickFromPool(pool), () => - this.advance(tiers, prefs, candidates) + const chosen = chooseTierWeighted( + tiers, + prefs, + (pool) => this.pickFromPool(pool), + () => this.advance(tiers, prefs, candidates) ); return chosen; } @@ -248,7 +251,7 @@ export function selectProvider( const pack = getModePack(config.modePack); if (pack) weights = pack; } - if (!validateWeights(weights)) weights = DEFAULT_WEIGHTS; + weights = normalizeScoringWeights(weights); // Filter out excluded providers const excluded: string[] = []; @@ -322,7 +325,10 @@ export function selectProvider( (a, b) => estimatedCostFor(a) - estimatedCostFor(b) )[0]; if (config.budgetFallback === "strict") { - throw new BudgetExceededError(config.budgetCap, cheapest ? estimatedCostFor(cheapest) : 0); + throw new BudgetExceededError( + config.budgetCap, + cheapest ? estimatedCostFor(cheapest) : 0 + ); } if (cheapest) selected = cheapest; } diff --git a/open-sse/services/autoCombo/routerStrategy.ts b/open-sse/services/autoCombo/routerStrategy.ts index b9923fe470..9bf207a28a 100644 --- a/open-sse/services/autoCombo/routerStrategy.ts +++ b/open-sse/services/autoCombo/routerStrategy.ts @@ -41,6 +41,7 @@ export interface RoutingDecision { reason: string; candidatesConsidered: number; finalScore: number; + connectionId?: string; } export interface RouterStrategy { @@ -103,6 +104,7 @@ class RulesStrategyImpl implements RouterStrategy { reason: `RulesStrategy: score=${best.score.toFixed(3)} (quota=${best.factors.quota.toFixed(2)}, health=${best.factors.health.toFixed(2)}, cost=${best.factors.costInv.toFixed(2)}, taskFit=${best.factors.taskFit.toFixed(2)})`, candidatesConsidered: ranked.length, finalScore: best.score, + connectionId: best.connectionId, }; } } diff --git a/open-sse/services/autoCombo/scoring.ts b/open-sse/services/autoCombo/scoring.ts index fccffa0191..ed410c36be 100644 --- a/open-sse/services/autoCombo/scoring.ts +++ b/open-sse/services/autoCombo/scoring.ts @@ -19,6 +19,7 @@ export interface ScoringFactors { tierAffinity: number; specificityMatch: number; contextAffinity: number; + cacheAffinity?: number; resetWindowAffinity: number; connectionDensity: number; } @@ -34,6 +35,7 @@ export interface ScoringWeights { tierAffinity: number; specificityMatch: number; contextAffinity: number; + cacheAffinity?: number; resetWindowAffinity: number; connectionDensity: number; } @@ -49,10 +51,30 @@ export const DEFAULT_WEIGHTS: ScoringWeights = { tierAffinity: 0.05, specificityMatch: 0.05, contextAffinity: 0.05, + cacheAffinity: 0, resetWindowAffinity: 0, connectionDensity: 0.05, }; +/** Normalize independently configured UI weights into a scoring distribution. */ +export function normalizeScoringWeights( + weights: Partial | null | undefined +): ScoringWeights { + if (!weights) return { ...DEFAULT_WEIGHTS }; + const entries = Object.keys(DEFAULT_WEIGHTS) as Array; + const sanitized = Object.fromEntries( + entries.map((key) => { + const value = Number(weights?.[key]); + return [key, Number.isFinite(value) && value >= 0 ? value : 0]; + }) + ) as unknown as ScoringWeights; + const total = entries.reduce((sum, key) => sum + Number(sanitized[key] ?? 0), 0); + if (total <= 0) return { ...DEFAULT_WEIGHTS }; + return Object.fromEntries( + entries.map((key) => [key, Number(sanitized[key] ?? 0) / total]) + ) as unknown as ScoringWeights; +} + export interface ProviderCandidate { provider: string; model: string; @@ -77,6 +99,8 @@ export interface ProviderCandidate { quotaResetIntervalSecs?: number; /** Score [0..1] for staying on the current session's provider/account/model path. */ contextAffinity?: number; + /** Score [0..1] for the account selected by the stable prompt-cache key. */ + cacheAffinity?: number; /** Score [0..1] for quota reset-window preference; sooner selected reset windows score higher. */ resetWindowAffinity?: number; connectionPoolSize?: number; @@ -110,6 +134,7 @@ export function calculateScore(factors: ScoringFactors, weights: ScoringWeights) (weights.tierAffinity ?? 0) * factors.tierAffinity + (weights.specificityMatch ?? 0) * factors.specificityMatch + (weights.contextAffinity ?? 0) * factors.contextAffinity + + (weights.cacheAffinity ?? 0) * (factors.cacheAffinity ?? 0) + (weights.resetWindowAffinity ?? 0) * factors.resetWindowAffinity + (weights.connectionDensity ?? 0) * factors.connectionDensity ); @@ -207,6 +232,7 @@ export function calculateFactors( tierAffinity: calculateTierAffinity(candidate, manifestHint), specificityMatch: calculateSpecificityMatch(candidate, manifestHint), contextAffinity: clamp01(candidate.contextAffinity ?? 0.5), + cacheAffinity: clamp01(candidate.cacheAffinity ?? 0), resetWindowAffinity: clamp01(candidate.resetWindowAffinity ?? 0.5), connectionDensity: clamp01(((candidate.connectionPoolSize ?? 1) - 1) / 10), }; diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index 6fd60b4cf6..1a4b88de30 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -83,6 +83,12 @@ import { selectQuotaShareTarget } from "./combo/quotaShareStrategy.ts"; import { makeConnectionConcurrencyResolver, lookupPositiveCap } from "./combo/concurrencyCaps.ts"; import { acquireQuotaShareConcurrencySlot } from "./combo/quotaShareConcurrency.ts"; import { orderTargetsByEvalScores } from "./evalRouting.ts"; +import { + applyPromptCacheAffinity, + expandPromptCacheAffinityTargets, + expandPromptCacheAffinityTargetsFromConnections, + resolvePromptCacheAffinityKey, +} from "./combo/promptCacheAffinity.ts"; import type { CompressionMode } from "./compression/types.ts"; import { getCachedProviderConnections } from "../../src/lib/db/readCache"; import { @@ -387,7 +393,10 @@ export async function buildAutoCandidates( await Promise.all( uniqueProviders.map(async (provider) => { try { - const connections = (await getCachedProviderConnections({ provider, isActive: true })) as Array>; + const connections = (await getCachedProviderConnections({ + provider, + isActive: true, + })) as Array>; const active = Array.isArray(connections) ? connections : []; connectionPoolCounts.set(provider, active.length); connectionsByProvider.set(provider, active); @@ -403,40 +412,10 @@ export async function buildAutoCandidates( }) ); - const expandedTargets: ResolvedComboTarget[] = []; - for (const target of targets) { - const provider = target.provider || parseModel(target.modelStr).provider || "unknown"; - const providerConnections = connectionsByProvider.get(provider) || []; - if (target.connectionId) { - expandedTargets.push(target); - continue; - } - const connectionIds = providerConnections - .map((c) => (c && typeof c === "object" && typeof c.id === "string" ? c.id : null)) - .filter((id): id is string => id !== null); - const allowedConnectionIds = Array.isArray(target.allowedConnectionIds) - ? new Set( - target.allowedConnectionIds.filter( - (connectionId): connectionId is string => - typeof connectionId === "string" && connectionId.trim().length > 0 - ) - ) - : null; - const scopedConnectionIds = allowedConnectionIds - ? connectionIds.filter((connectionId) => allowedConnectionIds.has(connectionId)) - : connectionIds; - if (scopedConnectionIds.length === 0) { - expandedTargets.push(target); - continue; - } - for (const connectionId of scopedConnectionIds) { - expandedTargets.push({ - ...target, - connectionId, - executionKey: `${target.executionKey}@${connectionId}`, - }); - } - } + const expandedTargets = expandPromptCacheAffinityTargetsFromConnections( + targets, + connectionsByProvider + ); // #5521: Expand fingerprint-based providers (mimocode, mcode, opencode) so each // fingerprint gets its own combo slot instead of being bundled into one connection. @@ -1298,14 +1277,21 @@ export async function handleComboChat({ apiKeyAllowedConnections, }); } + // An explicit cache-optimized combo outranks the global cache-affinity default, + // but only protects its ordering when this request actually produced a reusable + // cache key. Cache misses retain the normal session/eval routing behavior. + const cacheStrategyAffinityApplied = + strategy === "cache-optimized" && applyPromptCacheAffinity(orderedTargets, body).applied; // #6168: session stickiness opt-out. Per-combo `config.disableSessionStickiness` // overrides the global `settings.disableSessionStickiness` fallback (default false, // preserving the #3825 prompt-cache/504 fix). When disabled, skip the reorder and // treat the result as a no-op so the recordStickyBinding write-back below is skipped. - const disableSessionStickiness = resolveDisableSessionStickiness( - config as Record | null | undefined, - settings as Record | null | undefined - ); + const disableSessionStickiness = + cacheStrategyAffinityApplied || + resolveDisableSessionStickiness( + config as Record | null | undefined, + settings as Record | null | undefined + ); const _sticky = disableSessionStickiness ? ({ targets: orderedTargets, messageHash: null, stuck: false } as const) : await applySessionStickiness( @@ -1315,7 +1301,9 @@ export async function handleComboChat({ normalizeStickinessMessages(body as { messages?: unknown; input?: unknown }) ); orderedTargets = _sticky.targets; - orderedTargets = orderTargetsByEvalScores(orderedTargets, config.evalRouting, log); + if (!cacheStrategyAffinityApplied) { + orderedTargets = orderTargetsByEvalScores(orderedTargets, config.evalRouting, log); + } orderedTargets = filterTargetsByRequestCompatibility(orderedTargets, body, log); orderedTargets = applyContextRequirements(orderedTargets, config.contextRequirements, log); @@ -1347,6 +1335,62 @@ export async function handleComboChat({ orderedTargets = nextOrder; } + // Prompt-cache locality is applied after request eligibility and task routing. + // Session stickiness and explicit auto-router pins remain stronger continuity + // decisions; quota, health, and circuit-breaker gates still run per attempt. + const autoConfigForCacheWeight = + strategy === "auto" + ? ((combo.autoConfig || + ((config as Record).auto && + typeof (config as Record).auto === "object" + ? (config as Record).auto + : null) || + config) as Record) + : null; + const autoWeightsForCache = + autoConfigForCacheWeight?.weights && typeof autoConfigForCacheWeight.weights === "object" + ? (autoConfigForCacheWeight.weights as Record) + : null; + const autoUsesCacheScore = Number(autoWeightsForCache?.cacheAffinity) > 0; + const promptCacheAffinityEnabled = + settings?.promptCacheAffinityEnabled !== false && !autoUsesCacheScore; + const promptCacheAffinityTargets = + promptCacheAffinityEnabled && resolvePromptCacheAffinityKey(body) + ? await expandPromptCacheAffinityTargets(orderedTargets) + : orderedTargets; + const promptCacheAffinity = applyPromptCacheAffinity( + promptCacheAffinityTargets, + body, + promptCacheAffinityEnabled + ); + if (promptCacheAffinity.applied) { + const protectedOriginal = + (_sticky.stuck || + autoUsedExplicitRouter || + strategy === "quota-share" || + strategy === "weighted") && + orderedTargets[0]; + const protectedFirst = protectedOriginal + ? (promptCacheAffinity.targets.find( + (target) => + target === protectedOriginal || + target.executionKey === protectedOriginal.executionKey || + target.executionKey.startsWith(`${protectedOriginal.executionKey}@`) + ) ?? protectedOriginal) + : null; + orderedTargets = protectedFirst + ? [ + protectedFirst, + ...promptCacheAffinity.targets.filter((target) => target !== protectedFirst), + ] + : promptCacheAffinity.targets; + log.debug?.("COMBO", "Prompt-cache affinity applied", { + source: promptCacheAffinity.source, + fingerprint: promptCacheAffinity.fingerprint, + targetCount: orderedTargets.length, + }); + } + // Parallel pre-screen: check provider profiles and model availability for all targets // Only runs for priority strategy where sequential checking causes latency const preScreenMap = @@ -2251,7 +2295,11 @@ export async function handleComboChat({ }); recordedAttempts++; lastError = errorText || String(result.status); - comboErrors.push({ model: modelStr, status: result.status, error: errorText || String(result.status) }); + comboErrors.push({ + model: modelStr, + status: result.status, + error: errorText || String(result.status), + }); if (!lastStatus) lastStatus = result.status; if (i > 0) fallbackCount++; log.warn("COMBO", `Model ${modelStr} failed with body-specific error, stopping combo`); @@ -2345,7 +2393,11 @@ export async function handleComboChat({ }); recordedAttempts++; lastError = errorText || String(result.status); - comboErrors.push({ model: modelStr, status: result.status, error: errorText || String(result.status) }); + comboErrors.push({ + model: modelStr, + status: result.status, + error: errorText || String(result.status), + }); if (!lastStatus) lastStatus = result.status; if (i > 0) fallbackCount++; // Wire combo failures into the resilience dashboard (model-level lockout) @@ -2510,12 +2562,10 @@ export async function handleComboChat({ latencyMs, fallbackCount, }); - return errorResponseWithComboDiagnostics( - 504, - msg, - buildComboDiag("combo_timeout"), - { code: "COMBO_TIMEOUT", type: "server_error" } - ); + return errorResponseWithComboDiagnostics(504, msg, buildComboDiag("combo_timeout"), { + code: "COMBO_TIMEOUT", + type: "server_error", + }); } // All models failed in this set try @@ -2770,7 +2820,7 @@ async function handleRoundRobinCombo({ { code: "context_length_exceeded", type: "invalid_request_error" } ); } - const filteredTargets = filterTargetsByRequestCompatibility( + let filteredTargets = filterTargetsByRequestCompatibility( evalRankedTargets, body, log, @@ -2783,7 +2833,7 @@ async function handleRoundRobinCombo({ // permanently dropping a compat-rejected-but-healthy provider. const compatKeptSet = new Set(filteredTargets); const compatRejectedTargets = evalRankedTargets.filter((target) => !compatKeptSet.has(target)); - const modelCount = filteredTargets.length; + let modelCount = filteredTargets.length; if (modelCount === 0) { return comboModelNotFoundResponse("Round-robin combo has no executable targets"); } @@ -2889,6 +2939,11 @@ async function handleRoundRobinCombo({ config as Record | null | undefined, settings as Record | null | undefined ); + const rrAffinityEnabled = settings?.promptCacheAffinityEnabled !== false; + if (rrAffinityEnabled && resolvePromptCacheAffinityKey(body)) { + filteredTargets = await expandPromptCacheAffinityTargets(filteredTargets); + modelCount = filteredTargets.length; + } const _rrSessionSticky = disableSessionStickiness ? ({ targets: filteredTargets, messageHash: null, stuck: false } as const) : await applySessionStickiness( @@ -2897,7 +2952,22 @@ async function handleRoundRobinCombo({ // stickiness engages on the /v1/responses surface, not just Chat Completions. normalizeStickinessMessages(body as { messages?: unknown; input?: unknown }) ); + const rrAffinity = applyPromptCacheAffinity(filteredTargets, body, rrAffinityEnabled); + if (rrAffinity.applied) { + const stickyFirst = _rrSessionSticky.stuck ? _rrSessionSticky.targets[0] : null; + filteredTargets = stickyFirst + ? [stickyFirst, ...rrAffinity.targets.filter((target) => target !== stickyFirst)] + : rrAffinity.targets; + log.debug?.("COMBO-RR", "Prompt-cache affinity applied", { + source: rrAffinity.source, + fingerprint: rrAffinity.fingerprint, + targetCount: filteredTargets.length, + }); + } let rrStartIndex = startIndex; + if (rrAffinity.applied) { + rrStartIndex = 0; + } if (_rrSessionSticky.stuck) { const stickyIdx = filteredTargets.findIndex( (t) => t.connectionId === _rrSessionSticky.targets[0]?.connectionId diff --git a/open-sse/services/combo/applyStrategyOrdering.ts b/open-sse/services/combo/applyStrategyOrdering.ts index e8cb39ecf8..a2eba3a555 100644 --- a/open-sse/services/combo/applyStrategyOrdering.ts +++ b/open-sse/services/combo/applyStrategyOrdering.ts @@ -3,6 +3,11 @@ import { generateRoutingHints } from "../manifestAdapter"; import { resolveMaxConcurrentByConnection } from "./concurrencyCaps.ts"; import { sortTargetsByContextSize } from "./comboStructure.ts"; import { selectQuotaShareTarget } from "./quotaShareStrategy.ts"; +import { + applyPromptCacheAffinity, + expandPromptCacheAffinityTargets, + resolvePromptCacheAffinityKey, +} from "./promptCacheAffinity.ts"; import { orderTargetsByHeadroom, orderTargetsByResetAwareQuota, @@ -196,6 +201,16 @@ export async function applyStrategyOrdering( } else if (strategy === "context-optimized") { orderedTargets = sortTargetsByContextSize(orderedTargets); log.info("COMBO", `Context-optimized ordering: largest first (${orderedTargets[0]?.modelStr})`); + } else if (strategy === "cache-optimized") { + if (resolvePromptCacheAffinityKey(body)) { + orderedTargets = await expandPromptCacheAffinityTargets(orderedTargets); + } + const affinity = applyPromptCacheAffinity(orderedTargets, body); + orderedTargets = affinity.targets; + log.info( + "COMBO", + `Cache-optimized ordering: ${orderedTargets[0]?.modelStr}${orderedTargets[0]?.connectionId ? ` (${orderedTargets[0].connectionId})` : ""} first` + ); } else if (strategy === "headroom") { orderedTargets = await orderTargetsByHeadroom( orderedTargets, diff --git a/open-sse/services/combo/autoConfig.ts b/open-sse/services/combo/autoConfig.ts index ed56f11989..3d5cf8ffda 100644 --- a/open-sse/services/combo/autoConfig.ts +++ b/open-sse/services/combo/autoConfig.ts @@ -1,4 +1,8 @@ -import { DEFAULT_WEIGHTS, type ScoringWeights } from "../autoCombo/scoring.ts"; +import { + DEFAULT_WEIGHTS, + normalizeScoringWeights, + type ScoringWeights, +} from "../autoCombo/scoring.ts"; import { getModePack } from "../autoCombo/modePacks.ts"; import { isRecord } from "./comboData.ts"; import { resolveResetWindowConfig, resolveSlaRoutingPolicy } from "./quotaScoring.ts"; @@ -53,7 +57,9 @@ export function parseAutoConfig(combo: ComboLike, eligibleTargets: ResolvedCombo : undefined; const modePack = typeof autoConfigSource.modePack === "string" ? autoConfigSource.modePack : undefined; - const weights = modePack ? getModePack(modePack) || configuredWeights : configuredWeights; + const weights = normalizeScoringWeights( + modePack ? getModePack(modePack) || configuredWeights : configuredWeights + ); const resetWindowConfig = resolveResetWindowConfig(autoConfigSource); const slaPolicy = resolveSlaRoutingPolicy(autoConfigSource); diff --git a/open-sse/services/combo/promptCacheAffinity.ts b/open-sse/services/combo/promptCacheAffinity.ts new file mode 100644 index 0000000000..fdfe9e1352 --- /dev/null +++ b/open-sse/services/combo/promptCacheAffinity.ts @@ -0,0 +1,280 @@ +import { createHash } from "node:crypto"; +import { + analyzePrefix, + generatePromptCacheKey, +} from "../../../src/lib/promptCache/prefixAnalyzer.ts"; +import { getCachedProviderConnections } from "../../../src/lib/db/readCache"; +import { parseModel } from "../model.ts"; +import type { ResolvedComboTarget } from "./types.ts"; + +interface PromptCacheAffinityTarget { + executionKey: string; + connectionId?: string | null; +} + +export type PromptCacheAffinitySource = "explicit" | "prefix"; + +export interface PromptCacheAffinityResolution { + key: string; + source: PromptCacheAffinitySource; + fingerprint: string; +} + +export interface PromptCacheAffinityResult { + targets: ResolvedComboTarget[]; + applied: boolean; + source: PromptCacheAffinitySource | null; + fingerprint: string | null; +} + +function asRecord(value: unknown): Record | null { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : null; +} + +function normalizeMessageContent(value: unknown): string | unknown[] { + if (typeof value === "string" || Array.isArray(value)) return value; + try { + return JSON.stringify(value) || ""; + } catch { + return ""; + } +} + +function normalizeResponsesInput(body: Record): Array<{ + role: string; + content: string | unknown[]; +}> | null { + if (Array.isArray(body.messages) && body.messages.length > 0) { + return body.messages + .map((item) => { + const record = asRecord(item); + return record && typeof record.role === "string" + ? { role: record.role, content: normalizeMessageContent(record.content) } + : null; + }) + .filter((item): item is { role: string; content: string | unknown[] } => item !== null); + } + + if (typeof body.input === "string" && body.input.length > 0) { + return [{ role: "user", content: body.input }]; + } + + if (Array.isArray(body.input) && body.input.length > 0) { + return body.input + .map((item) => { + if (typeof item === "string") return { role: "user", content: item }; + const record = asRecord(item); + return record && typeof record.role === "string" + ? { role: record.role, content: normalizeMessageContent(record.content) } + : null; + }) + .filter((item): item is { role: string; content: string | unknown[] } => item !== null); + } + + return null; +} + +function readExplicitPromptCacheKey(body: Record): string | null { + const metadata = asRecord(body.metadata); + for (const value of [body.prompt_cache_key, metadata?.prompt_cache_key]) { + if (typeof value !== "string") continue; + const normalized = value.trim(); + if (normalized.length > 0 && normalized.length <= 4096) return normalized; + } + return null; +} + +/** + * Resolve a cache affinity key without exposing the key itself to callers that + * only need diagnostics. Explicit provider keys win; otherwise the existing + * prompt-prefix analyzer supplies a safe deterministic fallback. + */ +export function resolvePromptCacheAffinityKey( + body: Record | null | undefined +): PromptCacheAffinityResolution | null { + if (!body) return null; + + const explicit = readExplicitPromptCacheKey(body); + const messages = normalizeResponsesInput(body); + const prefixAnalysis = messages ? analyzePrefix(messages) : null; + // The analyzer intentionally returns a legacy empty-content hash for callers + // that need that historical value. Affinity must not use it: a request with + // only a first user turn has no reusable prompt prefix and would collapse + // otherwise distinct conversations onto one account. + const prefixKey = + prefixAnalysis && prefixAnalysis.prefixEndIdx >= 0 + ? generatePromptCacheKey(messages || []) + : ""; + const key = explicit ?? prefixKey; + if (!key) return null; + + const source: PromptCacheAffinitySource = explicit ? "explicit" : "prefix"; + const fingerprint = createHash("sha256").update(key).digest("hex").slice(0, 12); + return { key, source, fingerprint }; +} + +export function promptCacheTargetIdentity(target: PromptCacheAffinityTarget): string { + const connectionId = typeof target.connectionId === "string" ? target.connectionId.trim() : ""; + if (connectionId) return `connection:${connectionId}`; + return `execution:${target.executionKey}`; +} + +function rendezvousScore(key: string, identity: string): bigint { + const digest = createHash("sha256").update(key).update("\0").update(identity).digest("hex"); + return BigInt(`0x${digest.slice(0, 32)}`); +} + +/** + * Return a normalized cache-locality score for auto-combo scoring. The target + * selected by rendezvous hashing receives 1; all other accounts receive 0. + * Reusing the same prompt key therefore keeps selecting the same account. + */ +export function calculatePromptCacheAffinityScores( + targets: PromptCacheAffinityTarget[], + body: Record | null | undefined +): Map { + const resolution = resolvePromptCacheAffinityKey(body); + if (!resolution || targets.length === 0) return new Map(); + let winnerIdentity = ""; + let winnerScore = -1n; + for (const target of targets) { + const identity = promptCacheTargetIdentity(target); + const score = rendezvousScore(resolution.key, identity); + if (score > winnerScore || (score === winnerScore && identity < winnerIdentity)) { + winnerIdentity = identity; + winnerScore = score; + } + } + return new Map( + targets.map((target) => { + const identity = promptCacheTargetIdentity(target); + return [identity, identity === winnerIdentity ? 1 : 0]; + }) + ); +} + +/** + * Bind unscoped combo targets to concrete active provider accounts before + * rendezvous hashing. This keeps the selected cache identity identical to the + * account that credential resolution will execute, while preserving the + * original target as a fail-open fallback when no eligible account is known. + */ +export async function expandPromptCacheAffinityTargets( + targets: ResolvedComboTarget[] +): Promise { + const providers = Array.from( + new Set( + targets + .filter((target) => !target.connectionId) + .map( + (target) => + target.provider || + parseModel(target.modelStr).provider || + parseModel(target.modelStr).providerAlias || + "unknown" + ) + ) + ); + const connectionsByProvider = new Map>>(); + await Promise.all( + providers.map(async (provider) => { + try { + const connections = (await getCachedProviderConnections({ + provider, + isActive: true, + })) as Array>; + connectionsByProvider.set(provider, Array.isArray(connections) ? connections : []); + } catch { + connectionsByProvider.set(provider, []); + } + }) + ); + return expandPromptCacheAffinityTargetsFromConnections(targets, connectionsByProvider); +} + +export function expandPromptCacheAffinityTargetsFromConnections( + targets: ResolvedComboTarget[], + connectionsByProvider: Map>> +): ResolvedComboTarget[] { + const expandedTargets: ResolvedComboTarget[] = []; + for (const target of targets) { + if (target.connectionId) { + expandedTargets.push(target); + continue; + } + const parsed = parseModel(target.modelStr); + const provider = target.provider || parsed.provider || parsed.providerAlias || "unknown"; + const connectionIds = (connectionsByProvider.get(provider) || []) + .map((connection) => + connection && typeof connection.id === "string" ? connection.id.trim() : "" + ) + .filter((connectionId) => connectionId.length > 0); + const allowedConnectionIds = Array.isArray(target.allowedConnectionIds) + ? new Set( + target.allowedConnectionIds.filter( + (connectionId): connectionId is string => + typeof connectionId === "string" && connectionId.trim().length > 0 + ) + ) + : null; + const scopedConnectionIds = allowedConnectionIds + ? connectionIds.filter((connectionId) => allowedConnectionIds.has(connectionId)) + : connectionIds; + if (scopedConnectionIds.length === 0) { + expandedTargets.push(target); + continue; + } + for (const connectionId of scopedConnectionIds) { + expandedTargets.push({ + ...target, + connectionId, + executionKey: `${target.executionKey}@${connectionId}`, + }); + } + } + return expandedTargets; +} + +/** + * Order eligible targets using rendezvous hashing. The original order is used + * as the final tie-breaker, so targets sharing one account identity remain + * stable without using modelStr as the affinity identity. + */ +export function applyPromptCacheAffinity( + targets: ResolvedComboTarget[], + body: Record | null | undefined, + enabled: boolean = true +): PromptCacheAffinityResult { + const resolution = enabled ? resolvePromptCacheAffinityKey(body) : null; + if (!resolution || targets.length <= 1) { + return { + targets, + applied: false, + source: resolution?.source ?? null, + fingerprint: resolution?.fingerprint ?? null, + }; + } + + const ranked = targets.map((target, index) => ({ + target, + index, + identity: promptCacheTargetIdentity(target), + score: rendezvousScore(resolution.key, promptCacheTargetIdentity(target)), + })); + + ranked.sort((a, b) => { + if (a.score > b.score) return -1; + if (a.score < b.score) return 1; + const identityOrder = a.identity.localeCompare(b.identity); + return identityOrder !== 0 ? identityOrder : a.index - b.index; + }); + + return { + targets: ranked.map((entry) => entry.target), + applied: true, + source: resolution.source, + fingerprint: resolution.fingerprint, + }; +} diff --git a/open-sse/services/combo/resolveAutoStrategy.ts b/open-sse/services/combo/resolveAutoStrategy.ts index 9d38801b35..c0d05cf650 100644 --- a/open-sse/services/combo/resolveAutoStrategy.ts +++ b/open-sse/services/combo/resolveAutoStrategy.ts @@ -1,8 +1,5 @@ import { errorResponse, unavailableResponse } from "../../utils/error.ts"; -import { - BudgetExceededError, - selectProvider as selectAutoProvider, -} from "../autoCombo/engine.ts"; +import { BudgetExceededError, selectProvider as selectAutoProvider } from "../autoCombo/engine.ts"; import { resolveRequestModePack, parseRequestBudgetCap, @@ -21,6 +18,10 @@ import type { ResilienceSettings } from "../../../src/lib/resilience/settings"; import { parseAutoConfig } from "./autoConfig.ts"; import { dedupeTargetsByExecutionKey } from "./comboData.ts"; import { getModelContextLimitForModelString } from "./comboStructure.ts"; +import { + calculatePromptCacheAffinityScores, + promptCacheTargetIdentity, +} from "./promptCacheAffinity.ts"; import type { ResetWindowConfig } from "./quotaScoring.ts"; import { _registerExecutionCandidates, @@ -192,7 +193,11 @@ export async function resolveAutoStrategyOrder( // select-under-one-policy/rank-under-another bug this module's original fix // (parseAutoConfig honoring the combo's own stored modePack) set out to close. const weights = modePack ? getModePack(modePack) || configWeights : configWeights; - if (requestModePack.override || requestBudgetCap !== undefined || requestBudgetFallback !== undefined) { + if ( + requestModePack.override || + requestBudgetCap !== undefined || + requestBudgetFallback !== undefined + ) { log.debug?.( "COMBO", `Auto strategy: per-request controls applied (mode=${ @@ -227,6 +232,10 @@ export async function resolveAutoStrategyOrder( resetWindowConfig, autoCandidateResilienceSettings ); + const cacheAffinityScores = calculatePromptCacheAffinityScores(candidates, body); + for (const candidate of candidates) { + candidate.cacheAffinity = cacheAffinityScores.get(promptCacheTargetIdentity(candidate)) ?? 0; + } const routableCandidates = candidates.filter( (candidate) => candidate.quotaCutoffBlocked !== true ); @@ -250,6 +259,7 @@ export async function resolveAutoStrategyOrder( if (routableCandidates.length > 0) { let selectedProvider: string | null = null; let selectedModel: string | null = null; + let selectedConnectionId: string | null = null; let selectionReason = ""; if (routingStrategy !== "rules") { @@ -267,6 +277,7 @@ export async function resolveAutoStrategyOrder( ); selectedProvider = decision.provider; selectedModel = decision.model; + selectedConnectionId = decision.connectionId ?? null; selectionReason = decision.reason; autoUsedExplicitRouter = true; } catch (err) { @@ -306,6 +317,7 @@ export async function resolveAutoStrategyOrder( } selectedProvider = selection.provider; selectedModel = selection.model; + selectedConnectionId = selection.connectionId ?? null; selectionReason = `score=${selection.score.toFixed(3)}${selection.isExploration ? " (exploration)" : ""}`; } @@ -333,7 +345,11 @@ export async function resolveAutoStrategyOrder( scoredTargets.find((entry) => { const parsed = parseModel(entry.target.modelStr); const modelId = parsed.model || entry.target.modelStr; - return entry.target.provider === selectedProvider && modelId === selectedModel; + return ( + entry.target.provider === selectedProvider && + modelId === selectedModel && + (!selectedConnectionId || entry.target.connectionId === selectedConnectionId) + ); })?.target || rankedTargets[0] || eligibleTargets[0]; diff --git a/src/app/(dashboard)/dashboard/settings/components/ComboDefaultsTab.tsx b/src/app/(dashboard)/dashboard/settings/components/ComboDefaultsTab.tsx index 902a2fda31..dc79f5729e 100644 --- a/src/app/(dashboard)/dashboard/settings/components/ComboDefaultsTab.tsx +++ b/src/app/(dashboard)/dashboard/settings/components/ComboDefaultsTab.tsx @@ -104,6 +104,7 @@ export default function ComboDefaultsTab() { zeroLatencyOptimizationsEnabled: false, }); const [sessionAffinityTtlMs, setSessionAffinityTtlMs] = useState(0); + const [promptCacheAffinityEnabled, setPromptCacheAffinityEnabled] = useState(true); const [providerOverrides, setProviderOverrides] = useState({}); const [availableProviders, setAvailableProviders] = useState<{ id: string; provider: string }[]>( [] @@ -183,6 +184,7 @@ export default function ComboDefaultsTab() { ? Number(settingsData.sessionAffinityTtlMs) : 0 ); + setPromptCacheAffinityEnabled(settingsData.promptCacheAffinityEnabled !== false); }) .catch((err) => console.error("Failed to fetch combo defaults:", err)); }, []); @@ -228,6 +230,7 @@ export default function ComboDefaultsTab() { // #6168: global session-stickiness opt-out — persisted top-level on settings // (mirrors stickyRoundRobinLimit) so combo.ts resolution reads settings.disableSessionStickiness. disableSessionStickiness: disableSessionStickiness === true, + promptCacheAffinityEnabled, }; const comboDefaultsRes = await fetch("/api/settings/combo-defaults", { @@ -674,6 +677,24 @@ export default function ComboDefaultsTab() { } /> +
+
+

+ {translateOrFallback(t, "promptCacheAffinity", "Prompt-cache locality routing")} +

+

+ {translateOrFallback( + t, + "promptCacheAffinityDesc", + "Prefer the same provider account for matching prompt-cache keys while preserving health and quota failover." + )} +

+
+ setPromptCacheAffinityEnabled((enabled) => !enabled)} + /> +

diff --git a/src/i18n/messages/de.json b/src/i18n/messages/de.json index 60747b5af3..b03fd2d364 100644 --- a/src/i18n/messages/de.json +++ b/src/i18n/messages/de.json @@ -2294,6 +2294,7 @@ "randomDesc": "Einheitliche Zufallsauswahl, dann Rückgriff auf verbleibende Modelle", "leastUsedDesc": "Wählt das Modell mit den wenigsten Anfragen aus und gleicht die Last über die Zeit aus", "costOptimizedDesc": "Leitet basierend auf dem Preis zuerst zum günstigsten Modell weiter", + "cacheOptimizedDesc": "Leitet jeden wiederverwendbaren Prompt-Präfix konsistent zum selben Provider-Konto", "resetAware": "Reset-Aware RR", "resetAwareDesc": "Gewichtet Restquote gegen 5h- und Wochen-Resets und rotiert ähnliche Scores per Round Robin", "strictRandom": "Strict Random", @@ -2504,6 +2505,7 @@ "weightTaskFit": "Task Fit", "weightStability": "Stability", "weightTierPriority": "Tier", + "weightCacheAffinity": "Cache-Treffer-Affinität", "reviewIntelligentTitle": "Intelligent Routing Config", "strategyRecommendations": { "priority": { @@ -5112,6 +5114,8 @@ "purgeLogsFailed": "Failed to purge logs", "contextOpt": "Context Optimized", "contextOptDesc": "Routes based on context window requirements and conversation length", + "cacheOpt": "Cache-optimiert", + "cacheOptDesc": "Hält denselben wiederverwendbaren Prompt-Präfix auf demselben Provider-Konto", "priorityDesc": "Sequential fallback - tries provider 1 first, then provider 2, and so on", "weightedDesc": "Distributes traffic by percentage weights across providers", "modelRoutingTitle": "Model Routing Rules", diff --git a/src/i18n/messages/en.json b/src/i18n/messages/en.json index 8e19bfd593..caca9da959 100644 --- a/src/i18n/messages/en.json +++ b/src/i18n/messages/en.json @@ -2998,6 +2998,7 @@ "randomDesc": "Uniform random selection, then fallback to remaining models", "leastUsedDesc": "Picks the model with fewest requests, balancing load over time", "costOptimizedDesc": "Routes to the cheapest model first based on pricing", + "cacheOptimizedDesc": "Routes each reusable prompt prefix consistently to the same provider account", "resetAware": "Reset-Aware RR", "resetAwareDesc": "Balances remaining quota against 5h and weekly resets, then round-robins similar scores", "strictRandom": "Strict Random", @@ -3261,6 +3262,7 @@ "weightTaskFit": "Task Fit", "weightStability": "Stability", "weightTierPriority": "Tier", + "weightCacheAffinity": "Cache Hit Affinity", "reviewIntelligentTitle": "Intelligent Routing Config", "strategyRecommendations": { "priority": { @@ -6763,6 +6765,8 @@ "resetting": "Resetting...", "contextOpt": "Context Optimized", "contextOptDesc": "Routes based on context window requirements and conversation length", + "cacheOpt": "Cache Optimized", + "cacheOptDesc": "Keeps the same reusable prompt prefix on the same provider account", "priorityDesc": "Sequential fallback - tries provider 1 first, then provider 2, and so on", "weightedDesc": "Distributes traffic by percentage weights across providers", "modelRoutingTitle": "Model Routing Rules", @@ -7505,6 +7509,8 @@ "modelLockoutMaxBackoffStepsDescription": "Maximum number of backoff steps before the cooldown stops growing. The Max Cooldown cap is reached first in most configurations, making this a safety ceiling for when Max Cooldown is raised.", "disableSessionStickiness": "Disable session stickiness", "disableSessionStickinessDesc": "Round-robin and random combos rotate to a different connection on every request instead of pinning a whole conversation to one connection by the first-message hash. Leave off to preserve prompt-cache hits for multi-turn chats. Per-combo overrides take precedence.", + "promptCacheAffinity": "Prompt-cache locality routing", + "promptCacheAffinityDesc": "Prefer the same provider account for matching prompt-cache keys while preserving health and quota failover.", "credentialRedaction": "Credential Redaction", "credentialRedactionDesc": "Redact API keys, tokens, and secrets from context sent to providers and from responses.", "enableCredentialRedaction": "Enable credential redaction", diff --git a/src/lib/combos/intelligentRouting.ts b/src/lib/combos/intelligentRouting.ts index 628e1fe511..13cf531348 100644 --- a/src/lib/combos/intelligentRouting.ts +++ b/src/lib/combos/intelligentRouting.ts @@ -16,6 +16,7 @@ export type IntelligentRoutingWeights = { tierAffinity: number; specificityMatch: number; contextAffinity: number; + cacheAffinity: number; resetWindowAffinity: number; }; @@ -50,6 +51,7 @@ export const DEFAULT_INTELLIGENT_WEIGHTS: IntelligentRoutingWeights = { tierAffinity: 0.05, specificityMatch: 0.05, contextAffinity: 0.08, + cacheAffinity: 0, resetWindowAffinity: 0, }; @@ -79,6 +81,7 @@ export const FACTOR_LABELS: Record = { tierAffinity: "Tier Affinity", specificityMatch: "Specificity", contextAffinity: "Context Affinity", + cacheAffinity: "Cache Hit Affinity", resetWindowAffinity: "Reset Window", }; @@ -153,6 +156,8 @@ export function normalizeIntelligentRoutingConfig(config: unknown): IntelligentR toFiniteNumber(rawWeights.specificityMatch) ?? DEFAULT_INTELLIGENT_WEIGHTS.specificityMatch, contextAffinity: toFiniteNumber(rawWeights.contextAffinity) ?? DEFAULT_INTELLIGENT_WEIGHTS.contextAffinity, + cacheAffinity: + toFiniteNumber(rawWeights.cacheAffinity) ?? DEFAULT_INTELLIGENT_WEIGHTS.cacheAffinity, resetWindowAffinity: toFiniteNumber(rawWeights.resetWindowAffinity) ?? DEFAULT_INTELLIGENT_WEIGHTS.resetWindowAffinity, diff --git a/src/lib/db/settings.ts b/src/lib/db/settings.ts index 7cc6f499eb..5c997e4588 100644 --- a/src/lib/db/settings.ts +++ b/src/lib/db/settings.ts @@ -103,7 +103,9 @@ function withFamilyDefault(value: ProxyValue): ProxyValue { function applySessionAffinityLegacyFallback(settings: Record): void { if (settings.sessionAffinityTtlMs === undefined) { settings.sessionAffinityTtlMs = - typeof settings.codexSessionAffinityTtlMs === "number" ? settings.codexSessionAffinityTtlMs : 0; + typeof settings.codexSessionAffinityTtlMs === "number" + ? settings.codexSessionAffinityTtlMs + : 0; } } @@ -116,6 +118,7 @@ export async function getSettings() { tailscaleUrl: "", stickyRoundRobinLimit: 3, disableSessionStickiness: false, + promptCacheAffinityEnabled: true, comboStrategy: "fallback", comboStickyRoundRobinLimit: null, // null = inherit stickyRoundRobinLimit (a literal default here shadows the documented batched-rotation default of 3 — #6678 regression caught by the v3.8.47 release CI) providerStrategies: {}, diff --git a/src/shared/constants/routingStrategies.ts b/src/shared/constants/routingStrategies.ts index a1287af553..b14b91d8fe 100644 --- a/src/shared/constants/routingStrategies.ts +++ b/src/shared/constants/routingStrategies.ts @@ -15,6 +15,7 @@ export const ROUTING_STRATEGY_VALUES = [ "auto", "lkgp", "context-optimized", + "cache-optimized", "fusion", "pipeline", ] as const; @@ -197,6 +198,13 @@ export const ROUTING_STRATEGIES: RoutingStrategyOption[] = [ settingsDescKey: "contextOptDesc", icon: "text_snippet", }, + { + value: "cache-optimized", + labelKey: "cacheOpt", + combosDescKey: "cacheOptimizedDesc", + settingsDescKey: "cacheOptDesc", + icon: "cached", + }, { value: "fusion", labelKey: "fusion", diff --git a/src/shared/validation/schemas/combo.ts b/src/shared/validation/schemas/combo.ts index fbe95188cd..2d7b163be9 100644 --- a/src/shared/validation/schemas/combo.ts +++ b/src/shared/validation/schemas/combo.ts @@ -86,6 +86,7 @@ export const scoringWeightsSchema = z tierAffinity: z.number().min(0).max(1).optional().default(0.05), specificityMatch: z.number().min(0).max(1).optional().default(0.05), contextAffinity: z.number().min(0).max(1).optional().default(0.08), + cacheAffinity: z.number().min(0).max(1).optional().default(0), resetWindowAffinity: z.number().min(0).max(1).optional().default(0), }) .optional(); diff --git a/src/shared/validation/settingsSchemas.ts b/src/shared/validation/settingsSchemas.ts index 383e37075d..1ca0ff5f3f 100644 --- a/src/shared/validation/settingsSchemas.ts +++ b/src/shared/validation/settingsSchemas.ts @@ -218,12 +218,16 @@ export const updateSettingsSchema = z.object({ .optional(), // #6168: global session-stickiness opt-out (per-combo config overrides this). disableSessionStickiness: z.boolean().optional(), + /** Keep eligible combo targets close to the provider-side prompt cache. */ + promptCacheAffinityEnabled: z.boolean().optional(), /** * Per-operator quota row visibility on the usage dashboard, keyed by * provider id. Independent of the model catalog's isHidden/isDeleted flags. * Ported from upstream decolua/9router#2371. */ - quotaVisibility: z.record(z.string().trim().min(1), z.object({ hidden: z.array(z.string()).max(500).optional() })).optional(), + quotaVisibility: z + .record(z.string().trim().min(1), z.object({ hidden: z.array(z.string()).max(500).optional() })) + .optional(), requestRetry: z.number().int().min(0).max(10).optional(), maxRetryIntervalSec: z.number().int().min(0).max(300).optional(), maxBodySizeMb: z diff --git a/tests/unit/auto-combo-scoring-clamp.test.ts b/tests/unit/auto-combo-scoring-clamp.test.ts index 025c9c1aef..1e00aca021 100644 --- a/tests/unit/auto-combo-scoring-clamp.test.ts +++ b/tests/unit/auto-combo-scoring-clamp.test.ts @@ -16,6 +16,7 @@ import { calculateScore, calculateFactors, DEFAULT_WEIGHTS, + normalizeScoringWeights, } from "../../open-sse/services/autoCombo/scoring.ts"; import type { ScoringFactors, @@ -92,15 +93,52 @@ test("calculateFactors — out-of-range contextAffinity is clamped", () => { ); }); +test("calculateFactors — cache affinity is clamped and can be weighted", () => { + const factors = calculateFactors(candidate({ cacheAffinity: 4 }), [], "default", () => 0.5); + assert.equal(factors.cacheAffinity, 1); + const weights = Object.fromEntries( + Object.keys(DEFAULT_WEIGHTS).map((key) => [key, key === "cacheAffinity" ? 1 : 0]) + ) as typeof DEFAULT_WEIGHTS; + assert.equal(calculateScore(factors, weights), 1); +}); + +test("normalizeScoringWeights keeps independent UI values proportional", () => { + const normalized = normalizeScoringWeights({ + ...DEFAULT_WEIGHTS, + cacheAffinity: 0.5, + }); + const total = Object.values(normalized).reduce((sum, value) => sum + Number(value), 0); + assert.ok(Math.abs(total - 1) < 1e-9); + assert.ok((normalized.cacheAffinity ?? 0) > normalized.health); +}); + +test("normalizeScoringWeights does not inject hidden weights into saved configs", () => { + const normalized = normalizeScoringWeights({ health: 0.2, cacheAffinity: 0.5 }); + assert.equal(normalized.connectionDensity, 0); + assert.equal(normalized.quota, 0); + assert.ok(Math.abs(normalized.health - 2 / 7) < 1e-9); + assert.ok(Math.abs((normalized.cacheAffinity ?? 0) - 5 / 7) < 1e-9); +}); + test("calculateFactors — connectionDensity is clamped to [0,1] and NaN-safe", () => { // A large pool ((1000-1)/10 = 99.9) must not exceed 1 and skew the weighted score. - const big = calculateFactors(candidate({ connectionPoolSize: 1000 }), [candidate()], "default", () => 0.5); + const big = calculateFactors( + candidate({ connectionPoolSize: 1000 }), + [candidate()], + "default", + () => 0.5 + ); assert.ok( big.connectionDensity >= 0 && big.connectionDensity <= 1, `connectionDensity must be in [0,1], got ${big.connectionDensity}` ); // A non-finite pool size must map to 0 (clamp01), not propagate NaN into the score. - const nan = calculateFactors(candidate({ connectionPoolSize: NaN }), [candidate()], "default", () => 0.5); + const nan = calculateFactors( + candidate({ connectionPoolSize: NaN }), + [candidate()], + "default", + () => 0.5 + ); assert.ok( Number.isFinite(nan.connectionDensity), `connectionDensity must be finite (clamp01 maps NaN→0), got ${nan.connectionDensity}` diff --git a/tests/unit/combo-auto-config-split.test.ts b/tests/unit/combo-auto-config-split.test.ts index dd755b3a10..2df3158949 100644 --- a/tests/unit/combo-auto-config-split.test.ts +++ b/tests/unit/combo-auto-config-split.test.ts @@ -2,12 +2,21 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { parseAutoConfig } from "@omniroute/open-sse/services/combo/autoConfig.ts"; -import { DEFAULT_WEIGHTS } from "@omniroute/open-sse/services/autoCombo/scoring.ts"; +import { DEFAULT_WEIGHTS, normalizeScoringWeights } from "@omniroute/open-sse/services/autoCombo/scoring.ts"; import { MODE_PACKS } from "@omniroute/open-sse/services/autoCombo/modePacks.ts"; // Split guard for Block J Task 2: parseAutoConfig was extracted verbatim from // handleComboChat's inline auto-strategy config block. These assertions pin the // pure derivation so the extraction stays behavior-identical. +// +// #8008 (prompt-cache affinity) added a `cacheAffinity` scoring factor and made +// parseAutoConfig run configured/mode-pack weights through `normalizeScoringWeights()` +// so independently-tuned UI weights always sum to a valid distribution and always +// carry the new key. That is an intentional behavior change: `cfg.weights` is now a +// freshly normalized object rather than a reference to `DEFAULT_WEIGHTS` / +// `MODE_PACKS[...]` / the caller's raw weights object, so these assertions compare +// against `normalizeScoringWeights(...)` (structural equality) instead of the +// pre-#8008 reference-equality checks. const target = (provider: string, modelStr: string) => ({ provider, modelStr, executionKey: `${provider}>${modelStr}` }) as never; @@ -20,7 +29,7 @@ test("defaults: rules strategy, provider-derived pool, default weights", () => { ]); assert.equal(cfg.routingStrategy, "rules"); assert.deepEqual(cfg.candidatePool, ["openai", "anthropic"]); - assert.equal(cfg.weights, DEFAULT_WEIGHTS); + assert.deepEqual(cfg.weights, normalizeScoringWeights(DEFAULT_WEIGHTS)); assert.equal(cfg.explorationRate, 0.05); assert.equal(cfg.budgetCap, undefined); assert.equal(cfg.modePack, undefined); @@ -42,7 +51,11 @@ test("routerStrategy takes precedence over routingStrategy/strategyName", () => }); test("explicit candidatePool, weights, exploration and budget are honored", () => { - const customWeights = { latency: 1 } as never; + // A well-formed ScoringWeights object (not an arbitrary key) — since #8008, + // configured weights are run through normalizeScoringWeights(), which only + // recognizes the real ScoringWeights keys and zeroes out/ignores anything else, + // then re-normalizes the distribution to sum to 1. + const customWeights = { ...DEFAULT_WEIGHTS, latencyInv: 1 } as never; const cfg = parseAutoConfig( { name: "c", @@ -57,7 +70,7 @@ test("explicit candidatePool, weights, exploration and budget are honored", () = [target("ignored", "x")] ); assert.deepEqual(cfg.candidatePool, ["glm", "openai"]); - assert.equal(cfg.weights, customWeights); + assert.deepEqual(cfg.weights, normalizeScoringWeights(customWeights)); assert.equal(cfg.explorationRate, 0.3); assert.equal(cfg.budgetCap, 5); assert.equal(cfg.modePack, "coding"); @@ -76,7 +89,7 @@ test("valid modePack overrides configured weights for fallback scoring", () => { ); assert.equal(cfg.modePack, "ship-fast"); - assert.equal(cfg.weights, MODE_PACKS["ship-fast"]); + assert.deepEqual(cfg.weights, normalizeScoringWeights(MODE_PACKS["ship-fast"])); }); test("config.auto is preferred over top-level config", () => { diff --git a/tests/unit/combo-resolve-auto-strategy-split.test.ts b/tests/unit/combo-resolve-auto-strategy-split.test.ts index 84c0fe114e..690c41c25f 100644 --- a/tests/unit/combo-resolve-auto-strategy-split.test.ts +++ b/tests/unit/combo-resolve-auto-strategy-split.test.ts @@ -87,6 +87,55 @@ test("all candidates quota-cutoff-blocked -> early 429 Response", async () => { } }); +test("cache affinity scores expanded auto account candidates directly", async () => { + const candidates = [ + { + kind: "model", + stepId: "s1", + executionKey: "openai>gpt-4o@account-a", + modelStr: "gpt-4o", + provider: "openai", + model: "gpt-4o", + connectionId: "account-a", + quotaRemaining: 100, + quotaTotal: 100, + circuitBreakerState: "CLOSED", + costPer1MTokens: 1, + p95LatencyMs: 100, + latencyStdDev: 10, + errorRate: 0, + }, + { + kind: "model", + stepId: "s1", + executionKey: "openai>gpt-4o@account-b", + modelStr: "gpt-4o", + provider: "openai", + model: "gpt-4o", + connectionId: "account-b", + quotaRemaining: 100, + quotaTotal: 100, + circuitBreakerState: "CLOSED", + costPer1MTokens: 1, + p95LatencyMs: 100, + latencyStdDev: 10, + errorRate: 0, + }, + ]; + const deps = baseDeps((async () => candidates) as never); + deps.orderedTargets = [target("openai", "gpt-4o")]; + deps.body = { prompt_cache_key: "expanded-account-key", messages: [] }; + deps.combo.autoConfig = { + candidatePool: ["openai"], + explorationRate: 0, + weights: { cacheAffinity: 1 }, + }; + + await resolveAutoStrategyOrder(deps); + + assert.deepEqual(candidates.map((candidate) => candidate.cacheAffinity).sort(), [0, 1]); +}); + // #7008 follow-up: parseAutoConfig() (see combo-auto-config-split.test.ts) already // makes `weights` honor a combo's own STORED modePack. But resolveAutoStrategyOrder() // also supports a per-request `X-OmniRoute-Mode` override (relayOptions.mode) that can diff --git a/tests/unit/combo-routing-engine.test.ts b/tests/unit/combo-routing-engine.test.ts index a8f00c161e..41cda10850 100644 --- a/tests/unit/combo-routing-engine.test.ts +++ b/tests/unit/combo-routing-engine.test.ts @@ -15,6 +15,9 @@ const { resolveNestedComboModels, handleComboChat, } = await import("../../open-sse/services/combo.ts"); +const { resolveComboTargets } = await import("../../open-sse/services/combo/comboStructure.ts"); +const { applyPromptCacheAffinity } = + await import("../../open-sse/services/combo/promptCacheAffinity.ts"); const { resolveReasoningBufferedMaxTokens } = await import("../../open-sse/services/reasoningTokenBuffer.ts"); const { normalizeComboStep } = await import("../../src/lib/combos/steps.ts"); @@ -534,6 +537,50 @@ test("handleComboChat weighted strategy selects by weight and falls back in desc } }); +test("handleComboChat preserves the weighted primary before prompt-cache affinity reordering", async () => { + const combo = { + name: "weighted-cache-affinity-protection", + strategy: "weighted", + models: [ + { model: "openai/gpt-4o-mini", weight: 1 }, + { model: "claude/sonnet", weight: 9 }, + ], + config: { maxRetries: 0 }, + }; + const resolvedTargets = resolveComboTargets(combo, null); + assert.equal(resolvedTargets.length, 2); + + const cacheKey = Array.from({ length: 100 }, (_, index) => `weighted-cache-${index}`).find( + (key) => + applyPromptCacheAffinity(resolvedTargets, { prompt_cache_key: key }).targets[0] !== + resolvedTargets[0] + ); + assert.ok(cacheKey, "test fixture must exercise a different affinity winner"); + + const calls: string[] = []; + _setSecureRandomFloatSource(() => 0); + try { + const result = await handleComboChat({ + body: { prompt_cache_key: cacheKey }, + combo, + handleSingleModel: async (_body: Record, modelStr: string) => { + calls.push(modelStr); + return okResponse(); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: null, + relayOptions: null, + allCombos: null, + }); + + assert.equal(result.ok, true); + assert.deepEqual(calls, ["openai/gpt-4o-mini"]); + } finally { + _setSecureRandomFloatSource(null); + } +}); + test("handleComboChat weighted strategy falls back to uniform random when all weights are zero", async () => { const calls: any[] = []; _setSecureRandomFloatSource(() => 0.75); @@ -2011,6 +2058,57 @@ test("handleComboChat eval-driven routing prioritizes higher scoring evaluated t assert.deepEqual(calls, ["openai/eval-high"]); }); +test("cache-optimized preserves eval routing when no reusable cache key exists", async () => { + evalsDb.saveEvalRun({ + suiteId: "cache-miss-routing", + suiteName: "Cache Miss Routing", + target: { type: "model", id: "openai/cache-low", label: "Model: openai/cache-low" }, + summary: { total: 10, passed: 2, failed: 8, passRate: 20 }, + avgLatencyMs: 100, + results: [], + createdAt: new Date().toISOString(), + }); + evalsDb.saveEvalRun({ + suiteId: "cache-miss-routing", + suiteName: "Cache Miss Routing", + target: { type: "model", id: "openai/cache-high", label: "Model: openai/cache-high" }, + summary: { total: 10, passed: 10, failed: 0, passRate: 100 }, + avgLatencyMs: 100, + results: [], + createdAt: new Date().toISOString(), + }); + + const calls: string[] = []; + const result = await handleComboChat({ + body: { messages: [{ role: "user", content: "First turn without reusable prefix" }] }, + combo: { + name: "cache-optimized-miss", + strategy: "cache-optimized", + models: ["openai/cache-low", "openai/cache-high"], + config: { + evalRouting: { + enabled: true, + suiteIds: ["cache-miss-routing"], + qualityWeight: 1, + latencyWeight: 0, + }, + }, + }, + handleSingleModel: async (_body: any, modelStr: string) => { + calls.push(modelStr); + return okResponse(); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: { promptCacheAffinityEnabled: false }, + relayOptions: null as any, + allCombos: null, + }); + + assert.equal(result.ok, true); + assert.deepEqual(calls, ["openai/cache-high"]); +}); + test("handleComboChat eval-driven routing ignores stale and undersized eval runs", async () => { evalsDb.saveEvalRun({ suiteId: "routing-quality", @@ -2583,7 +2681,7 @@ test("handleComboChat context cache protection pins the model and tags tool-call }, isModelAvailable: async () => true, log: createLog(), - settings: null, + settings: { promptCacheAffinityEnabled: false }, relayOptions: null as any, allCombos: null, }); diff --git a/tests/unit/prompt-cache-affinity.test.ts b/tests/unit/prompt-cache-affinity.test.ts new file mode 100644 index 0000000000..4d9f8bc593 --- /dev/null +++ b/tests/unit/prompt-cache-affinity.test.ts @@ -0,0 +1,134 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + applyPromptCacheAffinity, + calculatePromptCacheAffinityScores, + expandPromptCacheAffinityTargetsFromConnections, + promptCacheTargetIdentity, + resolvePromptCacheAffinityKey, +} from "../../open-sse/services/combo/promptCacheAffinity.ts"; +import type { ResolvedComboTarget } from "../../open-sse/services/combo/types.ts"; +import { applyStrategyOrdering } from "../../open-sse/services/combo/applyStrategyOrdering.ts"; + +function target( + executionKey: string, + connectionId: string, + modelStr = "codex/gpt-5" +): ResolvedComboTarget { + return { + kind: "model", + stepId: executionKey, + executionKey, + modelStr, + provider: "codex", + providerId: connectionId, + connectionId, + weight: 1, + label: null, + }; +} + +test("uses explicit prompt_cache_key and never exposes it in the fingerprint", () => { + const body = { prompt_cache_key: "private-cache-key" }; + const resolution = resolvePromptCacheAffinityKey(body); + assert.equal(resolution?.source, "explicit"); + assert.notEqual(resolution?.fingerprint, body.prompt_cache_key); + assert.equal(resolution?.fingerprint?.length, 12); +}); + +test("derives a stable key from Responses input when explicit key is absent", () => { + const first = resolvePromptCacheAffinityKey({ + input: [ + { role: "system", content: "tools" }, + { role: "user", content: "hello" }, + ], + }); + const second = resolvePromptCacheAffinityKey({ + input: [ + { role: "system", content: "tools" }, + { role: "user", content: "hello" }, + ], + }); + assert.equal(first?.source, "prefix"); + assert.deepEqual(first, second); +}); + +test("rendezvous ordering is deterministic and distinguishes same-model accounts", () => { + const targets = [target("step-a", "account-a"), target("step-b", "account-b")]; + const body = { prompt_cache_key: "stable" }; + const first = applyPromptCacheAffinity(targets, body); + const second = applyPromptCacheAffinity([...targets].reverse(), body); + assert.deepEqual( + first.targets.map((item) => item.connectionId), + second.targets.map((item) => item.connectionId) + ); + assert.equal(first.applied, true); +}); + +test("disabled affinity and missing keys preserve the eligible order", () => { + const targets = [target("step-a", "account-a"), target("step-b", "account-b")]; + assert.deepEqual( + applyPromptCacheAffinity(targets, { input: [{ role: "user", content: "hello" }] }, true) + .targets, + targets + ); + assert.deepEqual( + applyPromptCacheAffinity([...targets], { prompt_cache_key: "stable" }, false).targets, + targets + ); +}); + +test("auto scoring assigns the cache winner to exactly one account", () => { + const targets = [target("step-a", "account-a"), target("step-b", "account-b")]; + const scores = calculatePromptCacheAffinityScores(targets, { + prompt_cache_key: "stable-auto-key", + }); + assert.equal(scores.size, 2); + assert.equal( + targets.reduce((sum, item) => sum + (scores.get(promptCacheTargetIdentity(item)) ?? 0), 0), + 1 + ); +}); + +test("expands unbound targets to active allowed accounts before cache routing", () => { + const unbound = { ...target("step-a", ""), connectionId: null }; + const expanded = expandPromptCacheAffinityTargetsFromConnections( + [{ ...unbound, allowedConnectionIds: ["account-b"] }], + new Map([["codex", [{ id: "account-a" }, { id: "account-b" }, { id: "account-c" }]]]) + ); + assert.deepEqual( + expanded.map((item) => item.connectionId), + ["account-b"] + ); + assert.equal(expanded[0].executionKey, "step-a@account-b"); +}); + +test("expanded auto candidates receive exactly one concrete account cache score", () => { + const unbound = { ...target("step-a", ""), connectionId: null }; + const expanded = expandPromptCacheAffinityTargetsFromConnections( + [unbound], + new Map([["codex", [{ id: "account-a" }, { id: "account-b" }]]]) + ); + const scores = calculatePromptCacheAffinityScores(expanded, { + prompt_cache_key: "expanded-auto-key", + }); + assert.equal( + expanded.reduce((sum, item) => sum + (scores.get(promptCacheTargetIdentity(item)) ?? 0), 0), + 1 + ); + assert.ok(expanded.every((item) => item.connectionId)); +}); + +test("cache-optimized strategy routes a stable prompt key to the same account", async () => { + const targets = [target("step-a", "account-a"), target("step-b", "account-b")]; + const deps = { + combo: { id: "cache-combo", name: "cache-combo" }, + config: {}, + body: { prompt_cache_key: "stable-strategy-key" }, + log: { info() {}, warn() {} }, + apiKeyAllowedConnections: null, + }; + const first = await applyStrategyOrdering("cache-optimized", targets, deps); + const second = await applyStrategyOrdering("cache-optimized", [...targets].reverse(), deps); + assert.equal(first[0].connectionId, second[0].connectionId); +});