diff --git a/open-sse/services/combo/autoStrategy.ts b/open-sse/services/combo/autoStrategy.ts index 7e22cc01f5..6f5ab16f47 100644 --- a/open-sse/services/combo/autoStrategy.ts +++ b/open-sse/services/combo/autoStrategy.ts @@ -39,6 +39,7 @@ import { calculateScore, computePoolMaxima, type ProviderCandidate, + type ScoringFactors, type ScoringWeights, } from "../autoCombo/scoring.ts"; import type { RoutingHint } from "../manifestAdapter"; @@ -407,10 +408,19 @@ export function scoreAutoTargets( } return { target, + factors, score, }; }) - .filter((entry): entry is { target: ResolvedComboTarget; score: number } => entry !== null) + .filter( + ( + entry + ): entry is { + target: ResolvedComboTarget; + factors: ScoringFactors; + score: number; + } => entry !== null + ) .sort((a, b) => b.score - a.score); } diff --git a/open-sse/services/combo/resolveAutoStrategy.ts b/open-sse/services/combo/resolveAutoStrategy.ts index 84ee43c43b..bafdd29502 100644 --- a/open-sse/services/combo/resolveAutoStrategy.ts +++ b/open-sse/services/combo/resolveAutoStrategy.ts @@ -4,6 +4,7 @@ import { errorResponseWithComboDiagnostics, } from "../../utils/error.ts"; import { BudgetExceededError, selectProvider as selectAutoProvider } from "../autoCombo/engine.ts"; +import type { ScoringWeights } from "../autoCombo/scoring.ts"; import { resolveRequestModePack, parseRequestBudgetCap, @@ -83,6 +84,53 @@ export type ResolveAutoStrategyResult = | { earlyResponse: Response } | { orderedTargets: ResolvedComboTarget[]; autoUsedExplicitRouter: boolean }; +export interface EvaluateAutoCandidatesOptions { + targets: ResolvedComboTarget[]; + comboName: string; + body: Record; + taskType: string; + weights: ScoringWeights; + sessionId?: string | null; + resetWindowConfig?: ResetWindowConfig; + resilienceSettings?: ResilienceSettings | null; + manifestHint?: RoutingHint | null; + buildAutoCandidates: BuildAutoCandidates; +} + +export async function evaluateAutoCandidates(options: EvaluateAutoCandidatesOptions) { + const builtCandidates = await options.buildAutoCandidates( + options.targets, + options.comboName, + options.sessionId, + options.resetWindowConfig, + options.resilienceSettings + ); + const cacheAffinityScores = calculatePromptCacheAffinityScores( + builtCandidates, + options.body, + options.sessionId + ); + const candidates = builtCandidates.map((candidate) => ({ + ...candidate, + cacheAffinity: cacheAffinityScores.get(promptCacheTargetIdentity(candidate)) ?? 0, + })); + const routableCandidates = candidates.filter( + (candidate) => candidate.quotaCutoffBlocked !== true + ); + return { + sourceCandidates: builtCandidates, + candidates, + routableCandidates, + scoredTargets: scoreAutoTargets( + options.targets, + routableCandidates, + options.taskType, + options.weights, + options.manifestHint + ), + }; +} + /** * Resolve target ordering for the `auto` combo strategy. * @@ -262,24 +310,34 @@ export async function resolveAutoStrategyOrder( }, } : resilienceSettings; - const candidates = await buildAutoCandidates( - eligibleTargets, - combo.name, - relayOptions?.sessionId, - resetWindowConfig, - autoCandidateResilienceSettings - ); - const cacheAffinityScores = calculatePromptCacheAffinityScores( - candidates, - body, - relayOptions?.sessionId - ); - for (const candidate of candidates) { - candidate.cacheAffinity = cacheAffinityScores.get(promptCacheTargetIdentity(candidate)) ?? 0; + // Complexity-aware routing (2026, opt-in): classify the request's + // difficulty and feed a tier hint into scoring so tierAffinity / + // specificityMatch favor candidates whose tier matches the request. + const autoManifestHint: RoutingHint | null = + config.complexityAwareRouting === true + ? buildComplexityRoutingHint( + eligibleTargets.filter((t) => t.kind === "model"), + body, + log + ) + : null; + + const { sourceCandidates, candidates, routableCandidates, scoredTargets } = + await evaluateAutoCandidates({ + targets: eligibleTargets, + comboName: combo.name, + body, + taskType, + weights, + sessionId: relayOptions?.sessionId, + resetWindowConfig, + resilienceSettings: autoCandidateResilienceSettings, + manifestHint: autoManifestHint, + buildAutoCandidates, + }); + for (let index = 0; index < sourceCandidates.length; index += 1) { + sourceCandidates[index].cacheAffinity = candidates[index]?.cacheAffinity; } - const routableCandidates = candidates.filter( - (candidate) => candidate.quotaCutoffBlocked !== true - ); const quotaBlockedCount = candidates.length - routableCandidates.length; if (quotaBlockedCount > 0) { log.info( @@ -368,25 +426,6 @@ export async function resolveAutoStrategyOrder( selectionReason = `score=${selection.score.toFixed(3)}${selection.isExploration ? " (exploration)" : ""}`; } - // Complexity-aware routing (2026, opt-in): classify the request's - // difficulty and feed a tier hint into scoring so tierAffinity / - // specificityMatch favor candidates whose tier matches the request. - const autoManifestHint: RoutingHint | null = - config.complexityAwareRouting === true - ? buildComplexityRoutingHint( - eligibleTargets.filter((t) => t.kind === "model"), - body, - log - ) - : null; - - const scoredTargets = scoreAutoTargets( - eligibleTargets, - routableCandidates, - taskType, - weights, - autoManifestHint - ); const rankedTargets = scoredTargets.map((entry) => entry.target); const selectedTarget = scoredTargets.find((entry) => { diff --git a/src/lib/usage/comboScoringInspector.ts b/src/lib/usage/comboScoringInspector.ts index 09846a1fb3..3fd12fe650 100644 --- a/src/lib/usage/comboScoringInspector.ts +++ b/src/lib/usage/comboScoringInspector.ts @@ -8,6 +8,11 @@ import { type ProviderConnectionView, } from "@/lib/usage/resilienceExplain"; import { getCircuitBreaker } from "@/shared/utils/circuitBreaker"; +import { buildAutoCandidates } from "@omniroute/open-sse/services/combo.ts"; +import { parseAutoConfig } from "@omniroute/open-sse/services/combo/autoConfig.ts"; +import { resolveComboTargets } from "@omniroute/open-sse/services/combo/comboStructure.ts"; +import { evaluateAutoCandidates } from "@omniroute/open-sse/services/combo/resolveAutoStrategy.ts"; +import type { AutoProviderCandidate, ComboLike } from "@omniroute/open-sse/services/combo/types.ts"; import { calculateFactors, calculateScore, @@ -315,6 +320,56 @@ function buildCandidate( }; } +function buildLiveCandidateContext( + candidate: AutoProviderCandidate, + target: TargetHealth, + forecastTarget: ComboForecastTarget | undefined +): CandidateContext { + const context = buildCandidate(target, forecastTarget).context; + for (const key of FACTOR_KEYS) context.sources[key] = "runtime"; + context.notes.quota = "Current credential-level quota from live candidate evaluation."; + context.notes.costInv = "Current model pricing from live candidate evaluation."; + context.notes.latencyInv = "Current credential-level latency from live candidate evaluation."; + context.notes.stability = "Current credential-level stability from live candidate evaluation."; + context.notes.contextAffinity = "Live candidate affinity evaluated without a preview session."; + if (candidate.cacheAffinity !== undefined) { + context.notes.cacheAffinity = "Prompt-cache affinity from live candidate evaluation."; + } + return context; +} + +function liveTargetHealth( + candidate: AutoProviderCandidate, + target: { + executionKey: string; + stepId: string; + modelStr: string; + provider: string; + connectionId?: string | null; + label?: string | null; + }, + historicalTarget: TargetHealth | undefined +): TargetHealth { + return { + executionKey: target.executionKey, + stepId: target.stepId, + model: target.modelStr, + provider: target.provider, + connectionId: target.connectionId ?? null, + label: target.label ?? null, + requests: historicalTarget?.requests ?? 0, + successRate: historicalTarget?.successRate ?? 0, + avgLatencyMs: historicalTarget?.avgLatencyMs ?? candidate.p95LatencyMs, + lastStatus: historicalTarget?.lastStatus ?? null, + lastUsedAt: historicalTarget?.lastUsedAt ?? null, + quotaRemainingPct: + candidate.quotaTotal > 0 ? (candidate.quotaRemaining / candidate.quotaTotal) * 100 : null, + quotaIsExhausted: candidate.quotaCutoffBlocked === true, + quotaTrend: historicalTarget?.quotaTrend ?? null, + quotaScope: candidate.connectionId ? "connection" : "provider", + }; +} + function factorBreakdown( factors: ScoringFactors, weights: ScoringWeights, @@ -362,7 +417,9 @@ async function buildInspectorCombo( forecastTargets: Map, autopilotCombo: ComboAutopilotCombo | undefined, taskType: string, - inspectorWeights: InspectorWeights + inspectorWeights: InspectorWeights, + configuredCombo?: ComboRecord, + configuredCombos: ComboRecord[] = [] ): Promise { const warnings: string[] = []; const { weights } = inspectorWeights; @@ -378,23 +435,67 @@ async function buildInspectorCombo( } const issueCounts = autopilotIssueCounts(autopilotCombo); - const contexts = targets.map((target) => - buildCandidate( - target, - forecastTargets.get(target.executionKey) || forecastTargets.get(target.stepId) - ) - ); - const pool = contexts.map((entry) => entry.candidate); + let scored: Array<{ + entry: { candidate: ProviderCandidate; context: CandidateContext }; + factors: ScoringFactors; + score: number; + }>; - const scored = contexts - .map((entry) => { - const factors = calculateFactors(entry.candidate, pool, taskType, getTaskFitness); - const score = calculateScore(factors, weights); - const issueCount = issueCounts.get(entry.context.target.executionKey) ?? 0; - entry.context.autopilotIssueCount = issueCount; - return { entry, factors, score }; - }) - .sort((left, right) => right.score - left.score); + if ( + combo.strategy === "auto" && + configuredCombo?.name && + Array.isArray(configuredCombo.models) && + configuredCombo.models.length > 0 + ) { + const comboLike = configuredCombo as ComboLike; + const resolvedTargets = resolveComboTargets(comboLike, configuredCombos as ComboLike[]); + const { resetWindowConfig } = parseAutoConfig(comboLike, resolvedTargets); + const evaluation = await evaluateAutoCandidates({ + targets: resolvedTargets, + comboName: comboLike.name, + body: {}, + taskType, + weights, + resetWindowConfig, + buildAutoCandidates, + }); + const candidatesByExecutionKey = new Map( + evaluation.routableCandidates.map((candidate) => [candidate.executionKey, candidate]) + ); + scored = evaluation.scoredTargets.map((item) => { + const candidate = candidatesByExecutionKey.get(item.target.executionKey); + if (!candidate) + throw new Error(`Missing evaluated candidate for ${item.target.executionKey}`); + const historicalTarget = targets.find( + (target) => + target.executionKey === item.target.executionKey || target.stepId === item.target.stepId + ); + const target = liveTargetHealth(candidate, item.target, historicalTarget); + const context = buildLiveCandidateContext( + candidate, + target, + forecastTargets.get(target.executionKey) ?? forecastTargets.get(target.stepId) + ); + context.autopilotIssueCount = issueCounts.get(target.executionKey) ?? 0; + return { entry: { candidate, context }, factors: item.factors, score: item.score }; + }); + } else { + const contexts = targets.map((target) => + buildCandidate( + target, + forecastTargets.get(target.executionKey) || forecastTargets.get(target.stepId) + ) + ); + const pool = contexts.map((entry) => entry.candidate); + scored = contexts + .map((entry) => { + const factors = calculateFactors(entry.candidate, pool, taskType, getTaskFitness); + const score = calculateScore(factors, weights); + entry.context.autopilotIssueCount = issueCounts.get(entry.context.target.executionKey) ?? 0; + return { entry, factors, score }; + }) + .sort((left, right) => right.score - left.score); + } const connectionsByProvider = new Map(); await Promise.allSettled( @@ -514,17 +615,18 @@ export async function buildComboScoringInspectorResponse( horizon: options.horizon, method: "read_only_recompute", combos: await Promise.all( - health.combos.map((combo) => - buildInspectorCombo( + health.combos.map((combo) => { + const configuredCombo = combosById.get(combo.comboId) ?? combosByName.get(combo.comboName); + return buildInspectorCombo( combo, targetForecastMap(forecastByComboId.get(combo.comboId)?.targets ?? []), autopilotByComboId.get(combo.comboId), taskType, - resolveInspectorWeights( - combosById.get(combo.comboId) ?? combosByName.get(combo.comboName) - ) - ) - ) + resolveInspectorWeights(configuredCombo), + configuredCombo, + configuredCombos + ); + }) ), }; } diff --git a/tests/unit/combo-resolve-auto-strategy-split.test.ts b/tests/unit/combo-resolve-auto-strategy-split.test.ts index 690c41c25f..afc6ab7d00 100644 --- a/tests/unit/combo-resolve-auto-strategy-split.test.ts +++ b/tests/unit/combo-resolve-auto-strategy-split.test.ts @@ -1,7 +1,11 @@ import { test, after } from "node:test"; import assert from "node:assert/strict"; -import { resolveAutoStrategyOrder } from "@omniroute/open-sse/services/combo/resolveAutoStrategy.ts"; +import { + evaluateAutoCandidates, + resolveAutoStrategyOrder, +} from "@omniroute/open-sse/services/combo/resolveAutoStrategy.ts"; +import { DEFAULT_WEIGHTS } from "@omniroute/open-sse/services/autoCombo/scoring.ts"; import { resetDbInstance } from "@/lib/db/core.ts"; // resolveAutoStrategyOrder loads the LKGP via the DB singleton (dynamic import); @@ -87,6 +91,44 @@ test("all candidates quota-cutoff-blocked -> early 429 Response", async () => { } }); +test("candidate evaluation is deterministic and does not mutate builder-owned candidates", async () => { + const candidates = [ + { + kind: "model", + stepId: "s1", + executionKey: "openai>gpt-4o@account-a", + modelStr: "gpt-4o", + provider: "openai", + model: "gpt-4o", + connectionId: "account-a", + quotaRemaining: 80, + quotaTotal: 100, + circuitBreakerState: "CLOSED", + costPer1MTokens: 1, + p95LatencyMs: 100, + latencyStdDev: 10, + errorRate: 0, + }, + ]; + const before = structuredClone(candidates); + const options = { + targets: [target("openai", "gpt-4o")], + comboName: "autoc", + body: { prompt_cache_key: "pure-evaluation", messages: [] }, + taskType: "default", + weights: DEFAULT_WEIGHTS, + buildAutoCandidates: (async () => candidates) as never, + }; + + const first = await evaluateAutoCandidates(options); + const second = await evaluateAutoCandidates(options); + + assert.deepEqual(first.scoredTargets, second.scoredTargets); + assert.deepEqual(candidates, before); + assert.notEqual(first.candidates[0], candidates[0]); + assert.equal(first.scoredTargets[0].factors.quota, 0.8); +}); + test("cache affinity scores expanded auto account candidates directly", async () => { const candidates = [ { diff --git a/tests/unit/combo-scoring-inspector.test.ts b/tests/unit/combo-scoring-inspector.test.ts index a8d4f4c9a2..335478787e 100644 --- a/tests/unit/combo-scoring-inspector.test.ts +++ b/tests/unit/combo-scoring-inspector.test.ts @@ -30,6 +30,11 @@ const { resetAllCircuitBreakers } = await import("../../src/shared/utils/circuit const { DEFAULT_WEIGHTS, normalizeScoringWeights } = await import("../../open-sse/services/autoCombo/scoring.ts"); const { MODE_PACKS } = await import("../../open-sse/services/autoCombo/modePacks.ts"); +const { buildAutoCandidates } = await import("../../open-sse/services/combo.ts"); +const { parseAutoConfig } = await import("../../open-sse/services/combo/autoConfig.ts"); +const { resolveComboTargets } = await import("../../open-sse/services/combo/comboStructure.ts"); +const { evaluateAutoCandidates } = + await import("../../open-sse/services/combo/resolveAutoStrategy.ts"); async function resetStorage() { comboMetrics.resetAllComboMetrics(); @@ -183,9 +188,7 @@ test("scoring inspector ranks targets and explains score contributions", async ( ); assert.ok(Math.abs(contributionSum - response.combos[0].targets[0].score) < 0.02); assert.ok(response.combos[0].targets[0].factors.some((factor) => factor.key === "quota")); - assert.ok( - response.combos[0].targets[0].factors.some((factor) => factor.source === "combo_health") - ); + assert.ok(response.combos[0].targets[0].factors.some((factor) => factor.source === "runtime")); }); test("scoring inspector reports mode packs over explicit auto weights", async () => { @@ -244,6 +247,122 @@ test("scoring inspector reports valid explicit auto weights", async () => { assert.equal(response.combos[0].modePack, null); assert.deepEqual(response.combos[0].weights, explicitWeights); }); +test("configured auto preview uses live evaluation scores and zero quota contribution", async () => { + const weights = normalizeScoringWeights({ + ...DEFAULT_WEIGHTS, + quota: 0, + latencyInv: DEFAULT_WEIGHTS.latencyInv + DEFAULT_WEIGHTS.quota, + }); + const combo = await combosDb.createCombo({ + name: "combo-scoring-live-parity", + strategy: "auto", + models: ["openai/gpt-4o-mini", "anthropic/claude-3-haiku"], + autoConfig: { weights }, + }); + + const response = await inspector.buildComboScoringInspectorResponse({ + range: "24h", + horizon: "7d", + comboId: String(combo.id), + combos: [combo], + skipAutopilot: true, + }); + const resolvedTargets = resolveComboTargets(combo as never, [combo] as never); + const evaluation = await evaluateAutoCandidates({ + targets: resolvedTargets, + comboName: combo.name, + body: {}, + taskType: "default", + weights, + resetWindowConfig: parseAutoConfig(combo as never, resolvedTargets).resetWindowConfig, + buildAutoCandidates, + }); + + assert.deepEqual( + response.combos[0].targets.map((target) => ({ + executionKey: target.executionKey, + score: target.score, + factors: Object.fromEntries(target.factors.map((factor) => [factor.key, factor.value])), + })), + evaluation.scoredTargets.map((target) => ({ + executionKey: target.target.executionKey, + score: Number(target.score.toFixed(4)), + factors: Object.fromEntries( + Object.entries(target.factors).map(([key, value]) => [key, Number(value.toFixed(4))]) + ), + })) + ); + for (const target of response.combos[0].targets) { + const quota = target.factors.find((factor) => factor.key === "quota"); + assert.equal(quota?.source, "runtime"); + assert.equal(quota?.weight, 0); + assert.equal(quota?.contribution, 0); + assert.equal( + target.score, + Number(target.factors.reduce((sum, factor) => sum + factor.contribution, 0).toFixed(4)) + ); + } +}); + +test("configured auto preview with no model steps preserves health-derived targets", async () => { + const combo = await combosDb.createCombo({ + name: "combo-scoring-empty-auto", + strategy: "auto", + models: [], + }); + const target = { + executionKey: "health-only-target", + stepId: "health-only-target", + model: "openai/gpt-4o-mini", + provider: "openai", + connectionId: null, + label: "Health-only target", + requests: 1, + successRate: 100, + avgLatencyMs: 100, + lastStatus: 200, + lastUsedAt: new Date().toISOString(), + quotaRemainingPct: 100, + quotaIsExhausted: false, + quotaTrend: null, + quotaScope: "provider" as const, + }; + + const response = await inspector.buildComboScoringInspectorResponse({ + range: "24h", + horizon: "7d", + comboId: String(combo.id), + combos: [combo], + skipAutopilot: true, + healthResponse: { + timeRange: "24h", + combos: [ + { + comboId: String(combo.id), + comboName: combo.name, + strategy: "auto", + models: [], + cost: { totalUsd: 0, avgPerRequestUsd: 0, byModel: [] }, + quotaHealth: { providers: [], worstRemainingPct: 100 }, + usageSkew: { modelDistribution: [], giniCoefficient: 0 }, + performance: { avgLatencyMs: 100, successRate: 100, totalRequests: 1 }, + targetHealth: [target], + }, + ], + }, + forecastResponse: { + asOf: new Date().toISOString(), + timeRange: "24h", + horizon: "7d", + method: "linear_history", + combos: [], + }, + }); + + assert.equal(response.combos[0].targets.length, 1); + assert.equal(response.combos[0].targets[0].executionKey, target.executionKey); +}); + test("scoring inspector normalizes partial explicit auto weights like runtime", async () => { const explicitWeights = { quota: 0.3,