diff --git a/open-sse/services/combo/autoConfig.ts b/open-sse/services/combo/autoConfig.ts index 2815444f25..ed56f11989 100644 --- a/open-sse/services/combo/autoConfig.ts +++ b/open-sse/services/combo/autoConfig.ts @@ -1,4 +1,5 @@ import { DEFAULT_WEIGHTS, type ScoringWeights } from "../autoCombo/scoring.ts"; +import { getModePack } from "../autoCombo/modePacks.ts"; import { isRecord } from "./comboData.ts"; import { resolveResetWindowConfig, resolveSlaRoutingPolicy } from "./quotaScoring.ts"; import type { ComboLike, ResolvedComboTarget } from "./types.ts"; @@ -34,7 +35,7 @@ export function parseAutoConfig(combo: ComboLike, eligibleTargets: ResolvedCombo ? autoConfigSource.candidatePool : [...new Set(eligibleTargets.map((target) => target.provider))]; - const weights = + const configuredWeights = autoConfigSource.weights && typeof autoConfigSource.weights === "object" ? (autoConfigSource.weights as ScoringWeights) : DEFAULT_WEIGHTS; @@ -52,6 +53,7 @@ export function parseAutoConfig(combo: ComboLike, eligibleTargets: ResolvedCombo : undefined; const modePack = typeof autoConfigSource.modePack === "string" ? autoConfigSource.modePack : undefined; + const weights = modePack ? getModePack(modePack) || configuredWeights : configuredWeights; const resetWindowConfig = resolveResetWindowConfig(autoConfigSource); const slaPolicy = resolveSlaRoutingPolicy(autoConfigSource); diff --git a/open-sse/services/combo/resolveAutoStrategy.ts b/open-sse/services/combo/resolveAutoStrategy.ts index ef6082f9af..9d38801b35 100644 --- a/open-sse/services/combo/resolveAutoStrategy.ts +++ b/open-sse/services/combo/resolveAutoStrategy.ts @@ -10,6 +10,7 @@ import { } from "../autoCombo/requestControls.ts"; import { selectWithStrategy } from "../autoCombo/routerStrategy.ts"; import { buildComplexityRoutingHint } from "../autoCombo/complexityRouter"; +import { getModePack } from "../autoCombo/modePacks.ts"; import { recordComboIntent } from "../comboMetrics.ts"; import { estimateTokens } from "../contextManager.ts"; import { classifyWithConfig } from "../intentClassifier.ts"; @@ -160,7 +161,7 @@ export async function resolveAutoStrategyOrder( const { routingStrategy, candidatePool, - weights, + weights: configWeights, explorationRate, budgetCap: configBudgetCap, budgetFallback: configBudgetFallback, @@ -180,6 +181,17 @@ export async function resolveAutoStrategyOrder( const budgetFallback = requestBudgetFallback ?? configBudgetFallback; const requestModePack = resolveRequestModePack(relayOptions?.mode); const modePack = requestModePack.override ? requestModePack.modePack : configModePack; + // #7008: `weights` must track the *effective* (post-override) modePack, not just + // the combo's stored one. `selectAutoProvider()` (engine.ts) already re-derives + // weights internally from the `modePack` it's given, so it correctly reacts to a + // per-request X-OmniRoute-Mode override — but `scoreAutoTargets()` (the fallback + // ranking below) has no such re-derivation and only ever sees whatever `weights` + // it's handed. Without this recompute, a request overriding e.g. `quality-first` + // to `ship-fast` would select its primary target under ship-fast weights but rank + // every fallback under the stale quality-first weights — the same + // select-under-one-policy/rank-under-another bug this module's original fix + // (parseAutoConfig honoring the combo's own stored modePack) set out to close. + const weights = modePack ? getModePack(modePack) || configWeights : configWeights; if (requestModePack.override || requestBudgetCap !== undefined || requestBudgetFallback !== undefined) { log.debug?.( "COMBO", diff --git a/tests/unit/combo-auto-config-split.test.ts b/tests/unit/combo-auto-config-split.test.ts index a833cae0c4..dd755b3a10 100644 --- a/tests/unit/combo-auto-config-split.test.ts +++ b/tests/unit/combo-auto-config-split.test.ts @@ -3,6 +3,7 @@ import assert from "node:assert/strict"; import { parseAutoConfig } from "@omniroute/open-sse/services/combo/autoConfig.ts"; import { DEFAULT_WEIGHTS } from "@omniroute/open-sse/services/autoCombo/scoring.ts"; +import { MODE_PACKS } from "@omniroute/open-sse/services/autoCombo/modePacks.ts"; // Split guard for Block J Task 2: parseAutoConfig was extracted verbatim from // handleComboChat's inline auto-strategy config block. These assertions pin the @@ -62,6 +63,22 @@ test("explicit candidatePool, weights, exploration and budget are honored", () = assert.equal(cfg.modePack, "coding"); }); +test("valid modePack overrides configured weights for fallback scoring", () => { + const cfg = parseAutoConfig( + { + name: "c", + autoConfig: { + weights: { ...DEFAULT_WEIGHTS, latencyInv: 0 }, + modePack: "ship-fast", + }, + } as never, + [] + ); + + assert.equal(cfg.modePack, "ship-fast"); + assert.equal(cfg.weights, MODE_PACKS["ship-fast"]); +}); + test("config.auto is preferred over top-level config", () => { const cfg = parseAutoConfig( { name: "c", config: { auto: { routerStrategy: "cost" }, routerStrategy: "rules" } } as never, diff --git a/tests/unit/combo-resolve-auto-strategy-split.test.ts b/tests/unit/combo-resolve-auto-strategy-split.test.ts index 7ffceeae16..84c0fe114e 100644 --- a/tests/unit/combo-resolve-auto-strategy-split.test.ts +++ b/tests/unit/combo-resolve-auto-strategy-split.test.ts @@ -86,3 +86,139 @@ test("all candidates quota-cutoff-blocked -> early 429 Response", async () => { assert.equal(result.earlyResponse.status, 429); } }); + +// #7008 follow-up: parseAutoConfig() (see combo-auto-config-split.test.ts) already +// makes `weights` honor a combo's own STORED modePack. But resolveAutoStrategyOrder() +// also supports a per-request `X-OmniRoute-Mode` override (relayOptions.mode) that can +// pick a *different* mode pack than the one stored on the combo for that single +// request — and `weights` must track that EFFECTIVE (post-override) modePack, not the +// stored one, so scoreAutoTargets' fallback ranking doesn't drift from the primary +// selection. These three synthetic candidates are built so every scoring factor is +// IDENTICAL between them except cost/latency/stability, and "dominant" clearly wins +// selection under any weight profile (so the engine's own randomized tier-rotation +// never affects which one becomes `orderedTargets[0]`) — isolating the assertion to +// the *fallback ranking order* of the remaining two, which is exactly what +// scoreAutoTargets (and only scoreAutoTargets) controls. +const weightSensitiveCandidates = () => + [ + { + kind: "model", + stepId: "dominant", + executionKey: "groq>dominant-model", + modelStr: "dominant-model", + provider: "groq", + model: "dominant-model", + quotaRemaining: 100, + quotaTotal: 100, + circuitBreakerState: "CLOSED", + costPer1MTokens: 0.01, + p95LatencyMs: 10, + latencyStdDev: 1, + errorRate: 0, + }, + { + // Wins under "quality-first" (high taskFit/stability weight): low latencyStdDev + // (=> high stability), but the highest cost and highest latency in the pool. + kind: "model", + stepId: "quality-leaning", + executionKey: "openai>model-alpha", + modelStr: "model-alpha", + provider: "openai", + model: "model-alpha", + quotaRemaining: 100, + quotaTotal: 100, + circuitBreakerState: "CLOSED", + costPer1MTokens: 20, + p95LatencyMs: 5000, + latencyStdDev: 1, + errorRate: 0, + }, + { + // Wins under "ship-fast" (high latencyInv/health weight): lowest latency and + // lowest cost in the pool, but the highest latencyStdDev (=> low stability). + kind: "model", + stepId: "speed-leaning", + executionKey: "anthropic>model-beta", + modelStr: "model-beta", + provider: "anthropic", + model: "model-beta", + quotaRemaining: 100, + quotaTotal: 100, + circuitBreakerState: "CLOSED", + costPer1MTokens: 1, + p95LatencyMs: 100, + latencyStdDev: 900, + errorRate: 0, + }, + ] as never; + +const weightSensitiveDeps = (autoConfig: Record, mode?: string) => + ({ + orderedTargets: [ + target("groq", "dominant-model"), + target("openai", "model-alpha"), + target("anthropic", "model-beta"), + ], + body: { messages: [{ role: "user", content: "hi" }] }, + combo: { + id: "auto-mode-override", + name: "auto-mode-override", + autoConfig: { + // Fixed candidatePool skips the DB-backed expandAutoComboCandidatePool + // path entirely (see the `candidatePool.length > 0` early-return there) — + // this test only cares about weight-driven ranking, not pool expansion. + candidatePool: ["groq", "openai", "anthropic"], + explorationRate: 0, + ...autoConfig, + }, + }, + settings: null, + config: {}, + relayOptions: mode ? { mode } : null, + resilienceSettings: { quotaPreflight: { enabled: false } }, + log: noopLog, + buildAutoCandidates: (async () => weightSensitiveCandidates()) as never, + }) as never; + +test("per-request X-OmniRoute-Mode override changes the EFFECTIVE weights used for fallback ranking, not just selection", async () => { + // Combo's own stored modePack is "quality-first" (would rank model-alpha before + // model-beta if the override were ignored), but the request overrides to + // "ship-fast" for this one call. + const overridden = await resolveAutoStrategyOrder( + weightSensitiveDeps({ modePack: "quality-first" }, "ship-fast") + ); + assert.ok("orderedTargets" in overridden, "expected a normal ordering result, not earlyResponse"); + if (!("orderedTargets" in overridden)) return; + + // A combo natively configured with "ship-fast" (no override at all) is the ground + // truth for what "effective modePack = ship-fast" should rank like. + const native = await resolveAutoStrategyOrder(weightSensitiveDeps({ modePack: "ship-fast" })); + assert.ok("orderedTargets" in native, "expected a normal ordering result, not earlyResponse"); + if (!("orderedTargets" in native)) return; + + // "dominant" overwhelms every weight profile tried here, so it is always the + // engine's selection (position 0) regardless of any exploration/tier-rotation + // randomness — the two asserted positions below are populated exclusively by + // scoreAutoTargets' deterministic weight-driven sort. + assert.equal(overridden.orderedTargets[0].provider, "groq"); + assert.equal(native.orderedTargets[0].provider, "groq"); + + // Ship-fast weights favor low-latency/high-health over stability, so the + // speed-leaning candidate outranks the quality-leaning one when the effective + // modePack is ship-fast — whether that's because it's natively configured that + // way, or because a per-request override made it so. + assert.equal( + overridden.orderedTargets[1].provider, + "anthropic", + "request-level ship-fast override should rank the speed-leaning candidate above the quality-leaning one" + ); + assert.equal(overridden.orderedTargets[2].provider, "openai"); + + // The override case must match the native ship-fast case EXACTLY — proving the + // override drives an identical effective weight vector for fallback ranking, not + // just for the initial selection. + assert.deepEqual( + overridden.orderedTargets.map((t) => t.provider), + native.orderedTargets.map((t) => t.provider) + ); +});