[codex] Keep mode-pack weights consistent in auto fallback ranking (#7008)

* fix(routing): honor modePack weights in combo fallback ranking

* fix(routing): make effective post-override modePack drive fallback weights

parseAutoConfig() already resolves `weights` from a combo's own STORED
modePack, but resolveAutoStrategyOrder() also supports a per-request
X-OmniRoute-Mode header override (#6024/#6025) that can select a
DIFFERENT mode pack than the one stored on the combo, for that single
request only. selectAutoProvider() (engine.ts) already re-derives
weights internally from the modePack it receives, so it correctly
reacts to the override -- but scoreAutoTargets(), which ranks the
fallback tail, had no such re-derivation and only ever saw the stale
pre-override weights from parseAutoConfig(). Net effect: a request
overriding e.g. "quality-first" to "ship-fast" would select its
primary target under ship-fast weights but rank every fallback under
quality-first weights -- the identical "select under one policy, rank
fallbacks under another" bug this module's original fix (honoring the
combo's own stored modePack) set out to close.

Recompute `weights` from the effective (post-override) `modePack`
right after it's resolved, so both selectAutoProvider and
scoreAutoTargets consume the same weight vector.

Adds a regression test proving a request-level X-OmniRoute-Mode
override produces IDENTICAL fallback-ranking weights to a combo
natively configured with that same modePack.

Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com>

---------

Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com>

open-sse/services/combo/autoConfig.ts

@@ -1,4 +1,5 @@
 import { DEFAULT_WEIGHTS, type ScoringWeights } from "../autoCombo/scoring.ts";
+import { getModePack } from "../autoCombo/modePacks.ts";
 import { isRecord } from "./comboData.ts";
 import { resolveResetWindowConfig, resolveSlaRoutingPolicy } from "./quotaScoring.ts";
 import type { ComboLike, ResolvedComboTarget } from "./types.ts";
@@ -34,7 +35,7 @@ export function parseAutoConfig(combo: ComboLike, eligibleTargets: ResolvedCombo
     ? autoConfigSource.candidatePool
     : [...new Set(eligibleTargets.map((target) => target.provider))];
 
-  const weights =
+  const configuredWeights =
     autoConfigSource.weights && typeof autoConfigSource.weights === "object"
       ? (autoConfigSource.weights as ScoringWeights)
       : DEFAULT_WEIGHTS;
@@ -52,6 +53,7 @@ export function parseAutoConfig(combo: ComboLike, eligibleTargets: ResolvedCombo
       : undefined;
   const modePack =
     typeof autoConfigSource.modePack === "string" ? autoConfigSource.modePack : undefined;
+  const weights = modePack ? getModePack(modePack) || configuredWeights : configuredWeights;
   const resetWindowConfig = resolveResetWindowConfig(autoConfigSource);
   const slaPolicy = resolveSlaRoutingPolicy(autoConfigSource);
 

open-sse/services/combo/resolveAutoStrategy.ts

@@ -10,6 +10,7 @@ import {
 } from "../autoCombo/requestControls.ts";
 import { selectWithStrategy } from "../autoCombo/routerStrategy.ts";
 import { buildComplexityRoutingHint } from "../autoCombo/complexityRouter";
+import { getModePack } from "../autoCombo/modePacks.ts";
 import { recordComboIntent } from "../comboMetrics.ts";
 import { estimateTokens } from "../contextManager.ts";
 import { classifyWithConfig } from "../intentClassifier.ts";
@@ -160,7 +161,7 @@ export async function resolveAutoStrategyOrder(
   const {
     routingStrategy,
     candidatePool,
-    weights,
+    weights: configWeights,
     explorationRate,
     budgetCap: configBudgetCap,
     budgetFallback: configBudgetFallback,
@@ -180,6 +181,17 @@ export async function resolveAutoStrategyOrder(
   const budgetFallback = requestBudgetFallback ?? configBudgetFallback;
   const requestModePack = resolveRequestModePack(relayOptions?.mode);
   const modePack = requestModePack.override ? requestModePack.modePack : configModePack;
+  // #7008: `weights` must track the *effective* (post-override) modePack, not just
+  // the combo's stored one. `selectAutoProvider()` (engine.ts) already re-derives
+  // weights internally from the `modePack` it's given, so it correctly reacts to a
+  // per-request X-OmniRoute-Mode override — but `scoreAutoTargets()` (the fallback
+  // ranking below) has no such re-derivation and only ever sees whatever `weights`
+  // it's handed. Without this recompute, a request overriding e.g. `quality-first`
+  // to `ship-fast` would select its primary target under ship-fast weights but rank
+  // every fallback under the stale quality-first weights — the same
+  // select-under-one-policy/rank-under-another bug this module's original fix
+  // (parseAutoConfig honoring the combo's own stored modePack) set out to close.
+  const weights = modePack ? getModePack(modePack) || configWeights : configWeights;
   if (requestModePack.override || requestBudgetCap !== undefined || requestBudgetFallback !== undefined) {
     log.debug?.(
       "COMBO",

tests/unit/combo-auto-config-split.test.ts

@@ -3,6 +3,7 @@ import assert from "node:assert/strict";
 
 import { parseAutoConfig } from "@omniroute/open-sse/services/combo/autoConfig.ts";
 import { DEFAULT_WEIGHTS } from "@omniroute/open-sse/services/autoCombo/scoring.ts";
+import { MODE_PACKS } from "@omniroute/open-sse/services/autoCombo/modePacks.ts";
 
 // Split guard for Block J Task 2: parseAutoConfig was extracted verbatim from
 // handleComboChat's inline auto-strategy config block. These assertions pin the
@@ -62,6 +63,22 @@ test("explicit candidatePool, weights, exploration and budget are honored", () =
   assert.equal(cfg.modePack, "coding");
 });
 
+test("valid modePack overrides configured weights for fallback scoring", () => {
+  const cfg = parseAutoConfig(
+    {
+      name: "c",
+      autoConfig: {
+        weights: { ...DEFAULT_WEIGHTS, latencyInv: 0 },
+        modePack: "ship-fast",
+      },
+    } as never,
+    []
+  );
+
+  assert.equal(cfg.modePack, "ship-fast");
+  assert.equal(cfg.weights, MODE_PACKS["ship-fast"]);
+});
+
 test("config.auto is preferred over top-level config", () => {
   const cfg = parseAutoConfig(
     { name: "c", config: { auto: { routerStrategy: "cost" }, routerStrategy: "rules" } } as never,

tests/unit/combo-resolve-auto-strategy-split.test.ts

@@ -86,3 +86,139 @@ test("all candidates quota-cutoff-blocked -> early 429 Response", async () => {
     assert.equal(result.earlyResponse.status, 429);
   }
 });
+
+// #7008 follow-up: parseAutoConfig() (see combo-auto-config-split.test.ts) already
+// makes `weights` honor a combo's own STORED modePack. But resolveAutoStrategyOrder()
+// also supports a per-request `X-OmniRoute-Mode` override (relayOptions.mode) that can
+// pick a *different* mode pack than the one stored on the combo for that single
+// request — and `weights` must track that EFFECTIVE (post-override) modePack, not the
+// stored one, so scoreAutoTargets' fallback ranking doesn't drift from the primary
+// selection. These three synthetic candidates are built so every scoring factor is
+// IDENTICAL between them except cost/latency/stability, and "dominant" clearly wins
+// selection under any weight profile (so the engine's own randomized tier-rotation
+// never affects which one becomes `orderedTargets[0]`) — isolating the assertion to
+// the *fallback ranking order* of the remaining two, which is exactly what
+// scoreAutoTargets (and only scoreAutoTargets) controls.
+const weightSensitiveCandidates = () =>
+  [
+    {
+      kind: "model",
+      stepId: "dominant",
+      executionKey: "groq>dominant-model",
+      modelStr: "dominant-model",
+      provider: "groq",
+      model: "dominant-model",
+      quotaRemaining: 100,
+      quotaTotal: 100,
+      circuitBreakerState: "CLOSED",
+      costPer1MTokens: 0.01,
+      p95LatencyMs: 10,
+      latencyStdDev: 1,
+      errorRate: 0,
+    },
+    {
+      // Wins under "quality-first" (high taskFit/stability weight): low latencyStdDev
+      // (=> high stability), but the highest cost and highest latency in the pool.
+      kind: "model",
+      stepId: "quality-leaning",
+      executionKey: "openai>model-alpha",
+      modelStr: "model-alpha",
+      provider: "openai",
+      model: "model-alpha",
+      quotaRemaining: 100,
+      quotaTotal: 100,
+      circuitBreakerState: "CLOSED",
+      costPer1MTokens: 20,
+      p95LatencyMs: 5000,
+      latencyStdDev: 1,
+      errorRate: 0,
+    },
+    {
+      // Wins under "ship-fast" (high latencyInv/health weight): lowest latency and
+      // lowest cost in the pool, but the highest latencyStdDev (=> low stability).
+      kind: "model",
+      stepId: "speed-leaning",
+      executionKey: "anthropic>model-beta",
+      modelStr: "model-beta",
+      provider: "anthropic",
+      model: "model-beta",
+      quotaRemaining: 100,
+      quotaTotal: 100,
+      circuitBreakerState: "CLOSED",
+      costPer1MTokens: 1,
+      p95LatencyMs: 100,
+      latencyStdDev: 900,
+      errorRate: 0,
+    },
+  ] as never;
+
+const weightSensitiveDeps = (autoConfig: Record<string, unknown>, mode?: string) =>
+  ({
+    orderedTargets: [
+      target("groq", "dominant-model"),
+      target("openai", "model-alpha"),
+      target("anthropic", "model-beta"),
+    ],
+    body: { messages: [{ role: "user", content: "hi" }] },
+    combo: {
+      id: "auto-mode-override",
+      name: "auto-mode-override",
+      autoConfig: {
+        // Fixed candidatePool skips the DB-backed expandAutoComboCandidatePool
+        // path entirely (see the `candidatePool.length > 0` early-return there) —
+        // this test only cares about weight-driven ranking, not pool expansion.
+        candidatePool: ["groq", "openai", "anthropic"],
+        explorationRate: 0,
+        ...autoConfig,
+      },
+    },
+    settings: null,
+    config: {},
+    relayOptions: mode ? { mode } : null,
+    resilienceSettings: { quotaPreflight: { enabled: false } },
+    log: noopLog,
+    buildAutoCandidates: (async () => weightSensitiveCandidates()) as never,
+  }) as never;
+
+test("per-request X-OmniRoute-Mode override changes the EFFECTIVE weights used for fallback ranking, not just selection", async () => {
+  // Combo's own stored modePack is "quality-first" (would rank model-alpha before
+  // model-beta if the override were ignored), but the request overrides to
+  // "ship-fast" for this one call.
+  const overridden = await resolveAutoStrategyOrder(
+    weightSensitiveDeps({ modePack: "quality-first" }, "ship-fast")
+  );
+  assert.ok("orderedTargets" in overridden, "expected a normal ordering result, not earlyResponse");
+  if (!("orderedTargets" in overridden)) return;
+
+  // A combo natively configured with "ship-fast" (no override at all) is the ground
+  // truth for what "effective modePack = ship-fast" should rank like.
+  const native = await resolveAutoStrategyOrder(weightSensitiveDeps({ modePack: "ship-fast" }));
+  assert.ok("orderedTargets" in native, "expected a normal ordering result, not earlyResponse");
+  if (!("orderedTargets" in native)) return;
+
+  // "dominant" overwhelms every weight profile tried here, so it is always the
+  // engine's selection (position 0) regardless of any exploration/tier-rotation
+  // randomness — the two asserted positions below are populated exclusively by
+  // scoreAutoTargets' deterministic weight-driven sort.
+  assert.equal(overridden.orderedTargets[0].provider, "groq");
+  assert.equal(native.orderedTargets[0].provider, "groq");
+
+  // Ship-fast weights favor low-latency/high-health over stability, so the
+  // speed-leaning candidate outranks the quality-leaning one when the effective
+  // modePack is ship-fast — whether that's because it's natively configured that
+  // way, or because a per-request override made it so.
+  assert.equal(
+    overridden.orderedTargets[1].provider,
+    "anthropic",
+    "request-level ship-fast override should rank the speed-leaning candidate above the quality-leaning one"
+  );
+  assert.equal(overridden.orderedTargets[2].provider, "openai");
+
+  // The override case must match the native ship-fast case EXACTLY — proving the
+  // override drives an identical effective weight vector for fallback ranking, not
+  // just for the initial selection.
+  assert.deepEqual(
+    overridden.orderedTargets.map((t) => t.provider),
+    native.orderedTargets.map((t) => t.provider)
+  );
+});