feat(sse): auto-promote successful combo model to position #1 (#4852)

Integrated into release/v3.8.37 — dropped the stale file-size-baseline.json hunk (re-derived against the rc17b rebaseline); code+test applied clean; 13/13 green.

CHANGELOG.md

@@ -10,6 +10,8 @@ _In development — bullets added per PR; finalized at release._
 
 ### 🔧 Bug Fixes
 
+- **feat(sse): auto-promote successful combo model**: a new opt-in `comboAutoPromoteEnabled` setting reorders a combo's persisted model list so that, when a combo model responds successfully, it is moved to position #1 for future requests. (thanks @arssnndr)
+
 - **fix(sse):** dense, deterministic `response.output` ordering in `response.completed` — items are now sorted by their actual `output_index` (via a recorded-as-emitted accumulator + stable sort) instead of being rebuilt from unordered state dicts; `normalizeOutputIndex` replaces fragile `parseInt` calls for robust index coercion; superseded tool calls (replaced at the same index mid-stream) are excluded from the final output array. (thanks @Marco9113)
 
 - **codex/translator**: normalize Codex custom/freeform tools (`apply_patch`, `type:"custom"` with no `parameters`) to a `{ input: string }` function schema instead of an empty schema — the empty schema made models invoke `apply_patch` with `{}`, breaking the Codex runtime which expects `{ input: string }`. Also map `custom_tool_call` / `custom_tool_call_output` input items and stream `apply_patch` tool calls via `custom_tool_call_input.delta`/`.done` events. (thanks @nstung463)

src/lib/combos/autoPromote.ts

@@ -0,0 +1,92 @@
+/**
+ * combos/autoPromote.ts — pure reorder helper for the "auto-promote successful
+ * combo model" feature.
+ *
+ * When the `comboAutoPromoteEnabled` setting is on and a combo model responds
+ * successfully, the winning model is moved to position #1 of the persisted
+ * combo so future requests try it first.
+ *
+ * OmniRoute stores `combo.models` as an array of `ComboStep` objects
+ * (`{ kind: "model", model, ... }`), unlike the upstream project which stores
+ * plain model strings. This helper accepts both shapes and reorders in place
+ * without mutating the input, returning `null` when no change is required
+ * (winner already first, winner absent, or empty list).
+ */
+
+type StepLike = { kind?: unknown; model?: unknown } | string;
+
+/** Extract the model id from a step entry (object or bare string). */
+export function comboStepModelId(step: unknown): string | null {
+  if (typeof step === "string") return step.trim().length > 0 ? step : null;
+  if (step && typeof step === "object") {
+    const model = (step as { model?: unknown }).model;
+    if (typeof model === "string" && model.trim().length > 0) return model;
+  }
+  return null;
+}
+
+/**
+ * Return a reordered copy of `models` with the entry matching `winningModel`
+ * moved to the front, or `null` if no reordering is needed/possible.
+ *
+ * Pure: never mutates the input array or its entries.
+ */
+export function promoteModelToFront<T extends StepLike>(
+  models: readonly T[] | null | undefined,
+  winningModel: string | null | undefined
+): T[] | null {
+  if (!Array.isArray(models) || models.length === 0) return null;
+  if (typeof winningModel !== "string" || winningModel.length === 0) return null;
+
+  const matchIndex = models.findIndex((step) => comboStepModelId(step) === winningModel);
+  // Winner not in the combo (e.g. global fallback) or already first → no-op.
+  if (matchIndex <= 0) return null;
+
+  const winner = models[matchIndex];
+  const rest = models.filter((_, index) => index !== matchIndex);
+  return [winner, ...rest];
+}
+
+interface PromoteComboDeps {
+  updateCombo: (id: string, data: { models: unknown[] }) => Promise<unknown>;
+  info?: (tag: string, msg: string) => void;
+  warn?: (tag: string, msg: string) => void;
+}
+
+/**
+ * Persist the auto-promotion of a successful combo model to position #1.
+ *
+ * Opt-in via the `comboAutoPromoteEnabled` setting. Best-effort: a DB failure is
+ * logged and swallowed so it never affects the already-successful response.
+ * No-op when the flag is off, the combo has no id, or the model is already first
+ * / absent. `updateCombo` is injected so this stays unit-testable without a DB.
+ */
+export async function promoteSuccessfulComboModel(
+  combo: { id?: unknown; name?: unknown; models?: unknown } | null | undefined,
+  winningModel: string | null | undefined,
+  settings: Record<string, unknown> | null | undefined,
+  deps: PromoteComboDeps
+): Promise<boolean> {
+  if (!combo || !settings || !settings.comboAutoPromoteEnabled) return false;
+  const comboId = typeof combo.id === "string" ? combo.id : null;
+  if (!comboId) return false;
+  const reordered = promoteModelToFront(
+    Array.isArray(combo.models) ? (combo.models as unknown[]) : null,
+    winningModel
+  );
+  if (!reordered) return false;
+  const label = typeof combo.name === "string" ? combo.name : comboId;
+  try {
+    await deps.updateCombo(comboId, { models: reordered });
+    deps.info?.("COMBO", `Model "${winningModel}" succeeded — promoted to #1 in combo "${label}"`);
+    return true;
+  } catch (dbErr: any) {
+    deps.warn?.(
+      "COMBO",
+      `Failed to promote model "${winningModel}" in combo "${label}": ${
+        dbErr?.message || "unknown error"
+      }`
+    );
+    return false;
+  }
+}

src/lib/db/settings.ts

@@ -124,6 +124,7 @@ export async function getSettings() {
     autoRefreshProviderQuota: false,
     autoRefreshProviderQuotaInterval: 180,
     comboConfigMode: "guided",
+    comboAutoPromoteEnabled: false,
     codexServiceTier: { enabled: false },
     claudeFastMode: {
       enabled: false,

src/sse/handlers/chat.ts

@@ -46,6 +46,8 @@ import * as log from "../utils/logger";
 import { checkAndRefreshToken } from "../services/tokenRefresh";
 import { createHookContext, runHooks, initPreRequestRegistry } from "@/lib/middleware/registry";
 import { deleteHandoff, getHandoff } from "@/lib/db/contextHandoffs";
+import { updateCombo } from "@/lib/db/combos";
+import { promoteSuccessfulComboModel } from "@/lib/combos/autoPromote";
 import {
   deleteSessionAccountAffinity,
   getCachedSettings,
@@ -192,6 +194,7 @@ function intersectAllowedConnectionIds(primary: unknown, secondary: unknown): st
 }
 
 const PROVIDER_BREAKER_FAILURE_STATUSES = new Set([408, 500, 502, 503, 504]);
+const comboPromoteDeps = { updateCombo, info: log.info, warn: log.warn };
 
 /**
  * Handle chat completion request
@@ -673,7 +676,17 @@ export async function handleChat(
           },
           target?.effectiveComboStrategy ?? combo.strategy,
           true
-        ),
+        ).then(async (res: Response) => {
+          // Auto-promote the winning combo model to position #1 (opt-in flag).
+          if (res?.ok)
+            await promoteSuccessfulComboModel(
+              combo,
+              m,
+              settings as Record<string, unknown>,
+              comboPromoteDeps
+            );
+          return res;
+        }),
       isModelAvailable: checkModelAvailable,
       log,
       settings,

tests/unit/combo-auto-promote.test.ts

@@ -0,0 +1,154 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+
+import {
+  promoteModelToFront,
+  comboStepModelId,
+  promoteSuccessfulComboModel,
+} from "@/lib/combos/autoPromote";
+
+function makeCombo() {
+  return {
+    id: "combo-1",
+    name: "my-combo",
+    models: [
+      { kind: "model", model: "a/1", weight: 1 },
+      { kind: "model", model: "b/2", weight: 1 },
+    ],
+  };
+}
+
+test("comboStepModelId extracts model id from ComboStep object", () => {
+  assert.equal(comboStepModelId({ kind: "model", model: "openai/gpt-4o" }), "openai/gpt-4o");
+});
+
+test("comboStepModelId extracts model id from bare string", () => {
+  assert.equal(comboStepModelId("anthropic/claude-3"), "anthropic/claude-3");
+});
+
+test("comboStepModelId returns null for empty/invalid entries", () => {
+  assert.equal(comboStepModelId(""), null);
+  assert.equal(comboStepModelId("   "), null);
+  assert.equal(comboStepModelId({ kind: "combo-ref", comboName: "x" }), null);
+  assert.equal(comboStepModelId(null), null);
+  assert.equal(comboStepModelId(42), null);
+});
+
+test("promoteModelToFront moves the winning ComboStep to position #1", () => {
+  const models = [
+    { kind: "model", model: "a/1", weight: 1 },
+    { kind: "model", model: "b/2", weight: 1 },
+    { kind: "model", model: "c/3", weight: 1 },
+  ];
+  const result = promoteModelToFront(models, "b/2");
+  assert.deepEqual(result, [
+    { kind: "model", model: "b/2", weight: 1 },
+    { kind: "model", model: "a/1", weight: 1 },
+    { kind: "model", model: "c/3", weight: 1 },
+  ]);
+});
+
+test("promoteModelToFront preserves order of the remaining models", () => {
+  const result = promoteModelToFront(["a", "b", "c", "d"], "d");
+  assert.deepEqual(result, ["d", "a", "b", "c"]);
+});
+
+test("promoteModelToFront does not mutate the input array or entries", () => {
+  const step = { kind: "model", model: "b/2", weight: 1 };
+  const models = [{ kind: "model", model: "a/1", weight: 1 }, step];
+  const snapshot = JSON.parse(JSON.stringify(models));
+  promoteModelToFront(models, "b/2");
+  assert.deepEqual(models, snapshot);
+  assert.equal(models[1], step); // same reference, untouched
+});
+
+test("promoteModelToFront returns null when winner is already first", () => {
+  assert.equal(promoteModelToFront(["a", "b", "c"], "a"), null);
+});
+
+test("promoteModelToFront returns null when winner is absent from the combo", () => {
+  assert.equal(promoteModelToFront(["a", "b"], "z"), null);
+});
+
+test("promoteModelToFront returns null for empty / missing inputs", () => {
+  assert.equal(promoteModelToFront([], "a"), null);
+  assert.equal(promoteModelToFront(null, "a"), null);
+  assert.equal(promoteModelToFront(undefined, "a"), null);
+  assert.equal(promoteModelToFront(["a", "b"], ""), null);
+  assert.equal(promoteModelToFront(["a", "b"], null), null);
+});
+
+test("promoteSuccessfulComboModel persists the reordered combo when flag is on", async () => {
+  const calls: Array<{ id: string; data: { models: unknown[] } }> = [];
+  const ok = await promoteSuccessfulComboModel(
+    makeCombo(),
+    "b/2",
+    { comboAutoPromoteEnabled: true },
+    {
+      updateCombo: async (id, data) => {
+        calls.push({ id, data });
+        return data;
+      },
+    }
+  );
+  assert.equal(ok, true);
+  assert.equal(calls.length, 1);
+  assert.equal(calls[0].id, "combo-1");
+  assert.deepEqual(
+    (calls[0].data.models as Array<{ model: string }>).map((m) => m.model),
+    ["b/2", "a/1"]
+  );
+});
+
+test("promoteSuccessfulComboModel is a no-op when the flag is off", async () => {
+  let called = false;
+  const ok = await promoteSuccessfulComboModel(
+    makeCombo(),
+    "b/2",
+    { comboAutoPromoteEnabled: false },
+    {
+      updateCombo: async (id, data) => {
+        called = true;
+        return data;
+      },
+    }
+  );
+  assert.equal(ok, false);
+  assert.equal(called, false);
+});
+
+test("promoteSuccessfulComboModel is a no-op when the winner is already first", async () => {
+  let called = false;
+  const ok = await promoteSuccessfulComboModel(
+    makeCombo(),
+    "a/1",
+    { comboAutoPromoteEnabled: true },
+    {
+      updateCombo: async (id, data) => {
+        called = true;
+        return data;
+      },
+    }
+  );
+  assert.equal(ok, false);
+  assert.equal(called, false);
+});
+
+test("promoteSuccessfulComboModel swallows DB errors and never throws", async () => {
+  let warned = "";
+  const ok = await promoteSuccessfulComboModel(
+    makeCombo(),
+    "b/2",
+    { comboAutoPromoteEnabled: true },
+    {
+      updateCombo: async () => {
+        throw new Error("db is down");
+      },
+      warn: (_tag, msg) => {
+        warned = msg;
+      },
+    }
+  );
+  assert.equal(ok, false);
+  assert.match(warned, /Failed to promote/);
+});