diff --git a/CHANGELOG.md b/CHANGELOG.md index 665ec40771..27ab3c617e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ _In development — bullets added per PR; finalized at release._ ### 🔧 Bug Fixes +- **feat(sse): auto-promote successful combo model**: a new opt-in `comboAutoPromoteEnabled` setting reorders a combo's persisted model list so that, when a combo model responds successfully, it is moved to position #1 for future requests. (thanks @arssnndr) + - **fix(sse):** dense, deterministic `response.output` ordering in `response.completed` — items are now sorted by their actual `output_index` (via a recorded-as-emitted accumulator + stable sort) instead of being rebuilt from unordered state dicts; `normalizeOutputIndex` replaces fragile `parseInt` calls for robust index coercion; superseded tool calls (replaced at the same index mid-stream) are excluded from the final output array. (thanks @Marco9113) - **codex/translator**: normalize Codex custom/freeform tools (`apply_patch`, `type:"custom"` with no `parameters`) to a `{ input: string }` function schema instead of an empty schema — the empty schema made models invoke `apply_patch` with `{}`, breaking the Codex runtime which expects `{ input: string }`. Also map `custom_tool_call` / `custom_tool_call_output` input items and stream `apply_patch` tool calls via `custom_tool_call_input.delta`/`.done` events. (thanks @nstung463) diff --git a/src/lib/combos/autoPromote.ts b/src/lib/combos/autoPromote.ts new file mode 100644 index 0000000000..e5928b3cc8 --- /dev/null +++ b/src/lib/combos/autoPromote.ts @@ -0,0 +1,92 @@ +/** + * combos/autoPromote.ts — pure reorder helper for the "auto-promote successful + * combo model" feature. + * + * When the `comboAutoPromoteEnabled` setting is on and a combo model responds + * successfully, the winning model is moved to position #1 of the persisted + * combo so future requests try it first. + * + * OmniRoute stores `combo.models` as an array of `ComboStep` objects + * (`{ kind: "model", model, ... }`), unlike the upstream project which stores + * plain model strings. This helper accepts both shapes and reorders in place + * without mutating the input, returning `null` when no change is required + * (winner already first, winner absent, or empty list). + */ + +type StepLike = { kind?: unknown; model?: unknown } | string; + +/** Extract the model id from a step entry (object or bare string). */ +export function comboStepModelId(step: unknown): string | null { + if (typeof step === "string") return step.trim().length > 0 ? step : null; + if (step && typeof step === "object") { + const model = (step as { model?: unknown }).model; + if (typeof model === "string" && model.trim().length > 0) return model; + } + return null; +} + +/** + * Return a reordered copy of `models` with the entry matching `winningModel` + * moved to the front, or `null` if no reordering is needed/possible. + * + * Pure: never mutates the input array or its entries. + */ +export function promoteModelToFront( + models: readonly T[] | null | undefined, + winningModel: string | null | undefined +): T[] | null { + if (!Array.isArray(models) || models.length === 0) return null; + if (typeof winningModel !== "string" || winningModel.length === 0) return null; + + const matchIndex = models.findIndex((step) => comboStepModelId(step) === winningModel); + // Winner not in the combo (e.g. global fallback) or already first → no-op. + if (matchIndex <= 0) return null; + + const winner = models[matchIndex]; + const rest = models.filter((_, index) => index !== matchIndex); + return [winner, ...rest]; +} + +interface PromoteComboDeps { + updateCombo: (id: string, data: { models: unknown[] }) => Promise; + info?: (tag: string, msg: string) => void; + warn?: (tag: string, msg: string) => void; +} + +/** + * Persist the auto-promotion of a successful combo model to position #1. + * + * Opt-in via the `comboAutoPromoteEnabled` setting. Best-effort: a DB failure is + * logged and swallowed so it never affects the already-successful response. + * No-op when the flag is off, the combo has no id, or the model is already first + * / absent. `updateCombo` is injected so this stays unit-testable without a DB. + */ +export async function promoteSuccessfulComboModel( + combo: { id?: unknown; name?: unknown; models?: unknown } | null | undefined, + winningModel: string | null | undefined, + settings: Record | null | undefined, + deps: PromoteComboDeps +): Promise { + if (!combo || !settings || !settings.comboAutoPromoteEnabled) return false; + const comboId = typeof combo.id === "string" ? combo.id : null; + if (!comboId) return false; + const reordered = promoteModelToFront( + Array.isArray(combo.models) ? (combo.models as unknown[]) : null, + winningModel + ); + if (!reordered) return false; + const label = typeof combo.name === "string" ? combo.name : comboId; + try { + await deps.updateCombo(comboId, { models: reordered }); + deps.info?.("COMBO", `Model "${winningModel}" succeeded — promoted to #1 in combo "${label}"`); + return true; + } catch (dbErr: any) { + deps.warn?.( + "COMBO", + `Failed to promote model "${winningModel}" in combo "${label}": ${ + dbErr?.message || "unknown error" + }` + ); + return false; + } +} diff --git a/src/lib/db/settings.ts b/src/lib/db/settings.ts index d94a3e9a99..10341b3860 100644 --- a/src/lib/db/settings.ts +++ b/src/lib/db/settings.ts @@ -124,6 +124,7 @@ export async function getSettings() { autoRefreshProviderQuota: false, autoRefreshProviderQuotaInterval: 180, comboConfigMode: "guided", + comboAutoPromoteEnabled: false, codexServiceTier: { enabled: false }, claudeFastMode: { enabled: false, diff --git a/src/sse/handlers/chat.ts b/src/sse/handlers/chat.ts index 3a5cbedc69..0856af5cb9 100644 --- a/src/sse/handlers/chat.ts +++ b/src/sse/handlers/chat.ts @@ -46,6 +46,8 @@ import * as log from "../utils/logger"; import { checkAndRefreshToken } from "../services/tokenRefresh"; import { createHookContext, runHooks, initPreRequestRegistry } from "@/lib/middleware/registry"; import { deleteHandoff, getHandoff } from "@/lib/db/contextHandoffs"; +import { updateCombo } from "@/lib/db/combos"; +import { promoteSuccessfulComboModel } from "@/lib/combos/autoPromote"; import { deleteSessionAccountAffinity, getCachedSettings, @@ -192,6 +194,7 @@ function intersectAllowedConnectionIds(primary: unknown, secondary: unknown): st } const PROVIDER_BREAKER_FAILURE_STATUSES = new Set([408, 500, 502, 503, 504]); +const comboPromoteDeps = { updateCombo, info: log.info, warn: log.warn }; /** * Handle chat completion request @@ -673,7 +676,17 @@ export async function handleChat( }, target?.effectiveComboStrategy ?? combo.strategy, true - ), + ).then(async (res: Response) => { + // Auto-promote the winning combo model to position #1 (opt-in flag). + if (res?.ok) + await promoteSuccessfulComboModel( + combo, + m, + settings as Record, + comboPromoteDeps + ); + return res; + }), isModelAvailable: checkModelAvailable, log, settings, diff --git a/tests/unit/combo-auto-promote.test.ts b/tests/unit/combo-auto-promote.test.ts new file mode 100644 index 0000000000..2b0f0b67d0 --- /dev/null +++ b/tests/unit/combo-auto-promote.test.ts @@ -0,0 +1,154 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +import { + promoteModelToFront, + comboStepModelId, + promoteSuccessfulComboModel, +} from "@/lib/combos/autoPromote"; + +function makeCombo() { + return { + id: "combo-1", + name: "my-combo", + models: [ + { kind: "model", model: "a/1", weight: 1 }, + { kind: "model", model: "b/2", weight: 1 }, + ], + }; +} + +test("comboStepModelId extracts model id from ComboStep object", () => { + assert.equal(comboStepModelId({ kind: "model", model: "openai/gpt-4o" }), "openai/gpt-4o"); +}); + +test("comboStepModelId extracts model id from bare string", () => { + assert.equal(comboStepModelId("anthropic/claude-3"), "anthropic/claude-3"); +}); + +test("comboStepModelId returns null for empty/invalid entries", () => { + assert.equal(comboStepModelId(""), null); + assert.equal(comboStepModelId(" "), null); + assert.equal(comboStepModelId({ kind: "combo-ref", comboName: "x" }), null); + assert.equal(comboStepModelId(null), null); + assert.equal(comboStepModelId(42), null); +}); + +test("promoteModelToFront moves the winning ComboStep to position #1", () => { + const models = [ + { kind: "model", model: "a/1", weight: 1 }, + { kind: "model", model: "b/2", weight: 1 }, + { kind: "model", model: "c/3", weight: 1 }, + ]; + const result = promoteModelToFront(models, "b/2"); + assert.deepEqual(result, [ + { kind: "model", model: "b/2", weight: 1 }, + { kind: "model", model: "a/1", weight: 1 }, + { kind: "model", model: "c/3", weight: 1 }, + ]); +}); + +test("promoteModelToFront preserves order of the remaining models", () => { + const result = promoteModelToFront(["a", "b", "c", "d"], "d"); + assert.deepEqual(result, ["d", "a", "b", "c"]); +}); + +test("promoteModelToFront does not mutate the input array or entries", () => { + const step = { kind: "model", model: "b/2", weight: 1 }; + const models = [{ kind: "model", model: "a/1", weight: 1 }, step]; + const snapshot = JSON.parse(JSON.stringify(models)); + promoteModelToFront(models, "b/2"); + assert.deepEqual(models, snapshot); + assert.equal(models[1], step); // same reference, untouched +}); + +test("promoteModelToFront returns null when winner is already first", () => { + assert.equal(promoteModelToFront(["a", "b", "c"], "a"), null); +}); + +test("promoteModelToFront returns null when winner is absent from the combo", () => { + assert.equal(promoteModelToFront(["a", "b"], "z"), null); +}); + +test("promoteModelToFront returns null for empty / missing inputs", () => { + assert.equal(promoteModelToFront([], "a"), null); + assert.equal(promoteModelToFront(null, "a"), null); + assert.equal(promoteModelToFront(undefined, "a"), null); + assert.equal(promoteModelToFront(["a", "b"], ""), null); + assert.equal(promoteModelToFront(["a", "b"], null), null); +}); + +test("promoteSuccessfulComboModel persists the reordered combo when flag is on", async () => { + const calls: Array<{ id: string; data: { models: unknown[] } }> = []; + const ok = await promoteSuccessfulComboModel( + makeCombo(), + "b/2", + { comboAutoPromoteEnabled: true }, + { + updateCombo: async (id, data) => { + calls.push({ id, data }); + return data; + }, + } + ); + assert.equal(ok, true); + assert.equal(calls.length, 1); + assert.equal(calls[0].id, "combo-1"); + assert.deepEqual( + (calls[0].data.models as Array<{ model: string }>).map((m) => m.model), + ["b/2", "a/1"] + ); +}); + +test("promoteSuccessfulComboModel is a no-op when the flag is off", async () => { + let called = false; + const ok = await promoteSuccessfulComboModel( + makeCombo(), + "b/2", + { comboAutoPromoteEnabled: false }, + { + updateCombo: async (id, data) => { + called = true; + return data; + }, + } + ); + assert.equal(ok, false); + assert.equal(called, false); +}); + +test("promoteSuccessfulComboModel is a no-op when the winner is already first", async () => { + let called = false; + const ok = await promoteSuccessfulComboModel( + makeCombo(), + "a/1", + { comboAutoPromoteEnabled: true }, + { + updateCombo: async (id, data) => { + called = true; + return data; + }, + } + ); + assert.equal(ok, false); + assert.equal(called, false); +}); + +test("promoteSuccessfulComboModel swallows DB errors and never throws", async () => { + let warned = ""; + const ok = await promoteSuccessfulComboModel( + makeCombo(), + "b/2", + { comboAutoPromoteEnabled: true }, + { + updateCombo: async () => { + throw new Error("db is down"); + }, + warn: (_tag, msg) => { + warned = msg; + }, + } + ); + assert.equal(ok, false); + assert.match(warned, /Failed to promote/); +});