From 879eda93798abc6bbcc9228a3bd239c6c5807a2d Mon Sep 17 00:00:00 2001 From: Raxxoor Date: Sun, 3 May 2026 20:12:52 +0100 Subject: [PATCH] fix(combos): align strategy contracts (#1892) Integrated into release/v3.7.9 --- open-sse/mcp-server/schemas/tools.ts | 30 +-- open-sse/mcp-server/tools/advancedTools.ts | 29 ++- open-sse/services/autoCombo/index.ts | 5 - open-sse/services/combo.ts | 54 ++++- .../settings/components/ComboDefaultsTab.tsx | 35 ++-- src/lib/db/jsonMigration.ts | 14 +- src/shared/constants/routingStrategies.ts | 81 +++++--- src/shared/schemas/validation.ts | 11 +- src/shared/validation/schemas.ts | 37 +--- src/shared/validation/settingsSchemas.ts | 19 +- src/types/settings.ts | 15 +- tests/unit/autocombo-unification.test.ts | 31 ++- tests/unit/combo-routing-engine.test.ts | 120 ++++++++++++ tests/unit/combo-strategies.test.ts | 185 +++++++++++------- tests/unit/json-migration-combos.test.ts | 51 +++++ ...settings-schema-routing-strategies.test.ts | 23 ++- 16 files changed, 494 insertions(+), 246 deletions(-) create mode 100644 tests/unit/json-migration-combos.test.ts diff --git a/open-sse/mcp-server/schemas/tools.ts b/open-sse/mcp-server/schemas/tools.ts index cacce07272..680639c02e 100644 --- a/open-sse/mcp-server/schemas/tools.ts +++ b/open-sse/mcp-server/schemas/tools.ts @@ -10,6 +10,10 @@ */ import { z } from "zod"; +import { + AUTO_ROUTING_STRATEGY_VALUES, + ROUTING_STRATEGY_VALUES, +} from "../../../src/shared/constants/routingStrategies.ts"; // ============ Shared Types ============ @@ -109,17 +113,7 @@ export const listCombosOutput = z.object({ priority: z.number(), }) ), - strategy: z.enum([ - "priority", - "weighted", - "round-robin", - "context-relay", - "strict-random", - "random", - "least-used", - "cost-optimized", - "auto", - ]), + strategy: z.enum(ROUTING_STRATEGY_VALUES), enabled: z.boolean(), metrics: z .object({ @@ -545,20 +539,10 @@ export const setBudgetGuardTool: McpToolDefinition< export const setRoutingStrategyInput = z.object({ comboId: z.string().describe("Combo ID or name to update"), strategy: z - .enum([ - "priority", - "weighted", - "round-robin", - "context-relay", - "strict-random", - "random", - "least-used", - "cost-optimized", - "auto", - ]) + .enum(ROUTING_STRATEGY_VALUES) .describe("Routing strategy to apply"), autoRoutingStrategy: z - .enum(["rules", "cost", "eco", "latency", "fast"]) + .enum(AUTO_ROUTING_STRATEGY_VALUES) .optional() .describe("Optional strategy used by auto mode (only used when strategy='auto')"), }); diff --git a/open-sse/mcp-server/tools/advancedTools.ts b/open-sse/mcp-server/tools/advancedTools.ts index 2275990db2..09041fa941 100644 --- a/open-sse/mcp-server/tools/advancedTools.ts +++ b/open-sse/mcp-server/tools/advancedTools.ts @@ -26,6 +26,11 @@ import { getComboModelString, getComboStepTarget, } from "../../../src/lib/combos/steps.ts"; +import type { + AutoRoutingStrategyValue, + RoutingStrategyValue, +} from "../../../src/shared/constants/routingStrategies.ts"; +import { normalizeRoutingStrategy } from "../../../src/shared/constants/routingStrategies.ts"; const OMNIROUTE_BASE_URL = resolveOmniRouteBaseUrl(); const OMNIROUTE_API_KEY = process.env.OMNIROUTE_API_KEY || ""; @@ -372,17 +377,8 @@ export async function handleSetBudgetGuard(args: { export async function handleSetRoutingStrategy(args: { comboId: string; - strategy: - | "priority" - | "weighted" - | "round-robin" - | "context-relay" - | "strict-random" - | "random" - | "least-used" - | "cost-optimized" - | "auto"; - autoRoutingStrategy?: "rules" | "cost" | "eco" | "latency" | "fast"; + strategy: RoutingStrategyValue; + autoRoutingStrategy?: AutoRoutingStrategyValue; }) { const start = Date.now(); try { @@ -424,8 +420,9 @@ export async function handleSetRoutingStrategy(args: { Object.keys(toRecord(combo.config)).length > 0 ? combo.config : comboData.config ); + const normalizedStrategy = normalizeRoutingStrategy(args.strategy); let nextConfig: JsonRecord | undefined = undefined; - if (args.strategy === "auto" && args.autoRoutingStrategy) { + if (normalizedStrategy === "auto" && args.autoRoutingStrategy) { const currentAutoConfig = toRecord(currentConfig.auto); nextConfig = { ...currentConfig, @@ -436,7 +433,7 @@ export async function handleSetRoutingStrategy(args: { }; } - const payload: JsonRecord = { strategy: args.strategy }; + const payload: JsonRecord = { strategy: normalizedStrategy }; if (nextConfig && Object.keys(nextConfig).length > 0) { payload.config = nextConfig; } @@ -451,16 +448,16 @@ export async function handleSetRoutingStrategy(args: { const updatedConfig = toRecord(updatedCombo.config); const resolvedAutoStrategy = toString(toRecord(updatedConfig.auto).routingStrategy) || - (args.strategy === "auto" ? (args.autoRoutingStrategy ?? "rules") : ""); + (normalizedStrategy === "auto" ? (args.autoRoutingStrategy ?? "rules") : ""); const result = { success: true, combo: { id: toString(updatedCombo.id, comboId), name: toString(updatedCombo.name, toString(combo.name, comboId)), - strategy: toString(updatedCombo.strategy, args.strategy), + strategy: toString(updatedCombo.strategy, normalizedStrategy), autoRoutingStrategy: - toString(updatedCombo.strategy, args.strategy) === "auto" ? resolvedAutoStrategy : null, + toString(updatedCombo.strategy, normalizedStrategy) === "auto" ? resolvedAutoStrategy : null, }, }; diff --git a/open-sse/services/autoCombo/index.ts b/open-sse/services/autoCombo/index.ts index 045ffaaf8d..4040c693f7 100644 --- a/open-sse/services/autoCombo/index.ts +++ b/open-sse/services/autoCombo/index.ts @@ -17,11 +17,6 @@ export { SelfHealingManager, getSelfHealingManager } from "./selfHealing"; export { MODE_PACKS, getModePack, getModePackNames } from "./modePacks"; export { selectProvider, - createAutoCombo, - getAutoCombo, - updateAutoCombo, - deleteAutoCombo, - listAutoCombos, type AutoComboConfig, type SelectionResult, } from "./engine"; diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index c84ca4f91d..df0c5a741f 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -48,6 +48,7 @@ import { resolveRequestRoutingTags, type RoutingTagMatchMode, } from "../../src/domain/tagRouter.ts"; +import { normalizeRoutingStrategy } from "../../src/shared/constants/routingStrategies.ts"; // Status codes that should mark round-robin target semaphores as cooling down. const TRANSIENT_FOR_SEMAPHORE = [429, 502, 503, 504]; @@ -630,17 +631,25 @@ function sortTargetsByUsage(targets: ResolvedComboTarget[], comboName: string) { */ function sortModelsByContextSize(models) { const withContext = models.map((modelStr) => { - const parsed = parseModel(modelStr); - const provider = parsed.provider || parsed.providerAlias || "unknown"; - const model = parsed.model || modelStr; - const limit = getModelContextLimit(provider, model); - return { modelStr, context: limit ?? 0 }; + return { modelStr, context: getModelContextLimitForModelString(modelStr) ?? 0 }; }); withContext.sort((a, b) => b.context - a.context); return withContext.map((e) => e.modelStr); } +function getModelContextLimitForModelString(modelStr: string) { + const parsed = parseModel(modelStr); + const provider = parsed.provider || parsed.providerAlias || "unknown"; + const model = parsed.model || modelStr; + return getModelContextLimit(provider, model); +} + function sortTargetsByContextSize(targets: ResolvedComboTarget[]) { + const hasKnownContext = targets.some( + (target) => getModelContextLimitForModelString(target.modelStr) != null + ); + if (!hasKnownContext) return targets; + const orderedModels = sortModelsByContextSize(targets.map((target) => target.modelStr)); const byModel = new Map(); for (const target of targets) { @@ -656,6 +665,34 @@ function sortTargetsByContextSize(targets: ResolvedComboTarget[]) { .filter((target): target is ResolvedComboTarget => target !== null); } +function getP2CTargetScore(target: ResolvedComboTarget, metrics: ReturnType): number { + const breakerState = getCircuitBreaker(target.provider)?.getStatus?.()?.state; + if (breakerState === "OPEN") return -Infinity; + const modelMetric = metrics?.byModel?.[target.modelStr] || null; + const successRate = Number(modelMetric?.successRate); + const avgLatency = Number(modelMetric?.avgLatencyMs); + const successScore = Number.isFinite(successRate) ? successRate / 100 : 0.5; + const latencyScore = Number.isFinite(avgLatency) && avgLatency > 0 ? 1 / Math.log10(avgLatency + 10) : 0.25; + const breakerPenalty = breakerState === "HALF_OPEN" ? 0.25 : 0; + return successScore + latencyScore - breakerPenalty; +} + +function orderTargetsByPowerOfTwoChoices(targets: ResolvedComboTarget[], comboName: string) { + if (targets.length <= 1) return targets; + const metrics = getComboMetrics(comboName); + const firstIndex = Math.floor(Math.random() * targets.length); + let secondIndex = Math.floor(Math.random() * (targets.length - 1)); + if (secondIndex >= firstIndex) secondIndex++; + + const first = targets[firstIndex]; + const second = targets[secondIndex]; + const selectedIndex = + getP2CTargetScore(second, metrics) > getP2CTargetScore(first, metrics) + ? secondIndex + : firstIndex; + return [targets[selectedIndex], ...targets.filter((_, index) => index !== selectedIndex)]; +} + function toTextContent(content) { if (typeof content === "string") return content; if (!Array.isArray(content)) return ""; @@ -1034,7 +1071,7 @@ export async function handleComboChat({ relayOptions, signal, }) { - const strategy = combo.strategy || "priority"; + const strategy = normalizeRoutingStrategy(combo.strategy || "priority"); const relayConfig = strategy === "context-relay" ? resolveContextRelayConfig(relayOptions?.config || null) : null; @@ -1427,6 +1464,11 @@ export async function handleComboChat({ } else if (strategy === "random") { orderedTargets = fisherYatesShuffle([...orderedTargets]); log.info("COMBO", `Random shuffle: ${orderedTargets.length} targets`); + } else if (strategy === "fill-first") { + log.info("COMBO", `Fill-first ordering: preserving priority order (${orderedTargets.length} targets)`); + } else if (strategy === "p2c") { + orderedTargets = orderTargetsByPowerOfTwoChoices(orderedTargets, combo.name); + log.info("COMBO", `Power-of-two-choices ordering: selected ${orderedTargets[0]?.modelStr}`); } else if (strategy === "least-used") { orderedTargets = sortTargetsByUsage(orderedTargets, combo.name); log.info("COMBO", `Least-used ordering: ${orderedTargets[0]?.modelStr} has fewest requests`); diff --git a/src/app/(dashboard)/dashboard/settings/components/ComboDefaultsTab.tsx b/src/app/(dashboard)/dashboard/settings/components/ComboDefaultsTab.tsx index 44f4c61f6c..4f0036b02e 100644 --- a/src/app/(dashboard)/dashboard/settings/components/ComboDefaultsTab.tsx +++ b/src/app/(dashboard)/dashboard/settings/components/ComboDefaultsTab.tsx @@ -3,7 +3,10 @@ import { useState, useEffect } from "react"; import { Card, Button, Input, Toggle } from "@/shared/components"; import { cn } from "@/shared/utils/cn"; -import { ROUTING_STRATEGIES } from "@/shared/constants/routingStrategies"; +import { + ROUTING_STRATEGIES, + SETTINGS_FALLBACK_STRATEGY_VALUES, +} from "@/shared/constants/routingStrategies"; import { useTranslations } from "next-intl"; const STRATEGY_LABEL_FALLBACKS: Record = { @@ -15,6 +18,7 @@ const LEGACY_COMBO_RESILIENCE_KEYS = new Set([ "healthCheckEnabled", "healthCheckTimeoutMs", ]); +const ACCOUNT_FALLBACK_STRATEGIES = new Set(SETTINGS_FALLBACK_STRATEGY_VALUES); function translateOrFallback( t: ReturnType, @@ -44,6 +48,17 @@ function sanitizeProviderOverrides(overrides?: Record | null) { ); } +function toGlobalRoutingPatch(strategy: string | undefined, stickyRoundRobinLimit?: number) { + const patch: Record = {}; + if (strategy && ACCOUNT_FALLBACK_STRATEGIES.has(strategy)) { + patch.fallbackStrategy = strategy; + } + if (strategy === "round-robin" && stickyRoundRobinLimit !== undefined) { + patch.stickyRoundRobinLimit = stickyRoundRobinLimit; + } + return patch; +} + export default function ComboDefaultsTab() { const [comboDefaults, setComboDefaults] = useState({ strategy: "priority", @@ -89,8 +104,7 @@ export default function ComboDefaultsTab() { setComboDefaults((prev) => ({ ...prev, ...sanitizeComboRuntimeConfig(comboData.comboDefaults), - strategy: - settingsData.fallbackStrategy ?? comboData.comboDefaults?.strategy ?? prev.strategy, + strategy: comboData.comboDefaults?.strategy ?? settingsData.fallbackStrategy ?? prev.strategy, stickyRoundRobinLimit: settingsData.stickyRoundRobinLimit ?? comboData.comboDefaults?.stickyRoundRobinLimit ?? @@ -109,8 +123,7 @@ export default function ComboDefaultsTab() { }; const syncGlobalRoutingSettings = async (patch: Record) => { - const keys = Object.keys(patch); - if (keys.length === 0) return true; + if (Object.keys(patch).length === 0) return; const res = await fetch("/api/settings", { method: "PATCH", @@ -121,21 +134,13 @@ export default function ComboDefaultsTab() { if (!res.ok) { throw new Error("Failed to sync global routing settings"); } - - return true; }; const saveComboDefaults = async () => { setSaving(true); try { const { stickyRoundRobinLimit, ...comboDefaultsPayload } = comboDefaults; - const settingsPatch: Record = {}; - if (comboDefaults.strategy) { - settingsPatch.fallbackStrategy = comboDefaults.strategy; - } - if (comboDefaults.strategy === "round-robin" && stickyRoundRobinLimit !== undefined) { - settingsPatch.stickyRoundRobinLimit = stickyRoundRobinLimit; - } + const settingsPatch = toGlobalRoutingPatch(comboDefaults.strategy, stickyRoundRobinLimit); const comboDefaultsRes = await fetch("/api/settings/combo-defaults", { method: "PATCH", @@ -243,7 +248,7 @@ export default function ComboDefaultsTab() { onClick={async () => { setComboDefaults((prev) => ({ ...prev, strategy: s.value })); try { - await syncGlobalRoutingSettings({ fallbackStrategy: s.value }); + await syncGlobalRoutingSettings(toGlobalRoutingPatch(s.value)); } catch (error) { console.error("Failed to sync fallback strategy:", error); showStatus("error", t("errorOccurred")); diff --git a/src/lib/db/jsonMigration.ts b/src/lib/db/jsonMigration.ts index b3128cdd9c..5e6e237485 100644 --- a/src/lib/db/jsonMigration.ts +++ b/src/lib/db/jsonMigration.ts @@ -12,6 +12,7 @@ */ import type Database from "better-sqlite3"; +import { normalizeRoutingStrategy } from "@/shared/constants/routingStrategies"; type SqliteDatabase = InstanceType; @@ -172,8 +173,19 @@ export function runJsonMigration( // 5. Combos for (const [index, combo] of (data.combos ?? []).entries()) { - const normalizedCombo = { + const config = + combo.config && typeof combo.config === "object" && !Array.isArray(combo.config) + ? { ...(combo.config as Record) } + : combo.config; + if (config && typeof config === "object" && !Array.isArray(config) && "strategy" in config) { + (config as Record).strategy = normalizeRoutingStrategy( + (config as Record).strategy + ); + } + const normalizedCombo: Record = { ...combo, + strategy: normalizeRoutingStrategy(combo.strategy), + config, sortOrder: typeof combo.sortOrder === "number" ? combo.sortOrder : index + 1, }; insertCombo.run({ diff --git a/src/shared/constants/routingStrategies.ts b/src/shared/constants/routingStrategies.ts index e2ebcdffb3..752683e7ca 100644 --- a/src/shared/constants/routingStrategies.ts +++ b/src/shared/constants/routingStrategies.ts @@ -1,17 +1,55 @@ -export type RoutingStrategyValue = - | "priority" - | "weighted" - | "round-robin" - | "context-relay" - | "fill-first" - | "p2c" - | "random" - | "least-used" - | "cost-optimized" - | "strict-random" - | "auto" - | "context-optimized" - | "lkgp"; +export const ROUTING_STRATEGY_VALUES = [ + "priority", + "weighted", + "round-robin", + "context-relay", + "fill-first", + "p2c", + "random", + "least-used", + "cost-optimized", + "strict-random", + "auto", + "lkgp", + "context-optimized", +] as const; + +export type RoutingStrategyValue = (typeof ROUTING_STRATEGY_VALUES)[number]; + +export const AUTO_ROUTING_STRATEGY_VALUES = [ + "rules", + "cost", + "eco", + "latency", + "fast", + "lkgp", +] as const; + +export type AutoRoutingStrategyValue = (typeof AUTO_ROUTING_STRATEGY_VALUES)[number]; + +export const ACCOUNT_FALLBACK_STRATEGY_VALUES = [ + "priority", + "weighted", + "fill-first", + "round-robin", + "p2c", + "random", + "least-used", + "cost-optimized", + "strict-random", +] as const; + +export type AccountFallbackStrategyValue = (typeof ACCOUNT_FALLBACK_STRATEGY_VALUES)[number]; + +export function normalizeRoutingStrategy(value: unknown): RoutingStrategyValue { + if (typeof value !== "string") return "priority"; + const normalized = value.trim().toLowerCase(); + if (normalized === "usage") return "least-used"; + if (normalized === "context") return "context-optimized"; + return (ROUTING_STRATEGY_VALUES as readonly string[]).includes(normalized) + ? (normalized as RoutingStrategyValue) + : "priority"; +} type RoutingStrategyOption = { value: RoutingStrategyValue; @@ -115,17 +153,4 @@ export const ROUTING_STRATEGIES: RoutingStrategyOption[] = [ }, ]; -export const SETTINGS_FALLBACK_STRATEGY_VALUES: RoutingStrategyValue[] = [ - "priority", - "weighted", - "fill-first", - "round-robin", - "p2c", - "random", - "least-used", - "cost-optimized", - "strict-random", - "auto", - "context-optimized", - "lkgp", -]; +export const SETTINGS_FALLBACK_STRATEGY_VALUES = ACCOUNT_FALLBACK_STRATEGY_VALUES; diff --git a/src/shared/schemas/validation.ts b/src/shared/schemas/validation.ts index 784518a4d0..628aeaeaf6 100644 --- a/src/shared/schemas/validation.ts +++ b/src/shared/schemas/validation.ts @@ -7,6 +7,7 @@ */ import { z } from "zod"; +import { ROUTING_STRATEGY_VALUES } from "@/shared/constants/routingStrategies"; // ─── Provider Connection ────────────────────────────────────────── @@ -39,15 +40,7 @@ export const comboSchema = z.object({ model: z.string().min(1, "Model pattern is required"), endpoint: z.enum(["chat", "embeddings", "images"]).default("chat"), strategy: z - .enum([ - "priority", - "weighted", - "round-robin", - "context-relay", - "random", - "least-used", - "cost-optimized", - ]) + .enum(ROUTING_STRATEGY_VALUES) .default("priority"), nodes: z.array(comboNodeSchema).min(1, "At least one node is required"), isActive: z.boolean().default(true), diff --git a/src/shared/validation/schemas.ts b/src/shared/validation/schemas.ts index 14cee73aaf..b28f347cb6 100644 --- a/src/shared/validation/schemas.ts +++ b/src/shared/validation/schemas.ts @@ -1,4 +1,8 @@ import { z } from "zod"; +import { + ACCOUNT_FALLBACK_STRATEGY_VALUES, + ROUTING_STRATEGY_VALUES, +} from "@/shared/constants/routingStrategies"; import { SUPPORTED_BATCH_ENDPOINTS } from "@/shared/constants/batchEndpoints"; import { COMBO_CONFIG_MODES } from "@/shared/constants/comboConfigMode"; import { isLocalProvider } from "@/shared/constants/providers"; @@ -324,22 +328,7 @@ const comboModelEntry = z.union([ comboRefStepInputSchema, ]); -export const comboStrategySchema = z.enum([ - "priority", - "weighted", - "round-robin", - "context-relay", - "random", - "least-used", - "cost-optimized", - "strict-random", - "auto", - "fill-first", - // #729 schema fixes for combo edit/save - "p2c", - "lkgp", - "context-optimized", -]); +export const comboStrategySchema = z.enum(ROUTING_STRATEGY_VALUES); const scoringWeightsSchema = z .object({ @@ -427,21 +416,7 @@ export const createComboSchema = z.object({ // ──── Settings Schemas ──── // FASE-01: Removed .passthrough() — only explicitly listed fields are accepted -const settingsFallbackStrategySchema = z.enum([ - "priority", - "weighted", - "round-robin", - "context-relay", - "fill-first", - "p2c", - "random", - "least-used", - "cost-optimized", - "strict-random", - "auto", - "context-optimized", - "lkgp", -]); +const settingsFallbackStrategySchema = z.enum(ACCOUNT_FALLBACK_STRATEGY_VALUES); export const updateSettingsSchema = z.object({ newPassword: z.string().min(1).max(200).optional(), diff --git a/src/shared/validation/settingsSchemas.ts b/src/shared/validation/settingsSchemas.ts index acec76d9b2..4d7928f8e5 100644 --- a/src/shared/validation/settingsSchemas.ts +++ b/src/shared/validation/settingsSchemas.ts @@ -8,22 +8,7 @@ import { z } from "zod"; import { COMBO_CONFIG_MODES } from "@/shared/constants/comboConfigMode"; import { HIDEABLE_SIDEBAR_ITEM_IDS } from "@/shared/constants/sidebarVisibility"; - -const fallbackStrategyValues = [ - "priority", - "weighted", - "round-robin", - "context-relay", - "fill-first", - "p2c", - "random", - "least-used", - "cost-optimized", - "strict-random", - "auto", - "context-optimized", - "lkgp", -] as const; +import { ACCOUNT_FALLBACK_STRATEGY_VALUES } from "@/shared/constants/routingStrategies"; const signatureCacheModeValues = ["enabled", "bypass", "bypass-strict"] as const; @@ -52,7 +37,7 @@ export const updateSettingsSchema = z.object({ hiddenSidebarItems: z.array(z.enum(HIDEABLE_SIDEBAR_ITEM_IDS)).optional(), comboConfigMode: z.enum(COMBO_CONFIG_MODES).optional(), // Routing settings (#134) - fallbackStrategy: z.enum(fallbackStrategyValues).optional(), + fallbackStrategy: z.enum(ACCOUNT_FALLBACK_STRATEGY_VALUES).optional(), wildcardAliases: z.array(z.object({ pattern: z.string(), target: z.string() })).optional(), stickyRoundRobinLimit: z.number().int().min(0).max(1000).optional(), requestRetry: z.number().int().min(0).max(10).optional(), diff --git a/src/types/settings.ts b/src/types/settings.ts index 43edc4064f..e8143246b2 100644 --- a/src/types/settings.ts +++ b/src/types/settings.ts @@ -1,5 +1,9 @@ import type { HideableSidebarItemId } from "@/shared/constants/sidebarVisibility"; import type { ResilienceSettings } from "@/lib/resilience/settings"; +import type { + AccountFallbackStrategyValue, + RoutingStrategyValue, +} from "@/shared/constants/routingStrategies"; /** * Application settings stored in SQLite key-value pairs. @@ -7,14 +11,7 @@ import type { ResilienceSettings } from "@/lib/resilience/settings"; export interface Settings { requireLogin: boolean; hasPassword: boolean; - fallbackStrategy: - | "fill-first" - | "round-robin" - | "p2c" - | "random" - | "least-used" - | "cost-optimized" - | "strict-random"; + fallbackStrategy: AccountFallbackStrategyValue; stickyRoundRobinLimit: number; requestRetry: number; maxRetryIntervalSec: number; @@ -31,7 +28,7 @@ export interface Settings { } export interface ComboDefaults { - strategy: "priority" | "weighted" | "round-robin" | "context-relay"; + strategy: RoutingStrategyValue; maxRetries: number; retryDelayMs: number; maxComboDepth: number; diff --git a/tests/unit/autocombo-unification.test.ts b/tests/unit/autocombo-unification.test.ts index 4ff04fa2dc..a242f83628 100644 --- a/tests/unit/autocombo-unification.test.ts +++ b/tests/unit/autocombo-unification.test.ts @@ -50,17 +50,19 @@ test("filterCombosByStrategyCategory returns expected combo subsets", () => { }); test("combo strategies stay aligned between UI metadata and schema validation", async () => { - const { ROUTING_STRATEGIES } = await import("../../src/shared/constants/routingStrategies.ts"); + const { ROUTING_STRATEGIES, ROUTING_STRATEGY_VALUES, normalizeRoutingStrategy } = + await import("../../src/shared/constants/routingStrategies.ts"); const { comboStrategySchema, createComboSchema } = await import("../../src/shared/validation/schemas.ts"); + const { comboSchema } = await import("../../src/shared/schemas/validation.ts"); + const { setRoutingStrategyInput } = await import("../../open-sse/mcp-server/schemas/tools.ts"); const strategyValues = ROUTING_STRATEGIES.map((strategy) => strategy.value); - assert.equal(strategyValues.length, 13); - assert.equal(new Set(strategyValues).size, 13); + assert.deepEqual(strategyValues, [...ROUTING_STRATEGY_VALUES]); + assert.equal(new Set(strategyValues).size, ROUTING_STRATEGY_VALUES.length); assert.equal(strategyValues.includes("auto"), true); assert.equal(strategyValues.includes("lkgp"), true); - assert.equal(comboStrategySchema.options.length, 13); - assert.equal(new Set(comboStrategySchema.options).size, 13); + assert.deepEqual(comboStrategySchema.options, [...ROUTING_STRATEGY_VALUES]); strategyValues.forEach((strategy) => { const parsed = createComboSchema.safeParse({ @@ -69,8 +71,27 @@ test("combo strategies stay aligned between UI metadata and schema validation", strategy, }); assert.equal(parsed.success, true, `schema should accept strategy ${strategy}`); + assert.equal( + comboSchema.safeParse({ + name: `legacy-combo-${strategy}`, + model: "openai/gpt-4o-mini", + strategy, + nodes: [{ connectionId: crypto.randomUUID() }], + }).success, + true, + `legacy combo schema should accept strategy ${strategy}` + ); + assert.equal( + setRoutingStrategyInput.safeParse({ comboId: "combo", strategy }).success, + true, + `MCP set strategy schema should accept ${strategy}` + ); }); + assert.equal(normalizeRoutingStrategy("usage"), "least-used"); + assert.equal(normalizeRoutingStrategy("context"), "context-optimized"); + assert.equal(normalizeRoutingStrategy("unknown"), "priority"); + const invalidParse = createComboSchema.safeParse({ name: "combo-invalid", models: ["openai/gpt-4o-mini"], diff --git a/tests/unit/combo-routing-engine.test.ts b/tests/unit/combo-routing-engine.test.ts index ca40725d0d..7079a800a4 100644 --- a/tests/unit/combo-routing-engine.test.ts +++ b/tests/unit/combo-routing-engine.test.ts @@ -432,6 +432,73 @@ test("handleComboChat random strategy uses shuffled model order", async () => { } }); +test("handleComboChat fill-first explicitly preserves priority order", async () => { + const calls: any[] = []; + + await handleComboChat({ + body: {}, + combo: { + name: "fill-first-order", + strategy: "fill-first", + models: ["model-a", "model-b"], + }, + handleSingleModel: async (_body: any, modelStr: any) => { + calls.push(modelStr); + return okResponse(); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: null, + relayOptions: null as any, + allCombos: null, + }); + + assert.deepEqual(calls, ["model-a"]); +}); + +test("handleComboChat p2c selects the better of two random choices by metrics", async () => { + const originalRandom = Math.random; + const calls: any[] = []; + const sequence = [0.0, 0.0]; + let idx = 0; + + recordComboRequest("p2c-combo", "model-a", { + success: true, + latencyMs: 2000, + strategy: "p2c", + }); + recordComboRequest("p2c-combo", "model-b", { + success: true, + latencyMs: 20, + strategy: "p2c", + }); + Math.random = () => sequence[idx++] ?? 0; + + try { + await handleComboChat({ + body: {}, + combo: { + name: "p2c-combo", + strategy: "p2c", + models: ["model-a", "model-b", "model-c"], + }, + handleSingleModel: async (_body: any, modelStr: any) => { + calls.push(modelStr); + return okResponse(); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: null, + relayOptions: null as any, + allCombos: null, + }); + + assert.deepEqual(calls, ["model-b"]); + } finally { + Math.random = originalRandom; + } +}); + test("handleComboChat least-used strategy prefers the model with fewer recorded requests", async () => { recordComboRequest("least-used-combo", "model-a", { success: true, @@ -1318,6 +1385,59 @@ test("handleComboChat context-optimized orders models by the largest synced cont assert.equal(calls[0], "openai/gpt-4o-max"); }); +test("handleComboChat context-optimized preserves order when all context limits are unknown", async () => { + const calls: any[] = []; + const result = await handleComboChat({ + body: {}, + combo: { + name: "context-optimized-unknown", + strategy: "context-optimized", + models: ["unknown/model-a", "unknown/model-b"], + }, + handleSingleModel: async (_body: any, modelStr: any) => { + calls.push(modelStr); + return okResponse(); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: null, + relayOptions: null as any, + allCombos: null, + }); + + assert.equal(result.ok, true); + assert.deepEqual(calls, ["unknown/model-a"]); +}); + +test("handleComboChat normalizes legacy strategy names at runtime", async () => { + const usageCalls: any[] = []; + recordComboRequest("legacy-usage-combo", "model-a", { + success: true, + latencyMs: 100, + strategy: "least-used", + }); + + await handleComboChat({ + body: {}, + combo: { + name: "legacy-usage-combo", + strategy: "usage", + models: ["model-a", "model-b"], + }, + handleSingleModel: async (_body: any, modelStr: any) => { + usageCalls.push(modelStr); + return okResponse(); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: null, + relayOptions: null as any, + allCombos: null, + }); + + assert.deepEqual(usageCalls, ["model-b"]); +}); + test("handleComboChat returns a 503 when every model is unavailable before execution", async () => { const result = await handleComboChat({ body: {}, diff --git a/tests/unit/combo-strategies.test.ts b/tests/unit/combo-strategies.test.ts index 3a4acc84d0..2f97b1a2de 100644 --- a/tests/unit/combo-strategies.test.ts +++ b/tests/unit/combo-strategies.test.ts @@ -1,21 +1,28 @@ -import test from "node:test"; +import test, { after } from "node:test"; import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; -import { handleComboChat } from "../../open-sse/services/combo.ts"; -import * as combosDb from "../../src/lib/db/combos.ts"; -import * as modelsDb from "../../src/lib/db/models.ts"; +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-combo-strategies-")); +const ORIGINAL_DATA_DIR = process.env.DATA_DIR; +process.env.DATA_DIR = TEST_DATA_DIR; -function stubConnection() { - return [ - { - id: "conn-openai", - provider: "openai", - authType: "oauth", - isActive: 1, - accessToken: "token", - }, - ]; -} +const dbCore = await import("../../src/lib/db/core.ts"); +const { handleComboChat } = await import("../../open-sse/services/combo.ts"); +const combosDb = await import("../../src/lib/db/combos.ts"); +const { recordComboRequest } = await import("../../open-sse/services/comboMetrics.ts"); +const { saveModelsDevCapabilities } = await import("../../src/lib/modelsDevSync.ts"); + +after(() => { + dbCore.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + if (ORIGINAL_DATA_DIR === undefined) { + delete process.env.DATA_DIR; + } else { + process.env.DATA_DIR = ORIGINAL_DATA_DIR; + } +}); const reqBodyNullContext = { model: "comboTest", @@ -29,78 +36,110 @@ const reqBodyTextArray = { stream: false, }; -test("handleComboChat with 'usage' strategy hits sortModelsByUsage", async () => { - const id = crypto.randomUUID(); - await combosDb.createCombo({ - id, - name: id, - strategy: "usage", - models: ["openai/gpt-3.5-turbo", "openai/gpt-4"], +function capability(limitContext: number) { + return { + tool_call: true, + reasoning: null, + attachment: null, + structured_output: null, + temperature: null, + modalities_input: "[]", + modalities_output: "[]", + knowledge_cutoff: null, + release_date: null, + last_updated: null, + status: null, + family: null, + open_weights: null, + limit_context: limitContext, + limit_input: null, + limit_output: null, + interleaved_field: null, + }; +} + +function okResponse(model: string) { + return Response.json({ choices: [{ message: { role: "assistant", content: model } }] }); +} + +function makeLog() { + return { + info() {}, + warn() {}, + debug() {}, + error() {}, + }; +} + +async function selectedModelFor(combo: Record, body: Record) { + const calls: string[] = []; + const response = await handleComboChat({ + body, + combo, + allCombos: [combo], + isModelAvailable: undefined, + relayOptions: undefined, + signal: undefined, + settings: {}, + log: makeLog(), + handleSingleModel: async (_body: unknown, modelStr: string) => { + calls.push(modelStr); + return okResponse(modelStr); + }, }); - // No proxy/http mock needed if we expect error or quick reject - const req = new Request("http://localhost", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ ...reqBodyTextArray, model: id }), + assert.equal(response.status, 200); + assert.equal(calls.length > 0, true); + return calls[0]; +} + +test("least-used strategy prefers the model with fewer recorded combo requests", async () => { + const name = `least-used-${crypto.randomUUID()}`; + const busyModel = "openai/gpt-4"; + const idleModel = "openai/gpt-3.5-turbo"; + const combo = await combosDb.createCombo({ + name, + strategy: "least-used", + models: [busyModel, idleModel], }); - try { - const res = await handleComboChat(id, req, stubConnection()); - assert.equal(res.status >= 200, true); - } catch (e: any) { - // Expect error as fetch is not globally mocked for this quick edge branch test, that's fine - } + recordComboRequest(name, busyModel, { + success: true, + latencyMs: 10, + strategy: "least-used", + }); + + assert.equal(await selectedModelFor(combo, reqBodyTextArray), idleModel); }); -test("handleComboChat with 'context' strategy hits sortModelsByContextSize", async () => { - const id = crypto.randomUUID(); - await combosDb.createCombo({ - id, - name: id, - strategy: "context", - models: ["openai/gpt-4-32k", "openai/gpt-4", "unknown/unknown"], +test("context-optimized strategy prefers the largest context window", async () => { + saveModelsDevCapabilities({ + "test-context": { + small: capability(8_000), + large: capability(64_000), + }, }); - const req = new Request("http://localhost", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ ...reqBodyNullContext, model: id }), + const combo = await combosDb.createCombo({ + name: `context-optimized-${crypto.randomUUID()}`, + strategy: "context-optimized", + models: ["test-context/small", "test-context/large", "unknown/unknown"], }); - try { - const res = await handleComboChat(id, req, stubConnection()); - } catch (e: any) {} + assert.equal(await selectedModelFor(combo, reqBodyNullContext), "test-context/large"); }); -test("handleComboChat hits extractPromptForIntent edge cases", async () => { - const id = crypto.randomUUID(); - await combosDb.createCombo({ - id, - name: id, +test("auto strategy handles null and empty prompt edge cases without throwing", async () => { + const combo = await combosDb.createCombo({ + name: `auto-${crypto.randomUUID()}`, strategy: "auto", - autoConfig: { intentConfig: { enabled: true } }, + config: { auto: { explorationRate: 0 } }, models: ["openai/gpt-4"], }); - // Null message content - const reqNull = new Request("http://localhost", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ model: id, messages: [{ role: "user", content: null }] }), - }); - - // Empty messages array - const reqEmpty = new Request("http://localhost", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ model: id, messages: [] }), - }); - - try { - await handleComboChat(id, reqNull, stubConnection()); - } catch (e: any) {} - try { - await handleComboChat(id, reqEmpty, stubConnection()); - } catch (e: any) {} + assert.equal( + await selectedModelFor(combo, { model: combo.name, messages: [{ role: "user", content: null }] }), + "openai/gpt-4" + ); + assert.equal(await selectedModelFor(combo, { model: combo.name, messages: [] }), "openai/gpt-4"); }); diff --git a/tests/unit/json-migration-combos.test.ts b/tests/unit/json-migration-combos.test.ts new file mode 100644 index 0000000000..b31d938068 --- /dev/null +++ b/tests/unit/json-migration-combos.test.ts @@ -0,0 +1,51 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-json-migration-")); +const ORIGINAL_DATA_DIR = process.env.DATA_DIR; +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../src/lib/db/core.ts"); +const { runJsonMigration } = await import("../../src/lib/db/jsonMigration.ts"); + +test.after(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + if (ORIGINAL_DATA_DIR === undefined) delete process.env.DATA_DIR; + else process.env.DATA_DIR = ORIGINAL_DATA_DIR; +}); + +test("runJsonMigration normalizes legacy combo strategy names at the import boundary", () => { + const db = core.getDbInstance(); + + runJsonMigration(db, { + combos: [ + { + id: "combo-usage", + name: "combo-usage", + strategy: "usage", + models: ["openai/gpt-4o-mini"], + config: { strategy: "context" }, + }, + { + id: "combo-unknown", + name: "combo-unknown", + strategy: "not-a-real-strategy", + models: ["openai/gpt-4o-mini"], + }, + ], + }); + + const rows = db.prepare("SELECT id, data FROM combos ORDER BY id ASC").all() as Array<{ + id: string; + data: string; + }>; + const byId = new Map(rows.map((row) => [row.id, JSON.parse(row.data)])); + + assert.equal(byId.get("combo-usage").strategy, "least-used"); + assert.equal(byId.get("combo-usage").config.strategy, "context-optimized"); + assert.equal(byId.get("combo-unknown").strategy, "priority"); +}); diff --git a/tests/unit/settings-schema-routing-strategies.test.ts b/tests/unit/settings-schema-routing-strategies.test.ts index fdacf9d37b..bb971d64cd 100644 --- a/tests/unit/settings-schema-routing-strategies.test.ts +++ b/tests/unit/settings-schema-routing-strategies.test.ts @@ -1,21 +1,28 @@ import test from "node:test"; import assert from "node:assert/strict"; -import { ROUTING_STRATEGIES } from "@/shared/constants/routingStrategies"; +import { SETTINGS_FALLBACK_STRATEGY_VALUES } from "@/shared/constants/routingStrategies"; import { updateSettingsSchema as settingsRouteSchema } from "@/shared/validation/settingsSchemas"; import { updateSettingsSchema as sharedSettingsSchema } from "@/shared/validation/schemas"; -for (const strategy of ROUTING_STRATEGIES) { - test(`settings route schema accepts fallbackStrategy=${strategy.value}`, () => { - const parsed = settingsRouteSchema.parse({ fallbackStrategy: strategy.value }); - assert.equal(parsed.fallbackStrategy, strategy.value); +for (const strategy of SETTINGS_FALLBACK_STRATEGY_VALUES) { + test(`settings route schema accepts fallbackStrategy=${strategy}`, () => { + const parsed = settingsRouteSchema.parse({ fallbackStrategy: strategy }); + assert.equal(parsed.fallbackStrategy, strategy); }); - test(`shared settings schema accepts fallbackStrategy=${strategy.value}`, () => { - const parsed = sharedSettingsSchema.parse({ fallbackStrategy: strategy.value }); - assert.equal(parsed.fallbackStrategy, strategy.value); + test(`shared settings schema accepts fallbackStrategy=${strategy}`, () => { + const parsed = sharedSettingsSchema.parse({ fallbackStrategy: strategy }); + assert.equal(parsed.fallbackStrategy, strategy); }); } +test("settings schemas reject combo-only strategies as account fallback strategies", () => { + for (const strategy of ["auto", "lkgp", "context-optimized"]) { + assert.equal(settingsRouteSchema.safeParse({ fallbackStrategy: strategy }).success, false); + assert.equal(sharedSettingsSchema.safeParse({ fallbackStrategy: strategy }).success, false); + } +}); + test("settings schemas accept cooldown-aware retry knobs", () => { const payload = { requestRetry: 3,