pjandro/OmniRoute
1
0
Fork 0
mirror of https://github.com/diegosouzapw/OmniRoute.git synced 2026-07-26 09:52:11 +03:00
Code Issues Packages Projects Releases Wiki Activity

fix(combos): align strategy contracts (#1892)

Browse Source 
Integrated into release/v3.7.9
This commit is contained in:
Raxxoor
2026-05-03 20:12:52 +01:00
committed by GitHub
parent 6b7f33183c
commit 879eda9379
16 changed files with 494 additions and 246 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
open-sse/mcp-server/schemas/tools.ts
View File

@@ -10,6 +10,10 @@
*/
import { z } from "zod";
import {
AUTO_ROUTING_STRATEGY_VALUES,
ROUTING_STRATEGY_VALUES,
} from "../../../src/shared/constants/routingStrategies.ts";
// ============ Shared Types ============
@@ -109,17 +113,7 @@ export const listCombosOutput = z.object({
priority: z.number(),
})
),
strategy: z.enum([
"priority",
"weighted",
"round-robin",
"context-relay",
"strict-random",
"random",
"least-used",
"cost-optimized",
"auto",
]),
strategy: z.enum(ROUTING_STRATEGY_VALUES),
enabled: z.boolean(),
metrics: z
.object({
@@ -545,20 +539,10 @@ export const setBudgetGuardTool: McpToolDefinition<
export const setRoutingStrategyInput = z.object({
comboId: z.string().describe("Combo ID or name to update"),
strategy: z
.enum([
"priority",
"weighted",
"round-robin",
"context-relay",
"strict-random",
"random",
"least-used",
"cost-optimized",
"auto",
])
.enum(ROUTING_STRATEGY_VALUES)
.describe("Routing strategy to apply"),
autoRoutingStrategy: z
.enum(["rules", "cost", "eco", "latency", "fast"])
.enum(AUTO_ROUTING_STRATEGY_VALUES)
.optional()
.describe("Optional strategy used by auto mode (only used when strategy='auto')"),
});

29
open-sse/mcp-server/tools/advancedTools.ts
View File

@@ -26,6 +26,11 @@ import {
getComboModelString,
getComboStepTarget,
} from "../../../src/lib/combos/steps.ts";
import type {
AutoRoutingStrategyValue,
RoutingStrategyValue,
} from "../../../src/shared/constants/routingStrategies.ts";
import { normalizeRoutingStrategy } from "../../../src/shared/constants/routingStrategies.ts";
const OMNIROUTE_BASE_URL = resolveOmniRouteBaseUrl();
const OMNIROUTE_API_KEY = process.env.OMNIROUTE_API_KEY || "";
@@ -372,17 +377,8 @@ export async function handleSetBudgetGuard(args: {
export async function handleSetRoutingStrategy(args: {
comboId: string;
strategy:
| "priority"
| "weighted"
| "round-robin"
| "context-relay"
| "strict-random"
| "random"
| "least-used"
| "cost-optimized"
| "auto";
autoRoutingStrategy?: "rules" | "cost" | "eco" | "latency" | "fast";
strategy: RoutingStrategyValue;
autoRoutingStrategy?: AutoRoutingStrategyValue;
}) {
const start = Date.now();
try {
@@ -424,8 +420,9 @@ export async function handleSetRoutingStrategy(args: {
Object.keys(toRecord(combo.config)).length > 0 ? combo.config : comboData.config
);
const normalizedStrategy = normalizeRoutingStrategy(args.strategy);
let nextConfig: JsonRecord | undefined = undefined;
if (args.strategy === "auto" && args.autoRoutingStrategy) {
if (normalizedStrategy === "auto" && args.autoRoutingStrategy) {
const currentAutoConfig = toRecord(currentConfig.auto);
nextConfig = {
...currentConfig,
@@ -436,7 +433,7 @@ export async function handleSetRoutingStrategy(args: {
};
}
const payload: JsonRecord = { strategy: args.strategy };
const payload: JsonRecord = { strategy: normalizedStrategy };
if (nextConfig && Object.keys(nextConfig).length > 0) {
payload.config = nextConfig;
}
@@ -451,16 +448,16 @@ export async function handleSetRoutingStrategy(args: {
const updatedConfig = toRecord(updatedCombo.config);
const resolvedAutoStrategy =
toString(toRecord(updatedConfig.auto).routingStrategy) ||
(args.strategy === "auto" ? (args.autoRoutingStrategy ?? "rules") : "");
(normalizedStrategy === "auto" ? (args.autoRoutingStrategy ?? "rules") : "");
const result = {
success: true,
combo: {
id: toString(updatedCombo.id, comboId),
name: toString(updatedCombo.name, toString(combo.name, comboId)),
strategy: toString(updatedCombo.strategy, args.strategy),
strategy: toString(updatedCombo.strategy, normalizedStrategy),
autoRoutingStrategy:
toString(updatedCombo.strategy, args.strategy) === "auto" ? resolvedAutoStrategy : null,
toString(updatedCombo.strategy, normalizedStrategy) === "auto" ? resolvedAutoStrategy : null,
},
};

5
open-sse/services/autoCombo/index.ts
View File

@@ -17,11 +17,6 @@ export { SelfHealingManager, getSelfHealingManager } from "./selfHealing";
export { MODE_PACKS, getModePack, getModePackNames } from "./modePacks";
export {
selectProvider,
createAutoCombo,
getAutoCombo,
updateAutoCombo,
deleteAutoCombo,
listAutoCombos,
type AutoComboConfig,
type SelectionResult,
} from "./engine";

54
open-sse/services/combo.ts
View File

@@ -48,6 +48,7 @@ import {
resolveRequestRoutingTags,
type RoutingTagMatchMode,
} from "../../src/domain/tagRouter.ts";
import { normalizeRoutingStrategy } from "../../src/shared/constants/routingStrategies.ts";
// Status codes that should mark round-robin target semaphores as cooling down.
const TRANSIENT_FOR_SEMAPHORE = [429, 502, 503, 504];
@@ -630,17 +631,25 @@ function sortTargetsByUsage(targets: ResolvedComboTarget[], comboName: string) {
*/
function sortModelsByContextSize(models) {
const withContext = models.map((modelStr) => {
const parsed = parseModel(modelStr);
const provider = parsed.provider || parsed.providerAlias || "unknown";
const model = parsed.model || modelStr;
const limit = getModelContextLimit(provider, model);
return { modelStr, context: limit ?? 0 };
return { modelStr, context: getModelContextLimitForModelString(modelStr) ?? 0 };
});
withContext.sort((a, b) => b.context - a.context);
return withContext.map((e) => e.modelStr);
}
function getModelContextLimitForModelString(modelStr: string) {
const parsed = parseModel(modelStr);
const provider = parsed.provider || parsed.providerAlias || "unknown";
const model = parsed.model || modelStr;
return getModelContextLimit(provider, model);
}
function sortTargetsByContextSize(targets: ResolvedComboTarget[]) {
const hasKnownContext = targets.some(
(target) => getModelContextLimitForModelString(target.modelStr) != null
);
if (!hasKnownContext) return targets;
const orderedModels = sortModelsByContextSize(targets.map((target) => target.modelStr));
const byModel = new Map<string, ResolvedComboTarget[]>();
for (const target of targets) {
@@ -656,6 +665,34 @@ function sortTargetsByContextSize(targets: ResolvedComboTarget[]) {
.filter((target): target is ResolvedComboTarget => target !== null);
}
function getP2CTargetScore(target: ResolvedComboTarget, metrics: ReturnType<typeof getComboMetrics>): number {
const breakerState = getCircuitBreaker(target.provider)?.getStatus?.()?.state;
if (breakerState === "OPEN") return -Infinity;
const modelMetric = metrics?.byModel?.[target.modelStr] || null;
const successRate = Number(modelMetric?.successRate);
const avgLatency = Number(modelMetric?.avgLatencyMs);
const successScore = Number.isFinite(successRate) ? successRate / 100 : 0.5;
const latencyScore = Number.isFinite(avgLatency) && avgLatency > 0 ? 1 / Math.log10(avgLatency + 10) : 0.25;
const breakerPenalty = breakerState === "HALF_OPEN" ? 0.25 : 0;
return successScore + latencyScore - breakerPenalty;
}
function orderTargetsByPowerOfTwoChoices(targets: ResolvedComboTarget[], comboName: string) {
if (targets.length <= 1) return targets;
const metrics = getComboMetrics(comboName);
const firstIndex = Math.floor(Math.random() * targets.length);
let secondIndex = Math.floor(Math.random() * (targets.length - 1));
if (secondIndex >= firstIndex) secondIndex++;
const first = targets[firstIndex];
const second = targets[secondIndex];
const selectedIndex =
getP2CTargetScore(second, metrics) > getP2CTargetScore(first, metrics)
? secondIndex
: firstIndex;
return [targets[selectedIndex], ...targets.filter((_, index) => index !== selectedIndex)];
}
function toTextContent(content) {
if (typeof content === "string") return content;
if (!Array.isArray(content)) return "";
@@ -1034,7 +1071,7 @@ export async function handleComboChat({
relayOptions,
signal,
}) {
const strategy = combo.strategy || "priority";
const strategy = normalizeRoutingStrategy(combo.strategy || "priority");
const relayConfig =
strategy === "context-relay" ? resolveContextRelayConfig(relayOptions?.config || null) : null;
@@ -1427,6 +1464,11 @@ export async function handleComboChat({
} else if (strategy === "random") {
orderedTargets = fisherYatesShuffle([...orderedTargets]);
log.info("COMBO", `Random shuffle: ${orderedTargets.length} targets`);
} else if (strategy === "fill-first") {
log.info("COMBO", `Fill-first ordering: preserving priority order (${orderedTargets.length} targets)`);
} else if (strategy === "p2c") {
orderedTargets = orderTargetsByPowerOfTwoChoices(orderedTargets, combo.name);
log.info("COMBO", `Power-of-two-choices ordering: selected ${orderedTargets[0]?.modelStr}`);
} else if (strategy === "least-used") {
orderedTargets = sortTargetsByUsage(orderedTargets, combo.name);
log.info("COMBO", `Least-used ordering: ${orderedTargets[0]?.modelStr} has fewest requests`);

35
src/app/(dashboard)/dashboard/settings/components/ComboDefaultsTab.tsx
View File

@@ -3,7 +3,10 @@
import { useState, useEffect } from "react";
import { Card, Button, Input, Toggle } from "@/shared/components";
import { cn } from "@/shared/utils/cn";
import { ROUTING_STRATEGIES } from "@/shared/constants/routingStrategies";
import {
ROUTING_STRATEGIES,
SETTINGS_FALLBACK_STRATEGY_VALUES,
} from "@/shared/constants/routingStrategies";
import { useTranslations } from "next-intl";
const STRATEGY_LABEL_FALLBACKS: Record<string, string> = {
@@ -15,6 +18,7 @@ const LEGACY_COMBO_RESILIENCE_KEYS = new Set([
"healthCheckEnabled",
"healthCheckTimeoutMs",
]);
const ACCOUNT_FALLBACK_STRATEGIES = new Set<string>(SETTINGS_FALLBACK_STRATEGY_VALUES);
function translateOrFallback(
t: ReturnType<typeof useTranslations>,
@@ -44,6 +48,17 @@ function sanitizeProviderOverrides(overrides?: Record<string, any> | null) {
);
}
function toGlobalRoutingPatch(strategy: string | undefined, stickyRoundRobinLimit?: number) {
const patch: Record<string, unknown> = {};
if (strategy && ACCOUNT_FALLBACK_STRATEGIES.has(strategy)) {
patch.fallbackStrategy = strategy;
}
if (strategy === "round-robin" && stickyRoundRobinLimit !== undefined) {
patch.stickyRoundRobinLimit = stickyRoundRobinLimit;
}
return patch;
}
export default function ComboDefaultsTab() {
const [comboDefaults, setComboDefaults] = useState<any>({
strategy: "priority",
@@ -89,8 +104,7 @@ export default function ComboDefaultsTab() {
setComboDefaults((prev) => ({
...prev,
...sanitizeComboRuntimeConfig(comboData.comboDefaults),
strategy:
settingsData.fallbackStrategy ?? comboData.comboDefaults?.strategy ?? prev.strategy,
strategy: comboData.comboDefaults?.strategy ?? settingsData.fallbackStrategy ?? prev.strategy,
stickyRoundRobinLimit:
settingsData.stickyRoundRobinLimit ??
comboData.comboDefaults?.stickyRoundRobinLimit ??
@@ -109,8 +123,7 @@ export default function ComboDefaultsTab() {
};
const syncGlobalRoutingSettings = async (patch: Record<string, unknown>) => {
const keys = Object.keys(patch);
if (keys.length === 0) return true;
if (Object.keys(patch).length === 0) return;
const res = await fetch("/api/settings", {
method: "PATCH",
@@ -121,21 +134,13 @@ export default function ComboDefaultsTab() {
if (!res.ok) {
throw new Error("Failed to sync global routing settings");
}
return true;
};
const saveComboDefaults = async () => {
setSaving(true);
try {
const { stickyRoundRobinLimit, ...comboDefaultsPayload } = comboDefaults;
const settingsPatch: Record<string, unknown> = {};
if (comboDefaults.strategy) {
settingsPatch.fallbackStrategy = comboDefaults.strategy;
}
if (comboDefaults.strategy === "round-robin" && stickyRoundRobinLimit !== undefined) {
settingsPatch.stickyRoundRobinLimit = stickyRoundRobinLimit;
}
const settingsPatch = toGlobalRoutingPatch(comboDefaults.strategy, stickyRoundRobinLimit);
const comboDefaultsRes = await fetch("/api/settings/combo-defaults", {
method: "PATCH",
@@ -243,7 +248,7 @@ export default function ComboDefaultsTab() {
onClick={async () => {
setComboDefaults((prev) => ({ ...prev, strategy: s.value }));
try {
await syncGlobalRoutingSettings({ fallbackStrategy: s.value });
await syncGlobalRoutingSettings(toGlobalRoutingPatch(s.value));
} catch (error) {
console.error("Failed to sync fallback strategy:", error);
showStatus("error", t("errorOccurred"));

14
src/lib/db/jsonMigration.ts
View File

@@ -12,6 +12,7 @@
*/
import type Database from "better-sqlite3";
import { normalizeRoutingStrategy } from "@/shared/constants/routingStrategies";
type SqliteDatabase = InstanceType<typeof Database>;
@@ -172,8 +173,19 @@ export function runJsonMigration(
// 5. Combos
for (const [index, combo] of (data.combos ?? []).entries()) {
const normalizedCombo = {
const config =
combo.config && typeof combo.config === "object" && !Array.isArray(combo.config)
? { ...(combo.config as Record<string, unknown>) }
: combo.config;
if (config && typeof config === "object" && !Array.isArray(config) && "strategy" in config) {
(config as Record<string, unknown>).strategy = normalizeRoutingStrategy(
(config as Record<string, unknown>).strategy
);
}
const normalizedCombo: Record<string, unknown> = {
...combo,
strategy: normalizeRoutingStrategy(combo.strategy),
config,
sortOrder: typeof combo.sortOrder === "number" ? combo.sortOrder : index + 1,
};
insertCombo.run({

81
src/shared/constants/routingStrategies.ts
View File

@@ -1,17 +1,55 @@
export type RoutingStrategyValue =
| "priority"
| "weighted"
| "round-robin"
| "context-relay"
| "fill-first"
| "p2c"
| "random"
| "least-used"
| "cost-optimized"
| "strict-random"
| "auto"
| "context-optimized"
| "lkgp";
export const ROUTING_STRATEGY_VALUES = [
"priority",
"weighted",
"round-robin",
"context-relay",
"fill-first",
"p2c",
"random",
"least-used",
"cost-optimized",
"strict-random",
"auto",
"lkgp",
"context-optimized",
] as const;
export type RoutingStrategyValue = (typeof ROUTING_STRATEGY_VALUES)[number];
export const AUTO_ROUTING_STRATEGY_VALUES = [
"rules",
"cost",
"eco",
"latency",
"fast",
"lkgp",
] as const;
export type AutoRoutingStrategyValue = (typeof AUTO_ROUTING_STRATEGY_VALUES)[number];
export const ACCOUNT_FALLBACK_STRATEGY_VALUES = [
"priority",
"weighted",
"fill-first",
"round-robin",
"p2c",
"random",
"least-used",
"cost-optimized",
"strict-random",
] as const;
export type AccountFallbackStrategyValue = (typeof ACCOUNT_FALLBACK_STRATEGY_VALUES)[number];
export function normalizeRoutingStrategy(value: unknown): RoutingStrategyValue {
if (typeof value !== "string") return "priority";
const normalized = value.trim().toLowerCase();
if (normalized === "usage") return "least-used";
if (normalized === "context") return "context-optimized";
return (ROUTING_STRATEGY_VALUES as readonly string[]).includes(normalized)
? (normalized as RoutingStrategyValue)
: "priority";
}
type RoutingStrategyOption = {
value: RoutingStrategyValue;
@@ -115,17 +153,4 @@ export const ROUTING_STRATEGIES: RoutingStrategyOption[] = [
},
];
export const SETTINGS_FALLBACK_STRATEGY_VALUES: RoutingStrategyValue[] = [
"priority",
"weighted",
"fill-first",
"round-robin",
"p2c",
"random",
"least-used",
"cost-optimized",
"strict-random",
"auto",
"context-optimized",
"lkgp",
];
export const SETTINGS_FALLBACK_STRATEGY_VALUES = ACCOUNT_FALLBACK_STRATEGY_VALUES;

11
src/shared/schemas/validation.ts
View File

@@ -7,6 +7,7 @@
*/
import { z } from "zod";
import { ROUTING_STRATEGY_VALUES } from "@/shared/constants/routingStrategies";
// ─── Provider Connection ──────────────────────────────────────────
@@ -39,15 +40,7 @@ export const comboSchema = z.object({
model: z.string().min(1, "Model pattern is required"),
endpoint: z.enum(["chat", "embeddings", "images"]).default("chat"),
strategy: z
.enum([
"priority",
"weighted",
"round-robin",
"context-relay",
"random",
"least-used",
"cost-optimized",
])
.enum(ROUTING_STRATEGY_VALUES)
.default("priority"),
nodes: z.array(comboNodeSchema).min(1, "At least one node is required"),
isActive: z.boolean().default(true),

37
src/shared/validation/schemas.ts
View File

@@ -1,4 +1,8 @@
import { z } from "zod";
import {
ACCOUNT_FALLBACK_STRATEGY_VALUES,
ROUTING_STRATEGY_VALUES,
} from "@/shared/constants/routingStrategies";
import { SUPPORTED_BATCH_ENDPOINTS } from "@/shared/constants/batchEndpoints";
import { COMBO_CONFIG_MODES } from "@/shared/constants/comboConfigMode";
import { isLocalProvider } from "@/shared/constants/providers";
@@ -324,22 +328,7 @@ const comboModelEntry = z.union([
comboRefStepInputSchema,
]);
export const comboStrategySchema = z.enum([
"priority",
"weighted",
"round-robin",
"context-relay",
"random",
"least-used",
"cost-optimized",
"strict-random",
"auto",
"fill-first",
// #729 schema fixes for combo edit/save
"p2c",
"lkgp",
"context-optimized",
]);
export const comboStrategySchema = z.enum(ROUTING_STRATEGY_VALUES);
const scoringWeightsSchema = z
.object({
@@ -427,21 +416,7 @@ export const createComboSchema = z.object({
// ──── Settings Schemas ────
// FASE-01: Removed .passthrough() — only explicitly listed fields are accepted
const settingsFallbackStrategySchema = z.enum([
"priority",
"weighted",
"round-robin",
"context-relay",
"fill-first",
"p2c",
"random",
"least-used",
"cost-optimized",
"strict-random",
"auto",
"context-optimized",
"lkgp",
]);
const settingsFallbackStrategySchema = z.enum(ACCOUNT_FALLBACK_STRATEGY_VALUES);
export const updateSettingsSchema = z.object({
newPassword: z.string().min(1).max(200).optional(),

19
src/shared/validation/settingsSchemas.ts
View File

@@ -8,22 +8,7 @@
import { z } from "zod";
import { COMBO_CONFIG_MODES } from "@/shared/constants/comboConfigMode";
import { HIDEABLE_SIDEBAR_ITEM_IDS } from "@/shared/constants/sidebarVisibility";
const fallbackStrategyValues = [
"priority",
"weighted",
"round-robin",
"context-relay",
"fill-first",
"p2c",
"random",
"least-used",
"cost-optimized",
"strict-random",
"auto",
"context-optimized",
"lkgp",
] as const;
import { ACCOUNT_FALLBACK_STRATEGY_VALUES } from "@/shared/constants/routingStrategies";
const signatureCacheModeValues = ["enabled", "bypass", "bypass-strict"] as const;
@@ -52,7 +37,7 @@ export const updateSettingsSchema = z.object({
hiddenSidebarItems: z.array(z.enum(HIDEABLE_SIDEBAR_ITEM_IDS)).optional(),
comboConfigMode: z.enum(COMBO_CONFIG_MODES).optional(),
// Routing settings (#134)
fallbackStrategy: z.enum(fallbackStrategyValues).optional(),
fallbackStrategy: z.enum(ACCOUNT_FALLBACK_STRATEGY_VALUES).optional(),
wildcardAliases: z.array(z.object({ pattern: z.string(), target: z.string() })).optional(),
stickyRoundRobinLimit: z.number().int().min(0).max(1000).optional(),
requestRetry: z.number().int().min(0).max(10).optional(),

15
src/types/settings.ts
View File

@@ -1,5 +1,9 @@
import type { HideableSidebarItemId } from "@/shared/constants/sidebarVisibility";
import type { ResilienceSettings } from "@/lib/resilience/settings";
import type {
AccountFallbackStrategyValue,
RoutingStrategyValue,
} from "@/shared/constants/routingStrategies";
/**
* Application settings stored in SQLite key-value pairs.
@@ -7,14 +11,7 @@ import type { ResilienceSettings } from "@/lib/resilience/settings";
export interface Settings {
requireLogin: boolean;
hasPassword: boolean;
fallbackStrategy:
| "fill-first"
| "round-robin"
| "p2c"
| "random"
| "least-used"
| "cost-optimized"
| "strict-random";
fallbackStrategy: AccountFallbackStrategyValue;
stickyRoundRobinLimit: number;
requestRetry: number;
maxRetryIntervalSec: number;
@@ -31,7 +28,7 @@ export interface Settings {
}
export interface ComboDefaults {
strategy: "priority" | "weighted" | "round-robin" | "context-relay";
strategy: RoutingStrategyValue;
maxRetries: number;
retryDelayMs: number;
maxComboDepth: number;

31
tests/unit/autocombo-unification.test.ts
View File

@@ -50,17 +50,19 @@ test("filterCombosByStrategyCategory returns expected combo subsets", () => {
});
test("combo strategies stay aligned between UI metadata and schema validation", async () => {
const { ROUTING_STRATEGIES } = await import("../../src/shared/constants/routingStrategies.ts");
const { ROUTING_STRATEGIES, ROUTING_STRATEGY_VALUES, normalizeRoutingStrategy } =
await import("../../src/shared/constants/routingStrategies.ts");
const { comboStrategySchema, createComboSchema } =
await import("../../src/shared/validation/schemas.ts");
const { comboSchema } = await import("../../src/shared/schemas/validation.ts");
const { setRoutingStrategyInput } = await import("../../open-sse/mcp-server/schemas/tools.ts");
const strategyValues = ROUTING_STRATEGIES.map((strategy) => strategy.value);
assert.equal(strategyValues.length, 13);
assert.equal(new Set(strategyValues).size, 13);
assert.deepEqual(strategyValues, [...ROUTING_STRATEGY_VALUES]);
assert.equal(new Set(strategyValues).size, ROUTING_STRATEGY_VALUES.length);
assert.equal(strategyValues.includes("auto"), true);
assert.equal(strategyValues.includes("lkgp"), true);
assert.equal(comboStrategySchema.options.length, 13);
assert.equal(new Set(comboStrategySchema.options).size, 13);
assert.deepEqual(comboStrategySchema.options, [...ROUTING_STRATEGY_VALUES]);
strategyValues.forEach((strategy) => {
const parsed = createComboSchema.safeParse({
@@ -69,8 +71,27 @@ test("combo strategies stay aligned between UI metadata and schema validation",
strategy,
});
assert.equal(parsed.success, true, `schema should accept strategy ${strategy}`);
assert.equal(
comboSchema.safeParse({
name: `legacy-combo-${strategy}`,
model: "openai/gpt-4o-mini",
strategy,
nodes: [{ connectionId: crypto.randomUUID() }],
}).success,
true,
`legacy combo schema should accept strategy ${strategy}`
);
assert.equal(
setRoutingStrategyInput.safeParse({ comboId: "combo", strategy }).success,
true,
`MCP set strategy schema should accept ${strategy}`
);
});
assert.equal(normalizeRoutingStrategy("usage"), "least-used");
assert.equal(normalizeRoutingStrategy("context"), "context-optimized");
assert.equal(normalizeRoutingStrategy("unknown"), "priority");
const invalidParse = createComboSchema.safeParse({
name: "combo-invalid",
models: ["openai/gpt-4o-mini"],

120
tests/unit/combo-routing-engine.test.ts
View File

@@ -432,6 +432,73 @@ test("handleComboChat random strategy uses shuffled model order", async () => {
}
});
test("handleComboChat fill-first explicitly preserves priority order", async () => {
const calls: any[] = [];
await handleComboChat({
body: {},
combo: {
name: "fill-first-order",
strategy: "fill-first",
models: ["model-a", "model-b"],
},
handleSingleModel: async (_body: any, modelStr: any) => {
calls.push(modelStr);
return okResponse();
},
isModelAvailable: async () => true,
log: createLog(),
settings: null,
relayOptions: null as any,
allCombos: null,
});
assert.deepEqual(calls, ["model-a"]);
});
test("handleComboChat p2c selects the better of two random choices by metrics", async () => {
const originalRandom = Math.random;
const calls: any[] = [];
const sequence = [0.0, 0.0];
let idx = 0;
recordComboRequest("p2c-combo", "model-a", {
success: true,
latencyMs: 2000,
strategy: "p2c",
});
recordComboRequest("p2c-combo", "model-b", {
success: true,
latencyMs: 20,
strategy: "p2c",
});
Math.random = () => sequence[idx++] ?? 0;
try {
await handleComboChat({
body: {},
combo: {
name: "p2c-combo",
strategy: "p2c",
models: ["model-a", "model-b", "model-c"],
},
handleSingleModel: async (_body: any, modelStr: any) => {
calls.push(modelStr);
return okResponse();
},
isModelAvailable: async () => true,
log: createLog(),
settings: null,
relayOptions: null as any,
allCombos: null,
});
assert.deepEqual(calls, ["model-b"]);
} finally {
Math.random = originalRandom;
}
});
test("handleComboChat least-used strategy prefers the model with fewer recorded requests", async () => {
recordComboRequest("least-used-combo", "model-a", {
success: true,
@@ -1318,6 +1385,59 @@ test("handleComboChat context-optimized orders models by the largest synced cont
assert.equal(calls[0], "openai/gpt-4o-max");
});
test("handleComboChat context-optimized preserves order when all context limits are unknown", async () => {
const calls: any[] = [];
const result = await handleComboChat({
body: {},
combo: {
name: "context-optimized-unknown",
strategy: "context-optimized",
models: ["unknown/model-a", "unknown/model-b"],
},
handleSingleModel: async (_body: any, modelStr: any) => {
calls.push(modelStr);
return okResponse();
},
isModelAvailable: async () => true,
log: createLog(),
settings: null,
relayOptions: null as any,
allCombos: null,
});
assert.equal(result.ok, true);
assert.deepEqual(calls, ["unknown/model-a"]);
});
test("handleComboChat normalizes legacy strategy names at runtime", async () => {
const usageCalls: any[] = [];
recordComboRequest("legacy-usage-combo", "model-a", {
success: true,
latencyMs: 100,
strategy: "least-used",
});
await handleComboChat({
body: {},
combo: {
name: "legacy-usage-combo",
strategy: "usage",
models: ["model-a", "model-b"],
},
handleSingleModel: async (_body: any, modelStr: any) => {
usageCalls.push(modelStr);
return okResponse();
},
isModelAvailable: async () => true,
log: createLog(),
settings: null,
relayOptions: null as any,
allCombos: null,
});
assert.deepEqual(usageCalls, ["model-b"]);
});
test("handleComboChat returns a 503 when every model is unavailable before execution", async () => {
const result = await handleComboChat({
body: {},

185
tests/unit/combo-strategies.test.ts
View File

@@ -1,21 +1,28 @@
import test from "node:test";
import test, { after } from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { handleComboChat } from "../../open-sse/services/combo.ts";
import * as combosDb from "../../src/lib/db/combos.ts";
import * as modelsDb from "../../src/lib/db/models.ts";
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-combo-strategies-"));
const ORIGINAL_DATA_DIR = process.env.DATA_DIR;
process.env.DATA_DIR = TEST_DATA_DIR;
function stubConnection() {
return [
{
id: "conn-openai",
provider: "openai",
authType: "oauth",
isActive: 1,
accessToken: "token",
},
];
}
const dbCore = await import("../../src/lib/db/core.ts");
const { handleComboChat } = await import("../../open-sse/services/combo.ts");
const combosDb = await import("../../src/lib/db/combos.ts");
const { recordComboRequest } = await import("../../open-sse/services/comboMetrics.ts");
const { saveModelsDevCapabilities } = await import("../../src/lib/modelsDevSync.ts");
after(() => {
dbCore.resetDbInstance();
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
if (ORIGINAL_DATA_DIR === undefined) {
delete process.env.DATA_DIR;
} else {
process.env.DATA_DIR = ORIGINAL_DATA_DIR;
}
});
const reqBodyNullContext = {
model: "comboTest",
@@ -29,78 +36,110 @@ const reqBodyTextArray = {
stream: false,
};
test("handleComboChat with 'usage' strategy hits sortModelsByUsage", async () => {
const id = crypto.randomUUID();
await combosDb.createCombo({
id,
name: id,
strategy: "usage",
models: ["openai/gpt-3.5-turbo", "openai/gpt-4"],
function capability(limitContext: number) {
return {
tool_call: true,
reasoning: null,
attachment: null,
structured_output: null,
temperature: null,
modalities_input: "[]",
modalities_output: "[]",
knowledge_cutoff: null,
release_date: null,
last_updated: null,
status: null,
family: null,
open_weights: null,
limit_context: limitContext,
limit_input: null,
limit_output: null,
interleaved_field: null,
};
}
function okResponse(model: string) {
return Response.json({ choices: [{ message: { role: "assistant", content: model } }] });
}
function makeLog() {
return {
info() {},
warn() {},
debug() {},
error() {},
};
}
async function selectedModelFor(combo: Record<string, unknown>, body: Record<string, unknown>) {
const calls: string[] = [];
const response = await handleComboChat({
body,
combo,
allCombos: [combo],
isModelAvailable: undefined,
relayOptions: undefined,
signal: undefined,
settings: {},
log: makeLog(),
handleSingleModel: async (_body: unknown, modelStr: string) => {
calls.push(modelStr);
return okResponse(modelStr);
},
});
// No proxy/http mock needed if we expect error or quick reject
const req = new Request("http://localhost", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ ...reqBodyTextArray, model: id }),
assert.equal(response.status, 200);
assert.equal(calls.length > 0, true);
return calls[0];
}
test("least-used strategy prefers the model with fewer recorded combo requests", async () => {
const name = `least-used-${crypto.randomUUID()}`;
const busyModel = "openai/gpt-4";
const idleModel = "openai/gpt-3.5-turbo";
const combo = await combosDb.createCombo({
name,
strategy: "least-used",
models: [busyModel, idleModel],
});
try {
const res = await handleComboChat(id, req, stubConnection());
assert.equal(res.status >= 200, true);
} catch (e: any) {
// Expect error as fetch is not globally mocked for this quick edge branch test, that's fine
}
recordComboRequest(name, busyModel, {
success: true,
latencyMs: 10,
strategy: "least-used",
});
assert.equal(await selectedModelFor(combo, reqBodyTextArray), idleModel);
});
test("handleComboChat with 'context' strategy hits sortModelsByContextSize", async () => {
const id = crypto.randomUUID();
await combosDb.createCombo({
id,
name: id,
strategy: "context",
models: ["openai/gpt-4-32k", "openai/gpt-4", "unknown/unknown"],
test("context-optimized strategy prefers the largest context window", async () => {
saveModelsDevCapabilities({
"test-context": {
small: capability(8_000),
large: capability(64_000),
},
});
const req = new Request("http://localhost", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ ...reqBodyNullContext, model: id }),
const combo = await combosDb.createCombo({
name: `context-optimized-${crypto.randomUUID()}`,
strategy: "context-optimized",
models: ["test-context/small", "test-context/large", "unknown/unknown"],
});
try {
const res = await handleComboChat(id, req, stubConnection());
} catch (e: any) {}
assert.equal(await selectedModelFor(combo, reqBodyNullContext), "test-context/large");
});
test("handleComboChat hits extractPromptForIntent edge cases", async () => {
const id = crypto.randomUUID();
await combosDb.createCombo({
id,
name: id,
test("auto strategy handles null and empty prompt edge cases without throwing", async () => {
const combo = await combosDb.createCombo({
name: `auto-${crypto.randomUUID()}`,
strategy: "auto",
autoConfig: { intentConfig: { enabled: true } },
config: { auto: { explorationRate: 0 } },
models: ["openai/gpt-4"],
});
// Null message content
const reqNull = new Request("http://localhost", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ model: id, messages: [{ role: "user", content: null }] }),
});
// Empty messages array
const reqEmpty = new Request("http://localhost", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ model: id, messages: [] }),
});
try {
await handleComboChat(id, reqNull, stubConnection());
} catch (e: any) {}
try {
await handleComboChat(id, reqEmpty, stubConnection());
} catch (e: any) {}
assert.equal(
await selectedModelFor(combo, { model: combo.name, messages: [{ role: "user", content: null }] }),
"openai/gpt-4"
);
assert.equal(await selectedModelFor(combo, { model: combo.name, messages: [] }), "openai/gpt-4");
});

51
tests/unit/json-migration-combos.test.ts Normal file
View File

@@ -0,0 +1,51 @@
import test from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-json-migration-"));
const ORIGINAL_DATA_DIR = process.env.DATA_DIR;
process.env.DATA_DIR = TEST_DATA_DIR;
const core = await import("../../src/lib/db/core.ts");
const { runJsonMigration } = await import("../../src/lib/db/jsonMigration.ts");
test.after(() => {
core.resetDbInstance();
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
if (ORIGINAL_DATA_DIR === undefined) delete process.env.DATA_DIR;
else process.env.DATA_DIR = ORIGINAL_DATA_DIR;
});
test("runJsonMigration normalizes legacy combo strategy names at the import boundary", () => {
const db = core.getDbInstance();
runJsonMigration(db, {
combos: [
{
id: "combo-usage",
name: "combo-usage",
strategy: "usage",
models: ["openai/gpt-4o-mini"],
config: { strategy: "context" },
},
{
id: "combo-unknown",
name: "combo-unknown",
strategy: "not-a-real-strategy",
models: ["openai/gpt-4o-mini"],
},
],
});
const rows = db.prepare("SELECT id, data FROM combos ORDER BY id ASC").all() as Array<{
id: string;
data: string;
}>;
const byId = new Map(rows.map((row) => [row.id, JSON.parse(row.data)]));
assert.equal(byId.get("combo-usage").strategy, "least-used");
assert.equal(byId.get("combo-usage").config.strategy, "context-optimized");
assert.equal(byId.get("combo-unknown").strategy, "priority");
});

23
tests/unit/settings-schema-routing-strategies.test.ts
View File

@@ -1,21 +1,28 @@
import test from "node:test";
import assert from "node:assert/strict";
import { ROUTING_STRATEGIES } from "@/shared/constants/routingStrategies";
import { SETTINGS_FALLBACK_STRATEGY_VALUES } from "@/shared/constants/routingStrategies";
import { updateSettingsSchema as settingsRouteSchema } from "@/shared/validation/settingsSchemas";
import { updateSettingsSchema as sharedSettingsSchema } from "@/shared/validation/schemas";
for (const strategy of ROUTING_STRATEGIES) {
test(`settings route schema accepts fallbackStrategy=${strategy.value}`, () => {
const parsed = settingsRouteSchema.parse({ fallbackStrategy: strategy.value });
assert.equal(parsed.fallbackStrategy, strategy.value);
for (const strategy of SETTINGS_FALLBACK_STRATEGY_VALUES) {
test(`settings route schema accepts fallbackStrategy=${strategy}`, () => {
const parsed = settingsRouteSchema.parse({ fallbackStrategy: strategy });
assert.equal(parsed.fallbackStrategy, strategy);
});
test(`shared settings schema accepts fallbackStrategy=${strategy.value}`, () => {
const parsed = sharedSettingsSchema.parse({ fallbackStrategy: strategy.value });
assert.equal(parsed.fallbackStrategy, strategy.value);
test(`shared settings schema accepts fallbackStrategy=${strategy}`, () => {
const parsed = sharedSettingsSchema.parse({ fallbackStrategy: strategy });
assert.equal(parsed.fallbackStrategy, strategy);
});
}
test("settings schemas reject combo-only strategies as account fallback strategies", () => {
for (const strategy of ["auto", "lkgp", "context-optimized"]) {
assert.equal(settingsRouteSchema.safeParse({ fallbackStrategy: strategy }).success, false);
assert.equal(sharedSettingsSchema.safeParse({ fallbackStrategy: strategy }).success, false);
}
});
test("settings schemas accept cooldown-aware retry knobs", () => {
const payload = {
requestRetry: 3,