From 72ee80649ca18085e4bdeded966b766181b59562 Mon Sep 17 00:00:00 2001 From: KooshaPari <42529354+KooshaPari@users.noreply.github.com> Date: Thu, 2 Jul 2026 21:38:51 -0700 Subject: [PATCH] feat(autoCombo): latency/speed-optimized routing mode + omniroute_pick_fastest_model MCP tool (#6011) * feat(autoCombo): latency/speed-optimized routing mode + omniroute_pick_fastest_model MCP tool * test(translator): split responses chat request coverage * refactor(mcp): extract fastest-model tool modules * fix(i18n): cover provider icon and cors labels * test(mutation): register latency coverage files * test(ci): collect executor unit tests * refactor(ci): reduce latency path complexity * fix(mcp): include models catalog module * feat(autoCombo): latency/speed-optimized routing + omniroute_pick_fastest_model MCP tool Re-cut onto release tip: keep speed-routing + MCP tool + supporting catalog split; drop out-of-scope translator split, en.json/ci.yml/package.json orphans, and unrelated proxyFetch/responsesStreamHelpers/tokenLimitCounter refactors. Co-authored-by: diegosouzapw --------- Co-authored-by: kooshapari Co-authored-by: Diego Rodrigues de Sa e Souza Co-authored-by: diegosouzapw --- CHANGELOG.md | 1 + open-sse/mcp-server/audit.ts | 90 ++--- open-sse/mcp-server/catalog.ts | 290 ++++++++++++++++ .../mcp-server/schemas/pickFastestModel.ts | 110 ++++++ open-sse/mcp-server/schemas/tools.ts | 5 +- open-sse/mcp-server/server.ts | 233 +------------ open-sse/mcp-server/tools/pickFastestModel.ts | 293 ++++++++++++++++ .../autoCombo/__tests__/speedRanking.test.ts | 226 ++++++++++++ open-sse/services/autoCombo/routerStrategy.ts | 147 ++++---- open-sse/services/autoCombo/speedRanking.ts | 327 ++++++++++++++++++ stryker.conf.json | 1 + 11 files changed, 1367 insertions(+), 356 deletions(-) create mode 100644 open-sse/mcp-server/catalog.ts create mode 100644 open-sse/mcp-server/schemas/pickFastestModel.ts create mode 100644 open-sse/mcp-server/tools/pickFastestModel.ts create mode 100644 open-sse/services/autoCombo/__tests__/speedRanking.test.ts create mode 100644 open-sse/services/autoCombo/speedRanking.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 27801ac2b2..b337fc9af1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - **Discovery tool (Phase 2):** add the `discoveryResults` DB module (CRUD over the `discovery_results` table, migration 074) and wire the opt-in provider-discovery service to persist and read findings through it (`persistDiscoveryResult`, `getDiscoveryResults`, `getDiscoveryResultById`, `markVerified`, `deleteDiscoveryResult`) with `(provider, method, endpoint)` upsert de-duplication. Adds the `/api/discovery/*` HTTP surface — `GET /results`, `GET|DELETE /results/:id`, `POST /scan`, `POST /verify/:id` — under **strict loopback-only** authorization (`/api/discovery/` is in `LOCAL_ONLY_API_PREFIXES` and is NOT manage-scope-bypassable, so the `scan` route's outbound probes can never be reached from a tunnel/remote origin). Adds a **dashboard UI tab** (Tools → Discovery, `/dashboard/discovery`) to run scans and review, verify, or delete findings. The service stays **opt-in / default-off**. - **feat(api):** expose a read-only provider plugin manifest at `GET /api/v1/provider-plugin-manifest` for sidecar/relay discovery. (thanks @KooshaPari) - **feat(sidecar):** advertise the provider manifest URL to Bifrost/CLIProxyAPI via the `X-OmniRoute-Provider-Manifest-Url` header (`OMNIROUTE_PROVIDER_MANIFEST_URL`). (thanks @KooshaPari) +- **feat(autoCombo):** add a latency/speed-optimized routing mode (shared `rankBySpeed` scoring core) plus the `omniroute_pick_fastest_model` MCP tool. (thanks @KooshaPari) ### 🔧 Bug Fixes diff --git a/open-sse/mcp-server/audit.ts b/open-sse/mcp-server/audit.ts index 259ded1c49..40e023212c 100644 --- a/open-sse/mcp-server/audit.ts +++ b/open-sse/mcp-server/audit.ts @@ -206,6 +206,49 @@ function toString(value: unknown): string { return typeof value === "string" ? value : ""; } +async function openBetterSqliteAuditDb(dbPath: string): Promise { + const Database = (await import("better-sqlite3")).default as unknown as new ( + dbPath: string + ) => AuditDatabase; + return new Database(dbPath); +} + +function nodeSqliteFallbackAvailable(): boolean { + const [maj, min] = (process.versions.node ?? "0.0").split(".").map(Number); + return maj > 22 || (maj === 22 && (min ?? 0) >= 5); +} + +async function openNodeSqliteAuditDb(dbPath: string): Promise { + const { DatabaseSync } = (await import("node:sqlite")) as { + DatabaseSync: new (p: string) => NodeSqliteDatabase; + }; + return createNodeSqliteAuditAdapter(new DatabaseSync(dbPath)); +} + +async function openFallbackAuditDb(dbPath: string, nativeMessage: string): Promise { + if (!nodeSqliteFallbackAvailable()) { + console.error( + `[MCP Audit] better-sqlite3 native binding unavailable and Node ${process.version} ` + + "has no built-in sqlite. Audit logging disabled. Fix: run " + + "`npm rebuild better-sqlite3` in the omniroute install root." + ); + return null; + } + + try { + const adapter = await openNodeSqliteAuditDb(dbPath); + console.warn( + `[MCP Audit] better-sqlite3 binding unavailable — fell back to node:sqlite ` + + `(${nativeMessage.split("\n")[0]})` + ); + return adapter; + } catch (nodeErr) { + const nodeMessage = nodeErr instanceof Error ? nodeErr.message : String(nodeErr); + console.error("[MCP Audit] Failed to connect to database:", nodeMessage); + return null; + } +} + /** * Lazy-load the database connection. * Uses the same SQLite database as the main OmniRoute app. @@ -238,58 +281,19 @@ async function getDb(): Promise { return null; } - // Try better-sqlite3 first (matches the main app's default driver). try { - const Database = (await import("better-sqlite3")).default as unknown as new ( - dbPath: string - ) => AuditDatabase; - const database = new Database(dbPath); + const database = await openBetterSqliteAuditDb(dbPath); setCachedAuditDb(database); return database; } catch (nativeErr) { - // Declared once at the top of the catch: nativeMessage is read both on - // the non-fallback bail-out and in the node:sqlite fallback warning - // further down. A block-scoped const inside the `if` below would be out - // of scope in the fallback path. const nativeMessage = nativeErr instanceof Error ? nativeErr.message : String(nativeErr); - // Reuse the canonical detection helper from the main app's DB layer - // so we cover every ABI/binding failure mode the rest of the codebase - // already knows about: missing MODULE_NOT_FOUND, ERR_DLOPEN_FAILED, - // "Module did not self-register", "Cannot find module 'better-sqlite3'", - // the standard V8 "was compiled against a different Node.js version" - // message, and the bindings-loader "Could not locate the bindings file". - // Real errors (corrupt db, permission denied) still surface to the operator. if (!isNativeSqliteLoadError(nativeErr)) { console.error("[MCP Audit] Failed to connect to database:", nativeMessage); return null; } - // Fall back to Node's built-in sqlite (Node 22.5+). - const [maj, min] = (process.versions.node ?? "0.0").split(".").map(Number); - if (maj < 22 || (maj === 22 && (min ?? 0) < 5)) { - console.error( - `[MCP Audit] better-sqlite3 native binding unavailable and Node ${process.version} ` + - "has no built-in sqlite. Audit logging disabled. Fix: run " + - "`npm rebuild better-sqlite3` in the omniroute install root." - ); - return null; - } - try { - const { DatabaseSync } = (await import("node:sqlite")) as { - DatabaseSync: new (p: string) => NodeSqliteDatabase; - }; - const nodeDb = new DatabaseSync(dbPath); - const adapter = createNodeSqliteAuditAdapter(nodeDb); - setCachedAuditDb(adapter); - console.warn( - `[MCP Audit] better-sqlite3 binding unavailable — fell back to node:sqlite ` + - `(${nativeMessage.split("\n")[0]})` - ); - return adapter; - } catch (nodeErr) { - const nodeMessage = nodeErr instanceof Error ? nodeErr.message : String(nodeErr); - console.error("[MCP Audit] Failed to connect to database:", nodeMessage); - return null; - } + const fallbackDb = await openFallbackAuditDb(dbPath, nativeMessage); + setCachedAuditDb(fallbackDb); + return fallbackDb; } } catch (err: unknown) { const message = err instanceof Error ? err.message : String(err); diff --git a/open-sse/mcp-server/catalog.ts b/open-sse/mcp-server/catalog.ts new file mode 100644 index 0000000000..d06a16f192 --- /dev/null +++ b/open-sse/mcp-server/catalog.ts @@ -0,0 +1,290 @@ +import { getCodexRequestDefaults } from "../../src/lib/providers/requestDefaults.ts"; +import { getProviderConnections } from "../../src/lib/db/providers.ts"; +import { AI_PROVIDERS, NOAUTH_PROVIDERS } from "../../src/shared/constants/providers.ts"; + +type JsonRecord = Record; +type McpCatalogStatus = "available" | "degraded" | "unavailable"; + +type McpCatalogResponse = { + models: Array<{ + id: string; + provider: string; + capabilities: string[]; + status: McpCatalogStatus; + thinkingEffort?: string; + pricing?: unknown; + }>; + source: string; + warning?: string; +}; + +type ProviderConnectionLike = { + id?: string; + provider?: string; + isActive?: boolean; + providerSpecificData?: unknown; +}; + +type McpCatalogRequestSpec = { + provider: string; + path: string; + thinkingEffort?: string; +}; + +function toRecord(value: unknown): JsonRecord { + return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {}; +} + +function toString(value: unknown, fallback = ""): string { + return typeof value === "string" ? value : fallback; +} + +function toStringArray(value: unknown, fallback: string[] = []): string[] { + return Array.isArray(value) ? value.map((item) => String(item)) : fallback; +} + +function buildProviderAliasMap(): Record { + const aliasMap: Record = {}; + + for (const provider of Object.values(AI_PROVIDERS)) { + if (!provider?.id) continue; + aliasMap[provider.id] = provider.id; + if (typeof provider.alias === "string" && provider.alias.length > 0) { + aliasMap[provider.alias] = provider.id; + } + } + + for (const provider of Object.values(NOAUTH_PROVIDERS)) { + if (!provider?.id) continue; + aliasMap[provider.id] = provider.id; + if ("alias" in provider && typeof provider.alias === "string" && provider.alias.length > 0) { + aliasMap[provider.alias] = provider.id; + } + } + + return aliasMap; +} + +function normalizeCapability(value: string): string { + switch (value) { + case "embeddings": + return "embedding"; + case "images": + return "image"; + case "videos": + return "video"; + case "moderations": + return "moderation"; + case "chat-completions": + return "chat"; + default: + return value; + } +} + +function getCatalogModelCapabilities(model: JsonRecord): string[] { + if (Array.isArray(model.capabilities) && model.capabilities.length > 0) { + return toStringArray(model.capabilities, ["chat"]).map(normalizeCapability); + } + + if (Array.isArray(model.supportedEndpoints) && model.supportedEndpoints.length > 0) { + return toStringArray(model.supportedEndpoints, ["chat"]).map(normalizeCapability); + } + + const type = toString(model.type); + if (type) return [normalizeCapability(type)]; + + return ["chat"]; +} + +function normalizeCatalogStatus( + model: JsonRecord, + source: string, + warning?: string +): McpCatalogStatus { + const explicitStatus = toString(model.status); + if ( + explicitStatus === "available" || + explicitStatus === "degraded" || + explicitStatus === "unavailable" + ) { + return explicitStatus; + } + + if (warning || source === "local_catalog") return "degraded"; + return "available"; +} + +function getConnectionThinkingEffort(connection: ProviderConnectionLike): string | undefined { + const provider = typeof connection.provider === "string" ? connection.provider : null; + const providerSpecificData = toRecord(connection.providerSpecificData); + + if (provider === "codex") { + return getCodexRequestDefaults(providerSpecificData).reasoningEffort || "medium"; + } + + const rawThinkingEffort = toString(providerSpecificData.thinkingEffort); + return rawThinkingEffort || undefined; +} + +function normalizeProviderModelRecord( + rawModel: unknown, + fallbackProvider: string, + source: string, + warning?: string, + thinkingEffort?: string +) { + const model = toRecord(rawModel); + const id = toString(model.id, ""); + + return { + id, + provider: toString(model.owned_by, toString(model.provider, fallbackProvider)), + capabilities: getCatalogModelCapabilities(model), + status: normalizeCatalogStatus(model, source, warning), + ...(thinkingEffort ? { thinkingEffort } : {}), + pricing: model.pricing, + }; +} + +function activeProviderConnections( + connections: ProviderConnectionLike[], + normalizeProviderId: (value: string) => string, + requestedProvider: string | null +): ProviderConnectionLike[] { + return connections.filter((connection) => { + const provider = + typeof connection?.provider === "string" ? normalizeProviderId(connection.provider) : null; + return !!provider && !!connection?.id && connection.isActive !== false && + (!requestedProvider || provider === requestedProvider); + }); +} + +function providerModelRequestSpecs( + connections: ProviderConnectionLike[], + normalizeProviderId: (value: string) => string +): McpCatalogRequestSpec[] { + return connections.map((connection) => ({ + provider: normalizeProviderId(String(connection.provider)), + path: `/api/providers/${encodeURIComponent(String(connection.id))}/models?excludeHidden=true`, + thinkingEffort: getConnectionThinkingEffort(connection), + })); +} + +function noAuthProviderSpec(requestedProvider: string): McpCatalogRequestSpec { + return { + provider: requestedProvider, + path: `/api/v1/providers/${encodeURIComponent(requestedProvider)}/models`, + thinkingEffort: undefined, + }; +} + +function emptyCatalogForProvider(requestedProvider: string): McpCatalogResponse { + return { + models: [], + source: "provider_connections", + warning: `No active connections found for provider '${requestedProvider}'.`, + }; +} + +function rawModelsFromCatalog(raw: JsonRecord): unknown[] { + if (Array.isArray(raw.models)) return raw.models; + if (Array.isArray(raw.data)) return raw.data; + return []; +} + +function maybeCatalogModel( + rawModel: unknown, + spec: McpCatalogRequestSpec, + source: string, + warning: string | undefined, + requestedCapability: string | null +): McpCatalogResponse["models"][number] | null { + const normalized = normalizeProviderModelRecord(rawModel, spec.provider, source, warning); + if (spec.thinkingEffort && !normalized.thinkingEffort) normalized.thinkingEffort = spec.thinkingEffort; + if (!normalized.id) return null; + if (requestedCapability && !normalized.capabilities.includes(requestedCapability)) return null; + return normalized; +} + +function addCatalogModels( + raw: JsonRecord, + spec: McpCatalogRequestSpec, + source: string, + warning: string | undefined, + requestedCapability: string | null, + collectedModels: Map +) { + for (const rawModel of rawModelsFromCatalog(raw)) { + const normalized = maybeCatalogModel(rawModel, spec, source, warning, requestedCapability); + if (normalized) collectedModels.set(`${normalized.provider}:${normalized.id}`, normalized); + } +} + +async function collectCatalogModels( + requestSpecs: McpCatalogRequestSpec[], + fetchJson: (path: string) => Promise, + requestedCapability: string | null +) { + const collectedModels = new Map(); + const warnings = new Set(); + const sources = new Set(); + + for (const spec of requestSpecs) { + const raw = toRecord(await fetchJson(spec.path)); + const source = toString(raw.source, spec.path.startsWith("/api/providers/") ? "api" : "v1_catalog"); + const warning = raw.warning ? String(raw.warning) : undefined; + if (warning) warnings.add(warning); + sources.add(source); + addCatalogModels(raw, spec, source, warning, requestedCapability, collectedModels); + } + + return { collectedModels, warnings, sources }; +} + +export async function getMcpModelsCatalog( + args: { provider?: string; capability?: string }, + deps: { + fetchJson?: (path: string) => Promise; + listProviderConnections?: () => Promise; + } = {} +): Promise { + const fetchJson = deps.fetchJson ?? ((path: string) => import("./server.ts").then((m) => m.omniRouteFetch(path))); + const listProviderConnections = deps.listProviderConnections ?? getProviderConnections; + const aliasMap = buildProviderAliasMap(); + const normalizeProviderId = (value: string) => aliasMap[value] || value; + const requestedProvider = args.provider ? normalizeProviderId(args.provider) : null; + const requestedCapability = args.capability ? normalizeCapability(args.capability) : null; + + let connections = await listProviderConnections(); + connections = Array.isArray(connections) ? connections : []; + const activeConnections = activeProviderConnections( + connections, + normalizeProviderId, + requestedProvider + ); + const requestSpecs = providerModelRequestSpecs(activeConnections, normalizeProviderId); + + if (requestedProvider && requestSpecs.length === 0) { + const isNoAuthProvider = Object.values(NOAUTH_PROVIDERS).some( + (provider) => provider.id === requestedProvider + ); + if (isNoAuthProvider) { + requestSpecs.push(noAuthProviderSpec(requestedProvider)); + } else { + return emptyCatalogForProvider(requestedProvider); + } + } + + const { collectedModels, warnings, sources } = await collectCatalogModels( + requestSpecs, + fetchJson, + requestedCapability + ); + + return { + models: [...collectedModels.values()], + source: sources.size === 1 ? [...sources][0] : "aggregated_provider_models", + ...(warnings.size > 0 ? { warning: [...warnings].join(" | ") } : {}), + }; +} diff --git a/open-sse/mcp-server/schemas/pickFastestModel.ts b/open-sse/mcp-server/schemas/pickFastestModel.ts new file mode 100644 index 0000000000..3434b02ec6 --- /dev/null +++ b/open-sse/mcp-server/schemas/pickFastestModel.ts @@ -0,0 +1,110 @@ +import { z } from "zod"; +import type { McpToolDefinition } from "./toolDefinition.ts"; + +export const pickFastestModelInput = z.object({ + comboId: z + .string() + .optional() + .describe( + "Optional combo id or name to scope the ranking to. Omit to rank across all enabled combos." + ), + includeUnhealthy: z + .boolean() + .optional() + .describe( + "When true, OPEN-circuit candidates are scored (sorted to the bottom) instead of filtered out." + ), + weights: z + .object({ + ttft: z.number().min(0).optional(), + tps: z.number().min(0).optional(), + e2e: z.number().min(0).optional(), + p95: z.number().min(0).optional(), + health: z.number().min(0).optional(), + reliability: z.number().min(0).optional(), + stability: z.number().min(0).optional(), + }) + .partial() + .optional() + .describe("Optional speed-ranking weight overrides merged onto the defaults."), + applyToCombo: z + .boolean() + .optional() + .describe("When true + comboId present, switches the combo to auto/latency routing."), + limit: z.number().int().min(1).max(50).optional().describe("Ranked result limit."), +}); + +export const pickFastestModelOutput = z.object({ + fastest: z + .object({ + provider: z.string(), + model: z.string(), + score: z.number(), + reason: z.string(), + }) + .nullable(), + ranked: z.array( + z.object({ + provider: z.string(), + model: z.string(), + score: z.number(), + factors: z.object({ + ttft: z.number(), + tps: z.number(), + e2e: z.number(), + p95: z.number(), + health: z.number(), + reliability: z.number(), + stability: z.number(), + }), + metrics: z.object({ + avgTtftMs: z.number().nullable(), + avgTokensPerSecond: z.number().nullable(), + avgE2ELatencyMs: z.number().nullable(), + p95LatencyMs: z.number().nullable(), + latencyStdDev: z.number().nullable(), + failureRate: z.number(), + circuitBreakerState: z.enum(["CLOSED", "OPEN", "HALF_OPEN"]), + }), + reason: z.string(), + }) + ), + weights: z.object({ + ttft: z.number(), + tps: z.number(), + e2e: z.number(), + p95: z.number(), + health: z.number(), + reliability: z.number(), + stability: z.number(), + }), + comboScope: z.object({ id: z.string(), name: z.string() }).nullable(), + appliedToCombo: z + .object({ + id: z.string(), + name: z.string(), + strategy: z.string(), + autoRoutingStrategy: z.string(), + }) + .nullable(), +}); + +export const pickFastestModelTool: McpToolDefinition< + typeof pickFastestModelInput, + typeof pickFastestModelOutput +> = { + name: "omniroute_pick_fastest_model", + description: + "Picks the fastest reliable provider-model pair from live telemetry and can apply latency routing to a combo.", + inputSchema: pickFastestModelInput, + outputSchema: pickFastestModelOutput, + scopes: ["read:combos", "read:health", "read:usage"], + auditLevel: "basic", + phase: 2, + sourceEndpoints: [ + "/api/combos", + "/api/monitoring/health", + "/api/usage/quota", + "/api/usage/analytics", + ], +}; diff --git a/open-sse/mcp-server/schemas/tools.ts b/open-sse/mcp-server/schemas/tools.ts index b7b0b84090..85b5c3edd7 100644 --- a/open-sse/mcp-server/schemas/tools.ts +++ b/open-sse/mcp-server/schemas/tools.ts @@ -1,5 +1,5 @@ /** - * MCP Tool Schemas — Contracts for all 22 core and advanced OmniRoute MCP tools. + * MCP Tool Schemas — Contracts for all 23 core and advanced OmniRoute MCP tools. * * Defines input/output Zod schemas, descriptions, scopes, and audit levels * for both essential (Phase 1) and advanced (Phase 2) MCP tools. @@ -11,6 +11,7 @@ import { z } from "zod"; import { toolSearchTool } from "./toolSearch.ts"; +import { pickFastestModelTool } from "./pickFastestModel.ts"; import { AUTO_ROUTING_STRATEGY_VALUES, ROUTING_STRATEGY_VALUES, @@ -22,6 +23,7 @@ import { // Re-exported here for backward compatibility (many modules import them from ./tools.ts). export type { AuditLevel, McpToolDefinition } from "./toolDefinition.ts"; import type { McpToolDefinition } from "./toolDefinition.ts"; +export { pickFastestModelInput, pickFastestModelOutput } from "./pickFastestModel.ts"; // ============ Phase 1: Essential Tools (8) ============ @@ -1466,6 +1468,7 @@ export const MCP_TOOLS = [ agentSkillsListTool, agentSkillsGetTool, agentSkillsCoverageTool, + pickFastestModelTool, ] as const; export const MCP_ESSENTIAL_TOOLS = MCP_TOOLS.filter((t) => t.phase === 1); diff --git a/open-sse/mcp-server/server.ts b/open-sse/mcp-server/server.ts index 554f1ab1d8..cbf95b1956 100644 --- a/open-sse/mcp-server/server.ts +++ b/open-sse/mcp-server/server.ts @@ -5,9 +5,7 @@ import { getComboModelString, getComboStepTarget, } from "../../src/lib/combos/steps.ts"; - import { registerToolSearchTool } from "./toolSearch/register.ts"; - import { MCP_TOOLS, getHealthInput, @@ -28,6 +26,7 @@ import { getProviderMetricsInput, bestComboForTaskInput, explainRouteInput, + pickFastestModelInput, getSessionSnapshotInput, dbHealthCheckInput, syncPricingInput, @@ -38,7 +37,6 @@ import { oneproxyStatsInput, } from "./schemas/tools.ts"; import { startMcpHeartbeat } from "./runtimeHeartbeat.ts"; - import { z } from "zod"; import { closeAuditDb, logToolCall } from "./audit.ts"; import { @@ -47,7 +45,6 @@ import { type McpToolExtraLike, } from "./scopeEnforcement.ts"; import { getMcpHttpAuthHeadersForInternalFetch } from "./httpAuthContext.ts"; - import { handleSimulateRoute, handleSetBudgetGuard, @@ -66,6 +63,7 @@ import { handleOneproxyRotate, handleOneproxyStats, } from "./tools/advancedTools.ts"; +import { handlePickFastestModel } from "./tools/pickFastestModel.ts"; import { memoryTools } from "./tools/memoryTools.ts"; import { skillTools } from "./tools/skillTools.ts"; import { agentSkillTools } from "./tools/agentSkillTools.ts"; @@ -86,11 +84,10 @@ import { type McpAccessibilityConfig, } from "../services/compression/engines/mcpAccessibility/constants.ts"; import { getDbInstance } from "../../src/lib/db/core.ts"; -import { getProviderConnections } from "../../src/lib/db/providers.ts"; -import { getCodexRequestDefaults } from "../../src/lib/providers/requestDefaults.ts"; import { normalizeQuotaResponse } from "../../src/shared/contracts/quota.ts"; -import { AI_PROVIDERS, NOAUTH_PROVIDERS } from "../../src/shared/constants/providers.ts"; import { resolveOmniRouteBaseUrl } from "../../src/shared/utils/resolveOmniRouteBaseUrl.ts"; +import { getMcpModelsCatalog } from "./catalog.ts"; +export { getMcpModelsCatalog } from "./catalog.ts"; const OMNIROUTE_BASE_URL = resolveOmniRouteBaseUrl(); const MCP_ENFORCE_SCOPES = process.env.OMNIROUTE_MCP_ENFORCE_SCOPES === "true"; @@ -144,28 +141,6 @@ type TextToolResult = { isError?: boolean; }; -type McpCatalogStatus = "available" | "degraded" | "unavailable"; - -type McpCatalogResponse = { - models: Array<{ - id: string; - provider: string; - capabilities: string[]; - status: McpCatalogStatus; - thinkingEffort?: string; - pricing?: unknown; - }>; - source: string; - warning?: string; -}; - -type ProviderConnectionLike = { - id?: string; - provider?: string; - isActive?: boolean; - providerSpecificData?: unknown; -}; - function toRecord(value: unknown): JsonRecord { return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {}; } @@ -210,7 +185,7 @@ function getOmniRouteApiKey(): string { return process.env.OMNIROUTE_API_KEY || ""; } -async function omniRouteFetch(path: string, options: RequestInit = {}): Promise { +export async function omniRouteFetch(path: string, options: RequestInit = {}): Promise { const url = `${OMNIROUTE_BASE_URL}${path}`; const apiKey = getOmniRouteApiKey(); const headers: Record = { @@ -233,202 +208,6 @@ async function omniRouteFetch(path: string, options: RequestInit = {}): Promise< return response.json(); } -function buildProviderAliasMap(): Record { - const aliasMap: Record = {}; - - for (const provider of Object.values(AI_PROVIDERS)) { - if (!provider?.id) continue; - aliasMap[provider.id] = provider.id; - if (typeof provider.alias === "string" && provider.alias.length > 0) { - aliasMap[provider.alias] = provider.id; - } - } - - for (const provider of Object.values(NOAUTH_PROVIDERS)) { - if (!provider?.id) continue; - aliasMap[provider.id] = provider.id; - if ("alias" in provider && typeof provider.alias === "string" && provider.alias.length > 0) { - aliasMap[provider.alias] = provider.id; - } - } - - return aliasMap; -} - -function normalizeCapability(value: string): string { - switch (value) { - case "embeddings": - return "embedding"; - case "images": - return "image"; - case "videos": - return "video"; - case "moderations": - return "moderation"; - case "chat-completions": - return "chat"; - default: - return value; - } -} - -function getCatalogModelCapabilities(model: JsonRecord): string[] { - if (Array.isArray(model.capabilities) && model.capabilities.length > 0) { - return toStringArray(model.capabilities, ["chat"]).map(normalizeCapability); - } - - if (Array.isArray(model.supportedEndpoints) && model.supportedEndpoints.length > 0) { - return toStringArray(model.supportedEndpoints, ["chat"]).map(normalizeCapability); - } - - const type = toString(model.type); - if (type) return [normalizeCapability(type)]; - - return ["chat"]; -} - -function normalizeCatalogStatus( - model: JsonRecord, - source: string, - warning?: string -): McpCatalogStatus { - const explicitStatus = toString(model.status); - if ( - explicitStatus === "available" || - explicitStatus === "degraded" || - explicitStatus === "unavailable" - ) { - return explicitStatus; - } - - if (warning || source === "local_catalog") return "degraded"; - return "available"; -} - -function getConnectionThinkingEffort(connection: ProviderConnectionLike): string | undefined { - const provider = typeof connection.provider === "string" ? connection.provider : null; - const providerSpecificData = toRecord(connection.providerSpecificData); - - if (provider === "codex") { - return getCodexRequestDefaults(providerSpecificData).reasoningEffort || "medium"; - } - - const rawThinkingEffort = toString(providerSpecificData.thinkingEffort); - return rawThinkingEffort || undefined; -} - -function normalizeProviderModelRecord( - rawModel: unknown, - fallbackProvider: string, - source: string, - warning?: string, - thinkingEffort?: string -) { - const model = toRecord(rawModel); - const id = toString(model.id, ""); - - return { - id, - provider: toString(model.owned_by, toString(model.provider, fallbackProvider)), - capabilities: getCatalogModelCapabilities(model), - status: normalizeCatalogStatus(model, source, warning), - ...(thinkingEffort ? { thinkingEffort } : {}), - pricing: model.pricing, - }; -} - -export async function getMcpModelsCatalog( - args: { provider?: string; capability?: string }, - deps: { - fetchJson?: (path: string) => Promise; - listProviderConnections?: () => Promise; - } = {} -): Promise { - const fetchJson = deps.fetchJson ?? ((path: string) => omniRouteFetch(path)); - const listProviderConnections = deps.listProviderConnections ?? getProviderConnections; - const aliasMap = buildProviderAliasMap(); - const normalizeProviderId = (value: string) => aliasMap[value] || value; - const requestedProvider = args.provider ? normalizeProviderId(args.provider) : null; - const requestedCapability = args.capability ? normalizeCapability(args.capability) : null; - - let connections = await listProviderConnections(); - connections = Array.isArray(connections) ? connections : []; - - const activeConnections = connections.filter((connection) => { - const provider = - typeof connection?.provider === "string" ? normalizeProviderId(connection.provider) : null; - if (!provider || !connection?.id || connection.isActive === false) return false; - if (requestedProvider && provider !== requestedProvider) return false; - return true; - }); - - const requestSpecs = activeConnections.map((connection) => ({ - provider: normalizeProviderId(String(connection.provider)), - path: `/api/providers/${encodeURIComponent(String(connection.id))}/models?excludeHidden=true`, - thinkingEffort: getConnectionThinkingEffort(connection), - })); - - if (requestedProvider && requestSpecs.length === 0) { - const isNoAuthProvider = Object.values(NOAUTH_PROVIDERS).some( - (provider) => provider.id === requestedProvider - ); - if (isNoAuthProvider) { - requestSpecs.push({ - provider: requestedProvider, - path: `/api/v1/providers/${encodeURIComponent(requestedProvider)}/models`, - thinkingEffort: undefined, - }); - } else { - return { - models: [], - source: "provider_connections", - warning: `No active connections found for provider '${requestedProvider}'.`, - }; - } - } - - const collectedModels = new Map(); - const warnings = new Set(); - const sources = new Set(); - - for (const spec of requestSpecs) { - const raw = toRecord(await fetchJson(spec.path)); - const source = toString( - raw.source, - spec.path.startsWith("/api/providers/") ? "api" : "v1_catalog" - ); - const warning = raw.warning ? String(raw.warning) : undefined; - if (warning) warnings.add(warning); - sources.add(source); - - const rawModels = Array.isArray(raw.models) - ? raw.models - : Array.isArray(raw.data) - ? raw.data - : []; - - for (const rawModel of rawModels) { - const normalized = normalizeProviderModelRecord(rawModel, spec.provider, source, warning); - if (spec.thinkingEffort && !normalized.thinkingEffort) { - normalized.thinkingEffort = spec.thinkingEffort; - } - if (!normalized.id) continue; - if (requestedCapability && !normalized.capabilities.includes(requestedCapability)) continue; - - const key = `${normalized.provider}:${normalized.id}`; - if (!collectedModels.has(key)) { - collectedModels.set(key, normalized); - } - } - } - - return { - models: [...collectedModels.values()], - source: sources.size === 1 ? [...sources][0] : "aggregated_provider_models", - ...(warnings.size > 0 ? { warning: [...warnings].join(" | ") } : {}), - }; -} - function withScopeEnforcement( toolName: string, handler: (args: unknown, extra?: McpToolExtraLike) => Promise, @@ -1085,6 +864,8 @@ export function createMcpServer(): McpServer { ) ); + server.registerTool("omniroute_pick_fastest_model", { description: "Picks the fastest reliable provider-model pair from live telemetry.", inputSchema: pickFastestModelInput }, withScopeEnforcement("omniroute_pick_fastest_model", (args) => handlePickFastestModel(pickFastestModelInput.parse(args)))); + server.registerTool( "omniroute_get_session_snapshot", { diff --git a/open-sse/mcp-server/tools/pickFastestModel.ts b/open-sse/mcp-server/tools/pickFastestModel.ts new file mode 100644 index 0000000000..2d22ab9e15 --- /dev/null +++ b/open-sse/mcp-server/tools/pickFastestModel.ts @@ -0,0 +1,293 @@ +import { logToolCall } from "../audit.ts"; +import { getMcpHttpAuthHeadersForInternalFetch } from "../httpAuthContext.ts"; +import { normalizeQuotaResponse } from "../../../src/shared/contracts/quota.ts"; +import { resolveOmniRouteBaseUrl } from "../../../src/shared/utils/resolveOmniRouteBaseUrl.ts"; +import { + getComboModelProvider, + getComboModelString, + getComboStepTarget, +} from "../../../src/lib/combos/steps.ts"; +import type { AutoRoutingStrategyValue } from "../../../src/shared/constants/routingStrategies.ts"; +import { rankBySpeed, DEFAULT_SPEED_WEIGHTS } from "../../services/autoCombo/speedRanking.ts"; +import type { SpeedCandidate } from "../../services/autoCombo/speedRanking.ts"; + +const OMNIROUTE_BASE_URL = resolveOmniRouteBaseUrl(); +const OMNIROUTE_API_KEY = process.env.OMNIROUTE_API_KEY || ""; + +async function apiFetch(path: string, options: RequestInit = {}): Promise { + const url = `${OMNIROUTE_BASE_URL}${path}`; + const headers: Record = { + "Content-Type": "application/json", + ...(OMNIROUTE_API_KEY ? { Authorization: `Bearer ${OMNIROUTE_API_KEY}` } : {}), + ...getMcpHttpAuthHeadersForInternalFetch(), + ...((options.headers as Record) || {}), + }; + const response = await fetch(url, { ...options, headers, signal: AbortSignal.timeout(30000) }); + if (!response.ok) { + const text = await response.text().catch(() => "Unknown error"); + throw new Error(`API [${response.status}]: ${text}`); + } + return response.json(); +} + +type JsonRecord = Record; +interface ComboModel { provider: string; model: string; inputCostPer1M: number; } +interface PickFastestModelArgs { + comboId?: string; + /** When true, OPEN-circuit candidates are still scored (sorted to the bottom). */ + includeUnhealthy?: boolean; + /** Optional weight overrides; merged onto DEFAULT_SPEED_WEIGHTS. */ + weights?: Partial<{ + ttft: number; + tps: number; + e2e: number; + p95: number; + health: number; + reliability: number; + stability: number; + }>; + /** When true + comboId present, sets the combo's autoRoutingStrategy to "latency". */ + applyToCombo?: boolean; + /** Max number of ranked entries to return (default 10). */ + limit?: number; +} +interface TelemetrySources { + combos: JsonRecord[]; + breakers: JsonRecord[]; + providers: ReturnType["providers"]; + analyticsByProvider: JsonRecord; + analyticsTop: JsonRecord; +} + +function isRecord(value: unknown): value is JsonRecord { return !!value && typeof value === "object" && !Array.isArray(value); } +function toRecord(value: unknown): JsonRecord { return isRecord(value) ? value : {}; } +function toArrayOfRecords(value: unknown): JsonRecord[] { return Array.isArray(value) ? value.filter(isRecord) : []; } +function toString(value: unknown, fallback = ""): string { return typeof value === "string" ? value : fallback; } +function toNumber(value: unknown, fallback = 0): number { + const parsed = typeof value === "number" ? value : typeof value === "string" && value.trim().length > 0 ? Number(value) : Number.NaN; + return Number.isFinite(parsed) ? parsed : fallback; +} +function getComboModels(combo: JsonRecord): ComboModel[] { + const directModels = toArrayOfRecords(combo.models); + const nestedModels = toArrayOfRecords(toRecord(combo.data).models); + const sourceModels = directModels.length > 0 ? directModels : nestedModels; + return sourceModels.map((model) => ({ + provider: getComboModelProvider(model) || (getComboModelString(model) ? "unknown" : "combo"), + model: getComboModelString(model) || getComboStepTarget(model) || "", + inputCostPer1M: toNumber(model.inputCostPer1M, 3.0), + })); +} +function normalizeCombosResponse(raw: unknown): JsonRecord[] { + if (Array.isArray(raw)) return raw.filter(isRecord); + const source = toRecord(raw); + return Array.isArray(source.combos) ? source.combos.filter(isRecord) : []; +} + +function settledValue(result: PromiseSettledResult): unknown { + return result.status === "fulfilled" ? result.value : undefined; +} + +async function fetchTelemetrySources(): Promise { + const [combosRaw, healthRaw, quotaRaw, analyticsRaw] = await Promise.allSettled([ + apiFetch("/api/combos"), + apiFetch("/api/monitoring/health"), + apiFetch("/api/usage/quota"), + apiFetch("/api/usage/analytics?period=session"), + ]); + + const analytics = toRecord(settledValue(analyticsRaw)); + return { + combos: normalizeCombosResponse(settledValue(combosRaw)), + breakers: toArrayOfRecords(toRecord(settledValue(healthRaw)).circuitBreakers), + providers: normalizeQuotaResponse(settledValue(quotaRaw) ?? {}).providers, + analyticsByProvider: toRecord(toRecord(analytics.byProvider)), + analyticsTop: analytics, + }; +} + +function selectComboScope(combos: JsonRecord[], comboId?: string) { + const targetCombo = comboId + ? combos.find((combo) => toString(combo.id) === comboId || toString(combo.name) === comboId) + : undefined; + return { + targetCombo, + scopedCombos: targetCombo ? [targetCombo] : combos.filter((combo) => combo.enabled !== false), + }; +} + +function noCandidatesResult(error: string) { + return { content: [{ type: "text" as const, text: JSON.stringify({ error }) }], isError: true }; +} + +function providerAnalytics(sources: TelemetrySources, provider: string) { + const perProvider = toRecord(sources.analyticsByProvider[provider]); + return perProvider.requests + ? perProvider + : toRecord(sources.analyticsTop.byProvider && toRecord(sources.analyticsTop.byProvider)[provider]); +} + +function buildCandidate(model: ComboModel, sources: TelemetrySources): SpeedCandidate { + const cb = sources.breakers.find((breaker) => toString(breaker.provider) === model.provider); + const q = sources.providers.find((providerEntry) => providerEntry.provider === model.provider); + const analytics = providerAnalytics(sources, model.provider); + const cbState = toString(cb?.state, "CLOSED") as SpeedCandidate["circuitBreakerState"]; + const p95 = toNumber(analytics.p95LatencyMs, NaN); + const errorRate = toNumber(analytics.errorRate, 0); + + return { + provider: model.provider, + model: model.model, + circuitBreakerState: cbState, + avgE2ELatencyMs: toNumber(analytics.avgLatencyMs, NaN), + p95LatencyMs: Number.isFinite(p95) ? p95 : 0, + avgTokensPerSecond: toNumber(analytics.avgTokensPerSecond ?? analytics.tps, NaN), + avgTtftMs: toNumber(analytics.avgTtftMs ?? analytics.ttftMs, NaN), + latencyStdDev: toNumber(analytics.latencyStdDev, NaN), + errorRate: Number.isFinite(errorRate) ? errorRate : 0, + failureRate: Number.isFinite(errorRate) ? errorRate : 0, + quotaRemaining: q?.quotaUsed != null && q?.quotaTotal + ? Math.max(0, 100 - q.quotaUsed / q.quotaTotal * 100) + : 100, + quotaTotal: q?.quotaTotal ?? 100, + costPer1MTokens: model.inputCostPer1M ?? 0, + }; +} + +function buildSpeedCandidates(scopedCombos: JsonRecord[], sources: TelemetrySources): SpeedCandidate[] { + const speedCandidates: SpeedCandidate[] = []; + for (const combo of scopedCombos) { + for (const model of getComboModels(combo)) { + if (model.provider && model.model) speedCandidates.push(buildCandidate(model, sources)); + } + } + return speedCandidates; +} + +function candidateCompletenessScore(candidate: SpeedCandidate): number { + return (candidate.circuitBreakerState ? 1 : 0) + (candidate.quotaRemaining != null ? 1 : 0); +} + +function dedupeCandidates(candidates: SpeedCandidate[]): SpeedCandidate[] { + const deduped = new Map(); + for (const candidate of candidates) { + const key = `${candidate.provider}::${candidate.model}`; + const existing = deduped.get(key); + if (!existing || candidateCompletenessScore(candidate) > candidateCompletenessScore(existing)) { + deduped.set(key, candidate); + } + } + return [...deduped.values()]; +} + +async function applyWinnerToCombo(targetCombo: JsonRecord, winner: { provider: string; model: string }) { + const comboId = toString(targetCombo.id); + const comboData = toRecord(targetCombo.data); + const baseConfig = toRecord(targetCombo.config); + const currentConfig = Object.keys(baseConfig).length > 0 ? baseConfig : toRecord(comboData.config); + const nextConfig = { + ...currentConfig, + auto: { + ...toRecord(currentConfig.auto), + routerStrategy: "latency" as AutoRoutingStrategyValue, + }, + }; + const updatedCombo = toRecord( + await apiFetch(`/api/combos/${encodeURIComponent(comboId)}`, { + method: "PUT", + body: JSON.stringify({ strategy: "auto", config: nextConfig }), + }) + ); + const updatedConfig = toRecord(updatedCombo.config); + return { + id: toString(updatedCombo.id, comboId), + name: toString(updatedCombo.name, toString(targetCombo.name, comboId)), + strategy: toString(updatedCombo.strategy, "auto"), + autoRoutingStrategy: toString(toRecord(updatedConfig.auto).routerStrategy, "latency"), + }; +} + +/** + * Speed-optimized "pick the fastest reliable provider×model" tool. + * + * Composes live telemetry from `/api/combos/metrics` (per-combo per-model + * avg latency + success rate), `/api/monitoring/health` (circuit-breaker + * state), `/api/usage/quota` (quota remaining) and `/api/usage/analytics` + * (per-provider p95 / errorRate) into SpeedCandidates, runs the same + * `rankBySpeed` engine that drives `LatencyStrategyImpl` and the latency- + * optimized playground preview, and returns: + * - the top-ranked (fastest) provider×model pair, + * - the full ranked list with per-factor scores (ttft / tps / e2e / + * p95 / health / reliability / stability) so callers can show + * "why this one wins" in dashboards, + * - optionally applies the choice to a target combo by flipping its + * strategy to "auto" + autoRoutingStrategy "latency", so the runtime + * router will keep using this ranking. + */ +export async function handlePickFastestModel(args: PickFastestModelArgs) { + const start = Date.now(); + try { + const sources = await fetchTelemetrySources(); + const { targetCombo, scopedCombos } = selectComboScope(sources.combos, args.comboId); + + if (scopedCombos.length === 0) { + return noCandidatesResult("No matching combos available"); + } + + const finalCandidates = dedupeCandidates(buildSpeedCandidates(scopedCombos, sources)); + if (finalCandidates.length === 0) { + return noCandidatesResult("No provider×model candidates available to rank"); + } + + const weights = args.weights ? { ...DEFAULT_SPEED_WEIGHTS, ...args.weights } : DEFAULT_SPEED_WEIGHTS; + const ranked = rankBySpeed(finalCandidates, weights, { + includeUnhealthy: args.includeUnhealthy === true, + }); + + const limit = Math.min(Math.max(toNumber(args.limit, 10), 1), 50); + const trimmed = ranked.slice(0, limit); + const winner = trimmed[0]; + + let appliedToCombo: JsonRecord | null = null; + if (args.applyToCombo && targetCombo && winner) { + appliedToCombo = await applyWinnerToCombo(targetCombo, winner); + } + + const result = { + fastest: winner + ? { + provider: winner.provider, + model: winner.model, + score: winner.score, + reason: winner.reason, + } + : null, + ranked: trimmed.map((entry) => ({ + provider: entry.provider, + model: entry.model, + score: entry.score, + factors: entry.factors, + metrics: entry.metrics, + reason: entry.reason, + })), + weights, + comboScope: targetCombo + ? { id: toString(targetCombo.id), name: toString(targetCombo.name) } + : null, + appliedToCombo, + }; + + await logToolCall("omniroute_pick_fastest_model", args, result, Date.now() - start, true); + return { content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }] }; + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + await logToolCall( + "omniroute_pick_fastest_model", + args, + null, + Date.now() - start, + false, + msg + ); + return { content: [{ type: "text" as const, text: `Error: ${msg}` }], isError: true }; + } +} diff --git a/open-sse/services/autoCombo/__tests__/speedRanking.test.ts b/open-sse/services/autoCombo/__tests__/speedRanking.test.ts new file mode 100644 index 0000000000..1599e36b82 --- /dev/null +++ b/open-sse/services/autoCombo/__tests__/speedRanking.test.ts @@ -0,0 +1,226 @@ +/** + * Unit tests for the speed-optimized provider×model ranking engine. + * + * Covers the pure `rankBySpeed` function used by: + * - the runtime `LatencyStrategyImpl` (routerStrategy.ts) + * - the `omniroute_pick_fastest_model` MCP tool + * - the latency-optimized playground preview (via the same shared core) + */ + +import { describe, it, expect } from "vitest"; +import { + rankBySpeed, + pickFastest, + DEFAULT_SPEED_WEIGHTS, +} from "../speedRanking"; +import type { SpeedCandidate } from "../speedRanking"; + +function candidate(overrides: Partial = {}): SpeedCandidate { + return { + provider: "anthropic", + model: "claude-sonnet", + circuitBreakerState: "CLOSED", + errorRate: 0, + failureRate: 0, + quotaRemaining: 100, + quotaTotal: 100, + costPer1MTokens: 3, + p95LatencyMs: 1000, + latencyStdDev: 100, + ...overrides, + }; +} + +describe("rankBySpeed — selection", () => { + it("returns an empty list when the pool is empty", () => { + expect(rankBySpeed([])).toEqual([]); + }); + + it("filters out OPEN circuit-breaker candidates by default", () => { + const pool: SpeedCandidate[] = [ + candidate({ provider: "broken", model: "x", circuitBreakerState: "OPEN" }), + candidate({ provider: "ok", model: "y", avgTtftMs: 200 }), + ]; + const ranked = rankBySpeed(pool); + expect(ranked).toHaveLength(1); + expect(ranked[0].provider).toBe("ok"); + }); + + it("keeps OPEN candidates when includeUnhealthy is set, sorted to the bottom", () => { + const pool: SpeedCandidate[] = [ + candidate({ provider: "broken", model: "x", circuitBreakerState: "OPEN" }), + candidate({ provider: "ok", model: "y", avgTtftMs: 200, avgE2ELatencyMs: 1000 }), + ]; + const ranked = rankBySpeed(pool, DEFAULT_SPEED_WEIGHTS, { includeUnhealthy: true }); + expect(ranked).toHaveLength(2); + expect(ranked[0].provider).toBe("ok"); + expect(ranked[1].provider).toBe("broken"); + }); +}); + +describe("rankBySpeed — metric weighting", () => { + it("picks the lower-TTFT provider×model when TTFT dominates", () => { + const fast: SpeedCandidate = candidate({ + provider: "fast", + model: "m", + avgTtftMs: 120, + avgE2ELatencyMs: 1000, + avgTokensPerSecond: 80, + p95LatencyMs: 1100, + }); + const slow: SpeedCandidate = candidate({ + provider: "slow", + model: "m", + avgTtftMs: 900, + avgE2ELatencyMs: 6000, + avgTokensPerSecond: 20, + p95LatencyMs: 7000, + }); + const ranked = rankBySpeed([slow, fast]); + expect(ranked[0].provider).toBe("fast"); + }); + + it("penalizes high failure rate so a flaky fast provider loses to a steady slower one", () => { + const flakyFast: SpeedCandidate = candidate({ + provider: "flaky", + model: "m", + avgTtftMs: 250, + avgE2ELatencyMs: 1500, + avgTokensPerSecond: 90, + p95LatencyMs: 1700, + errorRate: 0.3, + failureRate: 0.3, + }); + const steadySlow: SpeedCandidate = candidate({ + provider: "steady", + model: "m", + avgTtftMs: 400, + avgE2ELatencyMs: 2200, + avgTokensPerSecond: 70, + p95LatencyMs: 2500, + errorRate: 0.01, + failureRate: 0.01, + }); + const ranked = rankBySpeed([flakyFast, steadySlow]); + expect(ranked[0].provider).toBe("steady"); + }); + + it("penalizes high latency stdDev so a bursty fast provider loses to a steady slow one", () => { + const bursty: SpeedCandidate = candidate({ + provider: "bursty", + model: "m", + avgTtftMs: 100, + avgE2ELatencyMs: 900, + avgTokensPerSecond: 100, + p95LatencyMs: 950, + latencyStdDev: 1500, + }); + const steady: SpeedCandidate = candidate({ + provider: "steady", + model: "m", + avgTtftMs: 350, + avgE2ELatencyMs: 1500, + avgTokensPerSecond: 60, + p95LatencyMs: 1600, + latencyStdDev: 50, + }); + const ranked = rankBySpeed([bursty, steady]); + expect(ranked[0].provider).toBe("steady"); + }); + + it("rewards higher tokens-per-second when everything else ties", () => { + const base = { + avgTtftMs: 300, + avgE2ELatencyMs: 1500, + p95LatencyMs: 1600, + }; + const lowTps: SpeedCandidate = candidate({ provider: "low", model: "m", ...base, avgTokensPerSecond: 20 }); + const highTps: SpeedCandidate = candidate({ provider: "high", model: "m", ...base, avgTokensPerSecond: 200 }); + const ranked = rankBySpeed([lowTps, highTps]); + expect(ranked[0].provider).toBe("high"); + }); +}); + +describe("rankBySpeed — factor breakdown", () => { + it("emits per-factor values in [0..1] and an explanation", () => { + const ranked = rankBySpeed([ + candidate({ + provider: "a", + model: "m", + avgTtftMs: 100, + avgE2ELatencyMs: 1000, + avgTokensPerSecond: 100, + p95LatencyMs: 1100, + latencyStdDev: 50, + }), + ]); + expect(ranked).toHaveLength(1); + const factors = ranked[0].factors; + for (const value of Object.values(factors)) { + expect(value).toBeGreaterThanOrEqual(0); + expect(value).toBeLessThanOrEqual(1); + } + expect(ranked[0].reason).toMatch(/SpeedRanking\[/); + expect(ranked[0].reason).toMatch(/ttft=100ms/); + }); + + it("falls back to 0.5 per missing metric so new providers are not crushed", () => { + const ranked = rankBySpeed([candidate({ provider: "fresh", model: "m" })]); + expect(ranked).toHaveLength(1); + // No telemetry at all → weighted sum lands near 0.5 with reliability multiplier 1 + expect(ranked[0].factors.reliability).toBe(1); + expect(ranked[0].factors.health).toBe(1); + expect(ranked[0].factors.ttft).toBe(0.5); + expect(ranked[0].factors.tps).toBe(0.5); + }); +}); + +describe("rankBySpeed — weight overrides", () => { + it("respects caller weight overrides (e.g. heavy TTFT bias)", () => { + const fastTtft: SpeedCandidate = candidate({ + provider: "fast", + model: "m", + avgTtftMs: 100, + avgE2ELatencyMs: 5000, + avgTokensPerSecond: 5, + p95LatencyMs: 6000, + }); + const slowTtft: SpeedCandidate = candidate({ + provider: "slow", + model: "m", + avgTtftMs: 800, + avgE2ELatencyMs: 1200, + avgTokensPerSecond: 120, + p95LatencyMs: 1300, + }); + + const normal = rankBySpeed([fastTtft, slowTtft]); + // Normal weights still pick slowTtft because TPS/E2E dominate over TTFT gaps. + expect(normal[0].provider).toBe("slow"); + + const heavyTtft = rankBySpeed([fastTtft, slowTtft], { + ...DEFAULT_SPEED_WEIGHTS, + ttft: 0.7, + tps: 0.05, + e2e: 0.05, + p95: 0.05, + health: 0.05, + reliability: 0.05, + stability: 0.05, + }); + expect(heavyTtft[0].provider).toBe("fast"); + }); +}); + +describe("pickFastest", () => { + it("returns null on an empty pool", () => { + expect(pickFastest([])).toBeNull(); + }); + + it("returns the top-ranked candidate", () => { + const fast = candidate({ provider: "fast", model: "m", avgTtftMs: 100 }); + const slow = candidate({ provider: "slow", model: "m", avgTtftMs: 900 }); + const winner = pickFastest([slow, fast]); + expect(winner?.provider).toBe("fast"); + }); +}); \ No newline at end of file diff --git a/open-sse/services/autoCombo/routerStrategy.ts b/open-sse/services/autoCombo/routerStrategy.ts index 466d815607..b9923fe470 100644 --- a/open-sse/services/autoCombo/routerStrategy.ts +++ b/open-sse/services/autoCombo/routerStrategy.ts @@ -14,6 +14,8 @@ import type { ProviderCandidate, ScoredProvider } from "./scoring.ts"; import { scorePool } from "./scoring.ts"; import { getTaskFitness } from "./taskFitness.ts"; import { clamp01 } from "../../utils/number.ts"; +import { rankBySpeed } from "./speedRanking.ts"; +import type { SpeedCandidate } from "./speedRanking.ts"; export interface SlaRoutingPolicy { targetP95Ms?: number; @@ -49,6 +51,36 @@ export interface RouterStrategy { // ── RulesStrategy: wraps 6-factor scoring engine ──────────────────────────── +function toSpeedCandidate(c: ProviderCandidate): SpeedCandidate { + return { + // Identity + provider: c.provider, + model: c.model, + // Resource state + quotaRemaining: c.quotaRemaining, + quotaTotal: c.quotaTotal, + circuitBreakerState: c.circuitBreakerState, + // Costs + costPer1MTokens: c.costPer1MTokens, + // Latency metrics + p95LatencyMs: c.p95LatencyMs, + avgTtftMs: c.avgTtftMs, + avgE2ELatencyMs: c.avgE2ELatencyMs, + avgTokensPerSecond: c.avgTokensPerSecond, + latencyStdDev: c.latencyStdDev, + // Reliability + errorRate: c.errorRate, + failureRate: c.failureRate, + // Tier signals (forwarded so weights stay available for downstream tuning) + accountTier: c.accountTier, + quotaResetIntervalSecs: c.quotaResetIntervalSecs, + contextAffinity: c.contextAffinity, + resetWindowAffinity: c.resetWindowAffinity, + connectionPoolSize: c.connectionPoolSize, + connectionId: c.connectionId, + }; +} + class RulesStrategyImpl implements RouterStrategy { readonly name = "rules"; readonly description = @@ -100,105 +132,48 @@ class CostStrategyImpl implements RouterStrategy { // ── LatencyStrategy: prioritize low latency + reliability ─────────────────── -function positiveMetric(value: unknown): number | null { - const numericValue = Number(value); - return Number.isFinite(numericValue) && numericValue > 0 ? numericValue : null; -} - -function boundedRate(value: unknown): number { - const numericValue = Number(value); - return Number.isFinite(numericValue) && numericValue >= 0 ? Math.min(1, numericValue) : 0; -} - -function maxPositiveMetric( - candidates: ProviderCandidate[], - readMetric: (candidate: ProviderCandidate) => unknown, - fallback = 1 -): number { - return Math.max( - ...candidates.map((candidate) => positiveMetric(readMetric(candidate)) ?? 0), - fallback - ); -} - -function latencyMetricScore(value: number | null, maxValue: number): number { - if (value == null) return 0.5; - return inverseNormalized(value, maxValue); -} - -function throughputMetricScore(value: number | null, maxValue: number): number { - if (value == null) return 0.5; - return clamp01(value / Math.max(maxValue, 0.000_001)); -} - class LatencyStrategyImpl implements RouterStrategy { readonly name = "latency"; readonly description = "Prioritizes the fastest reliable provider-model pair using TTFT, TPS, E2E latency, health, fail rate, and stability"; select(pool: ProviderCandidate[], context: RoutingContext): RoutingDecision { - const healthy = pool.filter((c) => c.circuitBreakerState !== "OPEN"); - const candidates = healthy.length > 0 ? healthy : pool; - if (candidates.length === 0) throw new Error("[LatencyStrategy] No candidates available"); - - const maxP95 = maxPositiveMetric(candidates, (candidate) => candidate.p95LatencyMs); - const maxTtft = maxPositiveMetric( - candidates, - (candidate) => candidate.avgTtftMs ?? candidate.p95LatencyMs - ); - const maxE2E = maxPositiveMetric( - candidates, - (candidate) => candidate.avgE2ELatencyMs ?? candidate.p95LatencyMs - ); - const maxTps = maxPositiveMetric(candidates, (candidate) => candidate.avgTokensPerSecond); - const maxStdDev = maxPositiveMetric(candidates, (candidate) => candidate.latencyStdDev, 0.001); - - const scored = candidates - .map((candidate) => { - const p95 = positiveMetric(candidate.p95LatencyMs); - const ttft = positiveMetric(candidate.avgTtftMs) ?? p95; - const e2e = positiveMetric(candidate.avgE2ELatencyMs) ?? p95; - const tps = positiveMetric(candidate.avgTokensPerSecond); - const failureRate = boundedRate(candidate.failureRate ?? candidate.errorRate); - const healthScore = getHealthScore(candidate); - const p95Score = latencyMetricScore(p95, maxP95); - const ttftScore = latencyMetricScore(ttft, maxTtft); - const e2eScore = latencyMetricScore(e2e, maxE2E); - const throughputScore = throughputMetricScore(tps, maxTps); - const reliabilityScore = 1 - failureRate; - const stabilityScore = latencyMetricScore( - positiveMetric(candidate.latencyStdDev), - maxStdDev - ); - const rawScore = - ttftScore * 0.25 + - throughputScore * 0.2 + - e2eScore * 0.18 + - p95Score * 0.12 + - reliabilityScore * 0.15 + - healthScore * 0.05 + - stabilityScore * 0.05; - const reliabilityMultiplier = Math.max(0.05, reliabilityScore * reliabilityScore); - const score = rawScore * reliabilityMultiplier * Math.max(0.25, healthScore); - - return { candidate, score, ttft, e2e, tps, failureRate }; - }) - .sort((a, b) => b.score - a.score); - - const best = scored[0]; - if (!best) throw new Error("[LatencyStrategy] No candidates available"); + const ranked = rankBySpeed(pool.map(toSpeedCandidate)); + const winner = ranked[0]; + if (!winner) { + throw new Error("[LatencyStrategy] No candidates available after speed ranking"); + } return { - provider: best.candidate.provider, - model: best.candidate.model, + provider: winner.provider, + model: winner.model, strategy: this.name, - reason: `LatencyStrategy: ttft=${best.ttft ?? "n/a"}ms, tps=${best.tps ?? "n/a"}, e2e=${best.e2e ?? "n/a"}ms, p95=${best.candidate.p95LatencyMs}ms, failRate=${(best.failureRate * 100).toFixed(2)}%`, - candidatesConsidered: candidates.length, - finalScore: best.score, + reason: latencyDecisionReason(winner), + candidatesConsidered: ranked.length, + finalScore: winner.score, }; } } +function metricString(value: number | null | undefined, digits = 0): string { + return value == null ? "n/a" : value.toFixed(digits); +} + +function latencyDecisionReason(winner: ReturnType[number]): string { + const metrics = winner.metrics; + const e2e = metrics.avgE2ELatencyMs ?? metrics.p95LatencyMs; + return ( + `LatencyStrategy(score=${winner.score.toFixed(3)}): ` + + `ttft=${metricString(metrics.avgTtftMs)}ms ` + + `tps=${metricString(metrics.avgTokensPerSecond, 1)} ` + + `e2e=${metricString(e2e)}ms ` + + `p95=${metricString(metrics.p95LatencyMs)}ms ` + + `failRate=${((metrics.failureRate ?? 0) * 100).toFixed(2)}% ` + + `stability=${metricString(metrics.latencyStdDev)}ms ` + + `cb=${metrics.circuitBreakerState ?? "n/a"}` + ); +} + // ── SLAStrategy: favor targets that meet latency/error/cost SLOs ─────────── const DEFAULT_SLA_TARGET_P95_MS = 2_000; diff --git a/open-sse/services/autoCombo/speedRanking.ts b/open-sse/services/autoCombo/speedRanking.ts new file mode 100644 index 0000000000..514420a5a0 --- /dev/null +++ b/open-sse/services/autoCombo/speedRanking.ts @@ -0,0 +1,327 @@ +/** + * Speed-optimized Provider×Model Ranking + * + * Pure, framework-free scoring function that ranks provider×model candidates by + * the speed/reliability combination most likely to make a request feel "fast": + * + * - lower avg Time-To-First-Token (TTFT) — perceived responsiveness + * - higher avg tokens-per-second (TPS) — generation throughput + * - lower avg end-to-end (E2E) latency — full completion time + * - lower p95 latency — tail-risk backup metric + * - higher circuit-breaker health — must be reachable + * - lower failure / error rate — speed without flake + * - lower latency standard deviation (stability)— consistent, not bursty + * + * Every metric is optional; missing telemetry is treated as the pool median + * (i.e. 0.5 in [0..1]) so a brand-new provider/model does not get crushed, but + * also does not get a free pass on a metric we have no data for. This mirrors + * the behavior of the existing `LatencyStrategyImpl` and is intentional — the + * function is the canonical ranking for the "fastest reliable provider-model" + * UX in the playground + MCP `omniroute_pick_fastest_model` tool, and is + * reused by the runtime `LatencyStrategyImpl` so the runtime router picks the + * same winner as the user-facing preview. + * + * Optional weights override defaults so callers (tests, dashboards, future + * `modePacks.speed-first`) can rebalance without duplicating the math. + */ + +import type { ProviderCandidate } from "./scoring.ts"; + +/** Optional per-candidate telemetry surfaced by the ranking. */ +export interface SpeedCandidate extends ProviderCandidate { + /** Numeric health score [0..1]; computed from circuitBreakerState when missing. */ + health?: number; + /** Percentage quota remaining in [0..100]. Falls back to quotaRemaining. */ + quotaRemainingPct?: number; + /** Numeric capacity score [0..1] in case the caller wants to expose it. */ + capacityScore?: number; + /** Cached cost-per-1k tokens (denormalized from costPer1MTokens). */ + costPer1k?: number; + /** Optional quality score for the candidate (e.g. eval benchmark). */ + qualityScore?: number; + /** Optional strategic boost applied for premium/internal models. */ + strategicBoost?: number; + /** Optional SLO-violation penalty that should be subtracted from the raw score. */ + sloPenalty?: number; +} + +/** Per-factor contribution of a single candidate (each value clamped to [0..1]). */ +export interface SpeedFactors { + ttft: number; + tps: number; + e2e: number; + p95: number; + health: number; + reliability: number; + stability: number; +} + +/** Result of ranking a pool of candidates. */ +export interface SpeedRankedCandidate { + provider: string; + model: string; + /** Final composite score in [0..1]; higher is faster+more-reliable. */ + score: number; + factors: SpeedFactors; + /** Raw telemetry we observed for the candidate (in provider-native units). */ + metrics: { + avgTtftMs: number | null; + avgTokensPerSecond: number | null; + avgE2ELatencyMs: number | null; + p95LatencyMs?: number | null; + latencyStdDev: number | null; + failureRate: number; + circuitBreakerState: SpeedCandidate["circuitBreakerState"]; + }; + /** Human-readable explanation of why this candidate earned its score. */ + reason: string; +} + +/** Caller-tunable weights; defaults are documented at each property. */ +export interface SpeedRankingWeights { + /** Weight for TTFT (default 0.25). */ + ttft: number; + /** Weight for tokens/sec (default 0.20). */ + tps: number; + /** Weight for end-to-end latency (default 0.18). */ + e2e: number; + /** Weight for p95 latency fallback (default 0.12). */ + p95: number; + /** Weight for circuit-breaker health (default 0.05). */ + health: number; + /** Weight for reliability = 1 - failureRate (default 0.15). */ + reliability: number; + /** Weight for latency stability / low std-dev (default 0.05). */ + stability: number; +} + +/** + * Default weights — these sum to 1.0 and bias toward perceived speed (TTFT + + * TPS together account for 45% of the score) while still penalizing unsafe + * providers. They are deliberately exported so the playground UI can render + * the formula and tests can pin it. + */ +export const DEFAULT_SPEED_WEIGHTS: SpeedRankingWeights = { + ttft: 0.25, + tps: 0.2, + e2e: 0.18, + p95: 0.12, + health: 0.05, + reliability: 0.15, + stability: 0.05, +}; + +/** Human-readable label for each weight key — used in `reason` strings. */ +const FACTOR_LABEL: Record = { + ttft: "ttft", + tps: "tps", + e2e: "e2e", + p95: "p95", + health: "health", + reliability: "reliability", + stability: "stability", +}; + +function clamp01(value: number): number { + if (!Number.isFinite(value)) return 0; + if (value < 0) return 0; + if (value > 1) return 1; + return value; +} + +function positiveFinite(value: number | null | undefined): number | null { + return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : null; +} + +function toBoundedRate(value: number | null | undefined): number { + if (typeof value !== "number" || !Number.isFinite(value) || value < 0) return 0; + return Math.min(1, value); +} + +/** + * Pool-relative maximum for "lower is better" metrics. Returns at least + * `floor` so a candidate with the only positive measurement does not divide + * by zero. + */ +function poolMax( + values: ReadonlyArray, + readMetric: (value: T) => number | null, + floor = 1 +): number { + let max = floor; + for (const value of values) { + const v = readMetric(value); + if (v != null && v > max) max = v; + } + return max; +} + +/** + * Pool-relative maximum for "higher is better" metrics. Returns at least + * `floor` so a missing metric does not yield Infinity downstream. + */ +function poolMaxHigherBetter( + values: ReadonlyArray, + readMetric: (value: T) => number | null, + floor = 0.000_001 +): number { + let max = floor; + for (const value of values) { + const v = readMetric(value); + if (v != null && v > max) max = v; + } + return max; +} + +/** 1 - (value / max), clamped to [0..1]. Missing → 0.5 (pool median). */ +function lowerIsBetter(value: number | null | undefined, max: number): number { + if (value == null) return 0.5; + if (!Number.isFinite(value) || value < 0) return 0; + if (!Number.isFinite(max) || max <= 0) return 1; + return clamp01(1 - value / max); +} + +/** value / max, clamped to [0..1]. Missing → 0.5. */ +function higherIsBetter(value: number | null | undefined, max: number): number { + if (value == null) return 0.5; + if (!Number.isFinite(value) || value < 0) return 0; + if (!Number.isFinite(max) || max <= 0) return 0; + return clamp01(value / max); +} + +function healthScoreFor(state: SpeedCandidate["circuitBreakerState"]): number { + if (state === "CLOSED") return 1; + if (state === "HALF_OPEN") return 0.5; + return 0; // OPEN — caller is expected to filter these out beforehand, but be defensive. +} + +function speedPoolMaxima(pool: ReadonlyArray) { + return { + ttft: poolMax(pool, (c) => positiveFinite(c.avgTtftMs) ?? positiveFinite(c.p95LatencyMs)), + e2e: poolMax(pool, (c) => positiveFinite(c.avgE2ELatencyMs) ?? positiveFinite(c.p95LatencyMs)), + p95: poolMax(pool, (c) => positiveFinite(c.p95LatencyMs)), + tps: poolMaxHigherBetter(pool, (c) => positiveFinite(c.avgTokensPerSecond)), + stdDev: poolMax(pool, (c) => positiveFinite(c.latencyStdDev), 0.001), + }; +} + +function speedFactorsFor( + candidate: SpeedCandidate, + maxima: ReturnType, + failureRate: number +): SpeedFactors { + return { + ttft: lowerIsBetter(positiveFinite(candidate.avgTtftMs), maxima.ttft), + tps: higherIsBetter(positiveFinite(candidate.avgTokensPerSecond), maxima.tps), + e2e: lowerIsBetter(positiveFinite(candidate.avgE2ELatencyMs), maxima.e2e), + p95: lowerIsBetter(positiveFinite(candidate.p95LatencyMs), maxima.p95), + health: healthScoreFor(candidate.circuitBreakerState), + reliability: clamp01(1 - failureRate), + stability: lowerIsBetter(positiveFinite(candidate.latencyStdDev), maxima.stdDev), + }; +} + +function weightedSpeedScore(factors: SpeedFactors, weights: SpeedRankingWeights): number { + return ( + factors.ttft * weights.ttft + + factors.tps * weights.tps + + factors.e2e * weights.e2e + + factors.p95 * weights.p95 + + factors.health * weights.health + + factors.reliability * weights.reliability + + factors.stability * weights.stability + ); +} + +function applySpeedPenalties(weightedSum: number, factors: SpeedFactors): number { + const reliabilityMultiplier = Math.max(0.05, Math.pow(0.25 + 0.75 * factors.reliability, 2)); + const stabilityMultiplier = Math.max(0.05, Math.pow(0.25 + 0.75 * factors.stability, 2)); + return clamp01(weightedSum * reliabilityMultiplier * stabilityMultiplier * Math.max(0.25, factors.health)); +} + +function speedReason(candidate: SpeedCandidate, factors: SpeedFactors, metrics: SpeedRankedCandidate["metrics"]): string { + const reasonParts = [ + `ttft=${metrics.avgTtftMs == null ? "n/a" : `${Math.round(metrics.avgTtftMs)}ms`}`, + `tps=${metrics.avgTokensPerSecond == null ? "n/a" : metrics.avgTokensPerSecond.toFixed(1)}`, + `e2e=${metrics.avgE2ELatencyMs == null ? "n/a" : `${Math.round(metrics.avgE2ELatencyMs)}ms`}`, + `p95=${metrics.p95LatencyMs == null ? "n/a" : `${Math.round(metrics.p95LatencyMs)}ms`}`, + `failRate=${(metrics.failureRate * 100).toFixed(2)}%`, + `cb=${candidate.circuitBreakerState}`, + ]; + return `SpeedRanking[${FACTOR_LABEL.ttft}=${factors.ttft.toFixed(2)}, ${FACTOR_LABEL.tps}=${factors.tps.toFixed(2)}, ${FACTOR_LABEL.e2e}=${factors.e2e.toFixed(2)}, ${FACTOR_LABEL.p95}=${factors.p95.toFixed(2)}, ${FACTOR_LABEL.reliability}=${factors.reliability.toFixed(2)}, ${FACTOR_LABEL.health}=${factors.health.toFixed(2)}, ${FACTOR_LABEL.stability}=${factors.stability.toFixed(2)}] → ${reasonParts.join(", ")}`; +} + +/** + * Rank candidates for the speed-optimized routing mode. + * + * @param candidates Pool of provider×model candidates (typically the candidates + * inside an auto combo's provider pool, or any list assembled by the + * playground / MCP tool). + * @param weights Optional weight overrides — defaults to {@link DEFAULT_SPEED_WEIGHTS}. + * @param options.includeUnhealthy If false (default), OPEN circuit-breaker + * candidates are dropped before scoring. If true, they are scored with a + * health factor of 0 and a reliability factor of 0 so they sort to the + * bottom without changing the rest of the ranking. + * @returns The full ranked list, highest score first. The first entry is the + * "fastest reliable provider-model pair" for this pool. + */ +export function rankBySpeed( + candidates: ReadonlyArray, + weights: SpeedRankingWeights = DEFAULT_SPEED_WEIGHTS, + options: { includeUnhealthy?: boolean } = {} +): SpeedRankedCandidate[] { + if (candidates.length === 0) return []; + + const pool = options.includeUnhealthy + ? [...candidates] + : candidates.filter((c) => c.circuitBreakerState !== "OPEN"); + if (pool.length === 0) return []; + + const maxima = speedPoolMaxima(pool); + + const ranked = pool.map((candidate) => { + const p95 = positiveFinite(candidate.p95LatencyMs); + const ttft = positiveFinite(candidate.avgTtftMs); + const e2e = positiveFinite(candidate.avgE2ELatencyMs); + const tps = positiveFinite(candidate.avgTokensPerSecond); + const stdDev = positiveFinite(candidate.latencyStdDev); + const failureRate = toBoundedRate( + candidate.failureRate ?? (typeof candidate.errorRate === "number" ? candidate.errorRate : 0) + ); + const factors = speedFactorsFor(candidate, maxima, failureRate); + const score = applySpeedPenalties(weightedSpeedScore(factors, weights), factors); + const metrics = { + avgTtftMs: ttft, + avgTokensPerSecond: tps, + avgE2ELatencyMs: e2e, + p95LatencyMs: p95, + latencyStdDev: stdDev, + failureRate, + circuitBreakerState: candidate.circuitBreakerState, + }; + + return { + provider: candidate.provider, + model: candidate.model, + score, + factors, + metrics, + reason: speedReason(candidate, factors, metrics), + }; + }); + + return ranked.sort((a, b) => b.score - a.score); +} + +/** + * Convenience selector — returns the top-ranked candidate or `null` when the + * pool is empty. Used by the runtime `LatencyStrategyImpl` and the MCP + * `omniroute_pick_fastest_model` tool when only the winner is needed. + */ +export function pickFastest( + candidates: ReadonlyArray, + weights: SpeedRankingWeights = DEFAULT_SPEED_WEIGHTS +): SpeedRankedCandidate | null { + const ranked = rankBySpeed(candidates, weights); + return ranked.length > 0 ? ranked[0] : null; +} diff --git a/stryker.conf.json b/stryker.conf.json index 0aa2121634..2b4a90dec9 100644 --- a/stryker.conf.json +++ b/stryker.conf.json @@ -113,6 +113,7 @@ "tests/unit/combo-provider-cooldown.test.ts", "tests/unit/combo-provider-diversity-wiring.test.ts", "tests/unit/combo-quality-validator-reasoning.test.ts", + "tests/unit/combo-priority-quota-exhaustion-cutoff-5923.test.ts", "tests/unit/combo-quota-soft-penalty.test.ts", "tests/unit/combo-round-robin-streaming-lock-3811.test.ts", "tests/unit/combo-routing-engine.test.ts",