diff --git a/changelog.d/fixes/12730-vendor-route-effort-values.md b/changelog.d/fixes/12730-vendor-route-effort-values.md new file mode 100644 index 0000000000..4866681e06 --- /dev/null +++ b/changelog.d/fixes/12730-vendor-route-effort-values.md @@ -0,0 +1 @@ +- **fix(providers):** OpenAI-compatible model discovery now parses per-vendor-route `effort_values` (nested under `vendors..capabilities.reasoning` in `/v1/models`), intersected across vendor routes so a synced level is always honored on every route the model can land on; re-syncing a connection whose catalog declares this shape no longer silently resets the synced `supportedThinkingEfforts`/`defaultThinkingEffort` data ([#12730](https://github.com/diegosouzapw/OmniRoute/pull/12730)) diff --git a/src/lib/providerModels/modelDiscovery.ts b/src/lib/providerModels/modelDiscovery.ts index bfe9131cf4..3f79163d36 100644 --- a/src/lib/providerModels/modelDiscovery.ts +++ b/src/lib/providerModels/modelDiscovery.ts @@ -99,6 +99,61 @@ const effortListSchema = z.array(z.unknown()); const supportedReasoningLevelsSchema = z.object({ supported_reasoning_levels: z.unknown() }); const thinkingLevelsSchema = z.object({ thinking: z.object({ levels: z.unknown() }).partial() }); +// Vendor-route catalogs (e.g. Merge Gateway's `/v1/models`) nest per-route +// reasoning capability under `vendors..capabilities.reasoning` — the +// route's accepted effort levels live in `effort_values` (docs.merge.dev, +// "Effort levels per route"): the SAME canonical model lists different +// vocabularies per vendor route, and a request naming an effort level is +// served by a route that honors it when one exists (unpinned requests +// self-narrow). So the safe synced vocabulary is the INTERSECTION across +// vendor routes (a synced level must be honored on every route the model can +// land on), not the union. A route without `effort_values` (absent or empty) +// declares "no effort control" for that vendor and is excluded from the +// intersection. Like the other shapes in this file, detection is shape-gated, +// not provider-gated: a record that declares this structure is declaring its +// effort vocabulary. Validate with Zod (Hard Rule #7); a malformed ENTRY is +// dropped individually (never the whole route — discarding a route would +// WIDEN the intersection, fail-open). +const vendorRouteReasoningCapabilitySchema = z.object({ + effort_values: z.array(z.unknown()).optional(), +}); +const vendorRoutesSchema = z.record(z.string(), z.unknown()); + +function parseVendorRouteEffortValues(record: JsonRecord): string[][] { + const vendorsParsed = vendorRoutesSchema.safeParse(record.vendors); + if (!vendorsParsed.success) return []; + const perVendor: string[][] = []; + for (const vendorValue of Object.values(vendorsParsed.data)) { + const vendorRecord = asRecord(vendorValue); + const reasoningParsed = vendorRouteReasoningCapabilitySchema.safeParse( + asRecord(vendorRecord.capabilities).reasoning + ); + if (!reasoningParsed.success || !reasoningParsed.data) continue; + const efforts = Array.from( + new Set( + (reasoningParsed.data.effort_values ?? []) + .filter((effort): effort is string => typeof effort === "string" && effort.length > 0) + .map(normalizeSupportedEffort) + ) + ); + if (efforts.length > 0) perVendor.push(efforts); + } + return perVendor; +} + +/** + * Intersect `effort_values` across the record's vendor routes. Returns + * `undefined` when no vendor route declares a list (shape not present); + * returns an EMPTY array when routes declare disjoint vocabularies — that + * emptiness is authoritative (no tier works on every route) and must not + * fall through to lower-precedence generic shapes. + */ +function vendorRouteSharedEfforts(record: JsonRecord): string[] | undefined { + const perVendor = parseVendorRouteEffortValues(record); + if (perVendor.length === 0) return undefined; + return perVendor.reduce((acc, efforts) => acc.filter((effort) => efforts.includes(effort))); +} + // Maps common upstream synonyms onto OmniRoute's canonical effort vocabulary // (`src/shared/reasoning/effortStandardization.ts`). Values already in // `CANONICAL_EFFORT_VALUES`, and any unrecognized provider-native tier (e.g. @@ -167,9 +222,51 @@ export function detectDefaultThinkingEffort(record: JsonRecord): string | undefi const raw = parsed.data.default_effort; if (typeof raw === "string" && raw.length > 0) return normalizeSupportedEffort(raw); } + // Vendor-route fallback — only when the `vendors` shape IS the record's + // winning effort-vocabulary source. If a higher-precedence declared shape + // (`reasoning.supported_efforts`, `metadata.reasoning.supported_efforts`) + // produced a usable list, the record's default must never escape that + // winning list. Highest shared tier wins, ranked by the canonical order + // (vendor arrays are not guaranteed sorted). + const mergeShared = vendorRouteSharedEfforts(record); + if (mergeShared && mergeShared.length > 0 && !hasUsableDeclaredEffortList(record)) { + const ranked = mergeShared + .map((tier) => ({ tier, rank: CANONICAL_EFFORT_VALUES.indexOf(tier as never) })) + .filter((x) => x.rank >= 0) + .sort((a, b) => b.rank - a.rank); + if (ranked.length > 0) return ranked[0].tier; + } return undefined; } +/** + * Whether a higher-precedence declared shape (the flat import field, or either + * #7694 nested `supported_efforts` shape) yields a usable tier list — the + * exact "usable" semantics the vocabulary detection applies (non-empty after + * filtering + normalization). Used to decide whether the Merge vendors shape + * is the record's winning vocabulary source for default-effort derivation. + */ +function hasUsableDeclaredEffortList(record: JsonRecord): boolean { + if ( + Array.isArray(record.supportedThinkingEfforts) && + record.supportedThinkingEfforts.some((e) => typeof e === "string" && e.length > 0) + ) { + return true; + } + for (const holder of [record.reasoning, asRecord(record.metadata).reasoning]) { + const shapeParsed = reasoningSupportedEffortsSchema.safeParse(holder); + if (!shapeParsed.success || !shapeParsed.data) continue; + const rawEfforts = shapeParsed.data.supported_efforts; + if ( + Array.isArray(rawEfforts) && + rawEfforts.some((e) => typeof e === "string" && e.length > 0) + ) { + return true; + } + } + return false; +} + /** * #7694: read the nested `record.reasoning.supported_efforts` shape and normalize each * tier onto the canonical vocabulary. Returns `undefined` (never throws) when the field @@ -214,6 +311,19 @@ export function detectSupportedThinkingEfforts(record: JsonRecord): string[] | u } } + // Vendor-route catalogs: intersect `effort_values` across vendor routes. + // Placed after the flat import field handling (caller) and the + // #7694/#9160 nested shapes so those explicit per-model declarations keep + // precedence; runs before the generic `capabilities.effort_tiers` fallback + // because per-route vocabularies are strictly more specific than a flat + // tier list. An empty intersection (disjoint routes) is authoritative — + // nothing works on every route — and must not fall through to a generic + // tier list. + const mergeShared = vendorRouteSharedEfforts(record); + if (mergeShared !== undefined) { + return mergeShared.length > 0 ? mergeShared : []; + } + // #9160: fall back to `capabilities.effort_tiers` before the legacy fields. // OmniRoute's own catalog surfaces effort tiers inside `capabilities.effort_tiers`, // which the existing `parseEffortList` already handles (string arrays). @@ -247,7 +357,13 @@ function hasDeclaredEffortList(record: JsonRecord): boolean { if (Array.isArray(asRecord(record.reasoning).supported_efforts)) return true; if (Array.isArray(asRecord(record.capabilities).effort_tiers)) return true; if (Array.isArray(record.supported_reasoning_levels)) return true; - return Array.isArray(asRecord(record.thinking).levels); + if (Array.isArray(asRecord(record.thinking).levels)) return true; + // `vendors..capabilities.reasoning.effort_values` counts as a declared + // list so the fallback chain in `normalizeDiscoveredModels` stops here + // instead of applying provider-specific heuristics to a record that already + // declares its vocabulary explicitly (mirrors the other declared shapes: + // detect returning undefined means "declared, nothing usable"). + return parseVendorRouteEffortValues(record).length > 0; } export function isAutoFetchModelsEnabled(providerSpecificData: unknown): boolean { diff --git a/tests/unit/model-discovery-vendor-effort-values.test.ts b/tests/unit/model-discovery-vendor-effort-values.test.ts new file mode 100644 index 0000000000..438e63d4c1 --- /dev/null +++ b/tests/unit/model-discovery-vendor-effort-values.test.ts @@ -0,0 +1,169 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + detectDefaultThinkingEffort, + detectSupportedThinkingEfforts, + normalizeDiscoveredModels, +} from "@/lib/providerModels/modelDiscovery"; + +// Vendor-route catalogs (e.g. Merge Gateway `/v1/models`) nest per-route reasoning capability under +// `vendors..capabilities.reasoning` with the route's accepted effort +// levels in `effort_values` (docs.merge.dev, "Effort levels per route"). The +// same canonical model declares DIFFERENT vocabularies per vendor route, and +// unpinned requests self-narrow to a route honoring the requested level when +// one exists — so the synced vocabulary is the INTERSECTION across routes: +// a level outside the intersection can be silently adjusted (or 400 on pinned +// requests) on the route a request actually lands on. + +function mergeRecord(vendorEfforts: Record) { + return { + id: "zai/glm-5.3-flash", + vendors: Object.fromEntries( + Object.entries(vendorEfforts).map(([vendor, efforts]) => [ + vendor, + { capabilities: { reasoning: { effort_values: efforts } } }, + ]) + ), + }; +} + +test("vendor-route effort_values from a single vendor route is parsed into supportedThinkingEfforts", () => { + assert.deepEqual(detectSupportedThinkingEfforts(mergeRecord({ zai: ["low", "high", "max"] })), [ + "low", + "high", + "max", + ]); +}); + +test("vendor-route effort_values from multiple vendor routes intersects across routes", () => { + assert.deepEqual( + detectSupportedThinkingEfforts( + mergeRecord({ + zai: ["low", "high", "max"], + baseten: ["low", "medium", "high", "xhigh"], + makora: ["low", "high", "max"], + }) + ), + ["low", "high"] + ); +}); + +test("vendor routes without effort_values (no effort control) are excluded from the intersection", () => { + // fireworks: no effort_values key at all — "no effort control" per the docs. + const record = { + id: "moonshot/kimi-k3", + vendors: { + moonshot: { capabilities: { reasoning: { effort_values: ["low", "max"] } } }, + fireworks: { capabilities: { reasoning: {} } }, + }, + }; + assert.deepEqual(detectSupportedThinkingEfforts(record), ["low", "max"]); +}); + +test("a malformed vendor entry is dropped individually; no throw on garbage vendors", () => { + const record = { + id: "vendor/garbage", + vendors: { + good: { capabilities: { reasoning: { effort_values: ["low", "high"] } } }, + bad: "not-an-object", + worse: { capabilities: { reasoning: { effort_values: 42 } } }, + }, + }; + assert.doesNotThrow(() => detectSupportedThinkingEfforts(record)); + assert.deepEqual(detectSupportedThinkingEfforts(record), ["low", "high"]); +}); + +test("no vendors key degrades to undefined (legacy behavior preserved)", () => { + assert.equal(detectSupportedThinkingEfforts({ id: "plain/model" }), undefined); + assert.equal( + detectSupportedThinkingEfforts({ id: "plain/model", vendors: "not-a-map" }), + undefined + ); +}); + +test("effort synonyms are normalized inside effort_values", () => { + assert.deepEqual(detectSupportedThinkingEfforts(mergeRecord({ zai: ["low", "extra", "max"] })), [ + "low", + "xhigh", + "max", + ]); +}); + +test("vendor-route defaultThinkingEffort falls back to the intersected vocabulary's highest tier (rank-based)", () => { + // max survives both routes → default max. + const withMax = mergeRecord({ + zai: ["low", "high", "max"], + makora: ["none", "low", "high", "max"], + }); + assert.equal(detectDefaultThinkingEffort(withMax), "max"); + // max only on one route → the intersection's highest (high) is the default. + const withoutSharedMax = mergeRecord({ + zai: ["low", "high", "max"], + baseten: ["low", "medium", "high"], + }); + assert.equal(detectDefaultThinkingEffort(withoutSharedMax), "high"); + // Vendor arrays are not guaranteed sorted: first-route order must not + // decide the default — the canonical rank does. + const unsorted = mergeRecord({ + zai: ["max", "low", "high"], + makora: ["high", "low", "max"], + }); + assert.equal(detectDefaultThinkingEffort(unsorted), "max"); +}); + +test("vendor-route default is skipped when a higher-precedence declared shape wins the list", () => { + // Flat import field pins the vocabulary to ["low"]; the vendors + // intersection must not inject a default outside it. + const pinnedFlat = { + ...mergeRecord({ zai: ["low", "high", "max"] }), + supportedThinkingEfforts: ["low"], + }; + assert.equal(detectDefaultThinkingEffort(pinnedFlat), undefined); + // Same for the nested #7694 shape winning the list. + const pinnedNested = { + ...mergeRecord({ zai: ["low", "high", "max"] }), + reasoning: { supported_efforts: ["low"] }, + }; + assert.equal(detectDefaultThinkingEffort(pinnedNested), undefined); +}); + +test("disjoint vendor vocabularies produce an authoritative empty list (no fall-through)", () => { + const disjoint = mergeRecord({ a: ["low"], b: ["high"] }); + assert.deepEqual(detectSupportedThinkingEfforts(disjoint), []); + // And no default can be derived from an empty intersection. + assert.equal(detectDefaultThinkingEffort(disjoint), undefined); +}); + +test("a malformed entry inside effort_values is dropped individually, not the whole route", () => { + const record = { + id: "vendor/partial", + vendors: { + a: { capabilities: { reasoning: { effort_values: ["low", 42, "high"] } } }, + b: { capabilities: { reasoning: { effort_values: ["low", "high", "max"] } } }, + }, + }; + assert.deepEqual(detectSupportedThinkingEfforts(record), ["low", "high"]); +}); + +test("explicit default_effort keeps precedence over the vendor-route fallback", () => { + const record = { + ...mergeRecord({ zai: ["low", "high", "max"] }), + reasoning: { default_effort: "high" }, + }; + assert.equal(detectDefaultThinkingEffort(record), "high"); +}); + +test("normalizeDiscoveredModels threads the intersected vendor-route vocabulary into synced models", () => { + const models = normalizeDiscoveredModels([ + { + id: "zai/glm-5.3-flash", + vendors: { + zai: { capabilities: { reasoning: { effort_values: ["low", "high", "max"] } } }, + baseten: { capabilities: { reasoning: { effort_values: ["low", "high", "xhigh"] } } }, + }, + }, + ]); + const synced = models.find((model) => model.id === "zai/glm-5.3-flash"); + assert.deepEqual(synced?.supportedThinkingEfforts, ["low", "high"]); + assert.equal(synced?.defaultThinkingEffort, "high"); +});