diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d1ac83409..98340e1d5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ _In development — bullets added per PR; finalized at release._ - **fix(sse): restore MCP / third-party tool names on the native Claude path (MCP dispatch broken in Claude Code)** — since 3.8.27, every MCP tool call routed through OmniRoute to a native Claude OAuth provider failed client-side with `Error: No such tool available: `: tool schemas arrived fine but the streamed `tool_use.name` reached Claude Code in its cloaked form (e.g. `McpN8nMcpSearchWorkflows` instead of the registered `mcp__n8n-mcp__search_workflows`). The native-Claude tool-name cloak stashes its per-request alias→original map as a **non-enumerable** `_toolNameMap` on the request body; the request-inspector capture added in 3.8.27 rebuilds the captured body from its serialized form (`JSON.parse(JSON.stringify(...))`), which drops non-enumerable properties, so `finalBody._toolNameMap` was empty and the response-side un-cloak silently fell back to the static built-in map — never restoring dynamic MCP / snake_case names. Built-in tools (Bash/Read/…) were unaffected (static map); cross-format paths were unaffected (they attach the map enumerably). The provider-request capture now re-attaches the per-request map (kept non-enumerable, so it still never re-serializes upstream) when the captured copy lost it, restoring MCP tool dispatch. ([#4091](https://github.com/diegosouzapw/OmniRoute/issues/4091) — thanks @pedrotecinf, @NakHalal) - **fix(dashboard): Logs auto-refresh self-heals in embedded/proxied hosts that pin or mis-fire visibility** — a follow-up to #4054: the Request Logger still froze auto-refresh on some hosts (reported on 3.8.28 Docker, works on 3.8.24). #4054 made the initial visibility fail-open, but the pause is event-driven — a host that fires a one-shot `visibilitychange` → hidden and then keeps reporting `"hidden"` (or recovers without firing the event again) left the cached visibility flag stuck `false`, so the interval ticked but never polled (only the manual Refresh button worked). The poll tick now also re-checks the **live** `document.visibilityState`, and a **window `focus`** listener re-arms polling (a focused window is a reliable signal the page is actively viewed). A genuinely backgrounded browser tab still pauses (it reports `"hidden"` and never receives focus), preserving the #3109 network-saturation optimization. ([#4133](https://github.com/diegosouzapw/OmniRoute/issues/4133) — thanks @tjengbudi) +- **fix(capabilities): unify vision model-id detection into one shared source** — three code paths kept independent, drifting vision-model lists, so the same model id could get up to three different verdicts. Two concrete bugs: lite compression's gate was missing pixtral / llava / qwen-vl / glm-4v / kimi-vl / mistral-medium-3, so it **stripped images for those real vision models and blinded them** (same class as #4071 / #4012); and the `/v1/models` list was too broad, flagging text models (`gemma`, bare `kimi` like `kimi-k2`) as vision. All three (`modelCapabilities` routing fallback, `/v1/models` listing, lite image-strip gate) now delegate to a single conservative source `src/shared/constants/visionModels.ts`, which also restores `glm-4v` / `gemini-3` coverage and keeps the #3328 MiniMax M3 carve-out. ([#4072](https://github.com/diegosouzapw/OmniRoute/issues/4072) — thanks @diego-anselmo) --- diff --git a/open-sse/services/compression/lite.ts b/open-sse/services/compression/lite.ts index 41906fc2cb..3ed09beebf 100644 --- a/open-sse/services/compression/lite.ts +++ b/open-sse/services/compression/lite.ts @@ -1,3 +1,4 @@ +import { isVisionModelId } from "@/shared/constants/visionModels"; import type { CompressionResult, CompressionMode } from "./types.ts"; import { createCompressionStats } from "./stats.ts"; @@ -52,18 +53,13 @@ function normalizeMessageWhitespace(content: string): string { return collapseNewlineRuns(content).split("\n").map(trimTrailingHorizontalWhitespace).join("\n"); } +// Vision detection is centralized in `@/shared/constants/visionModels` (#4072) so +// the lite image-strip gate, the /v1/models listing, and the routing fallback can +// never disagree. The shared list keeps the #3328 MiniMax M3 carve-out and the +// pixtral/llava/qwen-vl/glm-4v/kimi-vl/mistral-medium-3 families this gate used to +// miss (stripping their images and blinding real vision models). function modelSupportsVision(model: string): boolean { - const normalized = model.toLowerCase(); - return ( - normalized.includes("vision") || - normalized.includes("gpt-4") || - normalized.includes("4o") || - normalized.includes("claude-3") || - normalized.includes("gemini") || - // #3328: MiniMax M3 is multimodal — verified it describes images via the opencode - // upstream. Without this, compression strips the image and the model goes "blind". - normalized.includes("minimax-m3") - ); + return isVisionModelId(model); } export function collapseWhitespace( diff --git a/src/app/api/v1/models/catalog.ts b/src/app/api/v1/models/catalog.ts index 0c3fbf70a4..6e3ba22da3 100644 --- a/src/app/api/v1/models/catalog.ts +++ b/src/app/api/v1/models/catalog.ts @@ -1,5 +1,6 @@ import { PROVIDER_MODELS, PROVIDER_ID_TO_ALIAS } from "@/shared/constants/models"; import { AI_PROVIDERS, NOAUTH_PROVIDERS } from "@/shared/constants/providers"; +import { isVisionModelId } from "@/shared/constants/visionModels"; import { getProviderConnections, getCombos, @@ -124,37 +125,10 @@ function minKnownNumber(values: Array): number | undefined { return Math.min(...knownValues); } -const VISION_MODEL_KEYWORDS = [ - "gpt-4o", - "gpt-4.1", - "gpt-4-vision", - "gpt-4-turbo", - "claude-3", - "claude-3.5", - "claude-3-5", - "claude-4", - "claude-opus", - "claude-sonnet", - "claude-haiku", - "gemini", - "gemma", - "llava", - "bakllava", - "pixtral", - "mistral-pixtral", - "qwen-vl", - "qvq", - "glm-4.6v", - "glm-4.5v", - "vision", - "multimodal", - "kimi", -]; -function isVisionModelId(modelId: string): boolean { - const normalized = String(modelId || "").toLowerCase(); - if (!normalized) return false; - return VISION_MODEL_KEYWORDS.some((keyword) => normalized.includes(keyword)); -} +// Vision detection is centralized in `@/shared/constants/visionModels` (#4072) so +// this listing path, the routing fallback, and lite compression share one verdict. +// Re-exported for callers/tests that imported it from here. +export { isVisionModelId }; function getVisionCapabilityFields(modelId: string) { if (!isVisionModelId(modelId)) return null; diff --git a/src/lib/modelCapabilities.ts b/src/lib/modelCapabilities.ts index 831edba9c8..7309196bc6 100644 --- a/src/lib/modelCapabilities.ts +++ b/src/lib/modelCapabilities.ts @@ -5,6 +5,7 @@ import { import { parseModel, resolveCanonicalProviderModel } from "@omniroute/open-sse/services/model.ts"; import { MODEL_SPECS, getModelSpec, type ModelSpec } from "@/shared/constants/modelSpecs"; import { getSyncedCapability } from "@/lib/modelsDevSync"; +import { isVisionModelId } from "@/shared/constants/visionModels"; const TOOL_CALLING_UNSUPPORTED_PATTERNS: string[] = []; const REASONING_UNSUPPORTED_PATTERNS = [ @@ -211,52 +212,16 @@ function getSyncedCapabilityForResolved( } /** - * High-precision vision model-id fragments, used ONLY as a last-resort fallback - * in resolveVisionCapability when there is no synced/registry/spec capability - * data (e.g. Mistral Pixtral, which ships no models.dev `attachment` flag and no - * registry `supportsVision`). Intentionally conservative: a false positive would - * let an image request route to a text-only model — the exact bug this guards - * against — so only unambiguously multimodal families are listed. Missing an - * exotic vision model is safe: it resolves to `null` and combo routing keeps it - * via the "no confirmed-vision target" fallback. + * Last-resort vision fallback in resolveVisionCapability when there is no + * synced/registry/spec capability data (e.g. Mistral Pixtral, which ships no + * models.dev `attachment` flag and no registry `supportsVision`). Delegates to + * the single shared source (`@/shared/constants/visionModels`, #4072) so routing, + * the `/v1/models` listing and lite compression can never disagree on whether a + * model is vision-capable. The list is intentionally conservative — a false + * positive would let an image request route to a text-only model. */ -const VISION_MODEL_ID_FRAGMENTS = [ - "pixtral", - "llava", - "bakllava", - "qwen-vl", - "qwen2-vl", - "qwen2.5-vl", - "qwen3-vl", - "qvq", - "internvl", - "minicpm-v", - "moondream", - "mimo-vl", - "kimi-vl", - "glm-4v", - "glm-4.5v", - "glm-4.6v", - "gpt-4o", - "gpt-4.1", - "gpt-4-turbo", - "gpt-4-vision", - "gemini-1.5", - "gemini-2", - "gemini-exp", - "claude-3", - "claude-opus-4", - "claude-sonnet-4", - "claude-haiku-4", - "mistral-medium-3", - "-vision", - "multimodal", -]; - -function modelIdLikelyVision(modelId: string | null | undefined): boolean { - if (!modelId) return false; - const normalized = modelId.toLowerCase(); - return VISION_MODEL_ID_FRAGMENTS.some((fragment) => normalized.includes(fragment)); +export function modelIdLikelyVision(modelId: string | null | undefined): boolean { + return isVisionModelId(modelId); } function resolveVisionCapability( diff --git a/src/shared/constants/visionModels.ts b/src/shared/constants/visionModels.ts new file mode 100644 index 0000000000..c6dda3a8e8 --- /dev/null +++ b/src/shared/constants/visionModels.ts @@ -0,0 +1,68 @@ +/** + * Single source of truth for the model-id vision heuristic (#4072). + * + * Three code paths used to keep their own drifting lists, so the same model id + * could get up to three different vision verdicts: + * - `src/lib/modelCapabilities.ts` — last-resort fallback in `resolveVisionCapability` (#4071) + * - `src/app/api/v1/models/catalog.ts` — `/v1/models` listing capability + * - `open-sse/services/compression/lite.ts` — gate that decides whether lite + * compression strips images + * + * Concrete bugs that caused: + * - `lite.ts` was missing pixtral / llava / qwen-vl / glm-4v / kimi-vl / + * mistral-medium-3, so lite compression stripped images for those real vision + * models and blinded them (same class as #4071 / #4012). + * - `catalog.ts` was too broad: bare `gemma` (text) and bare `kimi` (e.g. + * `kimi-k2`, text) produced false-positive `vision: true` in `/v1/models`. + * + * Keep this list CONSERVATIVE: a false positive in routing or compression + * re-creates #4071 (an image routed to / kept for a model that cannot see it). + * The zero-touch path for newly released vision models is the models.dev sync + * (`modalities` / `attachment`), not this fallback — this list only needs the + * stable, well-known vision families. + */ +export const VISION_MODEL_ID_FRAGMENTS = [ + "pixtral", + "llava", + "bakllava", + "qwen-vl", + "qwen2-vl", + "qwen2.5-vl", + "qwen3-vl", + "qvq", + "internvl", + "minicpm-v", + "moondream", + "mimo-vl", + "kimi-vl", + "glm-4v", + "glm-4.5v", + "glm-4.6v", + "gpt-4o", + "gpt-4.1", + "gpt-4-turbo", + "gpt-4-vision", + "gemini-1.5", + "gemini-2", + "gemini-3", + "gemini-exp", + "claude-3", + "claude-opus-4", + "claude-sonnet-4", + "claude-haiku-4", + "mistral-medium-3", + "minimax-m3", + "-vision", + "multimodal", +] as const; + +/** + * Whether a model id looks like a vision-capable model. Case-insensitive + * substring match against {@link VISION_MODEL_ID_FRAGMENTS}. Returns `false` for + * empty / nullish input. + */ +export function isVisionModelId(modelId: string | null | undefined): boolean { + if (!modelId) return false; + const normalized = String(modelId).toLowerCase(); + return VISION_MODEL_ID_FRAGMENTS.some((fragment) => normalized.includes(fragment)); +} diff --git a/tests/unit/vision-detection-consistency.test.ts b/tests/unit/vision-detection-consistency.test.ts new file mode 100644 index 0000000000..066274be21 --- /dev/null +++ b/tests/unit/vision-detection-consistency.test.ts @@ -0,0 +1,80 @@ +/** + * #4072 — one shared vision-detection source. + * + * Three code paths kept independent vision-model lists and drifted apart, giving + * the same model id up to three different verdicts: + * - `src/lib/modelCapabilities.ts` (`modelIdLikelyVision`) — routing fallback (#4071) + * - `src/app/api/v1/models/catalog.ts` (`isVisionModelId`) — /v1/models listing + * - `open-sse/services/compression/lite.ts` (`replaceImageUrls`) — lite image strip + * + * The two concrete bugs: + * - lite stripped images for real vision models it didn't know (pixtral, llava, + * qwen-vl, glm-4v, kimi-vl, mistral-medium-3) → blinded them; + * - catalog flagged text models as vision (`gemma`, bare `kimi` like `kimi-k2`). + * + * After unification all three delegate to `@/shared/constants/visionModels`. + */ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; + +import { isVisionModelId } from "../../src/shared/constants/visionModels.ts"; +import { isVisionModelId as catalogIsVisionModelId } from "../../src/app/api/v1/models/catalog.ts"; +import { modelIdLikelyVision } from "../../src/lib/modelCapabilities.ts"; +import { replaceImageUrls } from "../../open-sse/services/compression/lite.ts"; + +const VISION = [ + "mistral/pixtral-12b-latest", + "llava-1.5-7b", + "qwen-vl-max", + "gpt-4o", + "glm-4v", + "kimi-vl-a3b", + "mistral-medium-3", +]; +const NOT_VISION = ["ministral-14b-latest", "mistral-large-latest", "gemma-2-9b", "kimi-k2"]; + +function imageBody() { + return { + messages: [ + { + role: "user", + content: [ + { type: "image_url", image_url: { url: "data:image/png;base64,iVBOR" } }, + ], + }, + ], + }; +} + +// `replaceImageUrls` strips the image (applied=true) only when the model is NOT a +// vision model. So `applied === !isVision`. +function liteStripsImage(modelId: string): boolean { + return replaceImageUrls(imageBody(), modelId).applied; +} + +describe("#4072 vision detection is consistent across all three sources", () => { + for (const id of VISION) { + it(`treats ${id} as vision everywhere`, () => { + assert.equal(isVisionModelId(id), true, `shared isVisionModelId(${id})`); + assert.equal(catalogIsVisionModelId(id), true, `catalog isVisionModelId(${id})`); + assert.equal(modelIdLikelyVision(id), true, `modelCapabilities modelIdLikelyVision(${id})`); + assert.equal(liteStripsImage(id), false, `lite must KEEP the image for ${id}`); + }); + } + + for (const id of NOT_VISION) { + it(`treats ${id} as non-vision everywhere`, () => { + assert.equal(isVisionModelId(id), false, `shared isVisionModelId(${id})`); + assert.equal(catalogIsVisionModelId(id), false, `catalog isVisionModelId(${id})`); + assert.equal(modelIdLikelyVision(id), false, `modelCapabilities modelIdLikelyVision(${id})`); + assert.equal(liteStripsImage(id), true, `lite must STRIP the image for ${id}`); + }); + } + + it("preserves the MiniMax M3 #3328 carve-out and Gemini 3 multimodal", () => { + for (const id of ["minimax-m3", "minimax-m3-free", "oc/minimax-m3-free", "gemini-3-pro"]) { + assert.equal(isVisionModelId(id), true, `${id} should be vision`); + assert.equal(liteStripsImage(id), false, `lite must keep the image for ${id}`); + } + }); +});