From 0e6f844e1eddb9b1e6648d06162f067fcec17ba6 Mon Sep 17 00:00:00 2001 From: Markus Hartung Date: Wed, 19 Aug 2026 08:48:54 -0300 Subject: [PATCH] fix(fusion): apply vision-compatibility filter to fusion panel + judge (port from 9router#3378) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every non-fusion combo strategy runs candidates through filterTargetsByRequestCompatibility before dispatch, which excludes a target whose vision support cannot be confirmed `=== true` for an image-bearing request (#8332). tryFusionDispatch resolved its panel via the raw resolveComboTargets() and skipped that filter entirely, so a panel member (or explicit judge) with an unrecognized model id — capability lookup miss, supportsVision resolves to something other than true — still received the unmodified image body while a "confirmed vision" voice silently dropped out of the panel. Apply the same exclusion to the fusion panel and to an explicit judgeModel: when the request requires vision, drop targets without confirmed support (falling back to a capable panel member for the judge, same as an operator-hidden judge already does), and fail closed with capability_mismatch if the whole panel is excluded. Co-authored-by: anojndr --- CHANGELOG.md | 1 + open-sse/services/combo/dispatchPrelude.ts | 68 +++++++++- tests/unit/fusion-vision-panel-3378.test.ts | 142 ++++++++++++++++++++ 3 files changed, 206 insertions(+), 5 deletions(-) create mode 100644 tests/unit/fusion-vision-panel-3378.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 49668a6214..e3509df47d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -167,6 +167,7 @@ _Living section — regenerated 2026-08-12 from all cycle commits (cycle open `e ### 🐛 Bug Fixes +- **fusion**: fusion combos now apply the same vision-compatibility filter as every other combo strategy — a panel member or judge whose vision support cannot be confirmed is excluded from an image-bearing request instead of silently receiving it (port from upstream decolua/9router#3378) - **providers**: honor `PATCH /api/providers/[id]` so `omniroute providers rotate` stops 405ing (the OpenAPI spec and CLI already use PATCH) (PR #10366) - **executors**: fix internal timeout misclassified as client disconnect (499) for 7 niche executors — pass TimeoutError reason to controller.abort() (#8197 side-finding) - test(combo): guard auto/best-free never leaks the combo name as a model (#7754) diff --git a/open-sse/services/combo/dispatchPrelude.ts b/open-sse/services/combo/dispatchPrelude.ts index 2f5361a4cf..056016ae77 100644 --- a/open-sse/services/combo/dispatchPrelude.ts +++ b/open-sse/services/combo/dispatchPrelude.ts @@ -16,11 +16,18 @@ import { getCachedProviderConnections } from "../../../src/lib/db/readCache"; import { getCircuitBreaker } from "../../../src/shared/utils/circuitBreaker"; import { fisherYatesShuffle, getNextFromDeck } from "../../../src/shared/utils/shuffleDeck"; import { handleFusionChat, type FusionTuning } from "../fusion.ts"; +import { getResolvedModelCapabilities } from "../modelCapabilities.ts"; +import { errorResponseWithComboDiagnostics } from "../../utils/error.ts"; import { parseModel } from "../model.ts"; import { handlePipelineChat, type PipelineStep } from "../pipeline.ts"; import type { resolveComboSetupConfig } from "../comboConfig.ts"; import { clampComboDepth, MAX_GLOBAL_ATTEMPTS, resolveDelayMs } from "./comboPredicates.ts"; -import { resolveComboRuntimeUnits, resolveComboTargets } from "./comboStructure.ts"; +import { + deriveRequestCompatibilityRequirements, + isVisionIncompatibleTarget, + resolveComboRuntimeUnits, + resolveComboTargets, +} from "./comboStructure.ts"; import { isComboModelVisible } from "./comboVisibility.ts"; import { buildFusionHandleSingleModel, extractFusionPanelSpec } from "./fusionPanel.ts"; import { @@ -393,14 +400,27 @@ export async function tryFusionDispatch(args: { }): Promise { const { cfg, combo, config, strategy, log } = args; const configuredJudge = typeof cfg.judgeModel === "string" ? cfg.judgeModel : undefined; + const judgeFusionRequirements = deriveRequestCompatibilityRequirements(args.body); + // #3378: the judge stays in the original conversation (full history, including + // any image_url blocks) — a judge whose vision support cannot be confirmed is + // exactly as unsafe as an unconfirmed panel member (#8332). Drop it the same + // way an operator-hidden judge is dropped below, so fusion falls back to a + // (vision-confirmed) panel member instead of silently losing the image for + // the synthesis step. + const judgeLacksConfirmedVision = + judgeFusionRequirements.requiresVision && + !!configuredJudge && + getResolvedModelCapabilities(configuredJudge).supportsVision !== true; // The panel is filtered for hidden models by resolveComboTargets, but the // explicit judge is a bare string that never passes through it (#8878). Drop a // hidden judge so fusion falls back to a surviving panel member instead of // dispatching a model the operator hid. const judgeModel = - configuredJudge && !isComboModelVisible(configuredJudge, null, args.hiddenModelsByProvider) - ? undefined - : configuredJudge; + configuredJudge && + !judgeLacksConfirmedVision && + isComboModelVisible(configuredJudge, null, args.hiddenModelsByProvider) + ? configuredJudge + : undefined; const fusionTuning = cfg.fusionTuning && typeof cfg.fusionTuning === "object" ? (cfg.fusionTuning as FusionTuning) @@ -413,12 +433,50 @@ export async function tryFusionDispatch(args: { } if (strategy !== "fusion") return null; - const resolvedFusionTargets = resolveComboTargets( + const allResolvedFusionTargets = resolveComboTargets( combo, args.allCombos, clampComboDepth(config.maxComboDepth), args.hiddenModelsByProvider ); + // #3378 (ported from upstream decolua/9router): every non-fusion combo + // strategy runs candidates through filterTargetsByRequestCompatibility before + // dispatch, which excludes a target whose vision support cannot be *confirmed* + // `=== true` for an image-bearing request (#8332 — unknown is treated the same + // as unsupported, never silently forwarded). Fusion resolved its panel via the + // raw target list and skipped that filter entirely, so a panel member with an + // unrecognized model id (capability lookup misses -> supportsVision !== true) + // still received the unmodified image body while the panel silently lost a + // "confirmed vision" voice. Apply the same exclusion here so the fusion panel + // only fans an image request out to targets with confirmed vision support. + const fusionRequirements = judgeFusionRequirements; + const resolvedFusionTargets = fusionRequirements.requiresVision + ? allResolvedFusionTargets.filter( + (target) => !isVisionIncompatibleTarget(target, fusionRequirements) + ) + : allResolvedFusionTargets; + if (fusionRequirements.requiresVision && resolvedFusionTargets.length === 0) { + log.warn( + "COMBO", + `Combo "${combo.name}" fusion panel has no target with confirmed vision support for this image request — every candidate was excluded (#3378)` + ); + return errorResponseWithComboDiagnostics( + 400, + `No target in combo ${combo.name} has confirmed vision support for this image request`, + { + poolSize: allResolvedFusionTargets.length, + attempted: 0, + excluded: allResolvedFusionTargets.map((target) => ({ + provider: target.provider, + model: target.modelStr, + reason: "vision", + })), + attemptOrder: [], + terminalReason: "capability_mismatch", + }, + { code: "capability_mismatch", type: "invalid_request_error" } + ); + } // extractFusionPanelSpec only understands model strings / combo refs, so the // resolved targets have to be flattened before it runs. Keep them indexed so // the panel can be rehydrated below — dispatching the bare strings strips diff --git a/tests/unit/fusion-vision-panel-3378.test.ts b/tests/unit/fusion-vision-panel-3378.test.ts new file mode 100644 index 0000000000..572891d252 --- /dev/null +++ b/tests/unit/fusion-vision-panel-3378.test.ts @@ -0,0 +1,142 @@ +// Regression guard for upstream decolua/9router#3378: "Fusion combo sometimes +// can't see images even when all models support vision". +// +// Every non-fusion combo strategy runs the request through +// filterTargetsByRequestCompatibility (comboStructure.ts) before dispatch, which +// treats a target whose vision support is not *confirmed* `=== true` (unknown OR +// false) as vision-incompatible and excludes it (#8332). The fusion dispatch +// branch (dispatchPrelude.ts::tryFusionDispatch) resolves its panel via the raw +// resolveComboTargets() and skips that compat filter entirely — so a panel +// member whose model id is unrecognized by the capability registry (and thus +// resolves to supportsVision !== true) still receives the unmodified +// image-bearing body, without any signal that its capability could not be +// confirmed. +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-fusion-vision-3378-")); +const ORIGINAL_DATA_DIR = process.env.DATA_DIR; +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "fusion-vision-3378-test-secret"; + +const { handleComboChat } = await import("../../open-sse/services/combo.ts"); +const { saveModelsDevCapabilities, clearModelsDevCapabilities } = await import( + "../../src/lib/modelsDevSync.ts" +); +const { resetAllComboMetrics } = await import("../../open-sse/services/comboMetrics.ts"); +const { resetAllCircuitBreakers } = await import("../../src/shared/utils/circuitBreaker.ts"); +const { resetAll: resetAllSemaphores } = await import( + "../../open-sse/services/rateLimitSemaphore.ts" +); +const core = await import("../../src/lib/db/core.ts"); + +function createLog() { + return { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} }; +} + +function okResponse(content: string) { + return new Response(JSON.stringify({ choices: [{ message: { role: "assistant", content } }] }), { + status: 200, + headers: { "content-type": "application/json" }, + }); +} + +function capabilityEntry(overrides: Record = {}) { + return { + tool_call: true, + reasoning: false, + attachment: false, + structured_output: true, + temperature: true, + modalities_input: JSON.stringify(["text"]), + modalities_output: JSON.stringify(["text"]), + knowledge_cutoff: null, + release_date: null, + last_updated: null, + status: null, + family: null, + open_weights: false, + limit_context: 128000, + limit_input: 128000, + limit_output: 4096, + interleaved_field: null, + ...overrides, + }; +} + +const imageRequestBody = { + messages: [ + { + role: "user", + content: [ + { type: "text", text: "What is in this image?" }, + { type: "image_url", image_url: { url: "https://example.com/cat.png" } }, + ], + }, + ], +}; + +test.beforeEach(() => { + resetAllComboMetrics(); + resetAllCircuitBreakers(); + resetAllSemaphores(); + clearModelsDevCapabilities(); +}); + +test.after(() => { + resetAllComboMetrics(); + resetAllCircuitBreakers(); + resetAllSemaphores(); + clearModelsDevCapabilities(); + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + if (ORIGINAL_DATA_DIR === undefined) { + delete process.env.DATA_DIR; + } else { + process.env.DATA_DIR = ORIGINAL_DATA_DIR; + } +}); + +test( + "fusion panel must not dispatch an image_url request to a member whose vision " + + "support cannot be confirmed (#3378)", + async () => { + // fusion-vision-a is confirmed vision-capable. fusion-unknown has no + // capability entry at all (unrecognized id) -> getResolvedModelCapabilities + // resolves supportsVision to something other than `true`, exactly like the + // "unknown id silently treated as no vision" failure mode from the upstream + // report. + saveModelsDevCapabilities({ + openai: { + "fusion-vision-a": capabilityEntry({ attachment: true }), + }, + }); + + const dispatched: string[] = []; + const result = await handleComboChat({ + body: imageRequestBody, + combo: { + name: "fusion-vision-panel-3378", + strategy: "fusion", + models: ["openai/fusion-vision-a", "openai/fusion-unknown"], + config: { judgeModel: "openai/fusion-vision-a" }, + }, + handleSingleModel: async (_body, modelStr) => { + dispatched.push(modelStr); + return okResponse(`answer from ${modelStr}`); + }, + log: createLog(), + settings: {}, + allCombos: [], + }); + + assert.ok(result.status < 500, "combo call should not hard-fail"); + assert.ok( + !dispatched.includes("openai/fusion-unknown"), + "a panel member with unconfirmed vision support must never receive the raw image_url body" + ); + } +);