From 12d6d492e8e476a470438e334658c262239e8a7d Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza <8016841+diegosouzapw@users.noreply.github.com> Date: Fri, 17 Jul 2026 10:40:33 -0300 Subject: [PATCH] feat(sse): preserve tools/tool_choice for tool-bearing requests through fusion combos (#6771) (#7235) --- .../6771-fusion-preserve-tools-bypass.md | 1 + docs/routing/AUTO-COMBO.md | 11 +- open-sse/services/fusion.ts | 32 ++++ tests/unit/combo-fusion-strategy.test.ts | 1 - tests/unit/fusion-tools-bypass-6771.test.ts | 153 ++++++++++++++++++ 5 files changed, 195 insertions(+), 3 deletions(-) create mode 100644 changelog.d/features/6771-fusion-preserve-tools-bypass.md create mode 100644 tests/unit/fusion-tools-bypass-6771.test.ts diff --git a/changelog.d/features/6771-fusion-preserve-tools-bypass.md b/changelog.d/features/6771-fusion-preserve-tools-bypass.md new file mode 100644 index 0000000000..84215e4c4c --- /dev/null +++ b/changelog.d/features/6771-fusion-preserve-tools-bypass.md @@ -0,0 +1 @@ +- **feat(sse):** preserve `tools`/`tool_choice` for tool-bearing requests through fusion combos — bypass panel synthesis and route straight to the judge with tools intact (#6771 — thanks @chirag127). diff --git a/docs/routing/AUTO-COMBO.md b/docs/routing/AUTO-COMBO.md index a4967b4a08..6160420310 100644 --- a/docs/routing/AUTO-COMBO.md +++ b/docs/routing/AUTO-COMBO.md @@ -212,8 +212,15 @@ a single final answer from all panel responses. Ported from upstream `decolua/9r How it works: -1. **Fan-out** — the prompt is sent to every panel model at once, forced non-streaming - with tools stripped (the judge needs complete prose to synthesize). +0. **Tool-bearing bypass** — a request that carries a non-empty `tools` array with + `tool_choice` not explicitly `"none"` skips the panel entirely: it routes directly to + a single model (the configured judge, or `panel[0]`) with `tools`/`tool_choice` + passed through unmodified. Panel members have no tool access and the judge's + synthesis directive discourages tool-call emission, so agentic/tool-calling clients + get a real tool-call decision instead of synthesized prose (#6771). +1. **Fan-out** (non-tool-bearing requests only) — the prompt is sent to every panel + model at once, forced non-streaming with tools stripped (the judge needs complete + prose to synthesize). 2. **Quorum-grace collection** — as soon as `minPanel` answers arrive, a short grace timer starts for the stragglers, then fusion proceeds with whatever was collected. This caps the slowest model's penalty on wall time, bounded by a hard timeout. diff --git a/open-sse/services/fusion.ts b/open-sse/services/fusion.ts index d6e5f4fa1f..dd92c3bdbc 100644 --- a/open-sse/services/fusion.ts +++ b/open-sse/services/fusion.ts @@ -144,6 +144,18 @@ export function buildJudgePrompt(answers: Array<{ text: string }>): string { ].join("\n"); } +/** + * A request is "tool-bearing" when the client supplied tools AND did not + * explicitly opt out of tool use this turn (tool_choice: "none" is a valid + * way to declare available tools while opting out — that must NOT trigger + * the bypass, see issue #6771). + */ +export function isToolBearingRequest(body: Body): boolean { + const hasTools = Array.isArray(body.tools) && body.tools.length > 0; + if (!hasTools) return false; + return body.tool_choice !== "none"; +} + type Sentinel = { __timeout?: true; __error?: unknown }; // Resolve a Response (or sentinel) within ms; the loser keeps running but is ignored. @@ -230,6 +242,12 @@ export type HandleFusionChatOptions = { * complete prose to synthesize). The judge call keeps the client's original * stream flag + tools, so streaming and downstream tool use still work. * + * Tool-bearing requests (non-empty `tools` with `tool_choice` not "none") + * skip panel synthesis entirely and route straight to a single model (the + * configured judge, or panel[0]) with tools/tool_choice intact — panel + * members have no tool access and the judge's synthesis directive steers + * even a tools-capable judge away from emitting a tool call (#6771). + * * Speed: quorum-grace collection caps the straggler penalty. Quality: the judge * runs the consensus/contradiction/blind-spot analysis before writing. * @@ -284,6 +302,20 @@ export async function handleFusionChat({ `Combo "${comboName ?? ""}" | panel=${panel.length} [${panel.join(", ")}] | judge=${judge} | quorum=${minPanel}` ); + // Tool-bearing requests get no value from panel synthesis — panel members + // would answer with no tool access (degraded prose), and the judge's + // synthesis directive steers it away from emitting a tool call even though + // it technically still receives `tools`. Skip straight to a single model + // with the full, unmodified body (tools/tool_choice intact) so agentic + // clients get a real tool-call decision (#6771). + if (isToolBearingRequest(body)) { + log.info( + "FUSION", + `Combo "${comboName ?? ""}" received a tool-bearing request — bypassing panel synthesis, routing directly to ${judge} with tools intact` + ); + return handleSingleModel(body, judge); + } + // 1. Fan out to the panel in parallel: non-streaming, tools stripped (we want prose). const { tools: _tools, tool_choice: _tc, ...rest } = body; void _tools; diff --git a/tests/unit/combo-fusion-strategy.test.ts b/tests/unit/combo-fusion-strategy.test.ts index 7fb2d6148f..a6d1218641 100644 --- a/tests/unit/combo-fusion-strategy.test.ts +++ b/tests/unit/combo-fusion-strategy.test.ts @@ -79,7 +79,6 @@ test("fusion: fans out to the panel then routes a synthesis turn to the judge", body: { messages: [{ role: "user", content: "Q" }], stream: true, - tools: [{ name: "x" }], }, combo: fusionCombo(["p/a", "p/b", "p/c"], { judgeModel: "p/judge" }), handleSingleModel, diff --git a/tests/unit/fusion-tools-bypass-6771.test.ts b/tests/unit/fusion-tools-bypass-6771.test.ts new file mode 100644 index 0000000000..4b7402e898 --- /dev/null +++ b/tests/unit/fusion-tools-bypass-6771.test.ts @@ -0,0 +1,153 @@ +/** + * Regression guard for #6771 — fusion combos stripping tools/tool_choice for + * tool-bearing requests via panel-fan-out-then-judge synthesis. + * + * Root cause: panel members answer tool-shaped prompts with no tool access + * (degraded prose), and the judge's injected synthesis directive ("produce + * ONE authoritative final answer" from anonymized panel sources) steers even + * a tools-capable judge away from emitting a real tool call. + * + * Fix: detect a tool-bearing request up front (non-empty `tools`, and + * `tool_choice` not explicitly "none") and bypass the panel fan-out + + * judge-synthesis path entirely — route the full, unmodified body straight + * to a single model (the configured judgeModel, or panel[0]). + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-fusion-6771-")); +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "fusion-6771-test-secret"; + +const { handleComboChat } = await import("../../open-sse/services/combo.ts"); + +type Body = Record; + +function jsonResponse(model: string, content: string): Response { + const body = JSON.stringify({ model, choices: [{ message: { role: "assistant", content } }] }); + return new Response(body, { status: 200, headers: { "Content-Type": "application/json" } }); +} + +const noop = () => {}; +const log = { info: noop, warn: noop, debug: noop, error: noop }; + +function fusionCombo(models: string[], extra: Record = {}) { + return { + name: "fusion-tools", + strategy: "fusion", + models: models.map((m) => ({ model: m })), + config: extra, + }; +} + +const TOOLS = [ + { + type: "function", + function: { name: "get_weather", description: "Get the weather", parameters: {} }, + }, +]; + +test("6771: tool-bearing request bypasses panel fan-out — single call, tools intact, targets configured judge", async () => { + const calls: Array<{ model: string; body: Body }> = []; + const handleSingleModel = async (b: Body, m: string) => { + calls.push({ model: m, body: b }); + return jsonResponse(m, "tool decision"); + }; + + const requestBody: Body = { + messages: [{ role: "user", content: "what's the weather?" }], + tools: TOOLS, + tool_choice: "auto", + }; + + const res = await handleComboChat({ + body: requestBody, + combo: fusionCombo(["panel/a", "panel/b"], { judgeModel: "judge/model" }), + handleSingleModel, + log, + settings: {}, + allCombos: [], + }); + + // Exactly one call — not once per panel member + once for the judge. + assert.equal(calls.length, 1, `expected exactly 1 call, got: ${calls.map((c) => c.model).join(", ")}`); + assert.equal(calls[0].model, "judge/model"); + + // The forwarded body still contains the original tools/tool_choice unmodified. + assert.deepEqual(calls[0].body.tools, TOOLS); + assert.equal(calls[0].body.tool_choice, "auto"); + + assert.equal(res.status, 200); +}); + +test("6771: tool-bearing request with no explicit judgeModel targets panel[0]", async () => { + const calls: string[] = []; + const handleSingleModel = async (_b: Body, m: string) => { + calls.push(m); + return jsonResponse(m, "tool decision"); + }; + + await handleComboChat({ + body: { + messages: [{ role: "user", content: "what's the weather?" }], + tools: TOOLS, + }, + combo: fusionCombo(["panel/a", "panel/b"]), // no judgeModel — defaults to panel[0] + handleSingleModel, + log, + settings: {}, + allCombos: [], + }); + + assert.deepEqual(calls, ["panel/a"]); +}); + +test("6771: tools present but tool_choice:\"none\" still goes through normal fan-out+judge path", async () => { + const calls: string[] = []; + const handleSingleModel = async (_b: Body, m: string) => { + calls.push(m); + return jsonResponse(m, "prose answer"); + }; + + await handleComboChat({ + body: { + messages: [{ role: "user", content: "hi" }], + tools: TOOLS, + tool_choice: "none", + }, + combo: fusionCombo(["panel/a", "panel/b"], { judgeModel: "judge/model" }), + handleSingleModel, + log, + settings: {}, + allCombos: [], + }); + + // panel.length (2) fan-out calls + 1 judge call = 3. + assert.equal(calls.length, 3, `expected 3 calls (fan-out+judge), got: ${calls.join(", ")}`); + assert.deepEqual(calls.slice(0, 2).sort(), ["panel/a", "panel/b"]); + assert.equal(calls[2], "judge/model"); +}); + +test("6771: no regression — request without tools still goes through normal fan-out+judge path", async () => { + const calls: string[] = []; + const handleSingleModel = async (_b: Body, m: string) => { + calls.push(m); + return jsonResponse(m, "prose answer"); + }; + + await handleComboChat({ + body: { messages: [{ role: "user", content: "hi" }] }, + combo: fusionCombo(["panel/a", "panel/b"], { judgeModel: "judge/model" }), + handleSingleModel, + log, + settings: {}, + allCombos: [], + }); + + assert.equal(calls.length, 3, `expected 3 calls (fan-out+judge), got: ${calls.join(", ")}`); + assert.deepEqual(calls.slice(0, 2).sort(), ["panel/a", "panel/b"]); + assert.equal(calls[2], "judge/model"); +});