diff --git a/changelog.d/fixes/6764-fusion-combo-ref.md b/changelog.d/fixes/6764-fusion-combo-ref.md new file mode 100644 index 0000000000..403336afae --- /dev/null +++ b/changelog.d/fixes/6764-fusion-combo-ref.md @@ -0,0 +1 @@ +- **fix(routing):** fusion combos no longer silently drop `combo-ref` panel members — a referenced combo is now dispatched as one black-box panel voice instead of being dropped (#6764) diff --git a/docs/routing/AUTO-COMBO.md b/docs/routing/AUTO-COMBO.md index 6160420310..0130f5dd67 100644 --- a/docs/routing/AUTO-COMBO.md +++ b/docs/routing/AUTO-COMBO.md @@ -232,6 +232,11 @@ How it works: 4. **Graceful degradation** — 0 panel answers → `503`; exactly 1 survivor → that answer is returned directly (nothing to fuse); a single-model panel answers directly. +A panel member may also be a `combo-ref` step (`{kind: "combo-ref", comboName: "..."}`) referencing +another combo — it resolves as **one black-box panel voice** (a full recursive dispatch into the +referenced combo, not a fan-out of that combo's own targets), with the same depth/cycle protection +every other combo-ref-consuming strategy already uses (#6764). + ### Configuration Configured on the combo's `config` blob (no schema migration — it reuses the existing diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index 21735e31a4..a1f43236a6 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -146,6 +146,7 @@ import { } from "./combo/comboPredicates.ts"; import { applyComboTargetExhaustion } from "./combo/targetExhaustion.ts"; import { executeRuntimeUnitCombo } from "./combo/runtimeUnits.ts"; +import { extractFusionPanelSpec, buildFusionHandleSingleModel } from "./combo/fusionPanel.ts"; import { isRecord } from "./combo/comboData.ts"; import { expandProviderWildcardsInCombo, @@ -818,20 +819,47 @@ export async function handleComboChat({ ); } if (strategy === "fusion") { - const fusionModels = (combo.models || []) - .map((m) => { - if (typeof m === "string") return m; - if (m && typeof m === "object") { - const obj = m as Record; - if (typeof obj.model === "string") return obj.model; - } - return null; - }) - .filter((m): m is string => Boolean(m)); + const { panel: fusionModels, comboRefUnits } = extractFusionPanelSpec( + combo.models || [], + combo.name, + allCombos + ); + // Untyped like the existing `nestingContext` further down — `nesting` is + // already `ComboNestingContext | null` per HandleComboChatOptions, no new + // import needed. + const fusionNesting = nesting || { + depth: 0, + maxDepth: clampComboDepth(config.maxComboDepth), + visitedComboNames: [combo.name], + rootComboName: combo.name, + attemptBudget: { count: 0, limit: MAX_GLOBAL_ATTEMPTS }, + }; + const fusionHandleSingleModel = + comboRefUnits.size > 0 + ? buildFusionHandleSingleModel({ + handleSingleModel: handleSingleModelWithTimeout, + comboRefUnits, + allCombos, + nesting: fusionNesting, + baseOptions: { + body, + combo, + handleSingleModel, + isModelAvailable, + log, + settings, + allCombos, + relayOptions, + signal, + apiKeyAllowedConnections, + }, + runCombo: handleComboChat, + }) + : handleSingleModelWithTimeout; return handleFusionChat({ body, models: fusionModels, - handleSingleModel: handleSingleModelWithTimeout, + handleSingleModel: fusionHandleSingleModel, log, comboName: combo.name, judgeModel, diff --git a/open-sse/services/combo/fusionPanel.ts b/open-sse/services/combo/fusionPanel.ts new file mode 100644 index 0000000000..6397c5120c --- /dev/null +++ b/open-sse/services/combo/fusionPanel.ts @@ -0,0 +1,79 @@ +/** + * Fusion panel member extraction — resolves combo.models entries for the + * fusion strategy, including nested `combo-ref` steps (#6764). + * + * A combo-ref panel member is dispatched as ONE black-box panel voice (a full + * recursive handleComboChat call for the referenced combo, reusing the same + * executeComboRefUnit + cycle/depth guards every other combo-ref-consuming + * strategy already uses) — NOT a fan-out of the referenced combo's own + * targets. This keeps panel sizing and cost predictable and matches how a + * literal `auto/*` string panel member already behaves via the single- + * dispatch safety net in src/sse/handlers/chat.ts. + */ +import { normalizeComboStep } from "../../../src/lib/combos/steps.ts"; +import { executeComboRefUnit } from "./runtimeUnits.ts"; +import type { + ComboCollectionLike, + ComboNestingContext, + HandleComboChatOptions, + HandleSingleModel, + ResolvedComboRefTarget, +} from "./types.ts"; + +export type FusionPanelSpec = { + /** Dispatch keys handed to fusion.ts's `models` — comboName for combo-ref members, plain model string otherwise. */ + panel: string[]; + /** comboName -> resolved combo-ref unit, consumed by buildFusionHandleSingleModel. */ + comboRefUnits: Map; +}; + +export function extractFusionPanelSpec( + models: unknown[], + comboName: string, + allCombos: ComboCollectionLike +): FusionPanelSpec { + const panel: string[] = []; + const comboRefUnits = new Map(); + models.forEach((entry, index) => { + const step = normalizeComboStep(entry, { comboName, index, allCombos }); + if (!step) return; + if (step.kind === "combo-ref") { + if (!comboRefUnits.has(step.comboName)) { + comboRefUnits.set(step.comboName, { + kind: "combo-ref", + stepId: step.id, + executionKey: step.id, + comboName: step.comboName, + weight: step.weight, + label: step.label ?? null, + }); + } + panel.push(step.comboName); + return; + } + panel.push(step.model); + }); + return { panel, comboRefUnits }; +} + +export function buildFusionHandleSingleModel(args: { + handleSingleModel: HandleSingleModel; + comboRefUnits: Map; + allCombos: ComboCollectionLike; + nesting: ComboNestingContext; + baseOptions: HandleComboChatOptions; + runCombo: (options: HandleComboChatOptions) => Promise; +}): HandleSingleModel { + return (body, modelStr, target) => { + const unit = args.comboRefUnits.get(modelStr); + if (!unit) return args.handleSingleModel(body, modelStr, target); + return executeComboRefUnit({ + body, + unit, + allCombos: args.allCombos, + runCombo: args.runCombo, + baseOptions: args.baseOptions, + nesting: args.nesting, + }); + }; +} diff --git a/open-sse/services/combo/runtimeUnits.ts b/open-sse/services/combo/runtimeUnits.ts index d106ade336..e839fc01bb 100644 --- a/open-sse/services/combo/runtimeUnits.ts +++ b/open-sse/services/combo/runtimeUnits.ts @@ -98,7 +98,7 @@ function buildChildNestingContext(args: { }; } -async function executeComboRefUnit(args: { +export async function executeComboRefUnit(args: { body: Record; unit: ResolvedComboRefTarget; allCombos: ComboCollectionLike; diff --git a/tests/unit/combo-fusion-strategy-comboref-6764.test.ts b/tests/unit/combo-fusion-strategy-comboref-6764.test.ts new file mode 100644 index 0000000000..929ff8569a --- /dev/null +++ b/tests/unit/combo-fusion-strategy-comboref-6764.test.ts @@ -0,0 +1,190 @@ +/** + * #6764 — fusion strategy silently dropped `combo-ref` panel members. + * + * Every other combo strategy resolves a `{kind:"combo-ref", comboName}` panel + * member through the shared execute-mode machinery (see + * `open-sse/services/combo/runtimeUnits.ts::executeComboRefUnit`); the fusion + * branch in `open-sse/services/combo.ts` only recognized plain `string` or + * `{model: string}` entries, so a combo-ref member had neither field and was + * filtered out (`.filter(Boolean)`) — no error, no warning. This suite proves + * the fix: a combo-ref fusion panel member is dispatched as ONE black-box + * panel voice (a recursive `handleComboChat` call into the referenced combo), + * not dropped, not fanned out into the referenced combo's own targets. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-combo-fusion-ref-")); +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "combo-fusion-ref-test-secret"; + +const { handleComboChat } = await import("../../open-sse/services/combo.ts"); + +const noop = () => {}; +const log = { info: noop, warn: noop, debug: noop, error: noop }; + +type Body = Record; + +function okResponse(content: string): Response { + const body = JSON.stringify({ choices: [{ message: { role: "assistant", content } }] }); + return new Response(body, { status: 200, headers: { "Content-Type": "application/json" } }); +} + +function fusionCombo(models: unknown[], extra: Record = {}) { + return { + name: "test-fusion-combo-ref", + strategy: "fusion", + models, + config: extra, + }; +} + +test("fusion: a combo-ref panel member is dispatched, not silently dropped", async () => { + const seen: string[] = []; + const handleSingleModel = async (_b: Body, m: string) => { + seen.push(m); + if (m === "p/judge") return okResponse("FINAL"); + return okResponse(`ans-${m}`); + }; + const nestedPriority = { + name: "nested-priority", + strategy: "priority", + models: ["openai/nested-a"], + config: { maxRetries: 0, retryDelayMs: 0 }, + }; + const combo = fusionCombo( + [{ kind: "combo-ref", comboName: "nested-priority" }, { model: "p/plain" }], + { judgeModel: "p/judge" } + ); + + const res = await handleComboChat({ + body: { messages: [{ role: "user", content: "Q" }] }, + combo, + handleSingleModel, + log, + settings: {}, + allCombos: [combo, nestedPriority], + }); + + assert.equal(res.status, 200); + // Proves the combo-ref member was actually dispatched (its nested target + // model reached handleSingleModel) instead of being silently filtered out. + assert.ok( + seen.includes("openai/nested-a"), + `expected nested combo's target model to be dispatched, saw: ${seen.join(", ")}` + ); + assert.ok(seen.includes("p/plain"), "plain panel member must still dispatch alongside combo-ref"); +}); + +test("fusion: combo-ref-only panel resolves normally (not a 400 empty-panel error)", async () => { + const seen: string[] = []; + const handleSingleModel = async (_b: Body, m: string) => { + seen.push(m); + return okResponse(`ans-${m}`); + }; + const nestedPriority = { + name: "solo-nested", + strategy: "priority", + models: ["openai/solo-target"], + config: { maxRetries: 0, retryDelayMs: 0 }, + }; + const combo = fusionCombo([{ kind: "combo-ref", comboName: "solo-nested" }]); + + const res = await handleComboChat({ + body: { messages: [{ role: "user", content: "Q" }] }, + combo, + handleSingleModel, + log, + settings: {}, + allCombos: [combo, nestedPriority], + }); + + assert.notEqual(res.status, 400); + assert.ok(seen.includes("openai/solo-target")); +}); + +test("fusion: self-referencing combo-ref fails that panel member gracefully, not an infinite loop", async () => { + const seen: string[] = []; + const handleSingleModel = async (_b: Body, m: string) => { + seen.push(m); + return okResponse(`ans-${m}`); + }; + const combo = fusionCombo([ + { kind: "combo-ref", comboName: "test-fusion-combo-ref" }, + { model: "p/other" }, + ]); + + const res = await handleComboChat({ + body: { messages: [{ role: "user", content: "Q" }] }, + combo, + handleSingleModel, + log, + settings: {}, + allCombos: [combo], + }); + + // Overall request still degrades gracefully because "p/other" survives. + assert.equal(res.status, 200); + assert.ok(seen.includes("p/other")); + assert.ok(!seen.includes("test-fusion-combo-ref")); +}); + +test("fusion: combo-ref pointing at a nonexistent combo fails only that panel member", async () => { + const seen: string[] = []; + const handleSingleModel = async (_b: Body, m: string) => { + seen.push(m); + return okResponse(`ans-${m}`); + }; + const combo = fusionCombo([ + { kind: "combo-ref", comboName: "does-not-exist" }, + { model: "p/other" }, + ]); + + const res = await handleComboChat({ + body: { messages: [{ role: "user", content: "Q" }] }, + combo, + handleSingleModel, + log, + settings: {}, + allCombos: [combo], + }); + + assert.equal(res.status, 200); + assert.ok(seen.includes("p/other")); +}); + +test("fusion: mixed plain string / auto-style / combo-ref panel members all dispatch together", async () => { + const seen: string[] = []; + const handleSingleModel = async (_b: Body, m: string) => { + seen.push(m); + return okResponse(`ans-${m}`); + }; + const nestedPriority = { + name: "mixed-nested", + strategy: "priority", + models: ["openai/mixed-target"], + config: { maxRetries: 0, retryDelayMs: 0 }, + }; + const combo = fusionCombo([ + "auto/best-coding", + { model: "p/direct" }, + { kind: "combo-ref", comboName: "mixed-nested" }, + ]); + + const res = await handleComboChat({ + body: { messages: [{ role: "user", content: "Q" }] }, + combo, + handleSingleModel, + log, + settings: {}, + allCombos: [combo, nestedPriority], + }); + + assert.equal(res.status, 200); + assert.ok(seen.includes("auto/best-coding")); + assert.ok(seen.includes("p/direct")); + assert.ok(seen.includes("openai/mixed-target")); +});