mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-04 22:32:12 +03:00
feat(combo): Fusion strategy — parallel panel + judge synthesis (16th strategy) (#4652)
Integrated into release/v3.8.36 — Fusion combo strategy (16th, port 87e5c1c6); combo.ts baseline reconciled; release-green
This commit is contained in:
committed by
GitHub
parent
2d09f72b29
commit
122f6d3a19
@@ -140,7 +140,7 @@
|
||||
"open-sse/services/browserBackedChat.ts": 850,
|
||||
"open-sse/services/claudeCodeCompatible.ts": 1202,
|
||||
"_rebaseline_pr4592_exclude_exhausted_auto": "Reconcile #4592 already-merged growth: combo.ts 2991->3036 (+45, terminal-status quota-cutoff exclusion in buildAutoCandidates + opt-in gate). Fast-gate PR->release does not run check:file-size.",
|
||||
"open-sse/services/combo.ts": 3036,
|
||||
"open-sse/services/combo.ts": 3069,
|
||||
"open-sse/services/compression/strategySelector.ts": 848,
|
||||
"open-sse/services/rateLimitManager.ts": 1035,
|
||||
"open-sse/services/tokenRefresh.ts": 2070,
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
* Shared combo (model combo) handling with fallback support
|
||||
* Supports: priority, weighted, round-robin, random, least-used, cost-optimized,
|
||||
* reset-aware, reset-window, strict-random, auto, fill-first, p2c, lkgp,
|
||||
* context-optimized, and context-relay strategies
|
||||
* context-optimized, context-relay, and fusion strategies
|
||||
*/
|
||||
|
||||
import {
|
||||
@@ -118,6 +118,7 @@ import {
|
||||
recordStickyWeightedSuccess,
|
||||
} from "./combo/rrState.ts";
|
||||
import { validateResponseQuality, toRetryAfterDisplayValue } from "./combo/validateQuality.ts";
|
||||
import { handleFusionChat, type FusionTuning } from "./fusion.ts";
|
||||
import {
|
||||
TRANSIENT_FOR_SEMAPHORE,
|
||||
MAX_FALLBACK_WAIT_MS,
|
||||
@@ -737,6 +738,38 @@ export async function handleComboChat({
|
||||
return handleSingleModelWithTimeout(body, pinnedModel);
|
||||
}
|
||||
|
||||
// Fusion strategy: parallel panel + judge synthesis. Handled in a separate module
|
||||
// because it neither iterates targets in order nor needs the failover/retry/credential
|
||||
// gate machinery that follows — it fans out, then synthesizes once.
|
||||
if (strategy === "fusion") {
|
||||
const fusionModels = (combo.models || [])
|
||||
.map((m) => {
|
||||
if (typeof m === "string") return m;
|
||||
if (m && typeof m === "object") {
|
||||
const obj = m as Record<string, unknown>;
|
||||
if (typeof obj.model === "string") return obj.model;
|
||||
}
|
||||
return null;
|
||||
})
|
||||
.filter((m): m is string => Boolean(m));
|
||||
const cfg = config as Record<string, unknown>;
|
||||
const judgeModel =
|
||||
typeof cfg.judgeModel === "string" ? cfg.judgeModel : undefined;
|
||||
const tuning =
|
||||
cfg.fusionTuning && typeof cfg.fusionTuning === "object"
|
||||
? (cfg.fusionTuning as FusionTuning)
|
||||
: undefined;
|
||||
return handleFusionChat({
|
||||
body,
|
||||
models: fusionModels,
|
||||
handleSingleModel: handleSingleModelWithTimeout,
|
||||
log,
|
||||
comboName: combo.name,
|
||||
judgeModel,
|
||||
tuning,
|
||||
});
|
||||
}
|
||||
|
||||
const nestingContext = nesting || {
|
||||
depth: 0,
|
||||
maxDepth: clampComboDepth(config.maxComboDepth),
|
||||
|
||||
327
open-sse/services/fusion.ts
Normal file
327
open-sse/services/fusion.ts
Normal file
@@ -0,0 +1,327 @@
|
||||
/**
|
||||
* Fusion combo strategy — parallel panel + judge synthesis.
|
||||
*
|
||||
* A fusion combo fans the prompt out to every panel model in parallel, then a
|
||||
* configurable judge model synthesizes one final answer from all panel responses.
|
||||
*
|
||||
* - quorum-grace collection caps the straggler penalty (the slowest model
|
||||
* otherwise dominates wall time);
|
||||
* - anonymized sources prevent judge brand-bias ("Source N" rather than model name);
|
||||
* - degrades to a direct answer on a single survivor, 503 on total failure.
|
||||
*
|
||||
* Per OpenRouter's Fusion design, the judge does NOT merge — it analyzes
|
||||
* (consensus / contradictions / partial coverage / unique insights / blind spots)
|
||||
* then writes one answer grounded in that analysis. Most of fusion's quality lift
|
||||
* comes from this synthesis step.
|
||||
*
|
||||
* Ported from upstream decolua/9router (Daniil Schovkunov), adapted JS → TS and
|
||||
* wired through OmniRoute's existing combo schema (combo.config.judgeModel /
|
||||
* combo.config.fusionTuning).
|
||||
*/
|
||||
import { errorResponse, sanitizeErrorMessage } from "../utils/error.ts";
|
||||
import { extractTextContent } from "../translator/helpers/geminiHelper.ts";
|
||||
import type { ComboLogger, HandleSingleModel } from "./combo/types.ts";
|
||||
|
||||
// Fusion tuning. Overridable per-combo via combo.config.fusionTuning.
|
||||
export const FUSION_DEFAULTS = {
|
||||
minPanel: 2, // answers needed before stragglers get a grace window
|
||||
stragglerGraceMs: 8000, // wait this long for laggards once quorum is reached
|
||||
panelHardTimeoutMs: 90000, // absolute cap so one hung model can't stall forever
|
||||
} as const;
|
||||
|
||||
export type FusionTuning = {
|
||||
minPanel?: number;
|
||||
stragglerGraceMs?: number;
|
||||
panelHardTimeoutMs?: number;
|
||||
};
|
||||
|
||||
type Body = Record<string, unknown>;
|
||||
|
||||
/**
|
||||
* Extract assistant text from a non-stream completion across formats
|
||||
* (OpenAI chat, Claude messages, Gemini, OpenAI Responses). Returns "" if none.
|
||||
* Panel responses are already translated to the client format by chatCore, so the
|
||||
* leaf content → string step reuses the translator's own extractTextContent.
|
||||
*/
|
||||
export function extractPanelText(json: unknown): string {
|
||||
if (!json || typeof json !== "object") return "";
|
||||
const j = json as Record<string, unknown>;
|
||||
|
||||
// OpenAI chat completion
|
||||
const choices = j.choices as Array<Record<string, unknown>> | undefined;
|
||||
const choice = choices?.[0];
|
||||
if (choice) {
|
||||
const msg = (choice.message ?? choice.delta ?? {}) as Record<string, unknown>;
|
||||
const t = extractTextContent(msg.content);
|
||||
if (t.trim()) return t;
|
||||
if (typeof choice.text === "string" && choice.text.trim()) return choice.text;
|
||||
}
|
||||
|
||||
// Claude messages (text blocks share OpenAI's {type:"text"} shape)
|
||||
const claudeText = extractTextContent(j.content);
|
||||
if (claudeText.trim()) return claudeText;
|
||||
|
||||
// Gemini (parts carry .text without a type discriminator)
|
||||
const candidates = j.candidates as Array<Record<string, unknown>> | undefined;
|
||||
const parts = (candidates?.[0]?.content as Record<string, unknown> | undefined)?.parts as
|
||||
| Array<{ text?: unknown }>
|
||||
| undefined;
|
||||
if (Array.isArray(parts)) {
|
||||
const t = parts.map((p) => (typeof p?.text === "string" ? p.text : "")).join("");
|
||||
if (t.trim()) return t;
|
||||
}
|
||||
|
||||
// OpenAI Responses API
|
||||
const output = j.output as Array<Record<string, unknown>> | undefined;
|
||||
if (Array.isArray(output)) {
|
||||
const t = output
|
||||
.flatMap((o) =>
|
||||
Array.isArray(o.content)
|
||||
? (o.content as Array<{ text?: unknown }>).map((c) =>
|
||||
typeof c?.text === "string" ? c.text : ""
|
||||
)
|
||||
: []
|
||||
)
|
||||
.join("");
|
||||
if (t.trim()) return t;
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
/**
|
||||
* Append a synthesized user turn to whichever message array the request format uses.
|
||||
* Preserves the original conversation + system prompt so the judge has full context.
|
||||
*/
|
||||
export function appendUserTurn(body: Body, text: string): Body {
|
||||
const next: Body = { ...body };
|
||||
if (Array.isArray(body.messages)) {
|
||||
next.messages = [...(body.messages as unknown[]), { role: "user", content: text }];
|
||||
} else if (Array.isArray(body.input)) {
|
||||
next.input = [...(body.input as unknown[]), { role: "user", content: text }];
|
||||
} else if (Array.isArray(body.contents)) {
|
||||
next.contents = [
|
||||
...(body.contents as unknown[]),
|
||||
{ role: "user", parts: [{ text }] },
|
||||
];
|
||||
} else {
|
||||
next.messages = [{ role: "user", content: text }];
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the judge directive. Sources are anonymized ("Source N") so the judge
|
||||
* weighs substance, not the reputation of a model brand.
|
||||
*/
|
||||
export function buildJudgePrompt(answers: Array<{ text: string }>): string {
|
||||
const panel = answers.map((a, i) => `[Source ${i + 1}]\n${a.text}`).join("\n\n");
|
||||
|
||||
return [
|
||||
`You are the JUDGE in a model-fusion panel. ${answers.length} expert models independently answered the user's most recent request. Their responses are below, anonymized by source.`,
|
||||
"",
|
||||
"Do NOT mention that multiple models were used, and do NOT refer to the sources. Produce ONE authoritative final answer addressed directly to the user.",
|
||||
"",
|
||||
"First, internally analyze the panel along these dimensions: consensus (points most sources agree on — treat as higher-confidence), contradictions (where they disagree — resolve with your own judgment), partial coverage, unique insights only one source surfaced, and blind spots every source missed. Then write the best possible final answer grounded in that analysis — more complete and correct than any single response, with no filler.",
|
||||
"",
|
||||
"=== PANEL RESPONSES ===",
|
||||
panel,
|
||||
"=== END PANEL RESPONSES ===",
|
||||
"",
|
||||
"Now write the final answer to the user's original request.",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
type Sentinel = { __timeout?: true; __error?: unknown };
|
||||
|
||||
// Resolve a Response (or sentinel) within ms; the loser keeps running but is ignored.
|
||||
function withTimeout(
|
||||
promise: Promise<Response>,
|
||||
ms: number
|
||||
): Promise<Response | Sentinel> {
|
||||
return new Promise((resolve) => {
|
||||
const t = setTimeout(() => resolve({ __timeout: true }), ms);
|
||||
Promise.resolve(promise)
|
||||
.then((v) => {
|
||||
clearTimeout(t);
|
||||
resolve(v);
|
||||
})
|
||||
.catch((e) => {
|
||||
clearTimeout(t);
|
||||
resolve({ __error: e });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Collect panel responses with quorum-grace: as soon as `minPanel` calls succeed,
|
||||
* start a short grace timer for the rest, then proceed with whatever arrived. This
|
||||
* caps the straggler penalty while still preferring a full panel when everyone is
|
||||
* fast. Bounded by a hard timeout.
|
||||
*
|
||||
* Returns a sparse array aligned to `calls` (undefined = not yet / dropped).
|
||||
*/
|
||||
export function collectPanel(
|
||||
calls: Array<Promise<Response | Sentinel>>,
|
||||
cfg: { minPanel: number; stragglerGraceMs: number; panelHardTimeoutMs: number }
|
||||
): Promise<Array<Response | Sentinel | undefined>> {
|
||||
return new Promise((resolve) => {
|
||||
const out: Array<Response | Sentinel | undefined> = new Array(calls.length);
|
||||
let settled = 0;
|
||||
let ok = 0;
|
||||
let finished = false;
|
||||
let graceTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
const finish = () => {
|
||||
if (finished) return;
|
||||
finished = true;
|
||||
clearTimeout(hardTimer);
|
||||
if (graceTimer) clearTimeout(graceTimer);
|
||||
resolve(out);
|
||||
};
|
||||
const hardTimer = setTimeout(finish, cfg.panelHardTimeoutMs);
|
||||
calls.forEach((p, i) => {
|
||||
Promise.resolve(p)
|
||||
.then((v) => {
|
||||
out[i] = v;
|
||||
})
|
||||
.catch((e) => {
|
||||
out[i] = { __error: e };
|
||||
})
|
||||
.finally(() => {
|
||||
settled++;
|
||||
const slot = out[i] as Response | undefined;
|
||||
if (slot && (slot as Response).ok) ok++;
|
||||
if (settled === calls.length) return finish();
|
||||
if (ok >= cfg.minPanel && !graceTimer) {
|
||||
graceTimer = setTimeout(finish, cfg.stragglerGraceMs);
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export type HandleFusionChatOptions = {
|
||||
body: Body;
|
||||
models: string[];
|
||||
handleSingleModel: HandleSingleModel;
|
||||
log: ComboLogger;
|
||||
comboName?: string;
|
||||
judgeModel?: string | null;
|
||||
tuning?: FusionTuning | null;
|
||||
};
|
||||
|
||||
/**
|
||||
* Handle a fusion combo: fan the prompt out to every panel model in parallel,
|
||||
* then a judge model synthesizes one final answer from all panel responses.
|
||||
*
|
||||
* Panel calls are forced non-streaming with tools stripped (the judge needs
|
||||
* complete prose to synthesize). The judge call keeps the client's original
|
||||
* stream flag + tools, so streaming and downstream tool use still work.
|
||||
*
|
||||
* Speed: quorum-grace collection caps the straggler penalty. Quality: the judge
|
||||
* runs the consensus/contradiction/blind-spot analysis before writing.
|
||||
*
|
||||
* Degrades gracefully: 0 panel answers → 503, exactly 1 → return it directly.
|
||||
*/
|
||||
export async function handleFusionChat({
|
||||
body,
|
||||
models,
|
||||
handleSingleModel,
|
||||
log,
|
||||
comboName,
|
||||
judgeModel,
|
||||
tuning,
|
||||
}: HandleFusionChatOptions): Promise<Response> {
|
||||
const panel = Array.isArray(models) ? models.filter(Boolean) : [];
|
||||
if (panel.length === 0) {
|
||||
return errorResponse(400, "Fusion combo has no models");
|
||||
}
|
||||
|
||||
// A single-model fusion has nothing to fuse — just answer directly.
|
||||
if (panel.length === 1) {
|
||||
return handleSingleModel(body, panel[0]);
|
||||
}
|
||||
|
||||
const cfg = {
|
||||
minPanel: tuning?.minPanel ?? FUSION_DEFAULTS.minPanel,
|
||||
stragglerGraceMs: tuning?.stragglerGraceMs ?? FUSION_DEFAULTS.stragglerGraceMs,
|
||||
panelHardTimeoutMs: tuning?.panelHardTimeoutMs ?? FUSION_DEFAULTS.panelHardTimeoutMs,
|
||||
};
|
||||
const minPanel = Math.min(Math.max(2, cfg.minPanel), panel.length);
|
||||
const judge = judgeModel && judgeModel.trim() ? judgeModel.trim() : panel[0];
|
||||
log.info(
|
||||
"FUSION",
|
||||
`Combo "${comboName ?? ""}" | panel=${panel.length} [${panel.join(", ")}] | judge=${judge} | quorum=${minPanel}`
|
||||
);
|
||||
|
||||
// 1. Fan out to the panel in parallel: non-streaming, tools stripped (we want prose).
|
||||
const { tools: _tools, tool_choice: _tc, ...rest } = body;
|
||||
void _tools;
|
||||
void _tc;
|
||||
const panelBody: Body = { ...rest, stream: false };
|
||||
const t0 = Date.now();
|
||||
const calls = panel.map((m) =>
|
||||
withTimeout(handleSingleModel(panelBody, m), cfg.panelHardTimeoutMs)
|
||||
);
|
||||
const settled = await collectPanel(calls, { ...cfg, minPanel });
|
||||
log.info("FUSION", `fan-out collected in ${Date.now() - t0}ms`);
|
||||
|
||||
// 2. Collect successful answers.
|
||||
const answers: Array<{ model: string; text: string }> = [];
|
||||
for (let i = 0; i < settled.length; i++) {
|
||||
const res = settled[i];
|
||||
const model = panel[i];
|
||||
if (!res) {
|
||||
log.warn("FUSION", `Panel ${model} dropped (straggler/timeout)`);
|
||||
continue;
|
||||
}
|
||||
const sentinel = res as Sentinel;
|
||||
if (sentinel.__timeout) {
|
||||
log.warn("FUSION", `Panel ${model} timed out`);
|
||||
continue;
|
||||
}
|
||||
if (sentinel.__error) {
|
||||
log.warn("FUSION", `Panel ${model} threw`, {
|
||||
error: sanitizeErrorMessage(sentinel.__error as Error),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const resp = res as Response;
|
||||
if (!resp.ok) {
|
||||
log.warn("FUSION", `Panel ${model} failed`, { status: resp.status });
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const json = await resp.clone().json();
|
||||
const text = extractPanelText(json);
|
||||
if (text) {
|
||||
answers.push({ model, text });
|
||||
log.info("FUSION", `Panel ${model} ok (${text.length} chars)`);
|
||||
} else {
|
||||
log.warn("FUSION", `Panel ${model} returned empty content`);
|
||||
}
|
||||
} catch (e) {
|
||||
log.warn("FUSION", `Panel ${model} unparseable`, {
|
||||
error: sanitizeErrorMessage(e as Error),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Degrade gracefully when the panel is too thin to fuse.
|
||||
if (answers.length === 0) {
|
||||
log.warn("FUSION", "All panel models failed");
|
||||
return errorResponse(503, "All fusion panel models failed");
|
||||
}
|
||||
if (answers.length === 1) {
|
||||
log.info(
|
||||
"FUSION",
|
||||
`Only ${answers[0].model} succeeded — answering directly (no fusion)`
|
||||
);
|
||||
return handleSingleModel(body, answers[0].model);
|
||||
}
|
||||
|
||||
// 4. Judge analyzes + writes one final answer (streams to client if requested).
|
||||
const judgeBody = appendUserTurn(body, buildJudgePrompt(answers));
|
||||
log.info("FUSION", `Judging ${answers.length} answers with ${judge}`);
|
||||
return handleSingleModel(judgeBody, judge);
|
||||
}
|
||||
@@ -2384,6 +2384,8 @@
|
||||
"resetAwareDesc": "Balances remaining quota against 5h and weekly resets, then round-robins similar scores",
|
||||
"strictRandom": "Strict Random",
|
||||
"strictRandomDesc": "Shuffle deck — uses each model once before reshuffling",
|
||||
"fusion": "Fusion",
|
||||
"fusionDesc": "Fans the prompt to every panel model in parallel, then a judge model synthesizes one final answer",
|
||||
"models": "Models",
|
||||
"autoBalance": "Auto-balance",
|
||||
"advancedSettings": "Advanced Settings",
|
||||
|
||||
@@ -14,6 +14,7 @@ export const ROUTING_STRATEGY_VALUES = [
|
||||
"auto",
|
||||
"lkgp",
|
||||
"context-optimized",
|
||||
"fusion",
|
||||
] as const;
|
||||
|
||||
export type RoutingStrategyValue = (typeof ROUTING_STRATEGY_VALUES)[number];
|
||||
@@ -170,6 +171,13 @@ export const ROUTING_STRATEGIES: RoutingStrategyOption[] = [
|
||||
settingsDescKey: "contextOptDesc",
|
||||
icon: "text_snippet",
|
||||
},
|
||||
{
|
||||
value: "fusion",
|
||||
labelKey: "fusion",
|
||||
combosDescKey: "fusionDesc",
|
||||
settingsDescKey: "fusionDesc",
|
||||
icon: "hub",
|
||||
},
|
||||
];
|
||||
|
||||
export const SETTINGS_FALLBACK_STRATEGY_VALUES = ACCOUNT_FALLBACK_STRATEGY_VALUES;
|
||||
|
||||
209
tests/unit/combo-fusion-strategy.test.ts
Normal file
209
tests/unit/combo-fusion-strategy.test.ts
Normal file
@@ -0,0 +1,209 @@
|
||||
/**
|
||||
* Fusion combo strategy — parallel panel + judge synthesis.
|
||||
*
|
||||
* Ported from upstream decolua/9router (Daniil Schovkunov). Adds Fusion as the 16th
|
||||
* combo strategy: fan the prompt out to every panel model in parallel, then a judge
|
||||
* model synthesizes one final answer from all panel responses.
|
||||
*/
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-combo-fusion-"));
|
||||
process.env.DATA_DIR = TEST_DATA_DIR;
|
||||
process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "combo-fusion-test-secret";
|
||||
|
||||
const { handleComboChat } = await import("../../open-sse/services/combo.ts");
|
||||
|
||||
const noop = () => {};
|
||||
const log = { info: noop, warn: noop, debug: noop, error: noop };
|
||||
|
||||
type Body = Record<string, unknown>;
|
||||
|
||||
// Minimal OpenAI-chat Response-shaped object compatible with the engine's .ok + .clone().json() surface.
|
||||
function okResponse(content: string, { delayMs = 0 } = {}): Response | Promise<Response> {
|
||||
const body = JSON.stringify({ choices: [{ message: { role: "assistant", content } }] });
|
||||
const make = () =>
|
||||
new Response(body, { status: 200, headers: { "Content-Type": "application/json" } });
|
||||
return delayMs > 0 ? new Promise((r) => setTimeout(() => r(make()), delayMs)) : make();
|
||||
}
|
||||
|
||||
function errResponse(status = 500): Response {
|
||||
return new Response(JSON.stringify({ error: { message: "boom" } }), {
|
||||
status,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
}
|
||||
|
||||
function fusionCombo(models: string[], extra: Record<string, unknown> = {}) {
|
||||
return {
|
||||
name: "test-fusion-combo",
|
||||
strategy: "fusion",
|
||||
models: models.map((m) => ({ model: m })),
|
||||
config: extra,
|
||||
};
|
||||
}
|
||||
|
||||
test("fusion: single-model panel answers directly (nothing to fuse)", async () => {
|
||||
const calls: string[] = [];
|
||||
const handleSingleModel = async (_b: Body, m: string) => {
|
||||
calls.push(m);
|
||||
return okResponse("solo");
|
||||
};
|
||||
const res = await handleComboChat({
|
||||
body: { messages: [{ role: "user", content: "hi" }] },
|
||||
combo: fusionCombo(["p/only"]),
|
||||
handleSingleModel,
|
||||
log,
|
||||
settings: {},
|
||||
allCombos: [],
|
||||
});
|
||||
assert.equal(calls.length, 1);
|
||||
assert.equal(calls[0], "p/only");
|
||||
assert.equal(res.status, 200);
|
||||
});
|
||||
|
||||
test("fusion: fans out to the panel then routes a synthesis turn to the judge", async () => {
|
||||
const seen: string[] = [];
|
||||
const seenBodies: Body[] = [];
|
||||
const handleSingleModel = async (b: Body, m: string) => {
|
||||
seen.push(m);
|
||||
seenBodies.push(b);
|
||||
if (m === "p/judge") return okResponse("FINAL");
|
||||
return okResponse(`ans-${m}`);
|
||||
};
|
||||
|
||||
const res = await handleComboChat({
|
||||
body: {
|
||||
messages: [{ role: "user", content: "Q" }],
|
||||
stream: true,
|
||||
tools: [{ name: "x" }],
|
||||
},
|
||||
combo: fusionCombo(["p/a", "p/b", "p/c"], { judgeModel: "p/judge" }),
|
||||
handleSingleModel,
|
||||
log,
|
||||
settings: {},
|
||||
allCombos: [],
|
||||
});
|
||||
|
||||
// 3 panel calls + 1 judge call.
|
||||
assert.equal(seen.length, 4);
|
||||
assert.deepEqual(seen.slice(0, 3).sort(), ["p/a", "p/b", "p/c"]);
|
||||
assert.equal(seen[3], "p/judge");
|
||||
|
||||
// Panel calls are non-streaming with tools stripped.
|
||||
for (let i = 0; i < 3; i++) {
|
||||
const b = seenBodies[i];
|
||||
assert.equal(b.stream, false, "panel call should be non-streaming");
|
||||
assert.equal(b.tools, undefined, "panel call should have tools stripped");
|
||||
}
|
||||
|
||||
// Judge call carries every panel answer + keeps the client's stream flag.
|
||||
const judgeBody = seenBodies[3];
|
||||
const judgeMsgs = judgeBody.messages as Array<{ role: string; content: string }>;
|
||||
const judgeText = judgeMsgs[judgeMsgs.length - 1].content;
|
||||
assert.match(judgeText, /ans-p\/a/);
|
||||
assert.match(judgeText, /ans-p\/b/);
|
||||
assert.match(judgeText, /ans-p\/c/);
|
||||
assert.match(judgeText, /Source 1/);
|
||||
assert.equal(judgeBody.stream, true);
|
||||
|
||||
assert.equal(res.status, 200);
|
||||
});
|
||||
|
||||
test("fusion: defaults the judge to the first panel model when none is set", async () => {
|
||||
const seen: string[] = [];
|
||||
const handleSingleModel = async (_b: Body, m: string) => {
|
||||
seen.push(m);
|
||||
return okResponse(`ans-${m}`);
|
||||
};
|
||||
await handleComboChat({
|
||||
body: { messages: [{ role: "user", content: "Q" }] },
|
||||
combo: fusionCombo(["p/first", "p/second"]),
|
||||
handleSingleModel,
|
||||
log,
|
||||
settings: {},
|
||||
allCombos: [],
|
||||
});
|
||||
// Last call is the judge; defaults to panel[0].
|
||||
assert.equal(seen[seen.length - 1], "p/first");
|
||||
});
|
||||
|
||||
test("fusion: proceeds on quorum without waiting for a straggler (grace window)", async () => {
|
||||
const handleSingleModel = async (_b: Body, m: string) => {
|
||||
if (m === "p/slow") return okResponse("slow", { delayMs: 5000 });
|
||||
if (m === "p/judge") return okResponse("FINAL");
|
||||
return okResponse(`fast-${m}`);
|
||||
};
|
||||
|
||||
const t0 = Date.now();
|
||||
const seenBodies: Body[] = [];
|
||||
const wrapped = async (b: Body, m: string) => {
|
||||
seenBodies.push(b);
|
||||
return handleSingleModel(b, m);
|
||||
};
|
||||
await handleComboChat({
|
||||
body: { messages: [{ role: "user", content: "Q" }] },
|
||||
combo: fusionCombo(["p/x", "p/y", "p/slow"], {
|
||||
judgeModel: "p/judge",
|
||||
fusionTuning: { minPanel: 2, stragglerGraceMs: 50, panelHardTimeoutMs: 10000 },
|
||||
}),
|
||||
handleSingleModel: wrapped,
|
||||
log,
|
||||
settings: {},
|
||||
allCombos: [],
|
||||
});
|
||||
const elapsed = Date.now() - t0;
|
||||
|
||||
// Two fast answers reach quorum; grace is 50ms, so we never wait ~5s for p/slow.
|
||||
assert.ok(elapsed < 2000, `should not wait for straggler (took ${elapsed}ms)`);
|
||||
|
||||
const judgeBody = seenBodies[seenBodies.length - 1];
|
||||
const judgeMsgs = judgeBody.messages as Array<{ role: string; content: string }>;
|
||||
const judgeText = judgeMsgs[judgeMsgs.length - 1].content;
|
||||
assert.match(judgeText, /fast-p\/x/);
|
||||
assert.match(judgeText, /fast-p\/y/);
|
||||
assert.ok(!/slow/.test(judgeText), "straggler answer should not appear in the judge prompt");
|
||||
});
|
||||
|
||||
test("fusion: returns the lone survivor directly when only one panel model succeeds", async () => {
|
||||
const seen: string[] = [];
|
||||
const handleSingleModel = async (_b: Body, m: string) => {
|
||||
seen.push(m);
|
||||
if (m === "p/ok") return okResponse("lone");
|
||||
return errResponse(500);
|
||||
};
|
||||
await handleComboChat({
|
||||
body: { messages: [{ role: "user", content: "Q" }] },
|
||||
combo: fusionCombo(["p/ok", "p/bad"], {
|
||||
judgeModel: "p/judge",
|
||||
fusionTuning: { minPanel: 2, stragglerGraceMs: 50, panelHardTimeoutMs: 5000 },
|
||||
}),
|
||||
handleSingleModel,
|
||||
log,
|
||||
settings: {},
|
||||
allCombos: [],
|
||||
});
|
||||
// No judge call — single answer means there is nothing to fuse.
|
||||
assert.ok(
|
||||
!seen.includes("p/judge"),
|
||||
"judge should not be invoked when only one panel model survives"
|
||||
);
|
||||
});
|
||||
|
||||
test("fusion: returns 503 when the whole panel fails", async () => {
|
||||
const handleSingleModel = async () => errResponse(500);
|
||||
const res = await handleComboChat({
|
||||
body: { messages: [{ role: "user", content: "Q" }] },
|
||||
combo: fusionCombo(["p/a", "p/b"], {
|
||||
fusionTuning: { minPanel: 2, stragglerGraceMs: 50, panelHardTimeoutMs: 5000 },
|
||||
}),
|
||||
handleSingleModel,
|
||||
log,
|
||||
settings: {},
|
||||
allCombos: [],
|
||||
});
|
||||
assert.equal(res.status, 503);
|
||||
});
|
||||
Reference in New Issue
Block a user