diff --git a/Dockerfile b/Dockerfile index adf3cf5e91..98e0a3215b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -114,6 +114,12 @@ LABEL org.opencontainers.image.title="omniroute" \ ENV NODE_ENV=production ENV PORT=20128 ENV HOSTNAME=0.0.0.0 +# Runtime heap ceiling. 1024MB is enough for normal traffic but can be tight +# for large fusion-combo panels (many models fanned out in parallel, each +# response buffered in full — see open-sse/services/fusion.ts::FUSION_DEFAULTS +# .maxPanel, issue #1905). Override at `docker run` time with +# `-e OMNIROUTE_MEMORY_MB=2048` (or higher) if you raise fusionTuning.maxPanel +# above the default cap. ENV OMNIROUTE_MEMORY_MB=1024 ENV NODE_OPTIONS="--max-old-space-size=${OMNIROUTE_MEMORY_MB}" diff --git a/changelog.d/fixes/1905-fusion-panel-oom.md b/changelog.d/fixes/1905-fusion-panel-oom.md new file mode 100644 index 0000000000..68bc6dbfbc --- /dev/null +++ b/changelog.d/fixes/1905-fusion-panel-oom.md @@ -0,0 +1 @@ +- **fix(combos):** fusion combos now reject an oversized panel (>40 models by default, tunable via `fusionTuning.maxPanel`) with a clean 400 before fanning out, instead of buffering dozens of concurrent full responses in memory and OOM-crashing the whole container. (thanks @fontvu) diff --git a/open-sse/services/fusion.ts b/open-sse/services/fusion.ts index 000f5a21c5..d6e5f4fa1f 100644 --- a/open-sse/services/fusion.ts +++ b/open-sse/services/fusion.ts @@ -27,12 +27,20 @@ export const FUSION_DEFAULTS = { minPanel: 2, // answers needed before stragglers get a grace window stragglerGraceMs: 8000, // wait this long for laggards once quorum is reached panelHardTimeoutMs: 90000, // absolute cap so one hung model can't stall forever + // Hard cap on panel size (issue #1905). Every panel member is fanned out in + // parallel and its full response text buffered in memory simultaneously — + // with the runtime heap capped (Dockerfile OMNIROUTE_MEMORY_MB, default + // 1024MB), a large panel (reported: ~73 models) with sizable concurrent + // responses can exceed the heap ceiling and OOM-crash the whole process. + // Reject oversized panels up front with a clean 400 instead. + maxPanel: 40, } as const; export type FusionTuning = { minPanel?: number; stragglerGraceMs?: number; panelHardTimeoutMs?: number; + maxPanel?: number; }; type Body = Record; @@ -246,6 +254,21 @@ export async function handleFusionChat({ return handleSingleModel(body, panel[0]); } + // Reject an oversized panel BEFORE fan-out (issue #1905): fanning out N + // parallel calls and buffering N full response bodies at once is what + // drives the process into an OOM crash, not any one call in isolation. + const maxPanel = tuning?.maxPanel ?? FUSION_DEFAULTS.maxPanel; + if (panel.length > maxPanel) { + log.warn( + "FUSION", + `Combo "${comboName ?? ""}" panel=${panel.length} exceeds maxPanel=${maxPanel} — rejecting before fan-out (#1905)` + ); + return errorResponse( + 400, + `Fusion panel too large (${panel.length} models, max ${maxPanel}) — reduce the combo's target count or raise fusionTuning.maxPanel` + ); + } + const cfg = { minPanel: tuning?.minPanel ?? FUSION_DEFAULTS.minPanel, stragglerGraceMs: tuning?.stragglerGraceMs ?? FUSION_DEFAULTS.stragglerGraceMs, diff --git a/src/shared/validation/schemas/combo.ts b/src/shared/validation/schemas/combo.ts index 5c00a1be08..fbe95188cd 100644 --- a/src/shared/validation/schemas/combo.ts +++ b/src/shared/validation/schemas/combo.ts @@ -227,6 +227,11 @@ export const comboRuntimeConfigSchema = z minPanel: z.coerce.number().int().min(1).max(50).optional(), stragglerGraceMs: z.coerce.number().int().min(0).max(120_000).optional(), panelHardTimeoutMs: z.coerce.number().int().min(1000).max(600_000).optional(), + // Hard cap on panel size (issue #1905) — see FUSION_DEFAULTS.maxPanel in + // open-sse/services/fusion.ts. Bounds how many models can be fanned out + // and buffered in memory concurrently before the container's heap ceiling + // is at risk. + maxPanel: z.coerce.number().int().min(1).max(200).optional(), }) .strict() .optional(), diff --git a/tests/unit/fusion-panel-size-cap-1905.test.ts b/tests/unit/fusion-panel-size-cap-1905.test.ts new file mode 100644 index 0000000000..6b9ac1b935 --- /dev/null +++ b/tests/unit/fusion-panel-size-cap-1905.test.ts @@ -0,0 +1,79 @@ +/** + * Regression test for upstream issue decolua/9router#1905. + * + * Reported symptom: a fusion combo populated with ~70+ panel models fans every + * member out in parallel (`open-sse/services/fusion.ts::handleFusionChat` → + * `Promise.all`-style fan-out via `collectPanel`), buffering each model's full + * response text in memory at once. With the runtime heap capped at 1024MB + * (Dockerfile `OMNIROUTE_MEMORY_MB`), a large panel with sizable concurrent + * responses can exceed the heap ceiling and crash the whole container with + * "FATAL ERROR: Ineffective mark-compacts near heap limit — JavaScript heap + * out of memory" instead of failing one request gracefully. + * + * Fix: `handleFusionChat` now rejects panels above a configurable hard cap + * (`FUSION_DEFAULTS.maxPanel`, overridable via `fusionTuning.maxPanel`) with a + * clean 400 *before* fan-out, rather than let an unbounded panel size drive + * the process into an OOM crash. + */ +import test from "node:test"; +import assert from "node:assert/strict"; + +import { handleFusionChat, FUSION_DEFAULTS } from "../../open-sse/services/fusion.ts"; + +const noop = () => {}; +const log = { info: noop, warn: noop, debug: noop, error: noop }; + +type Body = Record; + +test("fusion #1905: an oversized panel (73 models) is rejected before fan-out instead of OOM-crashing", async () => { + let calls = 0; + const handleSingleModel = (_b: Body, _m: string) => { + calls++; + const body = JSON.stringify({ + choices: [{ message: { role: "assistant", content: "x".repeat(1000) } }], + }); + return Promise.resolve( + new Response(body, { status: 200, headers: { "Content-Type": "application/json" } }) + ); + }; + + const panel = Array.from({ length: 73 }, (_, i) => `provider/model-${i}`); + + const res = await handleFusionChat({ + body: { messages: [{ role: "user", content: "hi" }] }, + models: panel, + handleSingleModel, + log, + comboName: "auto", + }); + + assert.equal(res.status, 400); + // Must reject BEFORE fan-out — no per-model calls should have happened. + assert.equal(calls, 0, "panel fan-out must not start once the size cap is exceeded"); + + const json = (await res.json()) as { error?: { message?: string } }; + assert.match(json.error?.message ?? "", /panel/i); +}); + +test("fusion #1905: a panel at or under the cap still fans out normally", async () => { + const handleSingleModel = (_b: Body, _m: string) => { + const body = JSON.stringify({ + choices: [{ message: { role: "assistant", content: "ok" } }], + }); + return Promise.resolve( + new Response(body, { status: 200, headers: { "Content-Type": "application/json" } }) + ); + }; + + const panel = Array.from({ length: FUSION_DEFAULTS.maxPanel }, (_, i) => `provider/model-${i}`); + + const res = await handleFusionChat({ + body: { messages: [{ role: "user", content: "hi" }] }, + models: panel, + handleSingleModel, + log, + comboName: "auto", + }); + + assert.equal(res.status, 200); +});