fix(combos): reject oversized fusion panels before fan-out (port from 9router#1905) (#7120)

A fusion combo fans every panel model out in parallel and buffers each
model's full response text in memory simultaneously. With the runtime heap
capped by Dockerfile's OMNIROUTE_MEMORY_MB (default 1024MB), a large panel
(reported: ~73 models via an 'auto' combo with strategy: fusion) with
sizable concurrent responses can exceed the heap ceiling and OOM-crash the
whole container instead of failing one request.

handleFusionChat now rejects panels above a configurable hard cap
(FUSION_DEFAULTS.maxPanel = 40, overridable per-combo via
fusionTuning.maxPanel) with a clean 400 before fan-out begins.

Reported-by: Phong Vu (@fontvu) (https://github.com/decolua/9router/issues/1905)

Dockerfile

@@ -114,6 +114,12 @@ LABEL org.opencontainers.image.title="omniroute" \
 ENV NODE_ENV=production
 ENV PORT=20128
 ENV HOSTNAME=0.0.0.0
+# Runtime heap ceiling. 1024MB is enough for normal traffic but can be tight
+# for large fusion-combo panels (many models fanned out in parallel, each
+# response buffered in full — see open-sse/services/fusion.ts::FUSION_DEFAULTS
+# .maxPanel, issue #1905). Override at `docker run` time with
+# `-e OMNIROUTE_MEMORY_MB=2048` (or higher) if you raise fusionTuning.maxPanel
+# above the default cap.
 ENV OMNIROUTE_MEMORY_MB=1024
 ENV NODE_OPTIONS="--max-old-space-size=${OMNIROUTE_MEMORY_MB}"
 

changelog.d/fixes/1905-fusion-panel-oom.md

@@ -0,0 +1 @@
+- **fix(combos):** fusion combos now reject an oversized panel (>40 models by default, tunable via `fusionTuning.maxPanel`) with a clean 400 before fanning out, instead of buffering dozens of concurrent full responses in memory and OOM-crashing the whole container. (thanks @fontvu)

open-sse/services/fusion.ts

@@ -27,12 +27,20 @@ export const FUSION_DEFAULTS = {
   minPanel: 2, // answers needed before stragglers get a grace window
   stragglerGraceMs: 8000, // wait this long for laggards once quorum is reached
   panelHardTimeoutMs: 90000, // absolute cap so one hung model can't stall forever
+  // Hard cap on panel size (issue #1905). Every panel member is fanned out in
+  // parallel and its full response text buffered in memory simultaneously —
+  // with the runtime heap capped (Dockerfile OMNIROUTE_MEMORY_MB, default
+  // 1024MB), a large panel (reported: ~73 models) with sizable concurrent
+  // responses can exceed the heap ceiling and OOM-crash the whole process.
+  // Reject oversized panels up front with a clean 400 instead.
+  maxPanel: 40,
 } as const;
 
 export type FusionTuning = {
   minPanel?: number;
   stragglerGraceMs?: number;
   panelHardTimeoutMs?: number;
+  maxPanel?: number;
 };
 
 type Body = Record<string, unknown>;
@@ -246,6 +254,21 @@ export async function handleFusionChat({
     return handleSingleModel(body, panel[0]);
   }
 
+  // Reject an oversized panel BEFORE fan-out (issue #1905): fanning out N
+  // parallel calls and buffering N full response bodies at once is what
+  // drives the process into an OOM crash, not any one call in isolation.
+  const maxPanel = tuning?.maxPanel ?? FUSION_DEFAULTS.maxPanel;
+  if (panel.length > maxPanel) {
+    log.warn(
+      "FUSION",
+      `Combo "${comboName ?? ""}" panel=${panel.length} exceeds maxPanel=${maxPanel} — rejecting before fan-out (#1905)`
+    );
+    return errorResponse(
+      400,
+      `Fusion panel too large (${panel.length} models, max ${maxPanel}) — reduce the combo's target count or raise fusionTuning.maxPanel`
+    );
+  }
+
   const cfg = {
     minPanel: tuning?.minPanel ?? FUSION_DEFAULTS.minPanel,
     stragglerGraceMs: tuning?.stragglerGraceMs ?? FUSION_DEFAULTS.stragglerGraceMs,

src/shared/validation/schemas/combo.ts

@@ -227,6 +227,11 @@ export const comboRuntimeConfigSchema = z
         minPanel: z.coerce.number().int().min(1).max(50).optional(),
         stragglerGraceMs: z.coerce.number().int().min(0).max(120_000).optional(),
         panelHardTimeoutMs: z.coerce.number().int().min(1000).max(600_000).optional(),
+        // Hard cap on panel size (issue #1905) — see FUSION_DEFAULTS.maxPanel in
+        // open-sse/services/fusion.ts. Bounds how many models can be fanned out
+        // and buffered in memory concurrently before the container's heap ceiling
+        // is at risk.
+        maxPanel: z.coerce.number().int().min(1).max(200).optional(),
       })
       .strict()
       .optional(),

tests/unit/fusion-panel-size-cap-1905.test.ts

@@ -0,0 +1,79 @@
+/**
+ * Regression test for upstream issue decolua/9router#1905.
+ *
+ * Reported symptom: a fusion combo populated with ~70+ panel models fans every
+ * member out in parallel (`open-sse/services/fusion.ts::handleFusionChat` →
+ * `Promise.all`-style fan-out via `collectPanel`), buffering each model's full
+ * response text in memory at once. With the runtime heap capped at 1024MB
+ * (Dockerfile `OMNIROUTE_MEMORY_MB`), a large panel with sizable concurrent
+ * responses can exceed the heap ceiling and crash the whole container with
+ * "FATAL ERROR: Ineffective mark-compacts near heap limit — JavaScript heap
+ * out of memory" instead of failing one request gracefully.
+ *
+ * Fix: `handleFusionChat` now rejects panels above a configurable hard cap
+ * (`FUSION_DEFAULTS.maxPanel`, overridable via `fusionTuning.maxPanel`) with a
+ * clean 400 *before* fan-out, rather than let an unbounded panel size drive
+ * the process into an OOM crash.
+ */
+import test from "node:test";
+import assert from "node:assert/strict";
+
+import { handleFusionChat, FUSION_DEFAULTS } from "../../open-sse/services/fusion.ts";
+
+const noop = () => {};
+const log = { info: noop, warn: noop, debug: noop, error: noop };
+
+type Body = Record<string, unknown>;
+
+test("fusion #1905: an oversized panel (73 models) is rejected before fan-out instead of OOM-crashing", async () => {
+  let calls = 0;
+  const handleSingleModel = (_b: Body, _m: string) => {
+    calls++;
+    const body = JSON.stringify({
+      choices: [{ message: { role: "assistant", content: "x".repeat(1000) } }],
+    });
+    return Promise.resolve(
+      new Response(body, { status: 200, headers: { "Content-Type": "application/json" } })
+    );
+  };
+
+  const panel = Array.from({ length: 73 }, (_, i) => `provider/model-${i}`);
+
+  const res = await handleFusionChat({
+    body: { messages: [{ role: "user", content: "hi" }] },
+    models: panel,
+    handleSingleModel,
+    log,
+    comboName: "auto",
+  });
+
+  assert.equal(res.status, 400);
+  // Must reject BEFORE fan-out — no per-model calls should have happened.
+  assert.equal(calls, 0, "panel fan-out must not start once the size cap is exceeded");
+
+  const json = (await res.json()) as { error?: { message?: string } };
+  assert.match(json.error?.message ?? "", /panel/i);
+});
+
+test("fusion #1905: a panel at or under the cap still fans out normally", async () => {
+  const handleSingleModel = (_b: Body, _m: string) => {
+    const body = JSON.stringify({
+      choices: [{ message: { role: "assistant", content: "ok" } }],
+    });
+    return Promise.resolve(
+      new Response(body, { status: 200, headers: { "Content-Type": "application/json" } })
+    );
+  };
+
+  const panel = Array.from({ length: FUSION_DEFAULTS.maxPanel }, (_, i) => `provider/model-${i}`);
+
+  const res = await handleFusionChat({
+    body: { messages: [{ role: "user", content: "hi" }] },
+    models: panel,
+    handleSingleModel,
+    log,
+    comboName: "auto",
+  });
+
+  assert.equal(res.status, 200);
+});