From 6c8c87dbb65cbf0051420f8e8b80e0eae9c482ca Mon Sep 17 00:00:00 2001 From: Abhishek Divekar Date: Sun, 21 Jun 2026 17:04:44 +0530 Subject: [PATCH] feat(combo): per-combo stickyRoundRobinLimit override on the combos page (#4472) Thanks @adivekar-utexas! Rebased onto release/v3.8.32 (resolved the additive conflict with #3872's queueDepth). Per-combo stickyRoundRobinLimit override now ships. --- open-sse/services/combo.ts | 14 ++++++--- src/app/(dashboard)/dashboard/combos/page.tsx | 31 +++++++++++++++++++ src/shared/validation/schemas/combo.ts | 7 +++-- tests/unit/combo-config.test.ts | 18 +++++++++++ tests/unit/combo-strategy-fallbacks.test.ts | 26 ++++++++++++++++ 5 files changed, 90 insertions(+), 6 deletions(-) diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index 6e29860690..f93b4c43cd 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -1986,11 +1986,17 @@ async function handleRoundRobinCombo({ log ); - // Sticky batch size at the combo level. Reuses the global `stickyRoundRobinLimit` - // setting so a single knob controls sticky batching for both account fallback and - // combo targets. Values <= 1 preserve the historical one-request-per-target rotation. + // Sticky batch size at the combo level. A per-combo `stickyRoundRobinLimit` (in + // combo.config, resolved through the cascade) overrides the global setting so one + // combo can batch differently from the default. When the per-combo value is unset, + // fall back to the global `stickyRoundRobinLimit` so the existing knob still controls + // sticky batching for both account fallback and combo targets. Values <= 1 preserve + // the historical one-request-per-target rotation. + const perComboStickyLimit = (config as Record).stickyRoundRobinLimit; const stickyLimit = clampStickyRoundRobinTargetLimit( - (settings as Record | null)?.stickyRoundRobinLimit + perComboStickyLimit !== undefined && perComboStickyLimit !== null + ? perComboStickyLimit + : (settings as Record | null)?.stickyRoundRobinLimit ); const stickyRoundRobinEnabled = stickyLimit > 1; if ( diff --git a/src/app/(dashboard)/dashboard/combos/page.tsx b/src/app/(dashboard)/dashboard/combos/page.tsx index f6ec545a96..a2d1669661 100644 --- a/src/app/(dashboard)/dashboard/combos/page.tsx +++ b/src/app/(dashboard)/dashboard/combos/page.tsx @@ -157,6 +157,8 @@ const ADVANCED_FIELD_HELP_FALLBACK = { "Round-robin combo/model limit: max simultaneous requests sent to each model target. This is separate from any provider account-only cap.", queueTimeout: "How long a request can wait for a round-robin model slot before timing out. This queue is separate from any account-only concurrency cap.", + stickyLimit: + "Round-robin sticky batch size: consecutive successful requests sent to one target before rotating to the next. Empty inherits the global Sticky Limit setting; 1 disables batching (pure one-request rotation).", failoverBeforeRetry: "When enabled, a 429 from the upstream triggers immediate target failover instead of retrying the same URL first.", targetTimeoutMs: @@ -2699,6 +2701,8 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders, combo if (config.concurrencyPerModel !== undefined) configToSave.concurrencyPerModel = config.concurrencyPerModel; if (config.queueTimeoutMs !== undefined) configToSave.queueTimeoutMs = config.queueTimeoutMs; + if (config.stickyRoundRobinLimit !== undefined) + configToSave.stickyRoundRobinLimit = config.stickyRoundRobinLimit; } const hasConfigToSave = Object.keys(configToSave).length > 0; const hadExistingConfig = Object.keys(sanitizeComboRuntimeConfig(combo?.config)).length > 0; @@ -3789,6 +3793,33 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders, combo className="w-full text-xs py-1.5 px-2 rounded border border-black/10 dark:border-white/10 bg-transparent focus:border-primary focus:outline-none" /> +
+ + + setConfig({ + ...config, + stickyRoundRobinLimit: e.target.value + ? Number(e.target.value) + : undefined, + }) + } + className="w-full text-xs py-1.5 px-2 rounded border border-black/10 dark:border-white/10 bg-transparent focus:border-primary focus:outline-none" + /> +
)} {strategy === "context-relay" && ( diff --git a/src/shared/validation/schemas/combo.ts b/src/shared/validation/schemas/combo.ts index 68fc4830be..3412dad6f2 100644 --- a/src/shared/validation/schemas/combo.ts +++ b/src/shared/validation/schemas/combo.ts @@ -14,7 +14,6 @@ import { } from "@/shared/constants/upstreamHeaders"; import { MAX_TIMER_TIMEOUT_MS } from "@/shared/utils/runtimeTimeouts"; - // ──── Combo Schemas ──── export const comboStepMetaSchema = { @@ -135,6 +134,10 @@ export const comboRuntimeConfigSchema = z queueTimeoutMs: z.coerce.number().int().min(1000).max(120000).optional(), // #3872: pre-cascade semaphore queue depth (round-robin). 0 = fail over immediately. queueDepth: z.coerce.number().int().min(0).max(100).optional(), + // Per-combo sticky round-robin batch size. When unset, handleRoundRobinCombo + // falls back to the global `settings.stickyRoundRobinLimit` so the existing + // knob still controls the default. 0 clamps to 1 (no batching) upstream. + stickyRoundRobinLimit: z.coerce.number().int().min(0).max(1000).optional(), healthCheckEnabled: z.boolean().optional(), healthCheckTimeoutMs: z.coerce.number().int().min(100).max(30000).optional(), handoffThreshold: z.coerce.number().min(0.5).max(0.94).optional(), @@ -312,4 +315,4 @@ export const reorderCombosSchema = z export const testComboSchema = z.object({ comboName: z.string().trim().min(1, "comboName is required"), -}); \ No newline at end of file +}); diff --git a/tests/unit/combo-config.test.ts b/tests/unit/combo-config.test.ts index 2c8187b2b9..f9e574a0e1 100644 --- a/tests/unit/combo-config.test.ts +++ b/tests/unit/combo-config.test.ts @@ -531,6 +531,24 @@ test("createComboSchema accepts failoverBeforeRetry, maxSetRetries and setRetryD assert.equal(parsed.config.setRetryDelayMs, 1500); }); +test("createComboSchema accepts per-combo stickyRoundRobinLimit and rejects out-of-range", () => { + const parsed = createComboSchema.parse({ + name: "sticky-override", + models: ["openai/gpt-4o-mini"], + strategy: "round-robin", + config: { stickyRoundRobinLimit: 2 }, + }); + assert.equal(parsed.config.stickyRoundRobinLimit, 2); + + const tooHigh = createComboSchema.safeParse({ + name: "sticky-too-high", + models: ["openai/gpt-4o-mini"], + strategy: "round-robin", + config: { stickyRoundRobinLimit: 1001 }, + }); + assert.equal(tooHigh.success, false); +}); + test("createComboSchema coerces string numbers for maxSetRetries and setRetryDelayMs", () => { const parsed = createComboSchema.parse({ name: "coerce-test", diff --git a/tests/unit/combo-strategy-fallbacks.test.ts b/tests/unit/combo-strategy-fallbacks.test.ts index f8ab628faa..f835829ac4 100644 --- a/tests/unit/combo-strategy-fallbacks.test.ts +++ b/tests/unit/combo-strategy-fallbacks.test.ts @@ -330,6 +330,32 @@ test("round-robin uses existing stickyRoundRobinLimit for combo target batching" ]); }); +test("per-combo stickyRoundRobinLimit overrides the global setting", async () => { + const calls: string[] = []; + const combo = { + name: "rr-per-combo-sticky-override", + strategy: "round-robin", + models: ["openai/a", "claude/b", "gemini/c"], + config: { maxRetries: 0, retryDelayMs: 0, fallbackDelayMs: 0, stickyRoundRobinLimit: 2 }, + }; + for (let i = 0; i < 4; i += 1) { + const result = await handleComboChat({ + body: {}, + combo, + handleSingleModel: async (_body: any, m: string) => { + calls.push(m); + return okResponse(); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: { stickyRoundRobinLimit: 3 }, + allCombos: null, + }); + assert.equal(result.ok, true); + } + assert.deepEqual(calls, ["openai/a", "openai/a", "claude/b", "claude/b"]); +}); + test("round-robin sticky batching fallback success becomes sticky target", async () => { const calls: string[] = []; const combo = {