From a7c098e36cd3e648501f4bfafe3f0edbb8b5e28c Mon Sep 17 00:00:00 2001 From: "R.D." Date: Fri, 29 May 2026 15:51:44 -0400 Subject: [PATCH] Address zero-latency combo review feedback --- docs/guides/USER_GUIDE.md | 9 +- .../settings/components/ComboDefaultsTab.tsx | 2 +- src/shared/validation/schemas.ts | 24 ++++- tests/unit/combo-config.test.ts | 40 +++++++- tests/unit/combo-routing-engine.test.ts | 98 ++++++++++++++++++- 5 files changed, 162 insertions(+), 11 deletions(-) diff --git a/docs/guides/USER_GUIDE.md b/docs/guides/USER_GUIDE.md index 95442efe6e..bfb3c08368 100644 --- a/docs/guides/USER_GUIDE.md +++ b/docs/guides/USER_GUIDE.md @@ -981,10 +981,11 @@ Combo target timeouts inherit the current request timeout by default. Use **Targ (seconds)** on combo defaults or an individual combo only when a shorter per-target limit should trigger faster fallback. -Zero-latency combo optimizations are opt-in. Leave **Zero-latency optimizations** disabled when -fallback targets must receive the original request body exactly; enabling it allows configured -hedging, predictive TTFT skips, and proactive fallback compression to trade request fidelity for -lower tail latency. +Zero-latency combo optimizations are opt-in. Leave **Zero-latency optimizations** disabled to +prevent these latency features from racing fallback targets, skipping targets based on TTFT +history, or compressing fallback requests; enabling it allows configured hedging, predictive TTFT +skips, and proactive fallback compression to trade routing/request fidelity for lower tail +latency. --- diff --git a/src/app/(dashboard)/dashboard/settings/components/ComboDefaultsTab.tsx b/src/app/(dashboard)/dashboard/settings/components/ComboDefaultsTab.tsx index 7afc912c45..7e8c1537fd 100644 --- a/src/app/(dashboard)/dashboard/settings/components/ComboDefaultsTab.tsx +++ b/src/app/(dashboard)/dashboard/settings/components/ComboDefaultsTab.tsx @@ -565,7 +565,7 @@ export default function ComboDefaultsTab() { {translateOrFallback( t, "zeroLatencyOptimizationsDesc", - "Opt in to hedging, predictive TTFT skips, and proactive fallback compression. Leave off to preserve request bodies exactly." + "Opt in to hedging, predictive TTFT skips, and proactive fallback compression. Leave off to prevent these latency features from racing targets or compressing fallback requests." )}

diff --git a/src/shared/validation/schemas.ts b/src/shared/validation/schemas.ts index 1687034ece..2965c1d8a4 100644 --- a/src/shared/validation/schemas.ts +++ b/src/shared/validation/schemas.ts @@ -660,7 +660,29 @@ const comboRuntimeConfigSchema = z shadowRouting: shadowRoutingSchema.optional(), evalRouting: evalRoutingSchema.optional(), }) - .strict(); + .strict() + .superRefine((config, ctx) => { + if (config.zeroLatencyOptimizationsEnabled === true) return; + + const addZeroLatencyIssue = (path: string[]) => { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: + "zeroLatencyOptimizationsEnabled must be true to enable zero-latency combo features", + path, + }); + }; + + if (config.hedging === true) { + addZeroLatencyIssue(["hedging"]); + } + if (typeof config.predictiveTtftMs === "number" && config.predictiveTtftMs > 0) { + addZeroLatencyIssue(["predictiveTtftMs"]); + } + if (config.fallbackCompressionMode && config.fallbackCompressionMode !== "off") { + addZeroLatencyIssue(["fallbackCompressionMode"]); + } + }); const comboNameSchema = z .string() diff --git a/tests/unit/combo-config.test.ts b/tests/unit/combo-config.test.ts index 3c0bb73c78..69eac65af2 100644 --- a/tests/unit/combo-config.test.ts +++ b/tests/unit/combo-config.test.ts @@ -155,7 +155,7 @@ test("combo config schema accepts explicit zero-latency opt-in controls", () => zeroLatencyOptimizationsEnabled: true, hedging: true, hedgeDelayMs: 250, - fallbackCompressionMode: "off", + fallbackCompressionMode: "lite", fallbackCompressionThreshold: 2500, predictiveTtftMs: 1800, }, @@ -164,11 +164,47 @@ test("combo config schema accepts explicit zero-latency opt-in controls", () => assert.equal(parsed.config.zeroLatencyOptimizationsEnabled, true); assert.equal(parsed.config.hedging, true); assert.equal(parsed.config.hedgeDelayMs, 250); - assert.equal(parsed.config.fallbackCompressionMode, "off"); + assert.equal(parsed.config.fallbackCompressionMode, "lite"); assert.equal(parsed.config.fallbackCompressionThreshold, 2500); assert.equal(parsed.config.predictiveTtftMs, 1800); }); +test("combo config schema rejects enabled zero-latency subfeatures without opt-in", () => { + const result = createComboSchema.safeParse({ + name: "zero-latency-noop", + models: ["openai/gpt-4o-mini", "anthropic/claude-3-haiku"], + config: { + hedging: true, + fallbackCompressionMode: "lite", + predictiveTtftMs: 1800, + }, + }); + + assert.equal(result.success, false); + assert.deepEqual( + result.error.issues.map((issue) => issue.path.join(".")), + ["config.hedging", "config.predictiveTtftMs", "config.fallbackCompressionMode"] + ); +}); + +test("combo config schema allows zero-latency tuning fields when subfeatures stay disabled", () => { + const parsed = createComboSchema.parse({ + name: "zero-latency-disabled-tuning", + models: ["openai/gpt-4o-mini", "anthropic/claude-3-haiku"], + config: { + hedgeDelayMs: 250, + fallbackCompressionMode: "off", + fallbackCompressionThreshold: 2500, + predictiveTtftMs: 0, + }, + }); + + assert.equal(parsed.config.hedgeDelayMs, 250); + assert.equal(parsed.config.fallbackCompressionMode, "off"); + assert.equal(parsed.config.fallbackCompressionThreshold, 2500); + assert.equal(parsed.config.predictiveTtftMs, 0); +}); + test("resolveComboTargetTimeoutMs inherits the upstream timeout and only shortens it", () => { assert.equal(resolveComboTargetTimeoutMs({}, 600000), 600000); assert.equal(resolveComboTargetTimeoutMs({ targetTimeoutMs: 30000 }, 600000), 30000); diff --git a/tests/unit/combo-routing-engine.test.ts b/tests/unit/combo-routing-engine.test.ts index 800d2c53d8..bfc730bebb 100644 --- a/tests/unit/combo-routing-engine.test.ts +++ b/tests/unit/combo-routing-engine.test.ts @@ -1346,7 +1346,12 @@ test("handleComboChat preserves fallback request bodies when zero-latency optimi name: "zero-latency-disabled-preserves-body", strategy: "priority", models: ["provider-a/model-a", "provider-b/model-b"], - config: { maxRetries: 0, retryDelayMs: 1, fallbackCompressionThreshold: 1 }, + config: { + maxRetries: 0, + retryDelayMs: 1, + fallbackCompressionMode: "lite", + fallbackCompressionThreshold: 1, + }, }, handleSingleModel: async (requestBody: any, modelStr: any) => { if (modelStr === "provider-a/model-a") { @@ -1385,6 +1390,7 @@ test("handleComboChat applies fallback compression only after explicit zero-late maxRetries: 0, retryDelayMs: 1, zeroLatencyOptimizationsEnabled: true, + fallbackCompressionMode: "lite", fallbackCompressionThreshold: 1, }, }, @@ -1403,8 +1409,94 @@ test("handleComboChat applies fallback compression only after explicit zero-late }); assert.equal(result.status, 200); - assert.notEqual(fallbackBody.messages[1].content, longToolOutput); - assert.match(fallbackBody.messages[1].content, /\.\.\.\[truncated\]$/); + const fallbackToolContent = fallbackBody.messages[1].content; + assert.equal(typeof fallbackToolContent, "string"); + assert.ok(fallbackToolContent.length < longToolOutput.length); + assert.match(fallbackToolContent, /truncated/i); +}); + +test("handleComboChat suppresses hedging unless zero-latency optimizations are enabled", async () => { + const calls: any[] = []; + + const result = await handleComboChat({ + body: {}, + combo: { + name: "hedging-disabled-with-subfeature-set", + strategy: "priority", + models: ["model-a", "model-b"], + config: { + maxRetries: 0, + retryDelayMs: 1, + hedging: true, + hedgeDelayMs: 1, + }, + }, + handleSingleModel: async (_body: any, modelStr: any) => { + calls.push(modelStr); + await new Promise((resolve) => setTimeout(resolve, 20)); + return okResponse({ choices: [{ message: { content: modelStr } }] }); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: null, + relayOptions: null as any, + allCombos: null, + }); + + const payload = (await result.json()) as any; + + assert.equal(result.status, 200); + assert.equal(payload.choices[0].message.content, "model-a"); + assert.deepEqual(calls, ["model-a"]); +}); + +test("handleComboChat starts hedged fallback only after explicit zero-latency opt-in", async () => { + const calls: any[] = []; + + const result = await handleComboChat({ + body: {}, + combo: { + name: "hedging-enabled-with-zero-latency", + strategy: "priority", + models: ["model-a", "model-b"], + config: { + maxRetries: 0, + retryDelayMs: 1, + zeroLatencyOptimizationsEnabled: true, + hedging: true, + hedgeDelayMs: 1, + }, + }, + handleSingleModel: async (_body: any, modelStr: any, target: any) => { + calls.push(modelStr); + if (modelStr === "model-a") { + await new Promise((resolve) => { + const timer = setTimeout(resolve, 100); + target?.modelAbortSignal?.addEventListener( + "abort", + () => { + clearTimeout(timer); + resolve(undefined); + }, + { once: true } + ); + }); + return okResponse({ choices: [{ message: { content: "slow" } }] }); + } + return okResponse({ choices: [{ message: { content: "fast" } }] }); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: null, + relayOptions: null as any, + allCombos: null, + }); + + const payload = (await result.json()) as any; + + assert.equal(result.status, 200); + assert.equal(payload.choices[0].message.content, "fast"); + assert.deepEqual(calls, ["model-a", "model-b"]); }); test("handleComboChat round-robin falls through generic 400s when a later model succeeds", async () => {