From 744039403d953e1a2a18ed6e4ebf5bdbf07f5a64 Mon Sep 17 00:00:00 2001 From: Randi <55005611+rdself@users.noreply.github.com> Date: Wed, 27 May 2026 04:54:00 -0400 Subject: [PATCH] fix(combos): make target timeout configurable (#2775) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Merge PR #2775 — fix(combos): make target timeout configurable --- docs/guides/USER_GUIDE.md | 3 ++ docs/reference/ENVIRONMENT.md | 5 ++ open-sse/services/combo.ts | 35 ++++++++---- open-sse/services/comboConfig.ts | 22 ++++++++ src/app/(dashboard)/dashboard/combos/page.tsx | 54 +++++++++++++++++-- .../settings/components/ComboDefaultsTab.tsx | 35 ++++++++++++ src/shared/utils/runtimeTimeouts.ts | 1 + src/shared/validation/schemas.ts | 2 + tests/unit/combo-config.test.ts | 36 ++++++++++++- 9 files changed, 178 insertions(+), 15 deletions(-) diff --git a/docs/guides/USER_GUIDE.md b/docs/guides/USER_GUIDE.md index b7dd44ca52..2cf8b50ff5 100644 --- a/docs/guides/USER_GUIDE.md +++ b/docs/guides/USER_GUIDE.md @@ -973,6 +973,9 @@ Configure per-combo balancing in **Dashboard → Combos → Create/Edit → Stra | **Cost-Optimized** | Routes to the cheapest available model (uses pricing table) | Global combo defaults can be set in **Dashboard → Settings → Routing → Combo Defaults**. +Combo target timeouts inherit the current request timeout by default. Use **Target timeout +(seconds)** on combo defaults or an individual combo only when a shorter per-target limit should +trigger faster fallback. --- diff --git a/docs/reference/ENVIRONMENT.md b/docs/reference/ENVIRONMENT.md index d4dc0b6b83..c7d86cf066 100644 --- a/docs/reference/ENVIRONMENT.md +++ b/docs/reference/ENVIRONMENT.md @@ -549,6 +549,11 @@ REQUEST_TIMEOUT_MS (global override) | `OMNIROUTE_PPLX_TLS_TIMEOUT_MS` | `30000` | Wire-level timeout for the bogdanfinn/tls-client koffi binding (`perplexityTlsClient.ts`). | | `OMNIROUTE_PPLX_TLS_GRACE_MS` | `10000` | JS-side grace added on top of the wire timeout when the native binding is wedged. | +Combo target attempts inherit the resolved upstream request timeout (`FETCH_TIMEOUT_MS`, or +`REQUEST_TIMEOUT_MS` when it supplies the fetch default). Set `targetTimeoutMs` in a combo, +combo defaults, or provider override only to make combo fallback faster; values above the +current upstream timeout are capped to the upstream timeout. + ### Circuit Breaker Thresholds Provider-level circuit breaker tuning. Defaults reflect the scaled values used since v3.6 for 500+ connections. diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index 2ed8c81fcb..743d0cb96d 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -14,7 +14,7 @@ import { isProviderFailureCode, isProviderExhaustedReason, } from "./accountFallback.ts"; -import { RateLimitReason } from "../config/constants.ts"; +import { FETCH_TIMEOUT_MS, RateLimitReason } from "../config/constants.ts"; import { errorResponse, unavailableResponse } from "../utils/error.ts"; import { clamp01 } from "../utils/number.ts"; import { @@ -23,7 +23,11 @@ import { recordComboShadowRequest, getComboMetrics, } from "./comboMetrics.ts"; -import { resolveComboConfig, getDefaultComboConfig } from "./comboConfig.ts"; +import { + resolveComboConfig, + getDefaultComboConfig, + resolveComboTargetTimeoutMs, +} from "./comboConfig.ts"; import { maybeGenerateHandoff, resolveContextRelayConfig, @@ -106,7 +110,6 @@ function isAllAccountsRateLimitedResponse( const MAX_COMBO_DEPTH = 3; const MAX_FALLBACK_WAIT_MS = 5000; const MAX_GLOBAL_ATTEMPTS = 30; -const COMBO_MODEL_TIMEOUT_MS = 30_000; // 30s per model attempt within a combo (default FETCH_TIMEOUT_MS=600s) function resolveDelayMs(value: unknown, fallback: number): number { const numericValue = Number(value); @@ -2582,9 +2585,17 @@ export async function handleComboChat({ : handleSingleModel; // ───────────────────────────────────────────────────────────────────────── + // Use config cascade before dispatch so all strategies, pinned context routes, + // and round-robin targets share the same timeout policy. + const config = settings + ? resolveComboConfig(combo, settings) + : { ...getDefaultComboConfig(), ...(combo.config || {}) }; + const comboTargetTimeoutMs = resolveComboTargetTimeoutMs(config, FETCH_TIMEOUT_MS); + // ── Per-model timeout wrapper ──────────────────────────────────────────── - // Default FETCH_TIMEOUT_MS is 600s per model. For combos, we use a shorter - // per-model timeout so slow/hanging models don't block fallback. + // Combo target timeouts inherit FETCH_TIMEOUT_MS by default. Operators can + // configure targetTimeoutMs to shorten fallback latency, but never to extend + // beyond the current upstream request timeout. // // The timeoutController is forwarded to the inner caller via target.modelAbortSignal. // When the timeout fires we (a) resolve the race with a synthetic 524 and @@ -2596,6 +2607,12 @@ export async function handleComboChat({ modelStr: string, target?: SingleModelTarget ): Promise => { + if (comboTargetTimeoutMs <= 0) { + return handleSingleModelWrapped(b, modelStr, target).catch((err) => + errorResponse(502, err?.message ?? "Upstream model error") + ); + } + const timeoutController = new AbortController(); let timeoutId: ReturnType | undefined; let timedOut = false; @@ -2604,7 +2621,7 @@ export async function handleComboChat({ timedOut = true; log.warn( "COMBO", - `Model ${modelStr} exceeded ${COMBO_MODEL_TIMEOUT_MS}ms timeout — falling back` + `Model ${modelStr} exceeded ${comboTargetTimeoutMs}ms timeout — falling back` ); // Abort the inner request so its upstream fetch is cancelled and // downstream cooldown/breaker/usage mutations don't continue mutating @@ -2616,7 +2633,7 @@ export async function handleComboChat({ headers: { "Content-Type": "application/json" }, }) ); - }, COMBO_MODEL_TIMEOUT_MS); + }, comboTargetTimeoutMs); }); const targetWithSignal = { ...(target ?? {}), @@ -2663,10 +2680,6 @@ export async function handleComboChat({ }); } - // Use config cascade if settings provided - const config = settings - ? resolveComboConfig(combo, settings) - : { ...getDefaultComboConfig(), ...(combo.config || {}) }; const maxRetries = config.maxRetries ?? 1; const retryDelayMs = resolveDelayMs(config.retryDelayMs, 2000); const fallbackDelayMs = resolveDelayMs(config.fallbackDelayMs, 0); diff --git a/open-sse/services/comboConfig.ts b/open-sse/services/comboConfig.ts index 86887f64d6..6c2c1a6994 100644 --- a/open-sse/services/comboConfig.ts +++ b/open-sse/services/comboConfig.ts @@ -5,6 +5,8 @@ * Most specific wins. */ +import { MAX_TIMER_TIMEOUT_MS } from "../../src/shared/utils/runtimeTimeouts.ts"; + const DEFAULT_COMBO_CONFIG = { strategy: "priority", maxRetries: 1, @@ -79,6 +81,26 @@ function isRecord(value: unknown): value is ComboConfigRecord { return !!value && typeof value === "object" && !Array.isArray(value); } +function normalizePositiveTimeoutMs(value: unknown): number { + const numericValue = Number(value); + if (!Number.isFinite(numericValue) || numericValue <= 0) return 0; + return Math.min(Math.floor(numericValue), MAX_TIMER_TIMEOUT_MS); +} + +export function resolveComboTargetTimeoutMs( + config: Record | null | undefined, + upstreamTimeoutMs: number +): number { + const inheritedTimeoutMs = normalizePositiveTimeoutMs(upstreamTimeoutMs); + const configuredTimeoutMs = isRecord(config) + ? normalizePositiveTimeoutMs(config.targetTimeoutMs) + : 0; + + if (configuredTimeoutMs <= 0) return inheritedTimeoutMs; + if (inheritedTimeoutMs <= 0) return configuredTimeoutMs; + return Math.min(configuredTimeoutMs, inheritedTimeoutMs); +} + /** * Resolve effective config for a combo, applying cascade: * DEFAULT_COMBO_CONFIG → settings.comboDefaults → settings.providerOverrides[provider] → combo.config diff --git a/src/app/(dashboard)/dashboard/combos/page.tsx b/src/app/(dashboard)/dashboard/combos/page.tsx index 409b86ad19..b204bca1e4 100644 --- a/src/app/(dashboard)/dashboard/combos/page.tsx +++ b/src/app/(dashboard)/dashboard/combos/page.tsx @@ -159,6 +159,8 @@ const ADVANCED_FIELD_HELP_FALLBACK = { "How long a request can wait for a round-robin model slot before timing out. This queue is separate from any account-only concurrency cap.", failoverBeforeRetry: "When enabled, a 429 from the upstream triggers immediate target failover instead of retrying the same URL first.", + targetTimeoutMs: + "Optional combo target timeout. Empty inherits the current request timeout; larger values are capped to that timeout.", maxSetRetries: "Number of times to retry the full target set when every target fails. 0 = no set-level retry.", setRetryDelayMs: @@ -170,6 +172,20 @@ const LEGACY_COMBO_RESILIENCE_KEYS = new Set([ "healthCheckEnabled", "healthCheckTimeoutMs", ]); +const MS_PER_SECOND = 1000; + +function msToOptionalSecondsInput(value) { + const ms = Number(value); + if (!Number.isFinite(ms) || ms <= 0) return ""; + return String(Math.round(ms / MS_PER_SECOND)); +} + +function secondsInputToOptionalMs(value, maxSeconds = 86400) { + if (!value) return undefined; + const seconds = Number(value); + if (!Number.isFinite(seconds) || seconds <= 0) return undefined; + return Math.min(maxSeconds, Math.round(seconds)) * MS_PER_SECOND; +} function sanitizeComboRuntimeConfig(config) { if (!config || typeof config !== "object") return {}; @@ -1429,10 +1445,12 @@ function StrategyRecommendationsPanel({ strategy, onApply, showNudge }) { ); } -function FieldLabelWithHelp({ label, help, showHelp = true }) { +function FieldLabelWithHelp({ label, help, showHelp = true, htmlFor = undefined }) { return (
- + {showHelp && ( @@ -2691,7 +2709,9 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders, combo configToSave.concurrencyPerModel = config.concurrencyPerModel; if (config.queueTimeoutMs !== undefined) configToSave.queueTimeoutMs = config.queueTimeoutMs; } - if (Object.keys(configToSave).length > 0) { + const hasConfigToSave = Object.keys(configToSave).length > 0; + const hadExistingConfig = Object.keys(sanitizeComboRuntimeConfig(combo?.config)).length > 0; + if (hasConfigToSave || (isEdit && hadExistingConfig)) { saveData.config = configToSave; } @@ -3566,6 +3586,34 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders, combo className="w-full text-xs py-1.5 px-2 rounded border border-black/10 dark:border-white/10 bg-transparent focus:border-primary focus:outline-none" />
+
+ + + setConfig({ + ...config, + targetTimeoutMs: secondsInputToOptionalMs(e.target.value), + }) + } + className="w-full text-xs py-1.5 px-2 rounded border border-black/10 dark:border-white/10 bg-transparent focus:border-primary focus:outline-none" + /> +
{/* failoverBeforeRetry + maxSetRetries + setRetryDelayMs */}
diff --git a/src/app/(dashboard)/dashboard/settings/components/ComboDefaultsTab.tsx b/src/app/(dashboard)/dashboard/settings/components/ComboDefaultsTab.tsx index 82d97fd0b7..6f73726865 100644 --- a/src/app/(dashboard)/dashboard/settings/components/ComboDefaultsTab.tsx +++ b/src/app/(dashboard)/dashboard/settings/components/ComboDefaultsTab.tsx @@ -27,12 +27,24 @@ function msToSeconds(value: unknown): number { return Math.round(ms / MS_PER_SECOND); } +function msToOptionalSecondsInput(value: unknown): string { + const ms = Number(value); + if (!Number.isFinite(ms) || ms <= 0) return ""; + return String(Math.round(ms / MS_PER_SECOND)); +} + function secondsInputToMs(value: string, maxSeconds: number): number { const seconds = Number(value); if (!Number.isFinite(seconds) || seconds <= 0) return 0; return Math.min(maxSeconds, Math.round(seconds)) * MS_PER_SECOND; } +function secondsInputToOptionalMs(value: string, maxSeconds = 86400): number | undefined { + const seconds = Number(value); + if (!Number.isFinite(seconds) || seconds <= 0) return undefined; + return Math.min(maxSeconds, Math.round(seconds)) * MS_PER_SECOND; +} + function translateOrFallback( t: ReturnType, key: string, @@ -339,7 +351,30 @@ export default function ComboDefaultsTab() { className="text-sm" /> ))} + + setComboDefaults((prev) => ({ + ...prev, + targetTimeoutMs: secondsInputToOptionalMs(e.target.value), + })) + } + className="text-sm" + />
+

+ {translateOrFallback( + t, + "targetTimeoutHint", + "Combo targets inherit the current request timeout by default. Set a lower value here only when you want faster fallback." + )} +

diff --git a/src/shared/utils/runtimeTimeouts.ts b/src/shared/utils/runtimeTimeouts.ts index 3e656fd9a0..446f151c4b 100644 --- a/src/shared/utils/runtimeTimeouts.ts +++ b/src/shared/utils/runtimeTimeouts.ts @@ -8,6 +8,7 @@ type ReadTimeoutOptions = { export const DEFAULT_FETCH_TIMEOUT_MS = 600_000; export const DEFAULT_STREAM_IDLE_TIMEOUT_MS = 600_000; +export const MAX_TIMER_TIMEOUT_MS = 2_147_483_647; export const DEFAULT_SSE_HEARTBEAT_INTERVAL_MS = 15_000; export const DEFAULT_STREAM_READINESS_TIMEOUT_MS = 80_000; export const DEFAULT_FETCH_CONNECT_TIMEOUT_MS = 30_000; diff --git a/src/shared/validation/schemas.ts b/src/shared/validation/schemas.ts index 1ae332ade8..142abfe1d1 100644 --- a/src/shared/validation/schemas.ts +++ b/src/shared/validation/schemas.ts @@ -9,6 +9,7 @@ import { COMBO_CONFIG_MODES } from "@/shared/constants/comboConfigMode"; import { providerAllowsOptionalApiKey } from "@/shared/constants/providers"; import { HIDEABLE_SIDEBAR_ITEM_IDS } from "@/shared/constants/sidebarVisibility"; import { isForbiddenUpstreamHeaderName } from "@/shared/constants/upstreamHeaders"; +import { MAX_TIMER_TIMEOUT_MS } from "@/shared/utils/runtimeTimeouts"; function isHttpUrl(value: string): boolean { try { @@ -570,6 +571,7 @@ const comboRuntimeConfigSchema = z retryDelayMs: z.coerce.number().int().min(0).max(60000).optional(), fallbackDelayMs: z.coerce.number().int().min(0).max(60000).optional(), timeoutMs: z.coerce.number().int().min(1000).optional(), + targetTimeoutMs: z.coerce.number().int().min(0).max(MAX_TIMER_TIMEOUT_MS).optional(), concurrencyPerModel: z.coerce.number().int().min(1).max(20).optional(), queueTimeoutMs: z.coerce.number().int().min(1000).max(120000).optional(), healthCheckEnabled: z.boolean().optional(), diff --git a/tests/unit/combo-config.test.ts b/tests/unit/combo-config.test.ts index acc15cc0ec..65600fdb0a 100644 --- a/tests/unit/combo-config.test.ts +++ b/tests/unit/combo-config.test.ts @@ -1,10 +1,11 @@ import test from "node:test"; import assert from "node:assert/strict"; -const { resolveComboConfig, getDefaultComboConfig } = +const { resolveComboConfig, getDefaultComboConfig, resolveComboTargetTimeoutMs } = await import("../../open-sse/services/comboConfig.ts"); const { createComboSchema, updateComboDefaultsSchema } = await import("../../src/shared/validation/schemas.ts"); +const { MAX_TIMER_TIMEOUT_MS } = await import("../../src/shared/utils/runtimeTimeouts.ts"); test("getDefaultComboConfig returns a fresh copy of the defaults", () => { const first = getDefaultComboConfig(); @@ -41,10 +42,12 @@ test("resolveComboConfig applies the full cascade from defaults to combo overrid comboDefaults: { strategy: "round-robin", timeoutMs: 120000, + targetTimeoutMs: 90000, }, providerOverrides: { openai: { timeoutMs: 60000, + targetTimeoutMs: 45000, retryDelayMs: 500, fallbackDelayMs: 100, }, @@ -57,6 +60,7 @@ test("resolveComboConfig applies the full cascade from defaults to combo overrid assert.equal(result.retryDelayMs, 500); assert.equal(result.fallbackDelayMs, 100); assert.equal(result.maxRetries, 4); + assert.equal(result.targetTimeoutMs, 45000); assert.ok(!("timeoutMs" in result)); assert.ok(!("healthCheckEnabled" in result)); }); @@ -122,16 +126,46 @@ test("updateComboDefaultsSchema accepts arbitrarily large timeout defaults and p const parsed = updateComboDefaultsSchema.parse({ comboDefaults: { timeoutMs: 3600000, + targetTimeoutMs: 30000, }, providerOverrides: { anthropic: { timeoutMs: 5400000, + targetTimeoutMs: 45000, }, }, }); assert.equal(parsed.comboDefaults.timeoutMs, 3600000); + assert.equal(parsed.comboDefaults.targetTimeoutMs, 30000); assert.equal(parsed.providerOverrides.anthropic.timeoutMs, 5400000); + assert.equal(parsed.providerOverrides.anthropic.targetTimeoutMs, 45000); +}); + +test("resolveComboTargetTimeoutMs inherits the upstream timeout and only shortens it", () => { + assert.equal(resolveComboTargetTimeoutMs({}, 600000), 600000); + assert.equal(resolveComboTargetTimeoutMs({ targetTimeoutMs: 30000 }, 600000), 30000); + assert.equal(resolveComboTargetTimeoutMs({ targetTimeoutMs: 900000 }, 600000), 600000); + assert.equal(resolveComboTargetTimeoutMs({ targetTimeoutMs: 0 }, 600000), 600000); + assert.equal(resolveComboTargetTimeoutMs({ targetTimeoutMs: 30000 }, 0), 30000); + assert.equal(resolveComboTargetTimeoutMs({}, 0), 0); + assert.equal( + resolveComboTargetTimeoutMs({ targetTimeoutMs: 999999999999 }, 0), + MAX_TIMER_TIMEOUT_MS + ); + assert.equal(resolveComboTargetTimeoutMs({}, 999999999999), MAX_TIMER_TIMEOUT_MS); +}); + +test("combo timeout schema rejects values beyond the safe timer limit", () => { + const result = createComboSchema.safeParse({ + name: "unsafe-timeout", + models: ["openai/gpt-4"], + config: { + targetTimeoutMs: MAX_TIMER_TIMEOUT_MS + 1, + }, + }); + + assert.equal(result.success, false); }); test("resolveComboConfig preserves explicit empty handoffProviders overrides", () => {