fix(combos): make target timeout configurable (#2775)

Merge PR #2775 — fix(combos): make target timeout configurable
2026-07-26 09:52:11 +03:00 · 2026-05-27 04:54:00 -04:00
parent 07fd7f20cc
commit 744039403d
9 changed files with 178 additions and 15 deletions
--- a/docs/guides/USER_GUIDE.md
+++ b/docs/guides/USER_GUIDE.md
@@ -973,6 +973,9 @@ Configure per-combo balancing in **Dashboard → Combos → Create/Edit → Stra
 | **Cost-Optimized** | Routes to the cheapest available model (uses pricing table)              |

 Global combo defaults can be set in **Dashboard → Settings → Routing → Combo Defaults**.
+Combo target timeouts inherit the current request timeout by default. Use **Target timeout
+(seconds)** on combo defaults or an individual combo only when a shorter per-target limit should
+trigger faster fallback.

 ---

--- a/docs/reference/ENVIRONMENT.md
+++ b/docs/reference/ENVIRONMENT.md
@@ -549,6 +549,11 @@ REQUEST_TIMEOUT_MS (global override)
 | `OMNIROUTE_PPLX_TLS_TIMEOUT_MS`          | `30000`              | Wire-level timeout for the bogdanfinn/tls-client koffi binding (`perplexityTlsClient.ts`).  |
 | `OMNIROUTE_PPLX_TLS_GRACE_MS`            | `10000`              | JS-side grace added on top of the wire timeout when the native binding is wedged.           |

+Combo target attempts inherit the resolved upstream request timeout (`FETCH_TIMEOUT_MS`, or
+`REQUEST_TIMEOUT_MS` when it supplies the fetch default). Set `targetTimeoutMs` in a combo,
+combo defaults, or provider override only to make combo fallback faster; values above the
+current upstream timeout are capped to the upstream timeout.
+
 ### Circuit Breaker Thresholds

 Provider-level circuit breaker tuning. Defaults reflect the scaled values used since v3.6 for 500+ connections.
--- a/open-sse/services/combo.ts
+++ b/open-sse/services/combo.ts
@@ -14,7 +14,7 @@ import {
  isProviderFailureCode,
  isProviderExhaustedReason,
 } from "./accountFallback.ts";
-import { RateLimitReason } from "../config/constants.ts";
+import { FETCH_TIMEOUT_MS, RateLimitReason } from "../config/constants.ts";
 import { errorResponse, unavailableResponse } from "../utils/error.ts";
 import { clamp01 } from "../utils/number.ts";
 import {
@@ -23,7 +23,11 @@ import {
  recordComboShadowRequest,
  getComboMetrics,
 } from "./comboMetrics.ts";
-import { resolveComboConfig, getDefaultComboConfig } from "./comboConfig.ts";
+import {
+  resolveComboConfig,
+  getDefaultComboConfig,
+  resolveComboTargetTimeoutMs,
+} from "./comboConfig.ts";
 import {
  maybeGenerateHandoff,
  resolveContextRelayConfig,
@@ -106,7 +110,6 @@ function isAllAccountsRateLimitedResponse(
 const MAX_COMBO_DEPTH = 3;
 const MAX_FALLBACK_WAIT_MS = 5000;
 const MAX_GLOBAL_ATTEMPTS = 30;
-const COMBO_MODEL_TIMEOUT_MS = 30_000; // 30s per model attempt within a combo (default FETCH_TIMEOUT_MS=600s)

 function resolveDelayMs(value: unknown, fallback: number): number {
  const numericValue = Number(value);
@@ -2582,9 +2585,17 @@ export async function handleComboChat({
    : handleSingleModel;
  // ─────────────────────────────────────────────────────────────────────────

+  // Use config cascade before dispatch so all strategies, pinned context routes,
+  // and round-robin targets share the same timeout policy.
+  const config = settings
+    ? resolveComboConfig(combo, settings)
+    : { ...getDefaultComboConfig(), ...(combo.config || {}) };
+  const comboTargetTimeoutMs = resolveComboTargetTimeoutMs(config, FETCH_TIMEOUT_MS);
+
  // ── Per-model timeout wrapper ────────────────────────────────────────────
-  // Default FETCH_TIMEOUT_MS is 600s per model. For combos, we use a shorter
-  // per-model timeout so slow/hanging models don't block fallback.
+  // Combo target timeouts inherit FETCH_TIMEOUT_MS by default. Operators can
+  // configure targetTimeoutMs to shorten fallback latency, but never to extend
+  // beyond the current upstream request timeout.
  //
  // The timeoutController is forwarded to the inner caller via target.modelAbortSignal.
  // When the timeout fires we (a) resolve the race with a synthetic 524 and
@@ -2596,6 +2607,12 @@ export async function handleComboChat({
    modelStr: string,
    target?: SingleModelTarget
  ): Promise<Response> => {
+    if (comboTargetTimeoutMs <= 0) {
+      return handleSingleModelWrapped(b, modelStr, target).catch((err) =>
+        errorResponse(502, err?.message ?? "Upstream model error")
+      );
+    }
+
    const timeoutController = new AbortController();
    let timeoutId: ReturnType<typeof setTimeout> | undefined;
    let timedOut = false;
@@ -2604,7 +2621,7 @@ export async function handleComboChat({
        timedOut = true;
        log.warn(
          "COMBO",
-          `Model ${modelStr} exceeded ${COMBO_MODEL_TIMEOUT_MS}ms timeout — falling back`
+          `Model ${modelStr} exceeded ${comboTargetTimeoutMs}ms timeout — falling back`
        );
        // Abort the inner request so its upstream fetch is cancelled and
        // downstream cooldown/breaker/usage mutations don't continue mutating
@@ -2616,7 +2633,7 @@ export async function handleComboChat({
            headers: { "Content-Type": "application/json" },
          })
        );
-      }, COMBO_MODEL_TIMEOUT_MS);
+      }, comboTargetTimeoutMs);
    });
    const targetWithSignal = {
      ...(target ?? {}),
@@ -2663,10 +2680,6 @@ export async function handleComboChat({
    });
  }

-  // Use config cascade if settings provided
-  const config = settings
-    ? resolveComboConfig(combo, settings)
-    : { ...getDefaultComboConfig(), ...(combo.config || {}) };
  const maxRetries = config.maxRetries ?? 1;
  const retryDelayMs = resolveDelayMs(config.retryDelayMs, 2000);
  const fallbackDelayMs = resolveDelayMs(config.fallbackDelayMs, 0);
--- a/open-sse/services/comboConfig.ts
+++ b/open-sse/services/comboConfig.ts
@@ -5,6 +5,8 @@
 * Most specific wins.
 */

+import { MAX_TIMER_TIMEOUT_MS } from "../../src/shared/utils/runtimeTimeouts.ts";
+
 const DEFAULT_COMBO_CONFIG = {
  strategy: "priority",
  maxRetries: 1,
@@ -79,6 +81,26 @@ function isRecord(value: unknown): value is ComboConfigRecord {
  return !!value && typeof value === "object" && !Array.isArray(value);
 }

+function normalizePositiveTimeoutMs(value: unknown): number {
+  const numericValue = Number(value);
+  if (!Number.isFinite(numericValue) || numericValue <= 0) return 0;
+  return Math.min(Math.floor(numericValue), MAX_TIMER_TIMEOUT_MS);
+}
+
+export function resolveComboTargetTimeoutMs(
+  config: Record<string, unknown> | null | undefined,
+  upstreamTimeoutMs: number
+): number {
+  const inheritedTimeoutMs = normalizePositiveTimeoutMs(upstreamTimeoutMs);
+  const configuredTimeoutMs = isRecord(config)
+    ? normalizePositiveTimeoutMs(config.targetTimeoutMs)
+    : 0;
+
+  if (configuredTimeoutMs <= 0) return inheritedTimeoutMs;
+  if (inheritedTimeoutMs <= 0) return configuredTimeoutMs;
+  return Math.min(configuredTimeoutMs, inheritedTimeoutMs);
+}
+
 /**
 * Resolve effective config for a combo, applying cascade:
 *   DEFAULT_COMBO_CONFIG → settings.comboDefaults → settings.providerOverrides[provider] → combo.config
--- a/src/app/(dashboard)/dashboard/combos/page.tsx
+++ b/src/app/(dashboard)/dashboard/combos/page.tsx
@@ -159,6 +159,8 @@ const ADVANCED_FIELD_HELP_FALLBACK = {
    "How long a request can wait for a round-robin model slot before timing out. This queue is separate from any account-only concurrency cap.",
  failoverBeforeRetry:
    "When enabled, a 429 from the upstream triggers immediate target failover instead of retrying the same URL first.",
+  targetTimeoutMs:
+    "Optional combo target timeout. Empty inherits the current request timeout; larger values are capped to that timeout.",
  maxSetRetries:
    "Number of times to retry the full target set when every target fails. 0 = no set-level retry.",
  setRetryDelayMs:
@@ -170,6 +172,20 @@ const LEGACY_COMBO_RESILIENCE_KEYS = new Set([
  "healthCheckEnabled",
  "healthCheckTimeoutMs",
 ]);
+const MS_PER_SECOND = 1000;
+
+function msToOptionalSecondsInput(value) {
+  const ms = Number(value);
+  if (!Number.isFinite(ms) || ms <= 0) return "";
+  return String(Math.round(ms / MS_PER_SECOND));
+}
+
+function secondsInputToOptionalMs(value, maxSeconds = 86400) {
+  if (!value) return undefined;
+  const seconds = Number(value);
+  if (!Number.isFinite(seconds) || seconds <= 0) return undefined;
+  return Math.min(maxSeconds, Math.round(seconds)) * MS_PER_SECOND;
+}

 function sanitizeComboRuntimeConfig(config) {
  if (!config || typeof config !== "object") return {};
@@ -1429,10 +1445,12 @@ function StrategyRecommendationsPanel({ strategy, onApply, showNudge }) {
  );
 }

-function FieldLabelWithHelp({ label, help, showHelp = true }) {
+function FieldLabelWithHelp({ label, help, showHelp = true, htmlFor = undefined }) {
  return (
    <div className="flex items-center gap-1 mb-0.5">
-      <label className="text-[10px] text-text-muted">{label}</label>
+      <label htmlFor={htmlFor} className="text-[10px] text-text-muted">
+        {label}
+      </label>
      {showHelp && (
        <Tooltip position="bottom" content={help}>
          <span className="material-symbols-outlined text-[12px] text-text-muted cursor-help">
@@ -2691,7 +2709,9 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders, combo
        configToSave.concurrencyPerModel = config.concurrencyPerModel;
      if (config.queueTimeoutMs !== undefined) configToSave.queueTimeoutMs = config.queueTimeoutMs;
    }
-    if (Object.keys(configToSave).length > 0) {
+    const hasConfigToSave = Object.keys(configToSave).length > 0;
+    const hadExistingConfig = Object.keys(sanitizeComboRuntimeConfig(combo?.config)).length > 0;
+    if (hasConfigToSave || (isEdit && hadExistingConfig)) {
      saveData.config = configToSave;
    }

@@ -3566,6 +3586,34 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders, combo
                        className="w-full text-xs py-1.5 px-2 rounded border border-black/10 dark:border-white/10 bg-transparent focus:border-primary focus:outline-none"
                      />
                    </div>
+                    <div>
+                      <FieldLabelWithHelp
+                        label={getI18nOrFallback(t, "targetTimeout", "Target timeout (seconds)")}
+                        help={getI18nOrFallback(
+                          t,
+                          "advancedHelp.targetTimeoutMs",
+                          ADVANCED_FIELD_HELP_FALLBACK.targetTimeoutMs
+                        )}
+                        showHelp={!isExpertMode}
+                        htmlFor="combo-target-timeout-ms"
+                      />
+                      <input
+                        id="combo-target-timeout-ms"
+                        type="number"
+                        min="1"
+                        max="86400"
+                        step="1"
+                        value={msToOptionalSecondsInput(config.targetTimeoutMs)}
+                        placeholder={getI18nOrFallback(t, "inheritRequestTimeout", "inherit")}
+                        onChange={(e) =>
+                          setConfig({
+                            ...config,
+                            targetTimeoutMs: secondsInputToOptionalMs(e.target.value),
+                          })
+                        }
+                        className="w-full text-xs py-1.5 px-2 rounded border border-black/10 dark:border-white/10 bg-transparent focus:border-primary focus:outline-none"
+                      />
+                    </div>
                  </div>
                  {/* failoverBeforeRetry + maxSetRetries + setRetryDelayMs */}
                  <div className="grid grid-cols-2 gap-2 pt-2 border-t border-black/5 dark:border-white/5">
--- a/src/app/(dashboard)/dashboard/settings/components/ComboDefaultsTab.tsx
+++ b/src/app/(dashboard)/dashboard/settings/components/ComboDefaultsTab.tsx
@@ -27,12 +27,24 @@ function msToSeconds(value: unknown): number {
  return Math.round(ms / MS_PER_SECOND);
 }

+function msToOptionalSecondsInput(value: unknown): string {
+  const ms = Number(value);
+  if (!Number.isFinite(ms) || ms <= 0) return "";
+  return String(Math.round(ms / MS_PER_SECOND));
+}
+
 function secondsInputToMs(value: string, maxSeconds: number): number {
  const seconds = Number(value);
  if (!Number.isFinite(seconds) || seconds <= 0) return 0;
  return Math.min(maxSeconds, Math.round(seconds)) * MS_PER_SECOND;
 }

+function secondsInputToOptionalMs(value: string, maxSeconds = 86400): number | undefined {
+  const seconds = Number(value);
+  if (!Number.isFinite(seconds) || seconds <= 0) return undefined;
+  return Math.min(maxSeconds, Math.round(seconds)) * MS_PER_SECOND;
+}
+
 function translateOrFallback(
  t: ReturnType<typeof useTranslations>,
  key: string,
@@ -339,7 +351,30 @@ export default function ComboDefaultsTab() {
              className="text-sm"
            />
          ))}
+          <Input
+            label={translateOrFallback(t, "targetTimeout", "Target timeout (seconds)")}
+            type="number"
+            min={1}
+            max={86400}
+            step={1}
+            value={msToOptionalSecondsInput(comboDefaults.targetTimeoutMs)}
+            placeholder={translateOrFallback(t, "inheritRequestTimeout", "Inherit request timeout")}
+            onChange={(e) =>
+              setComboDefaults((prev) => ({
+                ...prev,
+                targetTimeoutMs: secondsInputToOptionalMs(e.target.value),
+              }))
+            }
+            className="text-sm"
+          />
        </div>
+        <p className="text-xs text-text-muted">
+          {translateOrFallback(
+            t,
+            "targetTimeoutHint",
+            "Combo targets inherit the current request timeout by default. Set a lower value here only when you want faster fallback."
+          )}
+        </p>

        <div className="grid grid-cols-1 gap-3 pt-3 border-t border-border/50">
          <div>
--- a/src/shared/utils/runtimeTimeouts.ts
+++ b/src/shared/utils/runtimeTimeouts.ts
@@ -8,6 +8,7 @@ type ReadTimeoutOptions = {

 export const DEFAULT_FETCH_TIMEOUT_MS = 600_000;
 export const DEFAULT_STREAM_IDLE_TIMEOUT_MS = 600_000;
+export const MAX_TIMER_TIMEOUT_MS = 2_147_483_647;
 export const DEFAULT_SSE_HEARTBEAT_INTERVAL_MS = 15_000;
 export const DEFAULT_STREAM_READINESS_TIMEOUT_MS = 80_000;
 export const DEFAULT_FETCH_CONNECT_TIMEOUT_MS = 30_000;
--- a/src/shared/validation/schemas.ts
+++ b/src/shared/validation/schemas.ts
@@ -9,6 +9,7 @@ import { COMBO_CONFIG_MODES } from "@/shared/constants/comboConfigMode";
 import { providerAllowsOptionalApiKey } from "@/shared/constants/providers";
 import { HIDEABLE_SIDEBAR_ITEM_IDS } from "@/shared/constants/sidebarVisibility";
 import { isForbiddenUpstreamHeaderName } from "@/shared/constants/upstreamHeaders";
+import { MAX_TIMER_TIMEOUT_MS } from "@/shared/utils/runtimeTimeouts";

 function isHttpUrl(value: string): boolean {
  try {
@@ -570,6 +571,7 @@ const comboRuntimeConfigSchema = z
    retryDelayMs: z.coerce.number().int().min(0).max(60000).optional(),
    fallbackDelayMs: z.coerce.number().int().min(0).max(60000).optional(),
    timeoutMs: z.coerce.number().int().min(1000).optional(),
+    targetTimeoutMs: z.coerce.number().int().min(0).max(MAX_TIMER_TIMEOUT_MS).optional(),
    concurrencyPerModel: z.coerce.number().int().min(1).max(20).optional(),
    queueTimeoutMs: z.coerce.number().int().min(1000).max(120000).optional(),
    healthCheckEnabled: z.boolean().optional(),
--- a/tests/unit/combo-config.test.ts
+++ b/tests/unit/combo-config.test.ts
@@ -1,10 +1,11 @@
 import test from "node:test";
 import assert from "node:assert/strict";

-const { resolveComboConfig, getDefaultComboConfig } =
+const { resolveComboConfig, getDefaultComboConfig, resolveComboTargetTimeoutMs } =
  await import("../../open-sse/services/comboConfig.ts");
 const { createComboSchema, updateComboDefaultsSchema } =
  await import("../../src/shared/validation/schemas.ts");
+const { MAX_TIMER_TIMEOUT_MS } = await import("../../src/shared/utils/runtimeTimeouts.ts");

 test("getDefaultComboConfig returns a fresh copy of the defaults", () => {
  const first = getDefaultComboConfig();
@@ -41,10 +42,12 @@ test("resolveComboConfig applies the full cascade from defaults to combo overrid
      comboDefaults: {
        strategy: "round-robin",
        timeoutMs: 120000,
+        targetTimeoutMs: 90000,
      },
      providerOverrides: {
        openai: {
          timeoutMs: 60000,
+          targetTimeoutMs: 45000,
          retryDelayMs: 500,
          fallbackDelayMs: 100,
        },
@@ -57,6 +60,7 @@ test("resolveComboConfig applies the full cascade from defaults to combo overrid
  assert.equal(result.retryDelayMs, 500);
  assert.equal(result.fallbackDelayMs, 100);
  assert.equal(result.maxRetries, 4);
+  assert.equal(result.targetTimeoutMs, 45000);
  assert.ok(!("timeoutMs" in result));
  assert.ok(!("healthCheckEnabled" in result));
 });
@@ -122,16 +126,46 @@ test("updateComboDefaultsSchema accepts arbitrarily large timeout defaults and p
  const parsed = updateComboDefaultsSchema.parse({
    comboDefaults: {
      timeoutMs: 3600000,
+      targetTimeoutMs: 30000,
    },
    providerOverrides: {
      anthropic: {
        timeoutMs: 5400000,
+        targetTimeoutMs: 45000,
      },
    },
  });

  assert.equal(parsed.comboDefaults.timeoutMs, 3600000);
+  assert.equal(parsed.comboDefaults.targetTimeoutMs, 30000);
  assert.equal(parsed.providerOverrides.anthropic.timeoutMs, 5400000);
+  assert.equal(parsed.providerOverrides.anthropic.targetTimeoutMs, 45000);
+});
+
+test("resolveComboTargetTimeoutMs inherits the upstream timeout and only shortens it", () => {
+  assert.equal(resolveComboTargetTimeoutMs({}, 600000), 600000);
+  assert.equal(resolveComboTargetTimeoutMs({ targetTimeoutMs: 30000 }, 600000), 30000);
+  assert.equal(resolveComboTargetTimeoutMs({ targetTimeoutMs: 900000 }, 600000), 600000);
+  assert.equal(resolveComboTargetTimeoutMs({ targetTimeoutMs: 0 }, 600000), 600000);
+  assert.equal(resolveComboTargetTimeoutMs({ targetTimeoutMs: 30000 }, 0), 30000);
+  assert.equal(resolveComboTargetTimeoutMs({}, 0), 0);
+  assert.equal(
+    resolveComboTargetTimeoutMs({ targetTimeoutMs: 999999999999 }, 0),
+    MAX_TIMER_TIMEOUT_MS
+  );
+  assert.equal(resolveComboTargetTimeoutMs({}, 999999999999), MAX_TIMER_TIMEOUT_MS);
+});
+
+test("combo timeout schema rejects values beyond the safe timer limit", () => {
+  const result = createComboSchema.safeParse({
+    name: "unsafe-timeout",
+    models: ["openai/gpt-4"],
+    config: {
+      targetTimeoutMs: MAX_TIMER_TIMEOUT_MS + 1,
+    },
+  });
+
+  assert.equal(result.success, false);
 });

 test("resolveComboConfig preserves explicit empty handoffProviders overrides", () => {