diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index 3ff14b78e9..699cac5199 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -72,6 +72,12 @@ const MAX_COMBO_DEPTH = 3; const MAX_FALLBACK_WAIT_MS = 5000; const MAX_GLOBAL_ATTEMPTS = 30; +function resolveDelayMs(value: unknown, fallback: number): number { + const numericValue = Number(value); + if (!Number.isFinite(numericValue) || numericValue < 0) return fallback; + return numericValue; +} + function comboModelNotFoundResponse(message: string) { return errorResponse(404, message); } @@ -1659,7 +1665,8 @@ export async function handleComboChat({ ? resolveComboConfig(combo, settings) : { ...getDefaultComboConfig(), ...(combo.config || {}) }; const maxRetries = config.maxRetries ?? 1; - const retryDelayMs = config.retryDelayMs ?? 2000; + const retryDelayMs = resolveDelayMs(config.retryDelayMs, 2000); + const fallbackDelayMs = resolveDelayMs(config.fallbackDelayMs, 0); let orderedTargets = strategy === "weighted" @@ -2027,17 +2034,19 @@ export async function handleComboChat({ // Record last known good provider (LKGP) for this combo/model (#919) if (provider) { - try { - const { setLKGP } = await import("../../src/lib/localDb"); - await Promise.all([ - setLKGP(combo.name, target.executionKey, provider), - setLKGP(combo.name, combo.id || combo.name, provider), - ]); - } catch (err) { - log.warn("COMBO", "Failed to record Last Known Good Provider. This is non-fatal.", { - err, - }); - } + void (async () => { + try { + const { setLKGP } = await import("../../src/lib/localDb"); + await Promise.all([ + setLKGP(combo.name, target.executionKey, provider), + setLKGP(combo.name, combo.id || combo.name, provider), + ]); + } catch (err) { + log.warn("COMBO", "Failed to record Last Known Good Provider. This is non-fatal.", { + err, + }); + } + })(); } return quality.clonedResponse ?? result; @@ -2141,8 +2150,8 @@ export async function handleComboChat({ log.warn("COMBO", `Model ${modelStr} failed, trying next`, { status: result.status }); const fallbackWaitMs = - retryDelayMs > 0 && cooldownMs > 0 && cooldownMs <= MAX_FALLBACK_WAIT_MS - ? Math.min(cooldownMs, retryDelayMs) + fallbackDelayMs > 0 && cooldownMs > 0 && cooldownMs <= MAX_FALLBACK_WAIT_MS + ? Math.min(cooldownMs, fallbackDelayMs) : 0; if ([502, 503, 504].includes(result.status) && fallbackWaitMs > 0) { log.info("COMBO", `Waiting ${fallbackWaitMs}ms before fallback to next model`); @@ -2229,7 +2238,8 @@ async function handleRoundRobinCombo({ const concurrency = config.concurrencyPerModel ?? 3; const queueTimeout = config.queueTimeoutMs ?? 30000; const maxRetries = config.maxRetries ?? 1; - const retryDelayMs = config.retryDelayMs ?? 2000; + const retryDelayMs = resolveDelayMs(config.retryDelayMs, 2000); + const fallbackDelayMs = resolveDelayMs(config.fallbackDelayMs, 0); const orderedTargets = resolveComboTargets(combo, allCombos); const filteredTargets = await applyRequestTagRouting(orderedTargets, body, log); @@ -2354,21 +2364,23 @@ async function handleRoundRobinCombo({ }); recordedAttempts++; if (provider) { - try { - const { setLKGP } = await import("../../src/lib/localDb"); - await Promise.all([ - setLKGP(combo.name, target.executionKey, provider), - setLKGP(combo.name, combo.id || combo.name, provider), - ]); - } catch (err) { - log.warn( - "COMBO-RR", - "Failed to record Last Known Good Provider. This is non-fatal.", - { - err, - } - ); - } + void (async () => { + try { + const { setLKGP } = await import("../../src/lib/localDb"); + await Promise.all([ + setLKGP(combo.name, target.executionKey, provider), + setLKGP(combo.name, combo.id || combo.name, provider), + ]); + } catch (err) { + log.warn( + "COMBO-RR", + "Failed to record Last Known Good Provider. This is non-fatal.", + { + err, + } + ); + } + })(); } return result; } @@ -2488,8 +2500,8 @@ async function handleRoundRobinCombo({ log.warn("COMBO-RR", `${modelStr} failed, trying next model`, { status: result.status }); const fallbackWaitMs = - retryDelayMs > 0 && cooldownMs > 0 && cooldownMs <= MAX_FALLBACK_WAIT_MS - ? Math.min(cooldownMs, retryDelayMs) + fallbackDelayMs > 0 && cooldownMs > 0 && cooldownMs <= MAX_FALLBACK_WAIT_MS + ? Math.min(cooldownMs, fallbackDelayMs) : 0; if ([502, 503, 504].includes(result.status) && fallbackWaitMs > 0) { log.info("COMBO-RR", `Waiting ${fallbackWaitMs}ms before fallback to next model`); diff --git a/open-sse/services/comboConfig.ts b/open-sse/services/comboConfig.ts index fa383c824e..d3e04daff3 100644 --- a/open-sse/services/comboConfig.ts +++ b/open-sse/services/comboConfig.ts @@ -9,6 +9,7 @@ const DEFAULT_COMBO_CONFIG = { strategy: "priority", maxRetries: 1, retryDelayMs: 2000, + fallbackDelayMs: 0, concurrencyPerModel: 3, // max simultaneous requests per model (round-robin) queueTimeoutMs: 30000, // max wait time in semaphore queue (round-robin) handoffThreshold: 0.85, diff --git a/src/app/api/settings/combo-defaults/route.ts b/src/app/api/settings/combo-defaults/route.ts index 5909d26d04..1840d5591e 100644 --- a/src/app/api/settings/combo-defaults/route.ts +++ b/src/app/api/settings/combo-defaults/route.ts @@ -49,6 +49,7 @@ export async function GET(request: Request) { strategy: "priority", maxRetries: 1, retryDelayMs: 2000, + fallbackDelayMs: 0, handoffThreshold: 0.85, handoffModel: "", maxMessagesForSummary: 30, diff --git a/src/shared/validation/schemas.ts b/src/shared/validation/schemas.ts index c0f30dc7d9..4548e3a784 100644 --- a/src/shared/validation/schemas.ts +++ b/src/shared/validation/schemas.ts @@ -375,6 +375,7 @@ const comboRuntimeConfigSchema = z strategy: comboStrategySchema.optional(), maxRetries: z.coerce.number().int().min(0).max(10).optional(), retryDelayMs: z.coerce.number().int().min(0).max(60000).optional(), + fallbackDelayMs: z.coerce.number().int().min(0).max(60000).optional(), timeoutMs: z.coerce.number().int().min(1000).optional(), concurrencyPerModel: z.coerce.number().int().min(1).max(20).optional(), queueTimeoutMs: z.coerce.number().int().min(1000).max(120000).optional(), diff --git a/src/sse/handlers/chat.ts b/src/sse/handlers/chat.ts index de643574ef..b2203786ed 100644 --- a/src/sse/handlers/chat.ts +++ b/src/sse/handlers/chat.ts @@ -26,7 +26,7 @@ import { import * as log from "../utils/logger"; import { checkAndRefreshToken } from "../services/tokenRefresh"; import { deleteHandoff, getHandoff } from "@/lib/db/contextHandoffs"; -import { getCachedSettings, getSettings, getCombos } from "@/lib/localDb"; +import { getCachedSettings, getCombos } from "@/lib/localDb"; import { ensureOpenAIStoreSessionFallback, isOpenAIResponsesStoreEnabled, @@ -91,6 +91,21 @@ registerBailianCodingPlanQuotaFetcher(); // opt-in) when the active bucket reaches zero. registerCrofUsageFetcher(); +let combosCachePromise: Promise | null = null; +let combosCacheTs = 0; +const COMBOS_CACHE_TTL_MS = 10_000; + +async function getCombosCachedForChat(): Promise { + const now = Date.now(); + if (combosCachePromise && now - combosCacheTs < COMBOS_CACHE_TTL_MS) { + return combosCachePromise; + } + + combosCacheTs = now; + combosCachePromise = getCombos().catch(() => []); + return combosCachePromise; +} + function normalizeAllowedConnectionIds(value: unknown): string[] | null { if (!Array.isArray(value)) return null; const ids = value.filter( @@ -336,8 +351,8 @@ export async function handleChat(request: any, clientRawRequest: any = null) { // Fetch settings and all combos for config cascade and nested resolution const [settings, allCombos] = await Promise.all([ - getSettings().catch(() => ({})), - getCombos().catch(() => []), + getCachedSettings().catch(() => ({})), + getCombosCachedForChat(), ]); const relayConfig = combo.strategy === "context-relay" ? resolveComboConfig(combo, settings) : null; diff --git a/src/sse/services/model.ts b/src/sse/services/model.ts index 55cbf9cb84..e1f2b64d23 100644 --- a/src/sse/services/model.ts +++ b/src/sse/services/model.ts @@ -1,6 +1,6 @@ // Re-export from open-sse with localDb integration import { getModelAliases, getComboByName, getProviderNodes, getCustomModels } from "@/lib/localDb"; -import { getSettings } from "@/lib/localDb"; +import { getCachedSettings } from "@/lib/localDb"; import { getComboStepTarget } from "@/lib/combos/steps"; import { parseModel, @@ -83,7 +83,7 @@ export async function getModelInfo(modelStr) { // stripModelPrefix: if enabled, strip provider prefix and re-resolve // the bare model name using existing heuristics (claude-* → anthropic, etc.) try { - const settings = await getSettings(); + const settings = await getCachedSettings(); if (settings.stripModelPrefix === true) { const strippedResult = await getModelInfoCore(parsed.model, getModelAliases); return { ...strippedResult, extendedContext }; diff --git a/src/types/combo.ts b/src/types/combo.ts index 9fbfa87cd4..b5befd8e57 100644 --- a/src/types/combo.ts +++ b/src/types/combo.ts @@ -10,6 +10,7 @@ export interface Combo { nodes: ComboNode[]; maxRetries: number; retryDelayMs: number; + fallbackDelayMs?: number; timeoutMs: number; healthCheckEnabled: boolean; createdAt: string; diff --git a/src/types/settings.ts b/src/types/settings.ts index 3a4c5ad202..0fb9956f68 100644 --- a/src/types/settings.ts +++ b/src/types/settings.ts @@ -32,6 +32,7 @@ export interface ComboDefaults { strategy: RoutingStrategyValue; maxRetries: number; retryDelayMs: number; + fallbackDelayMs?: number; maxComboDepth: number; trackMetrics: boolean; concurrencyPerModel?: number; diff --git a/tests/unit/combo-499-abort.test.ts b/tests/unit/combo-499-abort.test.ts index 0333fa72ac..58455d9511 100644 --- a/tests/unit/combo-499-abort.test.ts +++ b/tests/unit/combo-499-abort.test.ts @@ -133,7 +133,7 @@ test("signal abort during fallback wait interrupts immediately", async () => { combo: makeCombo("priority", ["a/m1", "b/m2"]), handleSingleModel, log, - settings: { retryDelayMs: 5000 }, // 5s delay would normally be slow + settings: { fallbackDelayMs: 5000 }, // 5s delay would normally be slow allCombos: [], signal: ac.signal, }); diff --git a/tests/unit/combo-config.test.ts b/tests/unit/combo-config.test.ts index 0c39a25993..abaa924f23 100644 --- a/tests/unit/combo-config.test.ts +++ b/tests/unit/combo-config.test.ts @@ -14,6 +14,7 @@ test("getDefaultComboConfig returns a fresh copy of the defaults", () => { assert.equal(first.strategy, "priority"); assert.equal(first.maxRetries, 1); assert.equal(first.retryDelayMs, 2000); + assert.equal(first.fallbackDelayMs, 0); assert.ok(!("timeoutMs" in first)); assert.ok(!("healthCheckEnabled" in first)); assert.equal(first.handoffThreshold, 0.85); @@ -40,6 +41,7 @@ test("resolveComboConfig applies the full cascade from defaults to combo overrid openai: { timeoutMs: 60000, retryDelayMs: 500, + fallbackDelayMs: 100, }, }, }, @@ -48,6 +50,7 @@ test("resolveComboConfig applies the full cascade from defaults to combo overrid assert.equal(result.strategy, "round-robin"); assert.equal(result.retryDelayMs, 500); + assert.equal(result.fallbackDelayMs, 100); assert.equal(result.maxRetries, 4); assert.ok(!("timeoutMs" in result)); assert.ok(!("healthCheckEnabled" in result));