pjandro/OmniRoute
1
0
Fork 0
mirror of https://github.com/diegosouzapw/OmniRoute.git synced 2026-07-26 09:52:11 +03:00
Code Issues Packages Projects Releases Wiki Activity

feat: add fallbackDelayMs to combo configuration and related settings

Browse Source
This commit is contained in:
Jan Leon
2026-05-08 19:21:34 +00:00
parent 23ec2f9fa8
commit 43553646ed
10 changed files with 73 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
open-sse/services/combo.ts
View File

@@ -72,6 +72,12 @@ const MAX_COMBO_DEPTH = 3;
const MAX_FALLBACK_WAIT_MS = 5000;
const MAX_GLOBAL_ATTEMPTS = 30;
function resolveDelayMs(value: unknown, fallback: number): number {
const numericValue = Number(value);
if (!Number.isFinite(numericValue) || numericValue < 0) return fallback;
return numericValue;
}
function comboModelNotFoundResponse(message: string) {
return errorResponse(404, message);
}
@@ -1659,7 +1665,8 @@ export async function handleComboChat({
? resolveComboConfig(combo, settings)
: { ...getDefaultComboConfig(), ...(combo.config || {}) };
const maxRetries = config.maxRetries ?? 1;
const retryDelayMs = config.retryDelayMs ?? 2000;
const retryDelayMs = resolveDelayMs(config.retryDelayMs, 2000);
const fallbackDelayMs = resolveDelayMs(config.fallbackDelayMs, 0);
let orderedTargets =
strategy === "weighted"
@@ -2027,17 +2034,19 @@ export async function handleComboChat({
// Record last known good provider (LKGP) for this combo/model (#919)
if (provider) {
try {
const { setLKGP } = await import("../../src/lib/localDb");
await Promise.all([
setLKGP(combo.name, target.executionKey, provider),
setLKGP(combo.name, combo.id || combo.name, provider),
]);
} catch (err) {
log.warn("COMBO", "Failed to record Last Known Good Provider. This is non-fatal.", {
err,
});
}
void (async () => {
try {
const { setLKGP } = await import("../../src/lib/localDb");
await Promise.all([
setLKGP(combo.name, target.executionKey, provider),
setLKGP(combo.name, combo.id || combo.name, provider),
]);
} catch (err) {
log.warn("COMBO", "Failed to record Last Known Good Provider. This is non-fatal.", {
err,
});
}
})();
}
return quality.clonedResponse ?? result;
@@ -2141,8 +2150,8 @@ export async function handleComboChat({
log.warn("COMBO", `Model ${modelStr} failed, trying next`, { status: result.status });
const fallbackWaitMs =
retryDelayMs > 0 && cooldownMs > 0 && cooldownMs <= MAX_FALLBACK_WAIT_MS
? Math.min(cooldownMs, retryDelayMs)
fallbackDelayMs > 0 && cooldownMs > 0 && cooldownMs <= MAX_FALLBACK_WAIT_MS
? Math.min(cooldownMs, fallbackDelayMs)
: 0;
if ([502, 503, 504].includes(result.status) && fallbackWaitMs > 0) {
log.info("COMBO", `Waiting ${fallbackWaitMs}ms before fallback to next model`);
@@ -2229,7 +2238,8 @@ async function handleRoundRobinCombo({
const concurrency = config.concurrencyPerModel ?? 3;
const queueTimeout = config.queueTimeoutMs ?? 30000;
const maxRetries = config.maxRetries ?? 1;
const retryDelayMs = config.retryDelayMs ?? 2000;
const retryDelayMs = resolveDelayMs(config.retryDelayMs, 2000);
const fallbackDelayMs = resolveDelayMs(config.fallbackDelayMs, 0);
const orderedTargets = resolveComboTargets(combo, allCombos);
const filteredTargets = await applyRequestTagRouting(orderedTargets, body, log);
@@ -2354,21 +2364,23 @@ async function handleRoundRobinCombo({
});
recordedAttempts++;
if (provider) {
try {
const { setLKGP } = await import("../../src/lib/localDb");
await Promise.all([
setLKGP(combo.name, target.executionKey, provider),
setLKGP(combo.name, combo.id || combo.name, provider),
]);
} catch (err) {
log.warn(
"COMBO-RR",
"Failed to record Last Known Good Provider. This is non-fatal.",
{
err,
}
);
}
void (async () => {
try {
const { setLKGP } = await import("../../src/lib/localDb");
await Promise.all([
setLKGP(combo.name, target.executionKey, provider),
setLKGP(combo.name, combo.id || combo.name, provider),
]);
} catch (err) {
log.warn(
"COMBO-RR",
"Failed to record Last Known Good Provider. This is non-fatal.",
{
err,
}
);
}
})();
}
return result;
}
@@ -2488,8 +2500,8 @@ async function handleRoundRobinCombo({
log.warn("COMBO-RR", `${modelStr} failed, trying next model`, { status: result.status });
const fallbackWaitMs =
retryDelayMs > 0 && cooldownMs > 0 && cooldownMs <= MAX_FALLBACK_WAIT_MS
? Math.min(cooldownMs, retryDelayMs)
fallbackDelayMs > 0 && cooldownMs > 0 && cooldownMs <= MAX_FALLBACK_WAIT_MS
? Math.min(cooldownMs, fallbackDelayMs)
: 0;
if ([502, 503, 504].includes(result.status) && fallbackWaitMs > 0) {
log.info("COMBO-RR", `Waiting ${fallbackWaitMs}ms before fallback to next model`);

1
open-sse/services/comboConfig.ts
View File

@@ -9,6 +9,7 @@ const DEFAULT_COMBO_CONFIG = {
strategy: "priority",
maxRetries: 1,
retryDelayMs: 2000,
fallbackDelayMs: 0,
concurrencyPerModel: 3, // max simultaneous requests per model (round-robin)
queueTimeoutMs: 30000, // max wait time in semaphore queue (round-robin)
handoffThreshold: 0.85,

1
src/app/api/settings/combo-defaults/route.ts
View File

@@ -49,6 +49,7 @@ export async function GET(request: Request) {
strategy: "priority",
maxRetries: 1,
retryDelayMs: 2000,
fallbackDelayMs: 0,
handoffThreshold: 0.85,
handoffModel: "",
maxMessagesForSummary: 30,

1
src/shared/validation/schemas.ts
View File

@@ -375,6 +375,7 @@ const comboRuntimeConfigSchema = z
strategy: comboStrategySchema.optional(),
maxRetries: z.coerce.number().int().min(0).max(10).optional(),
retryDelayMs: z.coerce.number().int().min(0).max(60000).optional(),
fallbackDelayMs: z.coerce.number().int().min(0).max(60000).optional(),
timeoutMs: z.coerce.number().int().min(1000).optional(),
concurrencyPerModel: z.coerce.number().int().min(1).max(20).optional(),
queueTimeoutMs: z.coerce.number().int().min(1000).max(120000).optional(),

21
src/sse/handlers/chat.ts
View File

@@ -26,7 +26,7 @@ import {
import * as log from "../utils/logger";
import { checkAndRefreshToken } from "../services/tokenRefresh";
import { deleteHandoff, getHandoff } from "@/lib/db/contextHandoffs";
import { getCachedSettings, getSettings, getCombos } from "@/lib/localDb";
import { getCachedSettings, getCombos } from "@/lib/localDb";
import {
ensureOpenAIStoreSessionFallback,
isOpenAIResponsesStoreEnabled,
@@ -91,6 +91,21 @@ registerBailianCodingPlanQuotaFetcher();
// opt-in) when the active bucket reaches zero.
registerCrofUsageFetcher();
let combosCachePromise: Promise<unknown[]> | null = null;
let combosCacheTs = 0;
const COMBOS_CACHE_TTL_MS = 10_000;
async function getCombosCachedForChat(): Promise<unknown[]> {
const now = Date.now();
if (combosCachePromise && now - combosCacheTs < COMBOS_CACHE_TTL_MS) {
return combosCachePromise;
}
combosCacheTs = now;
combosCachePromise = getCombos().catch(() => []);
return combosCachePromise;
}
function normalizeAllowedConnectionIds(value: unknown): string[] | null {
if (!Array.isArray(value)) return null;
const ids = value.filter(
@@ -336,8 +351,8 @@ export async function handleChat(request: any, clientRawRequest: any = null) {
// Fetch settings and all combos for config cascade and nested resolution
const [settings, allCombos] = await Promise.all([
getSettings().catch(() => ({})),
getCombos().catch(() => []),
getCachedSettings().catch(() => ({})),
getCombosCachedForChat(),
]);
const relayConfig =
combo.strategy === "context-relay" ? resolveComboConfig(combo, settings) : null;

4
src/sse/services/model.ts
View File

@@ -1,6 +1,6 @@
// Re-export from open-sse with localDb integration
import { getModelAliases, getComboByName, getProviderNodes, getCustomModels } from "@/lib/localDb";
import { getSettings } from "@/lib/localDb";
import { getCachedSettings } from "@/lib/localDb";
import { getComboStepTarget } from "@/lib/combos/steps";
import {
parseModel,
@@ -83,7 +83,7 @@ export async function getModelInfo(modelStr) {
// stripModelPrefix: if enabled, strip provider prefix and re-resolve
// the bare model name using existing heuristics (claude-* → anthropic, etc.)
try {
const settings = await getSettings();
const settings = await getCachedSettings();
if (settings.stripModelPrefix === true) {
const strippedResult = await getModelInfoCore(parsed.model, getModelAliases);
return { ...strippedResult, extendedContext };

1
src/types/combo.ts
View File

@@ -10,6 +10,7 @@ export interface Combo {
nodes: ComboNode[];
maxRetries: number;
retryDelayMs: number;
fallbackDelayMs?: number;
timeoutMs: number;
healthCheckEnabled: boolean;
createdAt: string;

1
src/types/settings.ts
View File

@@ -32,6 +32,7 @@ export interface ComboDefaults {
strategy: RoutingStrategyValue;
maxRetries: number;
retryDelayMs: number;
fallbackDelayMs?: number;
maxComboDepth: number;
trackMetrics: boolean;
concurrencyPerModel?: number;

2
tests/unit/combo-499-abort.test.ts
View File

@@ -133,7 +133,7 @@ test("signal abort during fallback wait interrupts immediately", async () => {
combo: makeCombo("priority", ["a/m1", "b/m2"]),
handleSingleModel,
log,
settings: { retryDelayMs: 5000 }, // 5s delay would normally be slow
settings: { fallbackDelayMs: 5000 }, // 5s delay would normally be slow
allCombos: [],
signal: ac.signal,
});

3
tests/unit/combo-config.test.ts
View File

@@ -14,6 +14,7 @@ test("getDefaultComboConfig returns a fresh copy of the defaults", () => {
assert.equal(first.strategy, "priority");
assert.equal(first.maxRetries, 1);
assert.equal(first.retryDelayMs, 2000);
assert.equal(first.fallbackDelayMs, 0);
assert.ok(!("timeoutMs" in first));
assert.ok(!("healthCheckEnabled" in first));
assert.equal(first.handoffThreshold, 0.85);
@@ -40,6 +41,7 @@ test("resolveComboConfig applies the full cascade from defaults to combo overrid
openai: {
timeoutMs: 60000,
retryDelayMs: 500,
fallbackDelayMs: 100,
},
},
},
@@ -48,6 +50,7 @@ test("resolveComboConfig applies the full cascade from defaults to combo overrid
assert.equal(result.strategy, "round-robin");
assert.equal(result.retryDelayMs, 500);
assert.equal(result.fallbackDelayMs, 100);
assert.equal(result.maxRetries, 4);
assert.ok(!("timeoutMs" in result));
assert.ok(!("healthCheckEnabled" in result));