pjandro/OmniRoute
1
0
Fork 0
mirror of https://github.com/diegosouzapw/OmniRoute.git synced 2026-08-05 14:52:09 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1473 from clousky2020/provider

Browse Source 
feat(fallback): add provider-level circuit breaker with configurable thresholds
This commit is contained in:
Diego Rodrigues de Sa e Souza
2026-04-21 10:08:44 -03:00
committed by GitHub
parent 1dae161823 f651a27418
commit 76b0f077a4
4 changed files with 163 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
open-sse/config/constants.ts
View File

@@ -170,6 +170,10 @@ export const PROVIDER_PROFILES = {
maxBackoffLevel: 8, // Higher ceiling (sessions may stay bad longer)
circuitBreakerThreshold: 3, // Opens fast (low limit providers)
circuitBreakerReset: 60000, // 1min reset
// Provider-level circuit breaker (entire provider cooldown after repeated failures)
providerFailureThreshold: 3, // 3 transient failures trigger provider cooldown
providerFailureWindowMs: 600000, // 10min window for counting failures
providerCooldownMs: 300000, // 5min cooldown when threshold reached
},
apikey: {
transientCooldown: 3000, // 3s (API providers recover faster)
@@ -177,6 +181,10 @@ export const PROVIDER_PROFILES = {
maxBackoffLevel: 5, // Lower ceiling (API quotas reset at known intervals)
circuitBreakerThreshold: 5, // More tolerant (occasional 502 is normal)
circuitBreakerReset: 30000, // 30s reset
// Provider-level circuit breaker (entire provider cooldown after repeated failures)
providerFailureThreshold: 5, // 5 transient failures trigger provider cooldown
providerFailureWindowMs: 1200000, // 20min window for counting failures
providerCooldownMs: 600000, // 10min cooldown when threshold reached
},
// Local providers (localhost inference backends like Ollama, LM Studio, oMLX).
// Not yet wired into getProviderProfile() — will be used when local provider_nodes
@@ -187,6 +195,10 @@ export const PROVIDER_PROFILES = {
maxBackoffLevel: 3, // Low ceiling (local either works or doesn't)
circuitBreakerThreshold: 2, // Opens fast (if local is down, it's down)
circuitBreakerReset: 15000, // 15s reset (check again quickly)
// Provider-level circuit breaker (entire provider cooldown after repeated failures)
providerFailureThreshold: 2, // 2 failures trigger provider cooldown
providerFailureWindowMs: 300000, // 5min window for counting failures
providerCooldownMs: 60000, // 1min cooldown when threshold reached
},
};

88
open-sse/services/accountFallback.ts
View File

@@ -2,6 +2,7 @@ import {
COOLDOWN_MS,
BACKOFF_CONFIG,
BACKOFF_STEPS_MS,
PROVIDER_PROFILES,
RateLimitReason,
HTTP_STATUS,
} from "../config/constants.ts";
@@ -27,6 +28,10 @@ type ProviderProfile = {
maxBackoffLevel: number;
circuitBreakerThreshold: number;
circuitBreakerReset: number;
// Provider-level circuit breaker fields
providerFailureThreshold: number;
providerFailureWindowMs: number;
providerCooldownMs: number;
};
type JsonRecord = Record<string, unknown>;
type ModelLockoutEntry = {
@@ -43,11 +48,25 @@ type ModelFailureState = {
resetAfterMs: number;
};
// Error codes that count toward provider-level failure threshold.
// Connection-scoped 429 rate limits stay in connection cooldown handling and
// do not contribute to the shared provider breaker.
// Provider-level failure tracking for circuit breaker behavior
type ProviderFailureEntry = {
failureCount: number;
lastFailureAt: number;
resetAfterMs: number;
cooldownUntil: number | null;
};
// Error codes that count toward provider-level failure threshold
const PROVIDER_FAILURE_ERROR_CODES = new Set([408, 500, 502, 503, 504]);
// Provider-level failure state map: providerId -> failure entry
const providerFailureState = new Map<string, ProviderFailureEntry>();
// Guard against synchronous re-entrant calls within the same event-loop tick.
// NOT a true mutex — Node.js is single-threaded, so different SSE streams
// can interleave across ticks. This Set prevents a single call from recursively
// re-entering recordProviderFailure within the same synchronous call stack.
const providerFailureLocks = new Set<string>();
// T06 (sub2api PR #1037): Signals that indicate permanent account deactivation.
// When a 401 body contains these strings, the account is permanently dead
// and should NOT be retried after token refresh.
@@ -173,6 +192,10 @@ function buildProviderProfile(
maxBackoffLevel: connectionCooldown.maxBackoffSteps,
circuitBreakerThreshold: providerBreaker.failureThreshold,
circuitBreakerReset: providerBreaker.resetTimeoutMs,
// Provider-level circuit breaker fields (not configurable via settings, use PROVIDER_PROFILES defaults)
providerFailureThreshold: PROVIDER_PROFILES[category].providerFailureThreshold,
providerFailureWindowMs: PROVIDER_PROFILES[category].providerFailureWindowMs,
providerCooldownMs: PROVIDER_PROFILES[category].providerCooldownMs,
} satisfies ProviderProfile;
}
@@ -511,12 +534,59 @@ export function recordProviderFailure(
provider: string | null | undefined,
log?: { warn?: (...args: unknown[]) => void }
): void {
const breaker = getProviderBreaker(provider);
if (!breaker || !provider) return;
breaker._onFailure();
const status = breaker.getStatus();
if (status.state === STATE.OPEN) {
log?.warn?.(`[ProviderBreaker] ${provider}: OPEN after ${status.failureCount} final failures`);
if (!provider) return;
// Guard against concurrent re-entrant calls within the same tick
if (providerFailureLocks.has(provider)) return;
providerFailureLocks.add(provider);
try {
const now = Date.now();
const entry = providerFailureState.get(provider);
// Check if we're in cooldown period
if (entry && entry.cooldownUntil !== null && now < entry.cooldownUntil) {
return; // Already in cooldown, don't record
}
// Check if failure window has expired
if (entry && now - entry.lastFailureAt > entry.resetAfterMs) {
// Window expired, reset count
providerFailureState.set(provider, {
failureCount: 1,
lastFailureAt: now,
resetAfterMs: PROVIDER_FAILURE_WINDOW_MS,
cooldownUntil: null,
});
return;
}
// Increment failure count
const newCount = entry ? entry.failureCount + 1 : 1;
if (newCount >= PROVIDER_FAILURE_THRESHOLD) {
// Threshold reached, enter cooldown
const cooldownUntil = now + PROVIDER_COOLDOWN_MS;
providerFailureState.set(provider, {
failureCount: newCount,
lastFailureAt: now,
resetAfterMs: PROVIDER_FAILURE_WINDOW_MS,
cooldownUntil,
});
log?.warn?.(
`[ProviderFailure] ${provider}: ${newCount} failures in ${PROVIDER_FAILURE_WINDOW_MS / 1000}s — entering ${PROVIDER_COOLDOWN_MS / 1000}s cooldown`
);
} else {
// Just increment counter
providerFailureState.set(provider, {
failureCount: newCount,
lastFailureAt: now,
resetAfterMs: PROVIDER_FAILURE_WINDOW_MS,
cooldownUntil: null,
});
}
} finally {
providerFailureLocks.delete(provider);
}
}

4
src/app/(dashboard)/dashboard/providers/[id]/page.tsx
View File

@@ -5839,7 +5839,7 @@ function EditConnectionModal({ isOpen, connection, onSave, onClose }: EditConnec
codexOpenaiStoreEnabled: false,
consoleApiKey: "",
ccCompatibleContext1m: false,
passthroughModels: connection.providerSpecificData?.passthroughModels === true,
passthroughModels: connection?.providerSpecificData?.passthroughModels === true,
});
const [testing, setTesting] = useState(false);
const [testResult, setTestResult] = useState(null);
@@ -5907,7 +5907,7 @@ function EditConnectionModal({ isOpen, connection, onSave, onClose }: EditConnec
codexOpenaiStoreEnabled: connection.providerSpecificData?.openaiStoreEnabled === true,
consoleApiKey: existingConsoleApiKey,
ccCompatibleContext1m: ccRequestDefaults.context1m,
passthroughModels: connection.providerSpecificData?.passthroughModels === true,
passthroughModels: connection?.providerSpecificData?.passthroughModels === true,
});
// Load existing extra keys from providerSpecificData
const existing = connection.providerSpecificData?.extraApiKeys;

86
tests/unit/settings-api.test.ts
View File

@@ -1,34 +1,88 @@
import { describe, test } from "node:test";
import { describe, test, beforeEach, afterEach, after } from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
// --- Create harness function (similar to _chatPipelineHarness pattern) ---
async function createSettingsApiHarness() {
const testDataDir = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-settings-api-"));
process.env.DATA_DIR = testDataDir;
process.env.REQUIRE_API_KEY = "false";
if (!process.env.API_KEY_SECRET) {
process.env.API_KEY_SECRET = "test-settings-api-secret-" + Date.now();
}
// --- Dynamic imports AFTER env setup ---
const core = await import("../../src/lib/db/core.ts");
const { getSettings, updateSettings } = await import("../../src/lib/db/settings.ts");
const settingsRoute = await import("../../src/app/api/settings/route.ts");
async function resetStorage() {
core.resetDbInstance();
fs.rmSync(testDataDir, { recursive: true, force: true });
fs.mkdirSync(testDataDir, { recursive: true });
}
function cleanup() {
core.resetDbInstance();
fs.rmSync(testDataDir, { recursive: true, force: true });
}
return {
testDataDir,
core,
getSettings,
updateSettings,
settingsRoute,
resetStorage,
cleanup,
};
}
// --- Initialize harness ---
const harness = await createSettingsApiHarness();
// --- Static import for helper (doesn't depend on DB) ---
import { makeManagementSessionRequest } from "../helpers/managementSession.ts";
import { getSettings, updateSettings } from "../../src/lib/db/settings.ts";
const settingsRoute = await import("../../src/app/api/settings/route.ts");
beforeEach(async () => {
await harness.resetStorage();
});
afterEach(async () => {
await harness.resetStorage();
});
after(() => {
harness.cleanup();
});
describe("Settings API - debugMode and hiddenSidebarItems", () => {
describe("debugMode", () => {
test("updateSettings with debugMode=true succeeds", async () => {
const result = await updateSettings({ debugMode: true });
const result = await harness.updateSettings({ debugMode: true });
assert.ok(result, "updateSettings should return truthy result");
const settings = await getSettings();
const settings = await harness.getSettings();
assert.strictEqual(settings.debugMode, true, "debugMode should be true");
});
test("updateSettings with debugMode=false succeeds", async () => {
const result = await updateSettings({ debugMode: false });
const result = await harness.updateSettings({ debugMode: false });
assert.ok(result, "updateSettings should return truthy result");
const settings = await getSettings();
const settings = await harness.getSettings();
assert.strictEqual(settings.debugMode, false, "debugMode should be false");
});
});
describe("hiddenSidebarItems", () => {
test("updateSettings with hiddenSidebarItems=['translator'] succeeds", async () => {
const result = await updateSettings({ hiddenSidebarItems: ["translator"] });
const result = await harness.updateSettings({ hiddenSidebarItems: ["translator"] });
assert.ok(result, "updateSettings should return truthy result");
const settings = await getSettings();
const settings = await harness.getSettings();
assert.deepStrictEqual(
settings.hiddenSidebarItems,
["translator"],
@@ -37,10 +91,10 @@ describe("Settings API - debugMode and hiddenSidebarItems", () => {
});
test("updateSettings with empty hiddenSidebarItems succeeds", async () => {
const result = await updateSettings({ hiddenSidebarItems: [] });
const result = await harness.updateSettings({ hiddenSidebarItems: [] });
assert.ok(result, "updateSettings should return truthy result");
const settings = await getSettings();
const settings = await harness.getSettings();
assert.deepStrictEqual(
settings.hiddenSidebarItems,
[],
@@ -51,13 +105,13 @@ describe("Settings API - debugMode and hiddenSidebarItems", () => {
describe("combined updates", () => {
test("updateSettings with both debugMode and hiddenSidebarItems succeeds", async () => {
const result = await updateSettings({
const result = await harness.updateSettings({
debugMode: true,
hiddenSidebarItems: ["translator"],
});
assert.ok(result, "updateSettings should return truthy result");
const settings = await getSettings();
const settings = await harness.getSettings();
assert.strictEqual(settings.debugMode, true, "debugMode should be true");
assert.deepStrictEqual(
settings.hiddenSidebarItems,
@@ -67,12 +121,12 @@ describe("Settings API - debugMode and hiddenSidebarItems", () => {
});
test("updateSettings persists antigravitySignatureCacheMode", async () => {
const result = await updateSettings({
const result = await harness.updateSettings({
antigravitySignatureCacheMode: "bypass-strict",
});
assert.ok(result, "updateSettings should return truthy result");
const settings = await getSettings();
const settings = await harness.getSettings();
assert.strictEqual(
settings.antigravitySignatureCacheMode,
"bypass-strict",
@@ -81,7 +135,7 @@ describe("Settings API - debugMode and hiddenSidebarItems", () => {
});
test("PUT /api/settings reuses the PATCH update flow", async () => {
const response = await settingsRoute.PUT(
const response = await harness.settingsRoute.PUT(
await makeManagementSessionRequest("http://localhost/api/settings", {
method: "PUT",
body: { antigravitySignatureCacheMode: "bypass" },