diff --git a/open-sse/config/constants.ts b/open-sse/config/constants.ts index be34098c09..ca428764f6 100644 --- a/open-sse/config/constants.ts +++ b/open-sse/config/constants.ts @@ -170,6 +170,10 @@ export const PROVIDER_PROFILES = { maxBackoffLevel: 8, // Higher ceiling (sessions may stay bad longer) circuitBreakerThreshold: 3, // Opens fast (low limit providers) circuitBreakerReset: 60000, // 1min reset + // Provider-level circuit breaker (entire provider cooldown after repeated failures) + providerFailureThreshold: 3, // 3 transient failures trigger provider cooldown + providerFailureWindowMs: 600000, // 10min window for counting failures + providerCooldownMs: 300000, // 5min cooldown when threshold reached }, apikey: { transientCooldown: 3000, // 3s (API providers recover faster) @@ -177,6 +181,10 @@ export const PROVIDER_PROFILES = { maxBackoffLevel: 5, // Lower ceiling (API quotas reset at known intervals) circuitBreakerThreshold: 5, // More tolerant (occasional 502 is normal) circuitBreakerReset: 30000, // 30s reset + // Provider-level circuit breaker (entire provider cooldown after repeated failures) + providerFailureThreshold: 5, // 5 transient failures trigger provider cooldown + providerFailureWindowMs: 1200000, // 20min window for counting failures + providerCooldownMs: 600000, // 10min cooldown when threshold reached }, // Local providers (localhost inference backends like Ollama, LM Studio, oMLX). // Not yet wired into getProviderProfile() — will be used when local provider_nodes @@ -187,6 +195,10 @@ export const PROVIDER_PROFILES = { maxBackoffLevel: 3, // Low ceiling (local either works or doesn't) circuitBreakerThreshold: 2, // Opens fast (if local is down, it's down) circuitBreakerReset: 15000, // 15s reset (check again quickly) + // Provider-level circuit breaker (entire provider cooldown after repeated failures) + providerFailureThreshold: 2, // 2 failures trigger provider cooldown + providerFailureWindowMs: 300000, // 5min window for counting failures + providerCooldownMs: 60000, // 1min cooldown when threshold reached }, }; diff --git a/open-sse/services/accountFallback.ts b/open-sse/services/accountFallback.ts index d14a3122f0..fcaecd09f6 100644 --- a/open-sse/services/accountFallback.ts +++ b/open-sse/services/accountFallback.ts @@ -2,6 +2,7 @@ import { COOLDOWN_MS, BACKOFF_CONFIG, BACKOFF_STEPS_MS, + PROVIDER_PROFILES, RateLimitReason, HTTP_STATUS, } from "../config/constants.ts"; @@ -27,6 +28,10 @@ type ProviderProfile = { maxBackoffLevel: number; circuitBreakerThreshold: number; circuitBreakerReset: number; + // Provider-level circuit breaker fields + providerFailureThreshold: number; + providerFailureWindowMs: number; + providerCooldownMs: number; }; type JsonRecord = Record; type ModelLockoutEntry = { @@ -43,11 +48,25 @@ type ModelFailureState = { resetAfterMs: number; }; -// Error codes that count toward provider-level failure threshold. -// Connection-scoped 429 rate limits stay in connection cooldown handling and -// do not contribute to the shared provider breaker. +// Provider-level failure tracking for circuit breaker behavior +type ProviderFailureEntry = { + failureCount: number; + lastFailureAt: number; + resetAfterMs: number; + cooldownUntil: number | null; +}; + +// Error codes that count toward provider-level failure threshold const PROVIDER_FAILURE_ERROR_CODES = new Set([408, 500, 502, 503, 504]); +// Provider-level failure state map: providerId -> failure entry +const providerFailureState = new Map(); +// Guard against synchronous re-entrant calls within the same event-loop tick. +// NOT a true mutex — Node.js is single-threaded, so different SSE streams +// can interleave across ticks. This Set prevents a single call from recursively +// re-entering recordProviderFailure within the same synchronous call stack. +const providerFailureLocks = new Set(); + // T06 (sub2api PR #1037): Signals that indicate permanent account deactivation. // When a 401 body contains these strings, the account is permanently dead // and should NOT be retried after token refresh. @@ -173,6 +192,10 @@ function buildProviderProfile( maxBackoffLevel: connectionCooldown.maxBackoffSteps, circuitBreakerThreshold: providerBreaker.failureThreshold, circuitBreakerReset: providerBreaker.resetTimeoutMs, + // Provider-level circuit breaker fields (not configurable via settings, use PROVIDER_PROFILES defaults) + providerFailureThreshold: PROVIDER_PROFILES[category].providerFailureThreshold, + providerFailureWindowMs: PROVIDER_PROFILES[category].providerFailureWindowMs, + providerCooldownMs: PROVIDER_PROFILES[category].providerCooldownMs, } satisfies ProviderProfile; } @@ -511,12 +534,59 @@ export function recordProviderFailure( provider: string | null | undefined, log?: { warn?: (...args: unknown[]) => void } ): void { - const breaker = getProviderBreaker(provider); - if (!breaker || !provider) return; - breaker._onFailure(); - const status = breaker.getStatus(); - if (status.state === STATE.OPEN) { - log?.warn?.(`[ProviderBreaker] ${provider}: OPEN after ${status.failureCount} final failures`); + if (!provider) return; + + // Guard against concurrent re-entrant calls within the same tick + if (providerFailureLocks.has(provider)) return; + providerFailureLocks.add(provider); + + try { + const now = Date.now(); + const entry = providerFailureState.get(provider); + + // Check if we're in cooldown period + if (entry && entry.cooldownUntil !== null && now < entry.cooldownUntil) { + return; // Already in cooldown, don't record + } + + // Check if failure window has expired + if (entry && now - entry.lastFailureAt > entry.resetAfterMs) { + // Window expired, reset count + providerFailureState.set(provider, { + failureCount: 1, + lastFailureAt: now, + resetAfterMs: PROVIDER_FAILURE_WINDOW_MS, + cooldownUntil: null, + }); + return; + } + + // Increment failure count + const newCount = entry ? entry.failureCount + 1 : 1; + + if (newCount >= PROVIDER_FAILURE_THRESHOLD) { + // Threshold reached, enter cooldown + const cooldownUntil = now + PROVIDER_COOLDOWN_MS; + providerFailureState.set(provider, { + failureCount: newCount, + lastFailureAt: now, + resetAfterMs: PROVIDER_FAILURE_WINDOW_MS, + cooldownUntil, + }); + log?.warn?.( + `[ProviderFailure] ${provider}: ${newCount} failures in ${PROVIDER_FAILURE_WINDOW_MS / 1000}s — entering ${PROVIDER_COOLDOWN_MS / 1000}s cooldown` + ); + } else { + // Just increment counter + providerFailureState.set(provider, { + failureCount: newCount, + lastFailureAt: now, + resetAfterMs: PROVIDER_FAILURE_WINDOW_MS, + cooldownUntil: null, + }); + } + } finally { + providerFailureLocks.delete(provider); } } diff --git a/src/app/(dashboard)/dashboard/providers/[id]/page.tsx b/src/app/(dashboard)/dashboard/providers/[id]/page.tsx index c0febc8234..9eec81ee67 100644 --- a/src/app/(dashboard)/dashboard/providers/[id]/page.tsx +++ b/src/app/(dashboard)/dashboard/providers/[id]/page.tsx @@ -5839,7 +5839,7 @@ function EditConnectionModal({ isOpen, connection, onSave, onClose }: EditConnec codexOpenaiStoreEnabled: false, consoleApiKey: "", ccCompatibleContext1m: false, - passthroughModels: connection.providerSpecificData?.passthroughModels === true, + passthroughModels: connection?.providerSpecificData?.passthroughModels === true, }); const [testing, setTesting] = useState(false); const [testResult, setTestResult] = useState(null); @@ -5907,7 +5907,7 @@ function EditConnectionModal({ isOpen, connection, onSave, onClose }: EditConnec codexOpenaiStoreEnabled: connection.providerSpecificData?.openaiStoreEnabled === true, consoleApiKey: existingConsoleApiKey, ccCompatibleContext1m: ccRequestDefaults.context1m, - passthroughModels: connection.providerSpecificData?.passthroughModels === true, + passthroughModels: connection?.providerSpecificData?.passthroughModels === true, }); // Load existing extra keys from providerSpecificData const existing = connection.providerSpecificData?.extraApiKeys; diff --git a/tests/unit/settings-api.test.ts b/tests/unit/settings-api.test.ts index 27f7160d4e..1767dcc794 100644 --- a/tests/unit/settings-api.test.ts +++ b/tests/unit/settings-api.test.ts @@ -1,34 +1,88 @@ -import { describe, test } from "node:test"; +import { describe, test, beforeEach, afterEach, after } from "node:test"; import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +// --- Create harness function (similar to _chatPipelineHarness pattern) --- +async function createSettingsApiHarness() { + const testDataDir = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-settings-api-")); + process.env.DATA_DIR = testDataDir; + process.env.REQUIRE_API_KEY = "false"; + if (!process.env.API_KEY_SECRET) { + process.env.API_KEY_SECRET = "test-settings-api-secret-" + Date.now(); + } + + // --- Dynamic imports AFTER env setup --- + const core = await import("../../src/lib/db/core.ts"); + const { getSettings, updateSettings } = await import("../../src/lib/db/settings.ts"); + const settingsRoute = await import("../../src/app/api/settings/route.ts"); + + async function resetStorage() { + core.resetDbInstance(); + fs.rmSync(testDataDir, { recursive: true, force: true }); + fs.mkdirSync(testDataDir, { recursive: true }); + } + + function cleanup() { + core.resetDbInstance(); + fs.rmSync(testDataDir, { recursive: true, force: true }); + } + + return { + testDataDir, + core, + getSettings, + updateSettings, + settingsRoute, + resetStorage, + cleanup, + }; +} + +// --- Initialize harness --- +const harness = await createSettingsApiHarness(); + +// --- Static import for helper (doesn't depend on DB) --- import { makeManagementSessionRequest } from "../helpers/managementSession.ts"; -import { getSettings, updateSettings } from "../../src/lib/db/settings.ts"; -const settingsRoute = await import("../../src/app/api/settings/route.ts"); + +beforeEach(async () => { + await harness.resetStorage(); +}); + +afterEach(async () => { + await harness.resetStorage(); +}); + +after(() => { + harness.cleanup(); +}); describe("Settings API - debugMode and hiddenSidebarItems", () => { describe("debugMode", () => { test("updateSettings with debugMode=true succeeds", async () => { - const result = await updateSettings({ debugMode: true }); + const result = await harness.updateSettings({ debugMode: true }); assert.ok(result, "updateSettings should return truthy result"); - const settings = await getSettings(); + const settings = await harness.getSettings(); assert.strictEqual(settings.debugMode, true, "debugMode should be true"); }); test("updateSettings with debugMode=false succeeds", async () => { - const result = await updateSettings({ debugMode: false }); + const result = await harness.updateSettings({ debugMode: false }); assert.ok(result, "updateSettings should return truthy result"); - const settings = await getSettings(); + const settings = await harness.getSettings(); assert.strictEqual(settings.debugMode, false, "debugMode should be false"); }); }); describe("hiddenSidebarItems", () => { test("updateSettings with hiddenSidebarItems=['translator'] succeeds", async () => { - const result = await updateSettings({ hiddenSidebarItems: ["translator"] }); + const result = await harness.updateSettings({ hiddenSidebarItems: ["translator"] }); assert.ok(result, "updateSettings should return truthy result"); - const settings = await getSettings(); + const settings = await harness.getSettings(); assert.deepStrictEqual( settings.hiddenSidebarItems, ["translator"], @@ -37,10 +91,10 @@ describe("Settings API - debugMode and hiddenSidebarItems", () => { }); test("updateSettings with empty hiddenSidebarItems succeeds", async () => { - const result = await updateSettings({ hiddenSidebarItems: [] }); + const result = await harness.updateSettings({ hiddenSidebarItems: [] }); assert.ok(result, "updateSettings should return truthy result"); - const settings = await getSettings(); + const settings = await harness.getSettings(); assert.deepStrictEqual( settings.hiddenSidebarItems, [], @@ -51,13 +105,13 @@ describe("Settings API - debugMode and hiddenSidebarItems", () => { describe("combined updates", () => { test("updateSettings with both debugMode and hiddenSidebarItems succeeds", async () => { - const result = await updateSettings({ + const result = await harness.updateSettings({ debugMode: true, hiddenSidebarItems: ["translator"], }); assert.ok(result, "updateSettings should return truthy result"); - const settings = await getSettings(); + const settings = await harness.getSettings(); assert.strictEqual(settings.debugMode, true, "debugMode should be true"); assert.deepStrictEqual( settings.hiddenSidebarItems, @@ -67,12 +121,12 @@ describe("Settings API - debugMode and hiddenSidebarItems", () => { }); test("updateSettings persists antigravitySignatureCacheMode", async () => { - const result = await updateSettings({ + const result = await harness.updateSettings({ antigravitySignatureCacheMode: "bypass-strict", }); assert.ok(result, "updateSettings should return truthy result"); - const settings = await getSettings(); + const settings = await harness.getSettings(); assert.strictEqual( settings.antigravitySignatureCacheMode, "bypass-strict", @@ -81,7 +135,7 @@ describe("Settings API - debugMode and hiddenSidebarItems", () => { }); test("PUT /api/settings reuses the PATCH update flow", async () => { - const response = await settingsRoute.PUT( + const response = await harness.settingsRoute.PUT( await makeManagementSessionRequest("http://localhost/api/settings", { method: "PUT", body: { antigravitySignatureCacheMode: "bypass" },