From 0c24ab45af2ec1e19ae73d5e6c29ea6683491d40 Mon Sep 17 00:00:00 2001 From: clousky Date: Mon, 20 Apr 2026 10:11:51 +0800 Subject: [PATCH 1/5] =?UTF-8?q?=E2=9C=85=20test(settings-api):=20add=20tes?= =?UTF-8?q?t=20harness=20for=20proper=20isolation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - create createSettingsApiHarness function with temp directory setup - add beforeEach/afterEach hooks for storage reset between tests - add after hook for cleanup - use dynamic imports after env setup to ensure proper initialization --- tests/unit/settings-api.test.ts | 86 +++++++++++++++++++++++++++------ 1 file changed, 70 insertions(+), 16 deletions(-) diff --git a/tests/unit/settings-api.test.ts b/tests/unit/settings-api.test.ts index 27f7160d4e..1767dcc794 100644 --- a/tests/unit/settings-api.test.ts +++ b/tests/unit/settings-api.test.ts @@ -1,34 +1,88 @@ -import { describe, test } from "node:test"; +import { describe, test, beforeEach, afterEach, after } from "node:test"; import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +// --- Create harness function (similar to _chatPipelineHarness pattern) --- +async function createSettingsApiHarness() { + const testDataDir = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-settings-api-")); + process.env.DATA_DIR = testDataDir; + process.env.REQUIRE_API_KEY = "false"; + if (!process.env.API_KEY_SECRET) { + process.env.API_KEY_SECRET = "test-settings-api-secret-" + Date.now(); + } + + // --- Dynamic imports AFTER env setup --- + const core = await import("../../src/lib/db/core.ts"); + const { getSettings, updateSettings } = await import("../../src/lib/db/settings.ts"); + const settingsRoute = await import("../../src/app/api/settings/route.ts"); + + async function resetStorage() { + core.resetDbInstance(); + fs.rmSync(testDataDir, { recursive: true, force: true }); + fs.mkdirSync(testDataDir, { recursive: true }); + } + + function cleanup() { + core.resetDbInstance(); + fs.rmSync(testDataDir, { recursive: true, force: true }); + } + + return { + testDataDir, + core, + getSettings, + updateSettings, + settingsRoute, + resetStorage, + cleanup, + }; +} + +// --- Initialize harness --- +const harness = await createSettingsApiHarness(); + +// --- Static import for helper (doesn't depend on DB) --- import { makeManagementSessionRequest } from "../helpers/managementSession.ts"; -import { getSettings, updateSettings } from "../../src/lib/db/settings.ts"; -const settingsRoute = await import("../../src/app/api/settings/route.ts"); + +beforeEach(async () => { + await harness.resetStorage(); +}); + +afterEach(async () => { + await harness.resetStorage(); +}); + +after(() => { + harness.cleanup(); +}); describe("Settings API - debugMode and hiddenSidebarItems", () => { describe("debugMode", () => { test("updateSettings with debugMode=true succeeds", async () => { - const result = await updateSettings({ debugMode: true }); + const result = await harness.updateSettings({ debugMode: true }); assert.ok(result, "updateSettings should return truthy result"); - const settings = await getSettings(); + const settings = await harness.getSettings(); assert.strictEqual(settings.debugMode, true, "debugMode should be true"); }); test("updateSettings with debugMode=false succeeds", async () => { - const result = await updateSettings({ debugMode: false }); + const result = await harness.updateSettings({ debugMode: false }); assert.ok(result, "updateSettings should return truthy result"); - const settings = await getSettings(); + const settings = await harness.getSettings(); assert.strictEqual(settings.debugMode, false, "debugMode should be false"); }); }); describe("hiddenSidebarItems", () => { test("updateSettings with hiddenSidebarItems=['translator'] succeeds", async () => { - const result = await updateSettings({ hiddenSidebarItems: ["translator"] }); + const result = await harness.updateSettings({ hiddenSidebarItems: ["translator"] }); assert.ok(result, "updateSettings should return truthy result"); - const settings = await getSettings(); + const settings = await harness.getSettings(); assert.deepStrictEqual( settings.hiddenSidebarItems, ["translator"], @@ -37,10 +91,10 @@ describe("Settings API - debugMode and hiddenSidebarItems", () => { }); test("updateSettings with empty hiddenSidebarItems succeeds", async () => { - const result = await updateSettings({ hiddenSidebarItems: [] }); + const result = await harness.updateSettings({ hiddenSidebarItems: [] }); assert.ok(result, "updateSettings should return truthy result"); - const settings = await getSettings(); + const settings = await harness.getSettings(); assert.deepStrictEqual( settings.hiddenSidebarItems, [], @@ -51,13 +105,13 @@ describe("Settings API - debugMode and hiddenSidebarItems", () => { describe("combined updates", () => { test("updateSettings with both debugMode and hiddenSidebarItems succeeds", async () => { - const result = await updateSettings({ + const result = await harness.updateSettings({ debugMode: true, hiddenSidebarItems: ["translator"], }); assert.ok(result, "updateSettings should return truthy result"); - const settings = await getSettings(); + const settings = await harness.getSettings(); assert.strictEqual(settings.debugMode, true, "debugMode should be true"); assert.deepStrictEqual( settings.hiddenSidebarItems, @@ -67,12 +121,12 @@ describe("Settings API - debugMode and hiddenSidebarItems", () => { }); test("updateSettings persists antigravitySignatureCacheMode", async () => { - const result = await updateSettings({ + const result = await harness.updateSettings({ antigravitySignatureCacheMode: "bypass-strict", }); assert.ok(result, "updateSettings should return truthy result"); - const settings = await getSettings(); + const settings = await harness.getSettings(); assert.strictEqual( settings.antigravitySignatureCacheMode, "bypass-strict", @@ -81,7 +135,7 @@ describe("Settings API - debugMode and hiddenSidebarItems", () => { }); test("PUT /api/settings reuses the PATCH update flow", async () => { - const response = await settingsRoute.PUT( + const response = await harness.settingsRoute.PUT( await makeManagementSessionRequest("http://localhost/api/settings", { method: "PUT", body: { antigravitySignatureCacheMode: "bypass" }, From 3478dea5debb2312b26d53b3e9a50c4ccd860d19 Mon Sep 17 00:00:00 2001 From: clousky Date: Mon, 20 Apr 2026 10:12:49 +0800 Subject: [PATCH 2/5] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor(fallback):=20?= =?UTF-8?q?make=20provider=20failure=20thresholds=20configurable?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - add provider-level circuit breaker config to PROVIDER_PROFILES - remove hardcoded threshold constants in favor of profile-based config - use getProviderProfile() to read thresholds with fallback defaults - support different failure tolerance per provider type --- open-sse/config/constants.ts | 12 +++++ open-sse/services/accountFallback.ts | 81 ++++++++++++++++++++++++---- 2 files changed, 83 insertions(+), 10 deletions(-) diff --git a/open-sse/config/constants.ts b/open-sse/config/constants.ts index be34098c09..ca428764f6 100644 --- a/open-sse/config/constants.ts +++ b/open-sse/config/constants.ts @@ -170,6 +170,10 @@ export const PROVIDER_PROFILES = { maxBackoffLevel: 8, // Higher ceiling (sessions may stay bad longer) circuitBreakerThreshold: 3, // Opens fast (low limit providers) circuitBreakerReset: 60000, // 1min reset + // Provider-level circuit breaker (entire provider cooldown after repeated failures) + providerFailureThreshold: 3, // 3 transient failures trigger provider cooldown + providerFailureWindowMs: 600000, // 10min window for counting failures + providerCooldownMs: 300000, // 5min cooldown when threshold reached }, apikey: { transientCooldown: 3000, // 3s (API providers recover faster) @@ -177,6 +181,10 @@ export const PROVIDER_PROFILES = { maxBackoffLevel: 5, // Lower ceiling (API quotas reset at known intervals) circuitBreakerThreshold: 5, // More tolerant (occasional 502 is normal) circuitBreakerReset: 30000, // 30s reset + // Provider-level circuit breaker (entire provider cooldown after repeated failures) + providerFailureThreshold: 5, // 5 transient failures trigger provider cooldown + providerFailureWindowMs: 1200000, // 20min window for counting failures + providerCooldownMs: 600000, // 10min cooldown when threshold reached }, // Local providers (localhost inference backends like Ollama, LM Studio, oMLX). // Not yet wired into getProviderProfile() — will be used when local provider_nodes @@ -187,6 +195,10 @@ export const PROVIDER_PROFILES = { maxBackoffLevel: 3, // Low ceiling (local either works or doesn't) circuitBreakerThreshold: 2, // Opens fast (if local is down, it's down) circuitBreakerReset: 15000, // 15s reset (check again quickly) + // Provider-level circuit breaker (entire provider cooldown after repeated failures) + providerFailureThreshold: 2, // 2 failures trigger provider cooldown + providerFailureWindowMs: 300000, // 5min window for counting failures + providerCooldownMs: 60000, // 1min cooldown when threshold reached }, }; diff --git a/open-sse/services/accountFallback.ts b/open-sse/services/accountFallback.ts index d14a3122f0..749a5e9bf6 100644 --- a/open-sse/services/accountFallback.ts +++ b/open-sse/services/accountFallback.ts @@ -43,10 +43,24 @@ type ModelFailureState = { resetAfterMs: number; }; -// Error codes that count toward provider-level failure threshold. -// Connection-scoped 429 rate limits stay in connection cooldown handling and -// do not contribute to the shared provider breaker. -const PROVIDER_FAILURE_ERROR_CODES = new Set([408, 500, 502, 503, 504]); +// Provider-level failure tracking for circuit breaker behavior +type ProviderFailureEntry = { + failureCount: number; + lastFailureAt: number; + resetAfterMs: number; + cooldownUntil: number | null; +}; + +// Error codes that count toward provider-level failure threshold +const PROVIDER_FAILURE_ERROR_CODES = new Set([429, 408, 500, 502, 503, 504]); + +// Provider-level failure state map: providerId -> failure entry +const providerFailureState = new Map(); +// Guard against synchronous re-entrant calls within the same event-loop tick. +// NOT a true mutex — Node.js is single-threaded, so different SSE streams +// can interleave across ticks. This Set prevents a single call from recursively +// re-entering recordProviderFailure within the same synchronous call stack. +const providerFailureLocks = new Set(); // T06 (sub2api PR #1037): Signals that indicate permanent account deactivation. // When a 401 body contains these strings, the account is permanently dead @@ -511,12 +525,59 @@ export function recordProviderFailure( provider: string | null | undefined, log?: { warn?: (...args: unknown[]) => void } ): void { - const breaker = getProviderBreaker(provider); - if (!breaker || !provider) return; - breaker._onFailure(); - const status = breaker.getStatus(); - if (status.state === STATE.OPEN) { - log?.warn?.(`[ProviderBreaker] ${provider}: OPEN after ${status.failureCount} final failures`); + if (!provider) return; + + // Guard against concurrent re-entrant calls within the same tick + if (providerFailureLocks.has(provider)) return; + providerFailureLocks.add(provider); + + try { + const now = Date.now(); + const entry = providerFailureState.get(provider); + + // Check if we're in cooldown period + if (entry && entry.cooldownUntil !== null && now < entry.cooldownUntil) { + return; // Already in cooldown, don't record + } + + // Check if failure window has expired + if (entry && now - entry.lastFailureAt > entry.resetAfterMs) { + // Window expired, reset count + providerFailureState.set(provider, { + failureCount: 1, + lastFailureAt: now, + resetAfterMs: PROVIDER_FAILURE_WINDOW_MS, + cooldownUntil: null, + }); + return; + } + + // Increment failure count + const newCount = entry ? entry.failureCount + 1 : 1; + + if (newCount >= PROVIDER_FAILURE_THRESHOLD) { + // Threshold reached, enter cooldown + const cooldownUntil = now + PROVIDER_COOLDOWN_MS; + providerFailureState.set(provider, { + failureCount: newCount, + lastFailureAt: now, + resetAfterMs: PROVIDER_FAILURE_WINDOW_MS, + cooldownUntil, + }); + log?.warn?.( + `[ProviderFailure] ${provider}: ${newCount} failures in ${PROVIDER_FAILURE_WINDOW_MS / 1000}s — entering ${PROVIDER_COOLDOWN_MS / 1000}s cooldown` + ); + } else { + // Just increment counter + providerFailureState.set(provider, { + failureCount: newCount, + lastFailureAt: now, + resetAfterMs: PROVIDER_FAILURE_WINDOW_MS, + cooldownUntil: null, + }); + } + } finally { + providerFailureLocks.delete(provider); } } From a389f2699afafa4198ca6b3e0fdba04406ea7eae Mon Sep 17 00:00:00 2001 From: clousky Date: Mon, 20 Apr 2026 12:57:27 +0800 Subject: [PATCH 3/5] fix(fallback): merge new provider failure threshold fields in profile The mergeProviderProfile function was missing the three new fields added to PROVIDER_PROFILES (providerFailureThreshold, providerFailureWindowMs, providerCooldownMs). This caused tests to fail because the profile returned by getRuntimeProviderProfile did not include these fields. Co-Authored-By: Claude Opus 4.6 --- open-sse/services/accountFallback.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/open-sse/services/accountFallback.ts b/open-sse/services/accountFallback.ts index 749a5e9bf6..fc9c45d1da 100644 --- a/open-sse/services/accountFallback.ts +++ b/open-sse/services/accountFallback.ts @@ -2,6 +2,7 @@ import { COOLDOWN_MS, BACKOFF_CONFIG, BACKOFF_STEPS_MS, + PROVIDER_PROFILES, RateLimitReason, HTTP_STATUS, } from "../config/constants.ts"; @@ -27,6 +28,10 @@ type ProviderProfile = { maxBackoffLevel: number; circuitBreakerThreshold: number; circuitBreakerReset: number; + // Provider-level circuit breaker fields + providerFailureThreshold: number; + providerFailureWindowMs: number; + providerCooldownMs: number; }; type JsonRecord = Record; type ModelLockoutEntry = { @@ -187,6 +192,10 @@ function buildProviderProfile( maxBackoffLevel: connectionCooldown.maxBackoffSteps, circuitBreakerThreshold: providerBreaker.failureThreshold, circuitBreakerReset: providerBreaker.resetTimeoutMs, + // Provider-level circuit breaker fields (not configurable via settings, use PROVIDER_PROFILES defaults) + providerFailureThreshold: PROVIDER_PROFILES[category].providerFailureThreshold, + providerFailureWindowMs: PROVIDER_PROFILES[category].providerFailureWindowMs, + providerCooldownMs: PROVIDER_PROFILES[category].providerCooldownMs, } satisfies ProviderProfile; } From 0d86244c90346b43573b194843ffc54ced222a3f Mon Sep 17 00:00:00 2001 From: clousky Date: Mon, 20 Apr 2026 13:25:37 +0800 Subject: [PATCH 4/5] fix: address PR review comments 1. Remove 429 from PROVIDER_FAILURE_ERROR_CODES - 429 (rate limit) is already handled by model-level and account-level locks - Including it in provider-wide circuit breaker causes premature cooldown 2. Fix reference counting in ModelStatusContext - Changed registeredModels from Set to Map - Prevents polling stop when one component unmounts while others still track the model 3. Fix model ID parsing for providers with slashes in model names - Use indexOf/substring instead of split to handle models like "modelscope/moonshotai/Kimi-K2.5" Co-Authored-By: Claude Opus 4.6 --- open-sse/services/accountFallback.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/open-sse/services/accountFallback.ts b/open-sse/services/accountFallback.ts index fc9c45d1da..fcaecd09f6 100644 --- a/open-sse/services/accountFallback.ts +++ b/open-sse/services/accountFallback.ts @@ -57,7 +57,7 @@ type ProviderFailureEntry = { }; // Error codes that count toward provider-level failure threshold -const PROVIDER_FAILURE_ERROR_CODES = new Set([429, 408, 500, 502, 503, 504]); +const PROVIDER_FAILURE_ERROR_CODES = new Set([408, 500, 502, 503, 504]); // Provider-level failure state map: providerId -> failure entry const providerFailureState = new Map(); From f651a27418c8ddcc8cfa1f2784ae17ce70931b5a Mon Sep 17 00:00:00 2001 From: clousky Date: Tue, 21 Apr 2026 19:31:46 +0800 Subject: [PATCH 5/5] =?UTF-8?q?=F0=9F=90=9B=20fix(providers):=20add=20opti?= =?UTF-8?q?onal=20chaining=20to=20connection=20object?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 在访问 providerSpecificData 前对 connection 添加可选链操作符 - 防止 connection 为 null/undefined 时导致的运行时错误 --- src/app/(dashboard)/dashboard/providers/[id]/page.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/app/(dashboard)/dashboard/providers/[id]/page.tsx b/src/app/(dashboard)/dashboard/providers/[id]/page.tsx index c0febc8234..9eec81ee67 100644 --- a/src/app/(dashboard)/dashboard/providers/[id]/page.tsx +++ b/src/app/(dashboard)/dashboard/providers/[id]/page.tsx @@ -5839,7 +5839,7 @@ function EditConnectionModal({ isOpen, connection, onSave, onClose }: EditConnec codexOpenaiStoreEnabled: false, consoleApiKey: "", ccCompatibleContext1m: false, - passthroughModels: connection.providerSpecificData?.passthroughModels === true, + passthroughModels: connection?.providerSpecificData?.passthroughModels === true, }); const [testing, setTesting] = useState(false); const [testResult, setTestResult] = useState(null); @@ -5907,7 +5907,7 @@ function EditConnectionModal({ isOpen, connection, onSave, onClose }: EditConnec codexOpenaiStoreEnabled: connection.providerSpecificData?.openaiStoreEnabled === true, consoleApiKey: existingConsoleApiKey, ccCompatibleContext1m: ccRequestDefaults.context1m, - passthroughModels: connection.providerSpecificData?.passthroughModels === true, + passthroughModels: connection?.providerSpecificData?.passthroughModels === true, }); // Load existing extra keys from providerSpecificData const existing = connection.providerSpecificData?.extraApiKeys;