From 881f59354d3bfc77eba31620d7c86a322a43e007 Mon Sep 17 00:00:00 2001 From: diegosouzapw Date: Sun, 19 Apr 2026 21:07:16 -0300 Subject: [PATCH 1/8] chore: bump version to 3.7.0 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index bb28534564..9387c47ca5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "omniroute", - "version": "3.6.9", + "version": "3.7.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "omniroute", - "version": "3.6.9", + "version": "3.7.0", "hasInstallScript": true, "license": "MIT", "workspaces": [ diff --git a/package.json b/package.json index 6b2cb413e8..fcc76e04a9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "omniroute", - "version": "3.6.9", + "version": "3.7.0", "description": "Smart AI Router with auto fallback โ€” route to FREE & cheap models, zero downtime. Works with Cursor, Cline, Claude Desktop, Codex, and any OpenAI-compatible tool.", "type": "module", "bin": { From 77f326e0ddcc9e5733b8264f70fa5546bc220b9a Mon Sep 17 00:00:00 2001 From: Benson K B Date: Mon, 20 Apr 2026 05:38:13 +0530 Subject: [PATCH 2/8] fix(dashboard): correct TOML round-trip corruption in codex config serializer (#1438) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Integrated into release/v3.7.0. Thanks @benzntech for this great contribution! ๐ŸŽ‰ We've removed the unrelated sync-fork.yml file and it's now merged into the release branch. --- src/app/api/cli-tools/codex-settings/route.ts | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/src/app/api/cli-tools/codex-settings/route.ts b/src/app/api/cli-tools/codex-settings/route.ts index deac929dae..5cf1151a1d 100644 --- a/src/app/api/cli-tools/codex-settings/route.ts +++ b/src/app/api/cli-tools/codex-settings/route.ts @@ -38,9 +38,16 @@ const parseToml = (content: string) => { // Key = value const kvMatch = trimmed.match(/^([^=]+)\s*=\s*(.+)$/); if (kvMatch) { - const key = kvMatch[1].trim(); + let key = kvMatch[1].trim(); let value = kvMatch[2].trim(); - // Remove quotes + // Strip quotes from key (TOML quoted keys like "gpt-5.3-codex") + if ( + (key.startsWith('"') && key.endsWith('"')) || + (key.startsWith("'") && key.endsWith("'")) + ) { + key = key.slice(1, -1); + } + // Remove quotes from string values only (not arrays, booleans, numbers) if ( (value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'")) @@ -58,13 +65,23 @@ const parseToml = (content: string) => { return result; }; +// Format a TOML value: arrays and booleans stay unquoted, strings get quoted +const formatTomlValue = (value: unknown): string => { + if (typeof value === "boolean") return value ? "true" : "false"; + if (typeof value === "number") return String(value); + // Preserve pre-formatted TOML arrays (e.g. ["a", "b"]) + if (typeof value === "string" && value.startsWith("[") && value.endsWith("]")) return value; + if (typeof value === "string") return `"${value}"`; + return `"${value}"`; +}; + // Convert parsed object back to TOML string const toToml = (parsed: Record) => { let lines: string[] = []; // Root level keys Object.entries(parsed._root).forEach(([key, value]) => { - lines.push(`${key} = "${value}"`); + lines.push(`${key} = ${formatTomlValue(value)}`); }); // Sections @@ -73,7 +90,7 @@ const toToml = (parsed: Record) => { lines.push(`[${section}]`); Object.entries(values).forEach(([key, value]) => { const formattedKey = key.includes(".") ? `"${key}"` : key; - lines.push(`${formattedKey} = "${value}"`); + lines.push(`${formattedKey} = ${formatTomlValue(value)}`); }); }); From d6cbdc158079ef547005cfbb4af485d66cab7dc8 Mon Sep 17 00:00:00 2001 From: clousky2020 <33016567+clousky2020@users.noreply.github.com> Date: Mon, 20 Apr 2026 08:13:45 +0800 Subject: [PATCH 3/8] feat: add ModelScope provider with circuit breaker and daily quota lock (#1430) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Integrated into release/v3.7.0. Thanks @clousky2020 for this massive and important contribution! ๐ŸŽ‰ We've translated the deprecation comments to English for consistency, and it is now officially merged into the release branch. Great work on the ModelScope integration and Circuit Breaker! --- next.config.mjs | 1 + open-sse/config/providerRegistry.ts | 28 +- open-sse/executors/antigravity.ts | 29 +- open-sse/executors/codex.ts | 2 + open-sse/handlers/chatCore.ts | 18 +- open-sse/index.ts | 3 + open-sse/services/accountFallback.ts | 249 +++++++++++++++++- open-sse/services/cloudCodeThinking.ts | 6 + open-sse/services/combo.ts | 60 +++++ open-sse/services/errorClassifier.ts | 5 + .../combos/IntelligentComboPanel.tsx | 3 + .../dashboard/providers/[id]/page.tsx | 28 ++ .../providers/components/ModelStatusBadge.tsx | 162 ++++++++++++ .../components/ModelStatusContext.tsx | 186 +++++++++++++ src/app/(dashboard)/layout.tsx | 7 +- src/i18n/messages/en.json | 11 +- src/i18n/messages/zh-CN.json | 13 +- src/sse/handlers/chat.ts | 6 +- src/sse/handlers/chatHelpers.ts | 7 +- src/sse/services/auth.ts | 26 +- tests/integration/chat-pipeline.test.ts | 12 + tests/unit/account-fallback-service.test.ts | 218 +++++++++++++++ tests/unit/chat-route-coverage.test.ts | 5 + tests/unit/combo-routing-engine.test.ts | 46 ++++ tests/unit/error-classifier.test.ts | 18 ++ 25 files changed, 1129 insertions(+), 20 deletions(-) create mode 100644 src/app/(dashboard)/dashboard/providers/components/ModelStatusBadge.tsx create mode 100644 src/app/(dashboard)/dashboard/providers/components/ModelStatusContext.tsx diff --git a/next.config.mjs b/next.config.mjs index db1017ae84..375b966e96 100644 --- a/next.config.mjs +++ b/next.config.mjs @@ -28,6 +28,7 @@ const nextConfig = { "./playwright-report/**/*", "./app.__qa_backup/**/*", "./tests/**/*", + "./logs/**/*", ], }, serverExternalPackages: [ diff --git a/open-sse/config/providerRegistry.ts b/open-sse/config/providerRegistry.ts index b48b3dc9d9..ede6e0cdff 100644 --- a/open-sse/config/providerRegistry.ts +++ b/open-sse/config/providerRegistry.ts @@ -1400,6 +1400,25 @@ export const REGISTRY: Record = { passthroughModels: true, }, + modelscope: { + id: "modelscope", + alias: "ms", + format: "openai", + executor: "default", + baseUrl: "https://api-inference.modelscope.cn/v1/chat/completions", + authType: "apikey", + authHeader: "bearer", + // ModelScope uses per-model quotas. Setting passthroughModels: true ensures 429/404 + // only locks the specific model, not the entire connection. This allows fallback + // to other models on the same API key. + passthroughModels: true, + models: [ + { id: "moonshotai/Kimi-K2.5", name: "Kimi K2.5" }, + { id: "ZhipuAI/GLM-5", name: "GLM-5" }, + { id: "stepfun-ai/Step-3.5-Flash", name: "Step-3.5-Flash" }, + ], + }, + // โ”€โ”€ New Free Providers (2026) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ longcat: { @@ -2128,8 +2147,14 @@ export function getUnsupportedParams(provider: string, modelId: string): readonl const cached = _unsupportedParamsMap.get(modelId); if (cached) return cached; - // 3. Handle prefixed model IDs (e.g., "openai/o3" โ†’ "o3") + // 3. Handle prefixed model IDs (e.g., "openai/o3" โ†’ "o3", "moonshotai/Kimi-K2.5" โ†’ "moonshotai/Kimi-K2.5") + // ModelScope models have slash in ID, check both full ID and bare ID if (modelId.includes("/")) { + // First check full model ID with provider prefix (e.g., "moonshotai/Kimi-K2.5") + const cachedWithPrefix = _unsupportedParamsMap.get(modelId); + if (cachedWithPrefix) return cachedWithPrefix; + + // Fall back to bare ID (e.g., "Kimi-K2.5") const bareId = modelId.split("/").pop() || ""; const bare = _unsupportedParamsMap.get(bareId); if (bare) return bare; @@ -2154,6 +2179,7 @@ export function getProviderCategory(provider: string): "oauth" | "apikey" { * Derive the latest opus/sonnet/haiku model IDs from the `claude` registry entry. * Picks the first model whose ID matches each family pattern โ€” registry order * determines precedence, so newer models should be listed first. + * @deprecated This function will be removed in v4.0, please use REGISTRY.claude?.models directly */ export function getClaudeCodeDefaultModels(): { opus: string; diff --git a/open-sse/executors/antigravity.ts b/open-sse/executors/antigravity.ts index b8dc191b7b..b987232735 100644 --- a/open-sse/executors/antigravity.ts +++ b/open-sse/executors/antigravity.ts @@ -735,18 +735,45 @@ export class AntigravityExecutor extends BaseExecutor { // consuming the stream. The client receives the unmodified SSE data. if (response.body) { let sseBuffer = ""; + const decoder = new TextDecoder(); // Singleton for correct streaming decode + const MAX_BUFFER_SIZE = 16 * 1024; // Limit to prevent OOM on large streams + const passThrough = new TransformStream({ transform(chunk, controller) { controller.enqueue(chunk); // Accumulate text to scan for remainingCredits try { - const text = new TextDecoder().decode(chunk, { stream: true }); + const text = decoder.decode(chunk, { stream: true }); sseBuffer += text; + // Limit buffer size to prevent unbounded growth + // Truncate only after a complete newline to avoid splitting SSE lines mid-payload + if (sseBuffer.length > MAX_BUFFER_SIZE) { + const lastNewline = sseBuffer.lastIndexOf( + "\n", + sseBuffer.length - MAX_BUFFER_SIZE + ); + if (lastNewline !== -1) { + sseBuffer = sseBuffer.slice(lastNewline + 1); + } else { + // No newline found in discard region โ€” buffer contains an incomplete SSE line. + // Discard it entirely to avoid returning malformed data; the remainingCredits + // parser won't find valid data in a truncated line anyway. + sseBuffer = ""; + } + } } catch { /* decoding best-effort */ } }, flush() { + // Final decode for any remaining bytes + try { + const text = decoder.decode(); // Flush pending bytes + sseBuffer += text; + } catch { + /* decoding best-effort */ + } + // Parse the accumulated SSE data for remainingCredits try { const lines = sseBuffer.split("\n"); diff --git a/open-sse/executors/codex.ts b/open-sse/executors/codex.ts index 899c4eb00d..fcbcbab1cd 100644 --- a/open-sse/executors/codex.ts +++ b/open-sse/executors/codex.ts @@ -271,6 +271,8 @@ function convertSystemToDeveloperRole(body: Record): void { * server-generated prefix (rs_, fc_, resp_, msg_) โ€” so the content is * preserved but the backend won't try to look it up * 4. Always deletes previous_response_id (endpoint doesn't persist responses) + * + * @deprecated This function will be removed in v4.0, Codex executor has updated processing logic */ function stripStoredItemReferences(body: Record): void { // Always strip previous_response_id โ€” the /codex/responses endpoint does not diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 49aefdbc8c..8206492282 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -14,7 +14,12 @@ import { createRequestLogger } from "../utils/requestLogger.ts"; import { getModelTargetFormat, PROVIDER_ID_TO_ALIAS } from "../config/providerModels.ts"; import { resolveModelAlias } from "../services/modelDeprecation.ts"; import { getUnsupportedParams } from "../config/providerRegistry.ts"; -import { hasPerModelQuota, lockModelIfPerModelQuota } from "../services/accountFallback.ts"; +import { + hasPerModelQuota, + lockModelIfPerModelQuota, + isDailyQuotaExhausted, + getMsUntilTomorrow, +} from "../services/accountFallback.ts"; import { COOLDOWN_MS } from "../config/constants.ts"; import { buildErrorBody, @@ -2109,17 +2114,22 @@ export async function handleChatCore({ } } else if (errorType === PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED) { // Providers with per-model quotas โ€” lock the model only, not the connection + // Daily quota exhausted: lock until tomorrow; otherwise use default cooldown + const isDailyQuota = isDailyQuotaExhausted(message); + const quotaCooldownMs = isDailyQuota + ? getMsUntilTomorrow() + : retryAfterMs || COOLDOWN_MS.rateLimit; if ( lockModelIfPerModelQuota( provider, connectionId, model, - "quota_exhausted", - retryAfterMs || COOLDOWN_MS.rateLimit + isDailyQuota ? "daily_quota_exhausted" : "quota_exhausted", + quotaCooldownMs ) ) { console.warn( - `[provider] Node ${connectionId} model-only quota exhausted (${statusCode}) for ${model} - ${Math.ceil((retryAfterMs || COOLDOWN_MS.rateLimit) / 1000)}s (connection stays active)` + `[provider] Node ${connectionId} ${isDailyQuota ? "daily " : ""}quota exhausted (${statusCode}) for ${model} - ${Math.ceil(quotaCooldownMs / 1000)}s (connection stays active)` ); } else { await updateProviderConnection(connectionId, { diff --git a/open-sse/index.ts b/open-sse/index.ts index fffb5ea17e..2c49410688 100644 --- a/open-sse/index.ts +++ b/open-sse/index.ts @@ -50,6 +50,9 @@ export { isAccountUnavailable, getUnavailableUntil, filterAvailableAccounts, + isProviderInCooldown, + getProviderCooldownRemainingMs, + getProvidersInCooldown, } from "./services/accountFallback.ts"; export { diff --git a/open-sse/services/accountFallback.ts b/open-sse/services/accountFallback.ts index 95bdf510df..efadc2c25d 100644 --- a/open-sse/services/accountFallback.ts +++ b/open-sse/services/accountFallback.ts @@ -24,6 +24,30 @@ type ModelFailureState = { resetAfterMs: number; }; +// Provider-level failure tracking for circuit breaker behavior +type ProviderFailureEntry = { + failureCount: number; + lastFailureAt: number; + resetAfterMs: number; + cooldownUntil: number | null; +}; + +// Error codes that count toward provider-level failure threshold +const PROVIDER_FAILURE_ERROR_CODES = new Set([429, 408, 500, 502, 503, 504]); + +// Configuration for provider-level failure tracking +const PROVIDER_FAILURE_THRESHOLD = 5; +const PROVIDER_FAILURE_WINDOW_MS = 20 * 60 * 1000; // 20 minutes +const PROVIDER_COOLDOWN_MS = 10 * 60 * 1000; // 10 minutes cooling + +// Provider-level failure state map: providerId -> failure entry +const providerFailureState = new Map(); +// Guard against synchronous re-entrant calls within the same event-loop tick. +// NOT a true mutex โ€” Node.js is single-threaded, so different SSE streams +// can interleave across ticks. This Set prevents a single call from recursively +// re-entering recordProviderFailure within the same synchronous call stack. +const providerFailureLocks = new Set(); + // T06 (sub2api PR #1037): Signals that indicate permanent account deactivation. // When a 401 body contains these strings, the account is permanently dead // and should NOT be retried after token refresh. @@ -82,8 +106,12 @@ const CONTEXT_OVERFLOW_PATTERNS = [ const MALFORMED_REQUEST_PATTERNS = [ /\bimproperly formed request\b/i, /\binvalid.*message.*format/i, - /\bmessages must alternate/i, - /\bempty (message|content)/i, + /\bmessages must alternate\b/i, + /\bempty (message|content)\b/i, + // Tool call function name errors + /\bfunction'?s? name (?:can't|can not|is|has) (?:blank|empty|missing)/i, + /function.*name.*(?:blank|empty|missing)/i, + /tool_call.*name.*(?:blank|empty|missing)/i, ]; /** @@ -355,11 +383,21 @@ export function clearModelLock(provider, connectionId, model) { * Compatible and passthrough providers multiplex multiple upstream models behind one * connection, so transient 404/429 responses should stay model-scoped instead of * poisoning the whole connection. + * + * @param provider - Provider ID + * @param _model - Model ID (reserved for future use) + * @param connectionPassthroughModels - Optional per-connection override from providerSpecificData. + * When provided, takes precedence over registry/provider-level logic. */ export function hasPerModelQuota( provider: string | null | undefined, - _model: string | null | undefined = null + _model: string | null | undefined = null, + connectionPassthroughModels?: boolean ): boolean { + // Connection-level override takes precedence (e.g., user-configured ModelScope) + if (typeof connectionPassthroughModels === "boolean") { + return connectionPassthroughModels; + } if (!provider) return false; if (provider === "gemini") return true; if (getPassthroughProviders().has(provider)) return true; @@ -376,18 +414,23 @@ export function lockModelIfPerModelQuota( connectionId: string, model: string | null, reason: string, - cooldownMs: number + cooldownMs: number, + connectionPassthroughModels?: boolean ): boolean { - if (!hasPerModelQuota(provider, model) || !model) return false; + if (!hasPerModelQuota(provider, model, connectionPassthroughModels) || !model) return false; + // Skip model-level lock if the entire provider is in circuit-breaker cooldown. + // The provider cooldown already prevents all requests, so a model lock is redundant. + if (isProviderInCooldown(provider)) return false; lockModel(provider, connectionId, model, reason, cooldownMs); return true; } export function shouldMarkAccountExhaustedFrom429( provider: string | null | undefined, - model: string | null | undefined = null + model: string | null | undefined = null, + connectionPassthroughModels?: boolean ): boolean { - return !hasPerModelQuota(provider, model); + return !hasPerModelQuota(provider, model, connectionPassthroughModels); } /** @@ -442,6 +485,145 @@ export function getAllModelLockouts() { return active; } +// โ”€โ”€โ”€ Provider-Level Failure Tracking โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +// Track failures at provider level: when a provider has too many transient failures +// across all its connections, cooldown the entire provider temporarily. + +/** + * Check if a provider is currently in cooldown due to too many failures + */ +export function isProviderInCooldown(provider: string | null | undefined): boolean { + if (!provider) return false; + const entry = providerFailureState.get(provider); + if (!entry) return false; + + // If in cooldown, check if it has expired + if (entry.cooldownUntil !== null && Date.now() >= entry.cooldownUntil) { + providerFailureState.delete(provider); + return false; + } + + return entry.cooldownUntil !== null; +} + +/** + * Get remaining cooldown time for a provider + */ +export function getProviderCooldownRemainingMs(provider: string | null | undefined): number | null { + if (!provider) return null; + const entry = providerFailureState.get(provider); + if (!entry || entry.cooldownUntil === null) return null; + + const remaining = entry.cooldownUntil - Date.now(); + return remaining > 0 ? remaining : null; +} + +/** + * Record a failure for a provider. When threshold is reached within the window, + * the provider enters cooldown. + */ +export function recordProviderFailure( + provider: string | null | undefined, + log?: { warn?: (...args: unknown[]) => void } +): void { + if (!provider) return; + + // Guard against concurrent re-entrant calls within the same tick + if (providerFailureLocks.has(provider)) return; + providerFailureLocks.add(provider); + + try { + const now = Date.now(); + const entry = providerFailureState.get(provider); + + // Check if we're in cooldown period + if (entry && entry.cooldownUntil !== null && now < entry.cooldownUntil) { + return; // Already in cooldown, don't record + } + + // Check if failure window has expired + if (entry && now - entry.lastFailureAt > entry.resetAfterMs) { + // Window expired, reset count + providerFailureState.set(provider, { + failureCount: 1, + lastFailureAt: now, + resetAfterMs: PROVIDER_FAILURE_WINDOW_MS, + cooldownUntil: null, + }); + return; + } + + // Increment failure count + const newCount = entry ? entry.failureCount + 1 : 1; + + if (newCount >= PROVIDER_FAILURE_THRESHOLD) { + // Threshold reached, enter cooldown + const cooldownUntil = now + PROVIDER_COOLDOWN_MS; + providerFailureState.set(provider, { + failureCount: newCount, + lastFailureAt: now, + resetAfterMs: PROVIDER_FAILURE_WINDOW_MS, + cooldownUntil, + }); + log?.warn?.( + `[ProviderFailure] ${provider}: ${newCount} failures in ${PROVIDER_FAILURE_WINDOW_MS / 1000}s โ€” entering ${PROVIDER_COOLDOWN_MS / 1000}s cooldown` + ); + } else { + // Just increment counter + providerFailureState.set(provider, { + failureCount: newCount, + lastFailureAt: now, + resetAfterMs: PROVIDER_FAILURE_WINDOW_MS, + cooldownUntil: null, + }); + } + } finally { + providerFailureLocks.delete(provider); + } +} + +/** + * Clear provider failure state (e.g., after successful request) + */ +export function clearProviderFailure(provider: string | null | undefined): void { + if (!provider) return; + providerFailureState.delete(provider); +} + +/** + * Get all providers currently in cooldown (for debugging/dashboard) + */ +export function getProvidersInCooldown(): Array<{ + provider: string; + failureCount: number; + cooldownRemainingMs: number | null; + lastFailureAt: number; +}> { + const result = []; + for (const [provider, entry] of providerFailureState) { + if (entry.cooldownUntil === null) continue; + const remaining = entry.cooldownUntil - Date.now(); + if (remaining <= 0) { + providerFailureState.delete(provider); + continue; + } + result.push({ + provider, + failureCount: entry.failureCount, + cooldownRemainingMs: remaining, + lastFailureAt: entry.lastFailureAt, + }); + } + return result; +} + +/** + * Check if a status code should be counted toward provider failure threshold + */ +export function isProviderFailureCode(status: number): boolean { + return PROVIDER_FAILURE_ERROR_CODES.has(status); +} + // โ”€โ”€โ”€ Retry-After Parsing โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ /** @@ -625,6 +807,40 @@ export function classifyError(status, errorText) { return RateLimitReason.UNKNOWN; } +// โ”€โ”€โ”€ Daily Quota Helpers โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +/** + * Calculate milliseconds from now until tomorrow at midnight (00:00:00). + * Used to lock a model until the next day when daily quota is exhausted. + * @returns {number} Milliseconds until tomorrow + */ +export function getMsUntilTomorrow(): number { + const now = new Date(); + const tomorrow = new Date(now); + tomorrow.setDate(tomorrow.getDate() + 1); + tomorrow.setHours(0, 0, 0, 0); + const ms = tomorrow.getTime() - now.getTime(); + // Guard against DST edge cases: if ms is negative (shouldn't happen) or + // unreasonably large (>25h due to spring-forward), cap at 24 hours. + return ms > 0 && ms <= 25 * 60 * 60 * 1000 ? ms : 24 * 60 * 60 * 1000; +} + +/** + * Check if error text indicates daily quota exhaustion (as opposed to rate limiting). + * Daily quota errors typically mention "today's quota" or "try again tomorrow". + * @param {string} errorText - Error message text + * @returns {boolean} True if daily quota is exhausted + */ +export function isDailyQuotaExhausted(errorText: string): boolean { + if (!errorText) return false; + const lower = errorText.toLowerCase(); + return ( + lower.includes("today's quota") || + lower.includes("daily quota") || + lower.includes("try again tomorrow") + ); +} + // โ”€โ”€โ”€ Configurable Backoff โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ /** @@ -681,6 +897,12 @@ export function checkFallbackError( const errorStr = (errorText || "").toString(); const profile = profileOverride ?? (provider ? getProviderProfile(provider) : null); + // Track provider-level failures for circuit breaker behavior + // Only count transient errors that are likely to recover + if (isProviderFailureCode(status)) { + recordProviderFailure(provider); + } + function parseResetFromHeaders(headers, errorStr = "") { if (!headers) return null; @@ -737,6 +959,19 @@ export function checkFallbackError( }; } + // Daily quota exhausted โ€” lock model until tomorrow + if (isDailyQuotaExhausted(errorStr)) { + const msUntilTomorrow = getMsUntilTomorrow(); + // Cap at 24 hours to handle timezone edge cases + const cooldownMs = Math.min(msUntilTomorrow, 24 * 60 * 60 * 1000); + return { + shouldFallback: true, + cooldownMs, + reason: RateLimitReason.QUOTA_EXHAUSTED, + dailyQuotaExhausted: true, + }; + } + if (lowerError.includes("no credentials")) { return { shouldFallback: true, diff --git a/open-sse/services/cloudCodeThinking.ts b/open-sse/services/cloudCodeThinking.ts index 769c414d7b..9d3b3b944a 100644 --- a/open-sse/services/cloudCodeThinking.ts +++ b/open-sse/services/cloudCodeThinking.ts @@ -23,6 +23,9 @@ function stripGeminiThinkingConfig(value: unknown): unknown { return next; } +/** + * @deprecated This function will be removed in v4.0, reasoning configuration processing has migrated to translateRequest + */ export function shouldStripCloudCodeThinking(provider: string, model: string): boolean { if (!provider || !model) return false; const normalizedModel = normalizeCloudCodeModel(model); @@ -33,6 +36,9 @@ export function shouldStripCloudCodeThinking(provider: string, model: string): b return CLOUD_CODE_REASONING_UNSUPPORTED_PATTERNS.some((pattern) => pattern.test(normalizedModel)); } +/** + * @deprecated This function will be removed in v4.0, reasoning configuration processing has migrated to translateRequest + */ export function stripCloudCodeThinkingConfig( body: Record ): Record { diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index dcbd63a457..796a8a58da 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -59,6 +59,47 @@ const COMBO_BAD_REQUEST_FALLBACK_PATTERNS = [ /unsupported content part type/i, /tool(?:_call|_use)? .* not (?:available|found)/i, /third-party apps/i, + // Context overflow โ€” model-specific, may succeed on a model with larger context window + /context overflow/i, + /context length exceeded/i, + /prompt too large/i, + /token limit/i, + /too many tokens/i, + /exceeds? context/i, + /maximum context/i, + /input too long/i, + /messages? exceed/i, + // Model not supported/found โ€” permanent model-level error, try next combo target + /no provider supported/i, + /model not found/i, + /model not available/i, + /unsupported model/i, + /model.*has no provider/i, + // Function calling format error โ€” model doesn't support this capability + /function\.?arguments.*(must be|should be|ๅฟ…้กป).*(json|JSON)/i, + /tool.*arguments.*invalid/i, + /function.*parameter.*(invalid|format)/i, + // Input length range error โ€” model-specific context limit + /range of input length/i, + /input length should be/i, + // Transient 400 errors from upstream โ€” should fallback to next combo target + /ๆœๅŠก้‡ๅˆฐไบ†ไธ€็‚นๅฐ็Šถๅ†ต/i, // ModelScope/Qwen transient error + /ๆŠฑๆญ‰.*?ๆ•ๆ„Ÿๅ†…ๅฎน.*?่ฏทๆฃ€ๆŸฅ/i, // ModelScope/Qwen content moderation with context + /ๅ†…ๅฎน.*?ๆ•ๆ„Ÿ.*?(?:ๆ— ๆณ•|่ฟ‡ๆปค)/i, // Content sensitivity block + /ๆ— ๆณ•ๅ“ๅบ”.*?่ฏทๆฑ‚/i, // "unable to respond to request" + /็จๅŽ้‡่ฏ•/i, // "retry later" in Chinese + /temporary.*error/i, + /transient.*error/i, + /service.*unavailable/i, + /please.*try.*again/i, + // Rate limit errors โ€” some providers return 400 instead of 429 + /\brate.?-?limit.?(?:exceeded|reached|hit)/i, + /too many requests/i, + /่ฏทๆฑ‚่ฟ‡ไบŽ้ข‘็น/i, // Chinese rate limit message + // Tool call function name errors โ€” model-specific, try next combo target + /\bfunction'?s? name (?:can't|can not|is|has) (?:blank|empty|missing)/i, + /function.*name.*(?:blank|empty|missing)/i, + /tool_call.*name.*(?:blank|empty|missing)/i, ]; // Patterns that signal all accounts for a provider are rate-limited / exhausted. @@ -77,6 +118,7 @@ function isAllAccountsRateLimitedResponse( const MAX_COMBO_DEPTH = 3; const MAX_FALLBACK_WAIT_MS = 5000; +const MAX_GLOBAL_ATTEMPTS = 30; function comboModelNotFoundResponse(message: string) { return errorResponse(404, message); @@ -1451,6 +1493,7 @@ export async function handleComboChat({ let earliestRetryAfter = null; let lastStatus = null; const startTime = Date.now(); + globalAttempts = 0; let fallbackCount = 0; let recordedAttempts = 0; @@ -1484,6 +1527,14 @@ export async function handleComboChat({ // Retry loop for transient errors for (let retry = 0; retry <= maxRetries; retry++) { + globalAttempts++; + if (globalAttempts > MAX_GLOBAL_ATTEMPTS) { + log.warn( + "COMBO", + `Maximum combo attempts (${MAX_GLOBAL_ATTEMPTS}) exceeded across all targets and fallbacks. Terminating loop to prevent runaway background requests.` + ); + return errorResponse(503, "Maximum combo retry limit reached"); + } if (retry > 0) { log.info( "COMBO", @@ -1799,6 +1850,7 @@ async function handleRoundRobinCombo({ let lastError = null; let lastStatus = null; let earliestRetryAfter = null; + let globalAttempts = 0; let fallbackCount = 0; let recordedAttempts = 0; @@ -1852,6 +1904,14 @@ async function handleRoundRobinCombo({ // Retry loop within this model try { for (let retry = 0; retry <= maxRetries; retry++) { + globalAttempts++; + if (globalAttempts > MAX_GLOBAL_ATTEMPTS) { + log.warn( + "COMBO-RR", + `Maximum combo attempts (${MAX_GLOBAL_ATTEMPTS}) exceeded. Terminating loop to prevent runaway requests.` + ); + return errorResponse(503, "Maximum combo retry limit reached"); + } if (retry > 0) { log.info( "COMBO-RR", diff --git a/open-sse/services/errorClassifier.ts b/open-sse/services/errorClassifier.ts index 39d513ffbe..4520c2a6e4 100644 --- a/open-sse/services/errorClassifier.ts +++ b/open-sse/services/errorClassifier.ts @@ -1,6 +1,7 @@ import { isAccountDeactivated, isCreditsExhausted, + isDailyQuotaExhausted, isOAuthInvalidToken, } from "./accountFallback.ts"; @@ -100,7 +101,11 @@ export function classifyProviderError(statusCode: number, responseBody: unknown) return PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED; } + // 429: Check if it's a daily quota exhaustion (lock until tomorrow) vs regular rate limit if (statusCode === 429) { + if (isDailyQuotaExhausted(bodyStr)) { + return PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED; + } return PROVIDER_ERROR_TYPES.RATE_LIMITED; } diff --git a/src/app/(dashboard)/dashboard/combos/IntelligentComboPanel.tsx b/src/app/(dashboard)/dashboard/combos/IntelligentComboPanel.tsx index 0a22d20e07..55013eab4d 100644 --- a/src/app/(dashboard)/dashboard/combos/IntelligentComboPanel.tsx +++ b/src/app/(dashboard)/dashboard/combos/IntelligentComboPanel.tsx @@ -11,6 +11,7 @@ import { extractIntelligentHealthState, normalizeIntelligentRoutingConfig, } from "@/lib/combos/intelligentRouting"; +import ModelStatusBadge from "@/app/(dashboard)/dashboard/providers/components/ModelStatusBadge"; function getI18nOrFallback(t: any, key: string, fallback: string) { if (typeof t?.has === "function" && t.has(key)) return t(key); @@ -283,6 +284,8 @@ export default function IntelligentComboPanel({

{entry.model}

+ {/* Model status badge - shows cooldown/error state */} + {percentage}% diff --git a/src/app/(dashboard)/dashboard/providers/[id]/page.tsx b/src/app/(dashboard)/dashboard/providers/[id]/page.tsx index 2b9d0abfc4..9164e05718 100644 --- a/src/app/(dashboard)/dashboard/providers/[id]/page.tsx +++ b/src/app/(dashboard)/dashboard/providers/[id]/page.tsx @@ -53,6 +53,7 @@ import { getCodexRequestDefaults as _getCodexRequestDefaults, } from "@/lib/providers/requestDefaults"; import { resolveDashboardProviderInfo } from "../providerPageUtils"; +import ModelStatusBadge from "@/app/(dashboard)/dashboard/providers/components/ModelStatusBadge"; type CompatByProtocolMap = Partial< Record< @@ -326,6 +327,7 @@ function anyUpstreamHeadersBadge( interface ModelRowProps { model: { id: string; name?: string; source?: string; isHidden?: boolean }; fullModel: string; + provider: string; copied?: string; onCopy: (text: string, key: string) => void; t: (key: string, values?: Record) => string; @@ -2460,6 +2462,7 @@ export default function ProviderDetailPage() { key={model.id} model={model} fullModel={`${providerDisplayAlias}/${model.id}`} + provider={providerId} copied={copied} onCopy={copy} t={t} @@ -3329,6 +3332,7 @@ export default function ProviderDetailPage() { function ModelRow({ model, fullModel, + provider, copied, onCopy, t, @@ -3358,6 +3362,7 @@ function ModelRow({ {fullModel} + - - {/* Expanded popover */} - {expanded && ( -
-
-
- - {t("modelStatus")} -
- -
- -
- {isHealthy ? ( -

{t("allModelsNormal")}

- ) : ( -
- {Object.entries(byProvider).map(([provider, provModels]) => ( -
-

- {provider} -

-
- {provModels.map((m) => { - const status = - STATUS_CONFIG[m.status as keyof typeof STATUS_CONFIG] || - STATUS_CONFIG.unknown; - const isClearing = clearing === `${m.provider}:${m.model}`; - return ( -
-
- - - {m.model} - -
- {m.status === "cooldown" && ( - - )} -
- ); - })} -
-
- ))} -
- )} -
-
- )} - - ); -} diff --git a/src/app/(dashboard)/dashboard/providers/components/ModelAvailabilityPanel.tsx b/src/app/(dashboard)/dashboard/providers/components/ModelAvailabilityPanel.tsx deleted file mode 100644 index 2ede152ba5..0000000000 --- a/src/app/(dashboard)/dashboard/providers/components/ModelAvailabilityPanel.tsx +++ /dev/null @@ -1,203 +0,0 @@ -"use client"; - -/** - * ModelAvailabilityPanel โ€” Batch B - * - * Shows real-time model availability and cooldown status. - * Fetched from /api/models/availability. - */ - -import { useState, useEffect, useCallback } from "react"; -import { useTranslations } from "next-intl"; -import { Card, Button } from "@/shared/components"; -import { useNotificationStore } from "@/store/notificationStore"; - -export default function ModelAvailabilityPanel() { - const t = useTranslations("providers"); - const tc = useTranslations("common"); - - const STATUS_CONFIG = { - available: { icon: "check_circle", color: "#22c55e", label: t("available") }, - cooldown: { icon: "schedule", color: "#f59e0b", label: t("cooldown") }, - unavailable: { icon: "error", color: "#ef4444", label: t("unavailable") }, - unknown: { icon: "help", color: "#6b7280", label: t("unknown") }, - }; - - const [data, setData] = useState(null); - const [loading, setLoading] = useState(true); - const [clearing, setClearing] = useState(null); - const notify = useNotificationStore(); - - const fetchStatus = useCallback(async () => { - try { - const res = await fetch("/api/models/availability"); - if (res.ok) { - const json = await res.json(); - setData(json); - } - } catch { - // silent fail โ€” will retry - } finally { - setLoading(false); - } - }, []); - - useEffect(() => { - fetchStatus(); - const interval = setInterval(fetchStatus, 30000); - return () => clearInterval(interval); - }, [fetchStatus]); - - const handleClearCooldown = async (provider: string, model: string) => { - setClearing(`${provider}:${model}`); - try { - const res = await fetch("/api/models/availability", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ action: "clearCooldown", provider, model }), - }); - if (res.ok) { - notify.success(t("cooldownCleared", { model })); - await fetchStatus(); - } else { - notify.error(t("failedClearCooldown")); - } - } catch { - notify.error(t("failedClearCooldown")); - } finally { - setClearing(null); - } - }; - - if (loading) { - return ( - -
- - {t("loadingAvailability")} -
- - ); - } - - const models = data?.models || []; - const unavailableCount = - data?.unavailableCount || models.filter((m: any) => m.status !== "available").length; - - if (models.length === 0 || unavailableCount === 0) { - return ( - -
-
- -
-
-

{t("modelAvailability")}

-

{t("allModelsOperational")}

-
-
- - ); - } - - // Group by provider - const byProvider: Record = {}; - models.forEach((m: any) => { - if (m.status === "available") return; - const key = m.provider || "unknown"; - if (!byProvider[key]) byProvider[key] = []; - byProvider[key].push(m); - }); - - return ( - -
-
-
- -
-
-

{t("modelAvailability")}

-

- {t("modelsWithIssues", { count: unavailableCount })} -

-
-
- -
- -
- {Object.entries(byProvider).map(([provider, provModels]) => ( -
-

{provider}

-
- {provModels.map((m) => { - const status = - STATUS_CONFIG[m.status as keyof typeof STATUS_CONFIG] || STATUS_CONFIG.unknown; - const isClearing = clearing === `${m.provider}:${m.model}`; - return ( -
-
- - {m.model} - - {status.label} - - {m.cooldownUntil && ( - - {t("until", { time: new Date(m.cooldownUntil).toLocaleTimeString() })} - - )} -
- {m.status === "cooldown" && ( - - )} -
- ); - })} -
-
- ))} -
- - ); -} diff --git a/src/app/(dashboard)/dashboard/providers/components/ModelStatusBadge.tsx b/src/app/(dashboard)/dashboard/providers/components/ModelStatusBadge.tsx deleted file mode 100644 index 82538a6d8d..0000000000 --- a/src/app/(dashboard)/dashboard/providers/components/ModelStatusBadge.tsx +++ /dev/null @@ -1,162 +0,0 @@ -"use client"; - -/** - * ModelStatusBadge โ€” compact single-model status indicator - * - * Shows a small badge with status icon (cooldown/unavailable/error) - * with a tooltip containing additional details like remaining cooldown time - * or last error message. - * Only renders for non-available models to keep the UI clean. - * - * Uses shared polling via ModelStatusContext to avoid redundant API calls. - */ - -import { useState, useEffect, useRef } from "react"; -import { useTranslations } from "next-intl"; -import Tooltip from "@/shared/components/Tooltip"; -import { useModelStatus } from "./ModelStatusContext"; - -interface ModelStatusBadgeProps { - provider: string; - model: string; - size?: "sm" | "md"; - className?: string; -} - -function formatRemainingTime(ms: number): string { - if (ms <= 0) return ""; - const seconds = Math.ceil(ms / 1000); - if (seconds < 60) return `${seconds}s`; - const minutes = Math.floor(seconds / 60); - const remainingSeconds = seconds % 60; - if (minutes < 60) { - return remainingSeconds > 0 ? `${minutes}m ${remainingSeconds}s` : `${minutes}m`; - } - const hours = Math.floor(minutes / 60); - const remainingMinutes = minutes % 60; - return remainingMinutes > 0 ? `${hours}h ${remainingMinutes}m` : `${hours}h`; -} - -export default function ModelStatusBadge({ - provider, - model, - size = "sm", - className = "", -}: ModelStatusBadgeProps) { - const t = useTranslations("providers"); - const status = useModelStatus(provider, model); - - // Store the latest remaining time calculated by the interval - const [displayRemainingMs, setDisplayRemainingMs] = useState(null); - - // Use ref to track server-provided initial cooldown duration for countdown calculation - const initialCooldownMsRef = useRef(null); - // Track when we started counting down - const countdownStartRef = useRef(null); - - // Set up countdown timer when cooldown begins - useEffect(() => { - if (status?.status !== "cooldown" || !status.remainingMs) { - // Reset refs when not in cooldown (no setState here) - initialCooldownMsRef.current = null; - countdownStartRef.current = null; - return; - } - - // Initialize timing refs (no setState here) - initialCooldownMsRef.current = status.remainingMs; - countdownStartRef.current = Date.now(); - - // Update countdown every second via interval callback (setState allowed here) - const interval = setInterval(() => { - if (!initialCooldownMsRef.current || !countdownStartRef.current) { - return; - } - - const elapsed = Date.now() - countdownStartRef.current; - const newRemaining = Math.max(0, initialCooldownMsRef.current - elapsed); - - setDisplayRemainingMs(newRemaining); - - // Stop updating when countdown reaches zero - if (newRemaining === 0) { - clearInterval(interval); - } - }, 1000); - - return () => clearInterval(interval); - }, [status?.status, status?.remainingMs]); - - // Derive the displayed remaining time: use countdown value if active, else use status value - const remainingMs = - status?.status === "cooldown" ? (displayRemainingMs ?? status.remainingMs ?? null) : null; - - // Don't render badge for available models (keep UI clean) - if (!status || status.status === "available" || status.status === "unknown") { - return null; - } - - const getStatusColor = () => { - switch (status.status) { - case "cooldown": - return "#f59e0b"; - case "unavailable": - case "error": - return "#ef4444"; - default: - return "#6b7280"; - } - }; - - const getStatusIcon = () => { - switch (status.status) { - case "cooldown": - return "schedule"; - case "unavailable": - case "error": - return "error"; - default: - return "help"; - } - }; - - const getTooltipText = () => { - switch (status.status) { - case "cooldown": { - const remaining = remainingMs !== null ? formatRemainingTime(remainingMs) : ""; - const reason = status.reason ? ` (${status.reason})` : ""; - const remainingText = remaining ? ` - ${remaining}` : ""; - return `${t("cooldown")}${reason}${remainingText}`; - } - case "unavailable": - return `${t("unavailable")}${status.reason ? `: ${status.reason}` : ""}`; - case "error": - return `${t("error")}${status.lastError ? `: ${status.lastError}` : ""}`; - default: - return ""; - } - }; - - const color = getStatusColor(); - const sizeClasses = size === "sm" ? "px-1.5 py-0.5" : "px-2 py-1"; - const iconSize = size === "sm" ? "text-[12px]" : "text-[14px]"; - - return ( - - - - {status.status === "cooldown" && remainingMs !== null && ( - {formatRemainingTime(remainingMs)} - )} - - - ); -} diff --git a/src/app/(dashboard)/dashboard/providers/components/ModelStatusContext.tsx b/src/app/(dashboard)/dashboard/providers/components/ModelStatusContext.tsx deleted file mode 100644 index 3605a9defd..0000000000 --- a/src/app/(dashboard)/dashboard/providers/components/ModelStatusContext.tsx +++ /dev/null @@ -1,186 +0,0 @@ -"use client"; - -/** - * ModelStatusContext โ€” shared polling for model availability - * - * Prevents redundant API calls by having all ModelStatusBadge components - * share a single polling interval. Only one request is made every 15 seconds - * regardless of how many badges are on the page. - */ - -import React, { createContext, useContext, useEffect, useRef, useCallback, useMemo } from "react"; - -export interface ModelStatus { - status: "available" | "cooldown" | "unavailable" | "error" | "unknown"; - reason?: string; - remainingMs?: number; - lastError?: string; -} - -interface ModelStatusContextValue { - getStatus: (provider: string, model: string) => ModelStatus | null; - registerModel: (key: string, provider: string, model: string) => void; - unregisterModel: (key: string) => void; -} - -const ModelStatusContext = createContext(null); - -// Global map of model key -> status -let modelStatusMap = new Map(); -// Global set of registered model keys -let registeredModels = new Set(); -// Polling interval ref (singleton) -let pollIntervalRef: NodeJS.Timeout | null = null; - -function getModelKey(provider: string, model: string): string { - return `${provider}/${model}`; -} - -async function fetchModelStatus(): Promise { - try { - const res = await fetch("/api/models/availability"); - if (!res.ok) return; - - const json = await res.json(); - const models = json?.models || []; - - // Update all registered models with fresh data - const now = Date.now(); - registeredModels.forEach((key) => { - const [provider, model] = key.split("/"); - // Use exact matching first to avoid gpt-4 matching gpt-4-turbo incorrectly - const modelEntry = - models.find((m: any) => m.provider === provider && m.model === model) || - models.find( - // Fallback to prefix matching only for models that contain the registered key - // This handles cases like "gpt-4o" matching badge for "gpt-4" - (m: any) => - m.provider === provider && - m.model && - model && - (m.model.startsWith(model + "-") || model.startsWith(m.model + "-")) - ); - - if (modelEntry) { - const newStatus: ModelStatus = { - status: modelEntry.status || "unknown", - reason: modelEntry.reason, - remainingMs: modelEntry.remainingMs, - lastError: modelEntry.lastError, - }; - - // For cooldown status, calculate remaining time based on server-provided value - if (modelEntry.status === "cooldown" && modelEntry.remainingMs) { - newStatus.remainingMs = modelEntry.remainingMs; - } - - modelStatusMap.set(key, newStatus); - } else { - modelStatusMap.set(key, { status: "available" }); - } - }); - - // Trigger re-render by dispatching custom event - window.dispatchEvent(new CustomEvent("model-status-update")); - } catch { - // Best-effort polling - } -} - -function ensurePolling(): void { - if (pollIntervalRef) return; - - // Initial fetch - fetchModelStatus(); - - // Poll every 15 seconds - pollIntervalRef = setInterval(fetchModelStatus, 15000); -} - -function stopPolling(): void { - if (pollIntervalRef) { - clearInterval(pollIntervalRef); - pollIntervalRef = null; - } -} - -export function ModelStatusProvider({ children }: { children: React.ReactNode }) { - const [, forceUpdate] = React.useState(0); - - // Listen for status updates from the global poller - useEffect(() => { - const handleUpdate = () => forceUpdate((n) => n + 1); - window.addEventListener("model-status-update", handleUpdate); - return () => window.removeEventListener("model-status-update", handleUpdate); - }, []); - - // Cleanup on unmount โ€” stop polling only when no models remain registered - useEffect(() => { - return () => { - if (registeredModels.size === 0) { - stopPolling(); - } - }; - }, []); - - const getStatus = useCallback((provider: string, model: string): ModelStatus | null => { - const key = getModelKey(provider, model); - return modelStatusMap.get(key) || null; - }, []); - - const registerModel = useCallback((key: string, provider: string, model: string): void => { - const wasEmpty = registeredModels.size === 0; - registeredModels.add(key); - - // Start polling when first model registers - if (wasEmpty) { - ensurePolling(); - } - - // Immediately fetch if no data yet - if (!modelStatusMap.has(key)) { - fetchModelStatus(); - } - }, []); - - const unregisterModel = useCallback((key: string): void => { - registeredModels.delete(key); - modelStatusMap.delete(key); - - // Stop polling when last model unregisters - if (registeredModels.size === 0) { - stopPolling(); - } - }, []); - - const value = useMemo( - () => ({ - getStatus, - registerModel, - unregisterModel, - }), - [getStatus, registerModel, unregisterModel] - ); - - return {children}; -} - -export function useModelStatus(provider: string, model: string): ModelStatus | null { - const context = useContext(ModelStatusContext); - - if (!context) { - throw new Error("useModelStatus must be used within a ModelStatusProvider"); - } - - const key = useMemo(() => getModelKey(provider, model), [provider, model]); - - // Register/unregister on mount/unmount - useEffect(() => { - context.registerModel(key, provider, model); - return () => context.unregisterModel(key); - }, [context, key, provider, model]); - - return context.getStatus(provider, model); -} - -export default ModelStatusContext; diff --git a/src/app/(dashboard)/dashboard/providers/page.tsx b/src/app/(dashboard)/dashboard/providers/page.tsx index de86e770cb..e509322fa6 100644 --- a/src/app/(dashboard)/dashboard/providers/page.tsx +++ b/src/app/(dashboard)/dashboard/providers/page.tsx @@ -26,7 +26,6 @@ import { getErrorCode, getRelativeTime } from "@/shared/utils"; import { pickDisplayValue } from "@/shared/utils/maskEmail"; import useEmailPrivacyStore from "@/store/emailPrivacyStore"; import { useNotificationStore } from "@/store/notificationStore"; -import ModelAvailabilityBadge from "./components/ModelAvailabilityBadge"; import { useTranslations } from "next-intl"; import { buildMergedOAuthProviderEntries, @@ -496,7 +495,6 @@ export default function ProvidersPage() {
- = { "context-relay": "Context Relay", }; +const LEGACY_COMBO_RESILIENCE_KEYS = new Set([ + "timeoutMs", + "healthCheckEnabled", + "healthCheckTimeoutMs", +]); + function translateOrFallback( t: ReturnType, key: string, @@ -18,14 +24,31 @@ function translateOrFallback( return typeof t.has === "function" && t.has(key) ? t(key) : fallback; } +function sanitizeComboRuntimeConfig(config?: Record | null) { + if (!config || typeof config !== "object") return {}; + return Object.fromEntries( + Object.entries(config).filter( + ([key, value]) => + value !== undefined && value !== null && !LEGACY_COMBO_RESILIENCE_KEYS.has(key) + ) + ); +} + +function sanitizeProviderOverrides(overrides?: Record | null) { + if (!overrides || typeof overrides !== "object") return {}; + return Object.fromEntries( + Object.entries(overrides).map(([providerId, config]) => [ + providerId, + sanitizeComboRuntimeConfig(config), + ]) + ); +} + export default function ComboDefaultsTab() { const [comboDefaults, setComboDefaults] = useState({ strategy: "priority", maxRetries: 1, retryDelayMs: 2000, - timeoutMs: 120000, - healthCheckEnabled: true, - healthCheckTimeoutMs: 3000, maxComboDepth: 3, trackMetrics: true, handoffThreshold: 0.85, @@ -54,7 +77,6 @@ export default function ComboDefaultsTab() { const numericSettings = [ { key: "maxRetries", label: t("maxRetriesLabel"), min: 0, max: 5 }, { key: "retryDelayMs", label: t("retryDelayLabel"), min: 500, max: 10000, step: 500 }, - { key: "timeoutMs", label: t("timeoutLabel"), min: 5000, step: 5000 }, { key: "maxComboDepth", label: t("maxNestingDepth"), min: 1, max: 10 }, ]; @@ -66,7 +88,7 @@ export default function ComboDefaultsTab() { .then(([comboData, settingsData]) => { setComboDefaults((prev) => ({ ...prev, - ...(comboData.comboDefaults || {}), + ...sanitizeComboRuntimeConfig(comboData.comboDefaults), strategy: settingsData.fallbackStrategy ?? comboData.comboDefaults?.strategy ?? prev.strategy, stickyRoundRobinLimit: @@ -74,7 +96,9 @@ export default function ComboDefaultsTab() { comboData.comboDefaults?.stickyRoundRobinLimit ?? prev.stickyRoundRobinLimit, })); - if (comboData.providerOverrides) setProviderOverrides(comboData.providerOverrides); + if (comboData.providerOverrides) { + setProviderOverrides(sanitizeProviderOverrides(comboData.providerOverrides)); + } }) .catch((err) => console.error("Failed to fetch combo defaults:", err)); }, []); @@ -116,7 +140,10 @@ export default function ComboDefaultsTab() { const comboDefaultsRes = await fetch("/api/settings/combo-defaults", { method: "PATCH", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ comboDefaults: comboDefaultsPayload, providerOverrides }), + body: JSON.stringify({ + comboDefaults: sanitizeComboRuntimeConfig(comboDefaultsPayload), + providerOverrides: sanitizeProviderOverrides(providerOverrides), + }), }); if (!comboDefaultsRes.ok) { @@ -136,7 +163,7 @@ export default function ComboDefaultsTab() { const addProviderOverride = () => { const name = newOverrideProvider.trim().toLowerCase(); if (!name || providerOverrides[name]) return; - setProviderOverrides((prev) => ({ ...prev, [name]: { maxRetries: 1, timeoutMs: 120000 } })); + setProviderOverrides((prev) => ({ ...prev, [name]: { maxRetries: 1 } })); setNewOverrideProvider(""); }; @@ -381,21 +408,6 @@ export default function ComboDefaultsTab() { {/* Toggles */}
-
-
-

{t("healthCheck")}

-

{t("healthCheckDesc")}

-
- - setComboDefaults((prev) => ({ - ...prev, - healthCheckEnabled: !prev.healthCheckEnabled, - })) - } - /> -

{t("trackMetrics")}

@@ -436,24 +448,6 @@ export default function ComboDefaultsTab() { aria-label={t("providerMaxRetriesAria", { provider })} /> {t("retries")} - - setProviderOverrides((prev) => ({ - ...prev, - [provider]: { - ...prev[provider], - timeoutMs: parseInt(e.target.value) || 120000, - }, - })) - } - className="text-xs w-24" - aria-label={t("providerTimeoutAria", { provider })} - /> - {t("ms")}
@@ -106,7 +98,7 @@ export default function PoliciesPanel() { gpp_maybe
-

{t("policiesCircuitBreakers")}

+

{t("policiesLocked")}

{t("activeIssuesDetected")}

@@ -115,50 +107,6 @@ export default function PoliciesPanel() {
- {/* Circuit Breakers */} - {circuitBreakers.filter((cb) => cb.state !== "closed").length > 0 && ( -
-

{t("circuitBreakers")}

-
- {circuitBreakers - .filter((cb) => cb.state !== "closed") - .map((cb, i) => { - const status = CB_STATUS[cb.state] || CB_STATUS.open; - return ( -
-
- - {status.icon} - - - {cb.name || cb.provider || "Unknown"} - - - {status.label} - - {cb.failures > 0 && ( - {cb.failures} failures - )} -
-
- ); - })} -
-
- )} - {/* Locked Identifiers */} {lockedIds.length > 0 && (
diff --git a/src/app/(dashboard)/dashboard/settings/components/ResilienceTab.tsx b/src/app/(dashboard)/dashboard/settings/components/ResilienceTab.tsx index e286af443b..f3ca6f7105 100644 --- a/src/app/(dashboard)/dashboard/settings/components/ResilienceTab.tsx +++ b/src/app/(dashboard)/dashboard/settings/components/ResilienceTab.tsx @@ -1,549 +1,316 @@ "use client"; -import { useState, useEffect, useCallback } from "react"; -import { Card, Button } from "@/shared/components"; +import { type ReactNode, useEffect, useState } from "react"; +import { Button, Card } from "@/shared/components"; import { useNotificationStore } from "@/store/notificationStore"; -import { useLocale, useTranslations } from "next-intl"; -import AutoDisableCard from "./AutoDisableCard"; +import { useTranslations } from "next-intl"; -// โ”€โ”€โ”€ State colors and labels โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ -const STATE_STYLES = { - CLOSED: { - bg: "bg-emerald-500/15", - text: "text-emerald-400", - border: "border-emerald-500/30", - icon: "check_circle", - }, - OPEN: { - bg: "bg-red-500/15", - text: "text-red-400", - border: "border-red-500/30", - icon: "error", - }, - HALF_OPEN: { - bg: "bg-amber-500/15", - text: "text-amber-400", - border: "border-amber-500/30", - icon: "warning", - }, +type RequestQueueSettings = { + autoEnableApiKeyProviders: boolean; + requestsPerMinute: number; + minTimeBetweenRequestsMs: number; + concurrentRequests: number; + maxWaitMs: number; }; -const CB_STATUS = { - closed: { icon: "check_circle", color: "#22c55e" }, - "half-open": { icon: "pending", color: "#f59e0b" }, - open: { icon: "error", color: "#ef4444" }, +type ConnectionCooldownProfileSettings = { + baseCooldownMs: number; + useUpstreamRetryHints: boolean; + maxBackoffSteps: number; }; -function getBreakerStateLabel(state, t) { - const normalized = String(state || "closed") - .toLowerCase() - .replaceAll("_", "-"); - if (normalized === "open") return t("breakerStateOpen"); - if (normalized === "half-open") return t("breakerStateHalfOpen"); - return t("breakerStateClosed"); +type ProviderBreakerProfileSettings = { + failureThreshold: number; + resetTimeoutMs: number; +}; + +type WaitForCooldownSettings = { + enabled: boolean; + maxRetries: number; + maxRetryWaitSec: number; +}; + +type ResilienceResponse = { + requestQueue: RequestQueueSettings; + connectionCooldown: { + oauth: ConnectionCooldownProfileSettings; + apikey: ConnectionCooldownProfileSettings; + }; + providerBreaker: { + oauth: ProviderBreakerProfileSettings; + apikey: ProviderBreakerProfileSettings; + }; + waitForCooldown: WaitForCooldownSettings; +}; + +function formatMs(value: number | null | undefined) { + if (typeof value !== "number") return "โ€”"; + return `${value}ms`; } -function formatMs(ms) { - if (!ms || ms <= 0) return "โ€”"; - if (ms < 1000) return `${ms}ms`; - if (ms < 60000) return `${(ms / 1000).toFixed(1)}s`; - return `${(ms / 60000).toFixed(1)}m`; +function SectionDescription({ + scope, + trigger, + effect, +}: { + scope: string; + trigger: string; + effect: string; +}) { + return ( +
+
+ Scope: {scope} +
+
+ Trigger: {trigger} +
+
+ Effect: {effect} +
+
+ ); } -function getErrorMessage(err, fallback) { - return err instanceof Error && err.message ? err.message : fallback; +function NumberField({ + label, + value, + suffix, + min = 0, + onChange, +}: { + label: string; + value: number; + suffix?: string; + min?: number; + onChange: (value: number) => void; +}) { + return ( + + ); } -// โ”€โ”€โ”€ Provider Profiles Card โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ -function ProviderProfilesCard({ profiles, onSave, saving }) { - const [editMode, setEditMode] = useState(false); - const [draft, setDraft] = useState(profiles); - const t = useTranslations("settings"); +function BooleanField({ + label, + description, + checked, + onChange, +}: { + label: string; + description: string; + checked: boolean; + onChange: (value: boolean) => void; +}) { + return ( + + ); +} + +function ProfileColumn({ + title, + icon, + children, +}: { + title: string; + icon: string; + children: ReactNode; +}) { + return ( +
+
+ {icon} +

{title}

+
+
{children}
+
+ ); +} + +function ActionRow({ + editing, + saving, + onEdit, + onCancel, + onSave, +}: { + editing: boolean; + saving: boolean; + onEdit: () => void; + onCancel: () => void; + onSave: () => void; +}) { const tc = useTranslations("common"); - - useEffect(() => { - setDraft(profiles); - }, [profiles]); - - const formatMsRaw = (value) => (value == null ? "โ€”" : `${value}${t("ms")}`); - const fields = [ - { key: "transientCooldown", label: t("transientCooldown"), format: formatMsRaw }, - { key: "rateLimitCooldown", label: t("rateLimitCooldown"), format: formatMsRaw }, - { key: "maxBackoffLevel", label: t("maxBackoffLevel") }, - { - key: "circuitBreakerThreshold", - label: t("cbThreshold"), - format: (value) => (value == null ? "โ€”" : t("failures", { count: value })), - }, - { key: "circuitBreakerReset", label: t("cbResetTime"), format: formatMsRaw }, - ]; - - const handleSave = () => { - // Only send 'oauth' and 'apikey' โ€” the API schema rejects any other keys (e.g. 'local') - const { oauth, apikey } = draft ?? {}; - onSave({ ...(oauth ? { oauth } : {}), ...(apikey ? { apikey } : {}) }); - setEditMode(false); - }; - - return ( - -
-
-
- -

{t("providerProfiles")}

-
- {editMode ? ( -
- - -
- ) : ( - - )} -
- -

{t("providerProfilesDesc")}

- -
- {["oauth", "apikey"].map((type) => ( -
-

- - {type === "oauth" ? t("oauthProviders") : t("apiKeyProviders")} -

-
- {fields.map(({ key, label, format }) => ( -
- {label} - {editMode ? ( - - setDraft({ - ...draft, - [type]: { ...draft[type], [key]: Number(e.target.value) }, - }) - } - className="w-24 px-2 py-1 text-xs rounded bg-white/10 border border-white/20 text-right" - /> - ) : ( - - {format - ? format(profiles?.[type]?.[key]) - : (profiles?.[type]?.[key] ?? "โ€”")} - - )} -
- ))} -
-
- ))} -
-
- - ); -} - -// โ”€โ”€โ”€ Editable Rate Limit Card โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ -function RateLimitCard({ rateLimitStatus, defaults, onSaveDefaults, saving }) { - const [editMode, setEditMode] = useState(false); - const [draft, setDraft] = useState(defaults || {}); - const t = useTranslations("settings"); - const tc = useTranslations("common"); - - // Sync draft when defaults change from parent (standard prop-to-state sync) - /* eslint-disable react-hooks/set-state-in-effect */ - useEffect(() => { - if (defaults) setDraft(defaults); - }, [defaults]); - /* eslint-enable react-hooks/set-state-in-effect */ - - const handleSave = () => { - onSaveDefaults(draft); - setEditMode(false); - }; - - return ( - -
-
-
- -

{t("rateLimiting")}

-
- {editMode ? ( -
- - -
- ) : ( - - )} -
- -

{t("rateLimitingDesc")}

- -
-

- {t("defaultSafetyNet")} -

-
- {[ - { key: "requestsPerMinute", label: t("rpm") }, - { key: "minTimeBetweenRequests", label: t("minGap"), format: formatMs }, - { key: "concurrentRequests", label: t("maxConcurrent") }, - ].map(({ key, label, format }) => ( -
- {editMode ? ( - - setDraft((prev) => ({ ...prev, [key]: parseInt(e.target.value) || 0 })) - } - className="w-full px-2 py-1 text-lg font-bold rounded bg-white/10 border border-white/20" - /> - ) : ( -
- {format ? format(defaults?.[key]) : (defaults?.[key] ?? "โ€”")} -
- )} -
{label}
-
- ))} -
-
- - {rateLimitStatus && rateLimitStatus.length > 0 ? ( -
-

- {t("activeLimiters")} -

- {rateLimitStatus.map((rl, i) => ( -
- {rl.provider || rl.key} -
- {rl.reservoir != null && ( - - {t("reservoir")}: {rl.reservoir} - - )} - {rl.running != null && ( - - {t("running")}: {rl.running} - - )} - {rl.queued != null && ( - - {t("queued")}: {rl.queued} - - )} -
-
- ))} -
- ) : ( -

{t("noActiveLimiters")}

- )} -
- - ); -} - -// โ”€โ”€โ”€ Circuit Breaker Card โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ -function CircuitBreakerCard({ breakers, onReset, loading }) { - const activeBreakers = breakers.filter((b) => b.state !== "CLOSED"); - const totalBreakers = breakers.length; - const t = useTranslations("settings"); - - return ( - -
-
-
- -

{t("circuitBreakers")}

-
-
- - {activeBreakers.length > 0 - ? t("tripped", { count: activeBreakers.length }) - : t("healthy", { count: totalBreakers })} - - {activeBreakers.length > 0 && ( - - )} -
-
- - {breakers.length === 0 ? ( -

{t("noCircuitBreakers")}

- ) : ( -
- {breakers.map((b) => { - const style = STATE_STYLES[b.state] || STATE_STYLES.CLOSED; - return ( -
-
- - {b.name.replace("combo:", "")} -
-
- {b.failureCount > 0 && ( - - {t("failures", { count: b.failureCount })} - - )} - - {getBreakerStateLabel(b.state, t)} - -
-
- ); - })} -
- )} -
- - ); -} - -// โ”€โ”€โ”€ Policies Panel (from Security tab) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ -function PoliciesCard() { - const [data, setData] = useState(null); - const [loading, setLoading] = useState(true); - const [unlocking, setUnlocking] = useState(null); - const notify = useNotificationStore(); - const locale = useLocale(); - const t = useTranslations("settings"); - - const fetchPolicies = useCallback(async () => { - try { - const res = await fetch("/api/policies"); - if (res.ok) { - const json = await res.json(); - setData(json); - } - } catch { - // silent - } finally { - setLoading(false); - } - }, []); - - useEffect(() => { - fetchPolicies(); - const interval = setInterval(fetchPolicies, 15000); - return () => clearInterval(interval); - }, [fetchPolicies]); - - const handleUnlock = async (identifier) => { - setUnlocking(identifier); - try { - const res = await fetch("/api/policies", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ action: "unlock", identifier }), - }); - if (res.ok) { - notify.success(t("unlockedIdentifier", { identifier })); - await fetchPolicies(); - } else { - notify.error(t("failedUnlock")); - } - } catch { - notify.error(t("failedUnlock")); - } finally { - setUnlocking(null); - } - }; - - const circuitBreakers = data?.circuitBreakers || []; - const lockedIds = data?.lockedIdentifiers || []; - const hasIssues = circuitBreakers.some((cb) => cb.state !== "closed") || lockedIds.length > 0; - - if (loading) { + if (editing) { return ( - -
- policy - {t("loadingPolicies")} -
- +
+ + +
); } return ( - -
-
-
- -

{t("policiesLocked")}

-
- {hasIssues && ( - - )} -
+ + ); +} - {!hasIssues ? ( -
-
- verified_user -
-
-

{t("allOperational")}

-
+function RequestQueueCard({ + value, + onSave, + saving, +}: { + value: RequestQueueSettings; + onSave: (next: RequestQueueSettings) => Promise; + saving: boolean; +}) { + const [editing, setEditing] = useState(false); + const [draft, setDraft] = useState(value); + + useEffect(() => { + setDraft(value); + }, [value]); + + return ( + +
+
+
+ speed +

Request Queue & Pacing

+ +
+ setEditing(true)} + onCancel={() => { + setDraft(value); + setEditing(false); + }} + onSave={async () => { + await onSave(draft); + setEditing(false); + }} + /> +
+ +

+ This layer only controls queueing and pacing. It does not write cooldowns or open breakers. +

+ +
+ {editing ? ( + <> + + setDraft((prev) => ({ ...prev, autoEnableApiKeyProviders })) + } + /> + setDraft((prev) => ({ ...prev, requestsPerMinute }))} + /> + + setDraft((prev) => ({ ...prev, minTimeBetweenRequestsMs })) + } + /> + + setDraft((prev) => ({ ...prev, concurrentRequests })) + } + /> + setDraft((prev) => ({ ...prev, maxWaitMs }))} + /> + ) : ( <> - {/* Circuit Breakers */} - {circuitBreakers.filter((cb) => cb.state !== "closed").length > 0 && ( -
-

{t("circuitBreakers")}

-
- {circuitBreakers - .filter((cb) => cb.state !== "closed") - .map((cb, i) => { - const status = CB_STATUS[cb.state] || CB_STATUS.open; - return ( -
-
- - {status.icon} - - - {cb.name || cb.provider || t("unknown")} - - - {getBreakerStateLabel(cb.state, t)} - - {cb.failures > 0 && ( - - {t("failures", { count: cb.failures })} - - )} -
-
- ); - })} -
+
+
Auto-enable for API key providers
+
+ {value.autoEnableApiKeyProviders ? "Enabled" : "Disabled"}
- )} - - {/* Locked Identifiers */} - {lockedIds.length > 0 && ( -
-

{t("lockedIdentifiers")}

-
- {lockedIds.map((id, i) => { - const identifier = typeof id === "string" ? id : id.identifier || id.id; - return ( -
-
- - lock - - {identifier} - {typeof id === "object" && id.lockedAt && ( - - {t("sinceDate", { - date: new Date(id.lockedAt).toLocaleString(locale), - })} - - )} -
- -
- ); - })} -
+
+
+
Requests per minute
+
+ {value.requestsPerMinute}
- )} +
+
+
Min time between requests
+
+ {formatMs(value.minTimeBetweenRequestsMs)} +
+
+
+
Concurrent requests
+
+ {value.concurrentRequests} +
+
+
+
Max queue wait
+
+ {formatMs(value.maxWaitMs)} +
+
)}
@@ -551,131 +318,435 @@ function PoliciesCard() { ); } -// โ”€โ”€โ”€ Main Resilience Tab โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ -export default function ResilienceTab() { - const [data, setData] = useState(null); - const [loading, setLoading] = useState(true); - const [saving, setSaving] = useState(false); - const [error, setError] = useState(null); - const t = useTranslations("settings"); - - const loadData = useCallback(async () => { - try { - setLoading(true); - const res = await fetch("/api/resilience"); - if (!res.ok) throw new Error(t("failedLoadWithStatus", { status: res.status })); - const json = await res.json(); - setData(json); - setError(null); - } catch (err) { - setError(getErrorMessage(err, t("failedLoadResilience"))); - } finally { - setLoading(false); - } - }, [t]); +function ConnectionCooldownCard({ + value, + onSave, + saving, +}: { + value: ResilienceResponse["connectionCooldown"]; + onSave: (next: ResilienceResponse["connectionCooldown"]) => Promise; + saving: boolean; +}) { + const [editing, setEditing] = useState(false); + const [draft, setDraft] = useState(value); useEffect(() => { - loadData(); - // Auto-refresh every 10s - const interval = setInterval(loadData, 10000); - return () => clearInterval(interval); - }, [loadData]); + setDraft(value); + }, [value]); - const handleResetBreakers = async () => { - try { - setLoading(true); - const res = await fetch("/api/resilience/reset", { method: "POST" }); - if (!res.ok) throw new Error(t("resetFailed")); - await loadData(); - } catch (err) { - setError(getErrorMessage(err, t("resetFailed"))); - } finally { - setLoading(false); - } + const renderProfile = (key: "oauth" | "apikey", title: string, icon: string) => { + const current = editing ? draft[key] : value[key]; + return ( + + {editing ? ( + <> + + setDraft((prev) => ({ ...prev, [key]: { ...prev[key], baseCooldownMs } })) + } + /> + + setDraft((prev) => ({ + ...prev, + [key]: { ...prev[key], useUpstreamRetryHints }, + })) + } + /> + + setDraft((prev) => ({ ...prev, [key]: { ...prev[key], maxBackoffSteps } })) + } + /> + + ) : ( + <> +
+ Base cooldown + {formatMs(current.baseCooldownMs)} +
+
+ Use upstream retry hints + + {current.useUpstreamRetryHints ? "Yes" : "No"} + +
+
+ Max backoff steps + {current.maxBackoffSteps} +
+ + )} + + ); }; - const handleSaveProfiles = async (profiles) => { - try { - setSaving(true); - const res = await fetch("/api/resilience", { - method: "PATCH", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ profiles }), - }); - if (!res.ok) throw new Error(t("saveFailed")); - await loadData(); - } catch (err) { - setError(getErrorMessage(err, t("saveFailed"))); - } finally { - setSaving(false); - } + return ( + +
+
+
+ timer_off +

Connection Cooldown

+
+ +
+ setEditing(true)} + onCancel={() => { + setDraft(value); + setEditing(false); + }} + onSave={async () => { + await onSave(draft); + setEditing(false); + }} + /> +
+ +

+ Base cooldown covers retryable connection failures. When upstream retry hints are enabled, + explicit provider wait windows override the local base cooldown. +

+ +
+ {renderProfile("oauth", "OAuth Providers", "lock")} + {renderProfile("apikey", "API Key Providers", "key")} +
+ + ); +} + +function ProviderBreakerCard({ + value, + onSave, + saving, +}: { + value: ResilienceResponse["providerBreaker"]; + onSave: (next: ResilienceResponse["providerBreaker"]) => Promise; + saving: boolean; +}) { + const [editing, setEditing] = useState(false); + const [draft, setDraft] = useState(value); + + useEffect(() => { + setDraft(value); + }, [value]); + + const renderProfile = (key: "oauth" | "apikey", title: string, icon: string) => { + const current = editing ? draft[key] : value[key]; + return ( + + {editing ? ( + <> + + setDraft((prev) => ({ ...prev, [key]: { ...prev[key], failureThreshold } })) + } + /> + + setDraft((prev) => ({ ...prev, [key]: { ...prev[key], resetTimeoutMs } })) + } + /> + + ) : ( + <> +
+ Failure threshold + {current.failureThreshold} +
+
+ Reset timeout + {formatMs(current.resetTimeoutMs)} +
+ + )} + + ); }; - const handleSaveDefaults = async (defaults) => { + return ( + +
+
+
+ + electrical_services + +

Provider Circuit Breaker

+
+ +
+ setEditing(true)} + onCancel={() => { + setDraft(value); + setEditing(false); + }} + onSave={async () => { + await onSave(draft); + setEditing(false); + }} + /> +
+ +

+ Breaker runtime state is shown only on the Health page. Connection-scoped 429 rate limits + stay in Connection Cooldown and do not trip the provider breaker. +

+ +
+ {renderProfile("oauth", "OAuth Providers", "lock")} + {renderProfile("apikey", "API Key Providers", "key")} +
+ + ); +} + +function WaitForCooldownCard({ + value, + onSave, + saving, +}: { + value: WaitForCooldownSettings; + onSave: (next: WaitForCooldownSettings) => Promise; + saving: boolean; +}) { + const [editing, setEditing] = useState(false); + const [draft, setDraft] = useState(value); + + useEffect(() => { + setDraft(value); + }, [value]); + + return ( + +
+
+
+ hourglass_top +

Wait For Cooldown

+
+ +
+ setEditing(true)} + onCancel={() => { + setDraft(value); + setEditing(false); + }} + onSave={async () => { + await onSave(draft); + setEditing(false); + }} + /> +
+ +

+ This only affects the current request. It does not write connection or provider state. +

+ +
+ {editing ? ( + <> + setDraft((prev) => ({ ...prev, enabled }))} + /> + setDraft((prev) => ({ ...prev, maxRetries }))} + /> + setDraft((prev) => ({ ...prev, maxRetryWaitSec }))} + /> + + ) : ( + <> +
+
Enable server-side waiting
+
+ {value.enabled ? "Enabled" : "Disabled"} +
+
+
+
Max retries
+
{value.maxRetries}
+
+
+
Max retry wait
+
+ {value.maxRetryWaitSec}s +
+
+ + )} +
+ + ); +} + +export default function ResilienceTab() { + const notify = useNotificationStore(); + const [data, setData] = useState(null); + const [loading, setLoading] = useState(true); + const [savingSection, setSavingSection] = useState(null); + + useEffect(() => { + let mounted = true; + + const load = async () => { + try { + const response = await fetch("/api/resilience"); + if (!response.ok) { + throw new Error(`HTTP ${response.status}`); + } + const json = await response.json(); + if (!mounted) return; + setData({ + requestQueue: json.requestQueue, + connectionCooldown: json.connectionCooldown, + providerBreaker: json.providerBreaker, + waitForCooldown: json.waitForCooldown, + }); + } catch (error) { + notify.error(error instanceof Error ? error.message : "Failed to load resilience settings"); + } finally { + if (mounted) setLoading(false); + } + }; + + void load(); + return () => { + mounted = false; + }; + }, [notify]); + + const savePatch = async (section: string, payload: Record) => { + setSavingSection(section); try { - setSaving(true); - const res = await fetch("/api/resilience", { + const response = await fetch("/api/resilience", { method: "PATCH", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ defaults }), + body: JSON.stringify(payload), }); - if (!res.ok) throw new Error(t("saveFailed")); - await loadData(); - } catch (err) { - setError(getErrorMessage(err, t("saveFailed"))); + const json = await response.json(); + if (!response.ok) { + throw new Error(json?.error?.message || json?.error || `HTTP ${response.status}`); + } + setData({ + requestQueue: json.requestQueue, + connectionCooldown: json.connectionCooldown, + providerBreaker: json.providerBreaker, + waitForCooldown: json.waitForCooldown, + }); + notify.success("Resilience settings updated."); + } catch (error) { + notify.error(error instanceof Error ? error.message : "Failed to save resilience settings"); + throw error; } finally { - setSaving(false); + setSavingSection(null); } }; if (loading && !data) { return ( -
- hourglass_empty - {t("loadingResilience")} -
+ +
+ progress_activity + Loading resilience settings... +
+ ); } - if (error && !data) { + if (!data) { return ( -
- error - {error} -
- +

Unable to load resilience settings.

 ); } return ( -
- {/* 1. Provider Profiles (resilience settings by auth type) */} - + +
+ info +
+

Resilience Structure

+

+ This page only configures behavior. Live breaker state is shown on the Health page. + Combo-specific retry and round-robin slot control remain on combo settings. +

+
+
+ + + savePatch("requestQueue", { requestQueue })} /> - {/* 1.5 Auto Disable Banned Accounts */} - - {/* 2. Rate Limiting (editable defaults + active limiters) */} - savePatch("connectionCooldown", { connectionCooldown })} /> - {/* 3. Circuit Breakers (combo pipeline) */} - savePatch("providerBreaker", { providerBreaker })} + /> + savePatch("waitForCooldown", { waitForCooldown })} /> - {/* 4. Policies & Locked Identifiers (from previous Security tab) */} -
); } diff --git a/src/app/(dashboard)/layout.tsx b/src/app/(dashboard)/layout.tsx index 8237e8eb34..e640e17628 100644 --- a/src/app/(dashboard)/layout.tsx +++ b/src/app/(dashboard)/layout.tsx @@ -1,10 +1,5 @@ import { DashboardLayout } from "@/shared/components"; -import { ModelStatusProvider } from "./dashboard/providers/components/ModelStatusContext"; export default function DashboardRootLayout({ children }) { - return ( - - {children} - - ); + return {children}; } diff --git a/src/app/api/models/availability/route.ts b/src/app/api/models/availability/route.ts deleted file mode 100644 index 9ef7418c54..0000000000 --- a/src/app/api/models/availability/route.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { NextResponse } from "next/server"; -import { - getAvailabilityReport, - clearModelUnavailability, - getUnavailableCount, -} from "@/domain/modelAvailability"; -import { clearModelAvailabilitySchema } from "@/shared/validation/schemas"; -import { isValidationFailure, validateBody } from "@/shared/validation/helpers"; - -export async function GET() { - try { - const report = getAvailabilityReport(); - const count = getUnavailableCount(); - return NextResponse.json({ unavailableCount: count, models: report }); - } catch (error) { - console.error("Error getting model availability:", error); - return NextResponse.json({ error: "Failed to get model availability" }, { status: 500 }); - } -} - -export async function POST(request) { - let rawBody; - try { - rawBody = await request.json(); - } catch { - return NextResponse.json( - { - error: { - message: "Invalid request", - details: [{ field: "body", message: "Invalid JSON body" }], - }, - }, - { status: 400 } - ); - } - - try { - const validation = validateBody(clearModelAvailabilitySchema, rawBody); - if (isValidationFailure(validation)) { - return NextResponse.json({ error: validation.error }, { status: 400 }); - } - const { provider, model } = validation.data; - - const removed = clearModelUnavailability(provider, model); - return NextResponse.json({ success: true, removed }); - } catch (error) { - console.error("Error clearing model availability:", error); - return NextResponse.json({ error: "Failed to clear model availability" }, { status: 500 }); - } -} diff --git a/src/app/api/policies/route.ts b/src/app/api/policies/route.ts index dc74eb76f6..47f0e3dfa5 100644 --- a/src/app/api/policies/route.ts +++ b/src/app/api/policies/route.ts @@ -1,14 +1,12 @@ import { NextResponse } from "next/server"; -import { getAllCircuitBreakerStatuses } from "@/shared/utils/circuitBreaker"; import { getLockedIdentifiers, forceUnlock } from "@/domain/lockoutPolicy"; import { policyActionSchema } from "@/shared/validation/schemas"; import { isValidationFailure, validateBody } from "@/shared/validation/helpers"; export async function GET() { try { - const circuitBreakers = getAllCircuitBreakerStatuses(); const lockedIdentifiers = getLockedIdentifiers(); - return NextResponse.json({ circuitBreakers, lockedIdentifiers }); + return NextResponse.json({ lockedIdentifiers }); } catch (error) { console.error("Error loading policies:", error); return NextResponse.json({ error: "Failed to load policies" }, { status: 500 }); diff --git a/src/app/api/resilience/reset/route.ts b/src/app/api/resilience/reset/route.ts index 3ee4e94aea..8c21960732 100644 --- a/src/app/api/resilience/reset/route.ts +++ b/src/app/api/resilience/reset/route.ts @@ -1,11 +1,10 @@ import { NextResponse } from "next/server"; /** - * POST /api/resilience/reset โ€” Reset all circuit breakers and clear cooldowns + * POST /api/resilience/reset โ€” Reset all provider circuit breakers */ export async function POST() { try { - // Reset all circuit breakers const { getAllCircuitBreakerStatuses, getCircuitBreaker } = await import("@/shared/utils/circuitBreaker"); diff --git a/src/app/api/resilience/route.ts b/src/app/api/resilience/route.ts index d1bea74e33..9436aae3cf 100644 --- a/src/app/api/resilience/route.ts +++ b/src/app/api/resilience/route.ts @@ -1,5 +1,12 @@ import { NextResponse } from "next/server"; import { getSettings, updateSettings } from "@/lib/localDb"; +import { + buildLegacyResilienceCompat, + mergeResilienceSettings, + resolveResilienceSettings, + type ResilienceSettings, + type ResilienceSettingsPatch, +} from "@/lib/resilience/settings"; import { updateResilienceSchema } from "@/shared/validation/schemas"; import { isValidationFailure, validateBody } from "@/shared/validation/helpers"; @@ -13,41 +20,130 @@ function getErrorMessage(error: unknown, fallback: string): string { return error instanceof Error && error.message ? error.message : fallback; } +function normalizeLegacyPatch(body: JsonRecord): ResilienceSettingsPatch { + const profiles = asRecord(body.profiles); + const defaults = asRecord(body.defaults); + const oauth = asRecord(profiles.oauth); + const apikey = asRecord(profiles.apikey); + + const patch: ResilienceSettingsPatch = {}; + + if (Object.keys(defaults).length > 0) { + patch.requestQueue = { + ...(typeof defaults.requestsPerMinute === "number" + ? { requestsPerMinute: defaults.requestsPerMinute } + : {}), + ...(typeof defaults.minTimeBetweenRequests === "number" + ? { minTimeBetweenRequestsMs: defaults.minTimeBetweenRequests } + : {}), + ...(typeof defaults.concurrentRequests === "number" + ? { concurrentRequests: defaults.concurrentRequests } + : {}), + }; + } + + if (Object.keys(oauth).length > 0 || Object.keys(apikey).length > 0) { + const buildLegacyCooldownPatch = (profile: JsonRecord) => { + const cooldownCandidates = [ + typeof profile.transientCooldown === "number" ? profile.transientCooldown : null, + typeof profile.rateLimitCooldown === "number" && profile.rateLimitCooldown > 0 + ? profile.rateLimitCooldown + : null, + ].filter((value): value is number => typeof value === "number"); + + return { + ...(cooldownCandidates.length > 0 + ? { baseCooldownMs: Math.max(...cooldownCandidates) } + : {}), + ...(typeof profile.rateLimitCooldown === "number" + ? { useUpstreamRetryHints: profile.rateLimitCooldown === 0 } + : {}), + ...(typeof profile.maxBackoffLevel === "number" + ? { maxBackoffSteps: profile.maxBackoffLevel } + : {}), + }; + }; + + patch.connectionCooldown = { + ...(Object.keys(oauth).length > 0 + ? { + oauth: buildLegacyCooldownPatch(oauth), + } + : {}), + ...(Object.keys(apikey).length > 0 + ? { + apikey: buildLegacyCooldownPatch(apikey), + } + : {}), + }; + + patch.providerBreaker = { + ...(Object.keys(oauth).length > 0 + ? { + oauth: { + ...(typeof oauth.circuitBreakerThreshold === "number" + ? { failureThreshold: oauth.circuitBreakerThreshold } + : {}), + ...(typeof oauth.circuitBreakerReset === "number" + ? { resetTimeoutMs: oauth.circuitBreakerReset } + : {}), + }, + } + : {}), + ...(Object.keys(apikey).length > 0 + ? { + apikey: { + ...(typeof apikey.circuitBreakerThreshold === "number" + ? { failureThreshold: apikey.circuitBreakerThreshold } + : {}), + ...(typeof apikey.circuitBreakerReset === "number" + ? { resetTimeoutMs: apikey.circuitBreakerReset } + : {}), + }, + } + : {}), + }; + } + + return patch; +} + +async function syncRuntimeSettings(resilienceSettings: ResilienceSettings) { + const { applyRequestQueueSettings } = + await import("@omniroute/open-sse/services/rateLimitManager"); + applyRequestQueueSettings(resilienceSettings.requestQueue); +} + /** - * GET /api/resilience โ€” Get current resilience configuration and status + * GET /api/resilience โ€” Get current resilience configuration */ export async function GET() { try { - // Dynamic imports for open-sse modules - const { getAllCircuitBreakerStatuses } = await import("@/shared/utils/circuitBreaker"); - const { getAllRateLimitStatus } = await import("@omniroute/open-sse/services/rateLimitManager"); - const { PROVIDER_PROFILES, DEFAULT_API_LIMITS } = - await import("@omniroute/open-sse/config/constants"); - const settings = await getSettings(); - const circuitBreakers = getAllCircuitBreakerStatuses(); - const rateLimitStatus = getAllRateLimitStatus(); + const resilience = resolveResilienceSettings(settings); return NextResponse.json({ - profiles: settings.providerProfiles || PROVIDER_PROFILES, - defaults: { - ...DEFAULT_API_LIMITS, - ...asRecord(settings.rateLimitDefaults), + requestQueue: resilience.requestQueue, + connectionCooldown: resilience.connectionCooldown, + providerBreaker: resilience.providerBreaker, + waitForCooldown: { + enabled: resilience.waitForCooldown.enabled, + maxRetries: resilience.waitForCooldown.maxRetries, + maxRetryWaitSec: resilience.waitForCooldown.maxRetryWaitSec, }, - circuitBreakers, - rateLimitStatus, + legacy: buildLegacyResilienceCompat(resilience), }); } catch (err: unknown) { console.error("[API] GET /api/resilience error:", err); return NextResponse.json( - { error: getErrorMessage(err, "Failed to load resilience status") }, + { error: getErrorMessage(err, "Failed to load resilience settings") }, { status: 500 } ); } } /** - * PATCH /api/resilience โ€” Update provider resilience profiles and/or rate limit defaults + * PATCH /api/resilience โ€” Update resilience configuration */ export async function PATCH(request) { let rawBody; @@ -70,18 +166,47 @@ export async function PATCH(request) { if (isValidationFailure(validation)) { return NextResponse.json({ error: validation.error }, { status: 400 }); } - const { profiles, defaults } = validation.data; - const updates: Record = {}; - if (profiles) updates.providerProfiles = profiles; - if (defaults) updates.rateLimitDefaults = defaults; + const body = validation.data as JsonRecord; + const currentSettings = await getSettings(); + const currentResilience = resolveResilienceSettings(currentSettings); + const nextResilience = mergeResilienceSettings(currentResilience, { + ...(body.requestQueue + ? { requestQueue: body.requestQueue as ResilienceSettingsPatch["requestQueue"] } + : {}), + ...(body.connectionCooldown + ? { + connectionCooldown: + body.connectionCooldown as ResilienceSettingsPatch["connectionCooldown"], + } + : {}), + ...(body.providerBreaker + ? { providerBreaker: body.providerBreaker as ResilienceSettingsPatch["providerBreaker"] } + : {}), + ...(body.waitForCooldown + ? { waitForCooldown: body.waitForCooldown as ResilienceSettingsPatch["waitForCooldown"] } + : {}), + ...normalizeLegacyPatch(body), + }); - await updateSettings(updates); + await updateSettings({ + resilienceSettings: nextResilience, + requestRetry: nextResilience.waitForCooldown.maxRetries, + maxRetryIntervalSec: nextResilience.waitForCooldown.maxRetryWaitSec, + }); + await syncRuntimeSettings(nextResilience); return NextResponse.json({ ok: true, - ...(profiles ? { profiles } : {}), - ...(defaults ? { defaults } : {}), + requestQueue: nextResilience.requestQueue, + connectionCooldown: nextResilience.connectionCooldown, + providerBreaker: nextResilience.providerBreaker, + waitForCooldown: { + enabled: nextResilience.waitForCooldown.enabled, + maxRetries: nextResilience.waitForCooldown.maxRetries, + maxRetryWaitSec: nextResilience.waitForCooldown.maxRetryWaitSec, + }, + legacy: buildLegacyResilienceCompat(nextResilience), }); } catch (err: unknown) { console.error("[API] PATCH /api/resilience error:", err); diff --git a/src/app/api/settings/combo-defaults/route.ts b/src/app/api/settings/combo-defaults/route.ts index fb2df51cda..49c01285b4 100644 --- a/src/app/api/settings/combo-defaults/route.ts +++ b/src/app/api/settings/combo-defaults/route.ts @@ -3,6 +3,32 @@ import { getSettings, updateSettings } from "@/lib/localDb"; import { updateComboDefaultsSchema } from "@/shared/validation/schemas"; import { isValidationFailure, validateBody } from "@/shared/validation/helpers"; +const LEGACY_COMBO_RESILIENCE_KEYS = new Set([ + "timeoutMs", + "healthCheckEnabled", + "healthCheckTimeoutMs", +]); + +function sanitizeComboRuntimeConfig(config?: Record | null) { + if (!config || typeof config !== "object") return {}; + return Object.fromEntries( + Object.entries(config).filter( + ([key, value]) => + value !== undefined && value !== null && !LEGACY_COMBO_RESILIENCE_KEYS.has(key) + ) + ); +} + +function sanitizeProviderOverrides(overrides?: Record | null) { + if (!overrides || typeof overrides !== "object") return {}; + return Object.fromEntries( + Object.entries(overrides).map(([providerId, config]) => [ + providerId, + sanitizeComboRuntimeConfig(config), + ]) + ); +} + /** * GET /api/settings/combo-defaults * Returns the current combo global defaults and provider overrides @@ -10,21 +36,23 @@ import { isValidationFailure, validateBody } from "@/shared/validation/helpers"; export async function GET() { try { const settings: any = await getSettings(); + const comboDefaults = sanitizeComboRuntimeConfig(settings.comboDefaults); + const providerOverrides = sanitizeProviderOverrides(settings.providerOverrides); return NextResponse.json({ - comboDefaults: settings.comboDefaults || { - strategy: "priority", - maxRetries: 1, - retryDelayMs: 2000, - timeoutMs: 120000, - healthCheckEnabled: true, - healthCheckTimeoutMs: 3000, - handoffThreshold: 0.85, - handoffModel: "", - maxMessagesForSummary: 30, - maxComboDepth: 3, - trackMetrics: true, - }, - providerOverrides: settings.providerOverrides || {}, + comboDefaults: + Object.keys(comboDefaults).length > 0 + ? comboDefaults + : { + strategy: "priority", + maxRetries: 1, + retryDelayMs: 2000, + handoffThreshold: 0.85, + handoffModel: "", + maxMessagesForSummary: 30, + maxComboDepth: 3, + trackMetrics: true, + }, + providerOverrides, }); } catch (error) { console.log("Error fetching combo defaults:", error); @@ -63,16 +91,16 @@ export async function PATCH(request) { const updates: Record = {}; if (body.comboDefaults) { - updates.comboDefaults = body.comboDefaults; + updates.comboDefaults = sanitizeComboRuntimeConfig(body.comboDefaults); } if (body.providerOverrides) { - updates.providerOverrides = body.providerOverrides; + updates.providerOverrides = sanitizeProviderOverrides(body.providerOverrides); } const settings: any = await updateSettings(updates); return NextResponse.json({ - comboDefaults: settings.comboDefaults || {}, - providerOverrides: settings.providerOverrides || {}, + comboDefaults: sanitizeComboRuntimeConfig(settings.comboDefaults), + providerOverrides: sanitizeProviderOverrides(settings.providerOverrides), }); } catch (error) { console.log("Error updating combo defaults:", error); diff --git a/src/domain/modelAvailability.ts b/src/domain/modelAvailability.ts deleted file mode 100644 index 9510c0ea73..0000000000 --- a/src/domain/modelAvailability.ts +++ /dev/null @@ -1,319 +0,0 @@ -/** - * Model Availability โ€” Domain Layer (T-19) - * - * Tracks model availability per provider with TTL-based cooldowns. - * When a model becomes unavailable (rate-limited, erroring), it is - * marked with a cooldown period. The availability report powers - * the dashboard health view. - * - * @module domain/modelAvailability - */ - -/** - * @typedef {Object} UnavailableEntry - * @property {string} provider - * @property {string} model - * @property {number} unavailableSince - timestamp - * @property {number} cooldownMs - * @property {string} [reason] - */ - -/** @type {Map} */ -const unavailable = new Map(); - -/** - * @typedef {Object} FailureState - * @property {number} failureCount - * @property {number} lastFailureAt - * @property {number} resetAfterMs - */ - -/** - * @typedef {Object} ProviderProfile - * @property {number} [transientCooldown] - * @property {number} [rateLimitCooldown] - * @property {number} [maxBackoffLevel] - * @property {number} [circuitBreakerThreshold] - * @property {number} [circuitBreakerReset] - */ - -/** @type {Map} */ -const failureState = new Map(); - -const FAILURE_WINDOW_MS = 30 * 60 * 1000; - -const PROBLEMATIC_STATUS_COOLDOWNS = { - 429: 5 * 60 * 1000, - 408: 60 * 1000, - 500: 2 * 60 * 1000, - 502: 2 * 60 * 1000, - 503: 2 * 60 * 1000, - 504: 2 * 60 * 1000, -}; - -const MIN_PROBLEMATIC_COOLDOWN_MS = 60 * 1000; -const MAX_PROBLEMATIC_COOLDOWN_MS = 30 * 60 * 1000; - -function toPositiveNumber(value) { - return Number.isFinite(value) && Number(value) > 0 ? Number(value) : null; -} - -function toNonNegativeNumber(value) { - return Number.isFinite(value) && Number(value) >= 0 ? Number(value) : null; -} - -/** - * The first layer already reacts immediately to authoritative model/account failures. - * Global provider/model quarantine is the escalation layer, so its failure window and - * threshold are only customized when a runtime provider profile is supplied. - * - * @param {ProviderProfile | null | undefined} profile - * @returns {number} - */ -function getFailureWindowMs(profile) { - return toPositiveNumber(profile?.circuitBreakerReset) ?? FAILURE_WINDOW_MS; -} - -/** - * Without a runtime profile we preserve legacy behavior: quarantine on the first failure. - * - * @param {ProviderProfile | null | undefined} profile - * @returns {number} - */ -function getFailureThreshold(profile) { - return toPositiveNumber(profile?.circuitBreakerThreshold) ?? 1; -} - -function getLegacyStatusCooldown(status) { - return status && Object.prototype.hasOwnProperty.call(PROBLEMATIC_STATUS_COOLDOWNS, status) - ? PROBLEMATIC_STATUS_COOLDOWNS[status] - : 0; -} - -/** - * @param {number | null} status - * @param {ProviderProfile | null | undefined} profile - * @returns {number} - */ -function getProfileStatusCooldown(status, profile) { - if (!profile) return 0; - if (status === 429) { - return toPositiveNumber(profile.rateLimitCooldown) ?? 0; - } - return toPositiveNumber(profile.transientCooldown) ?? 0; -} - -/** - * @param {number} baseCooldownMs - * @param {number} failureCount - * @param {ProviderProfile | null | undefined} profile - * @returns {number} - */ -function getScaledCooldown(baseCooldownMs, failureCount, profile) { - const safeBase = toPositiveNumber(baseCooldownMs) ?? 1000; - if (!profile) { - return Math.min( - Math.max(safeBase, MIN_PROBLEMATIC_COOLDOWN_MS) * Math.pow(2, Math.max(0, failureCount - 1)), - MAX_PROBLEMATIC_COOLDOWN_MS - ); - } - - const maxBackoffLevel = Math.max( - 0, - Math.trunc(toNonNegativeNumber(profile.maxBackoffLevel) ?? 0) - ); - const exponent = Math.min(Math.max(0, failureCount - 1), maxBackoffLevel); - return safeBase * Math.pow(2, exponent); -} - -/** - * Build a composite key for provider+model. - * @param {string} provider - * @param {string} model - * @returns {string} - */ -function makeKey(provider, model) { - return `${provider}::${model}`; -} - -/** - * Check if a model is currently available. - * - * @param {string} provider - Provider ID (e.g. "openai", "anthropic") - * @param {string} model - Model ID (e.g. "gpt-4o", "claude-sonnet-4-20250514") - * @returns {boolean} true if model is available (not in cooldown) - */ -export function isModelAvailable(provider, model) { - const key = makeKey(provider, model); - const entry = unavailable.get(key); - if (!entry) return true; - - // Check if cooldown has expired - if (Date.now() - entry.unavailableSince >= entry.cooldownMs) { - unavailable.delete(key); - return true; - } - - return false; -} - -/** - * Get remaining cooldown information for a model, if it is currently unavailable. - * - * @param {string} provider - * @param {string} model - * @returns {{ provider: string, model: string, reason: string, remainingMs: number, unavailableSince: string } | null} - */ -export function getModelCooldownInfo(provider, model) { - const key = makeKey(provider, model); - const entry = unavailable.get(key); - if (!entry) return null; - - const elapsed = Date.now() - entry.unavailableSince; - if (elapsed >= entry.cooldownMs) { - unavailable.delete(key); - return null; - } - - return { - provider: entry.provider, - model: entry.model, - reason: entry.reason || "unknown", - remainingMs: entry.cooldownMs - elapsed, - unavailableSince: new Date(entry.unavailableSince).toISOString(), - }; -} - -/** - * Mark a model as temporarily unavailable. - * - * @param {string} provider - * @param {string} model - * @param {number} [cooldownMs=60000] - Cooldown in milliseconds (default 60s) - * @param {string} [reason] - Optional reason for unavailability - */ -export function setModelUnavailable(provider, model, cooldownMs = 60000, reason) { - const key = makeKey(provider, model); - const now = Date.now(); - const safeCooldownMs = Number.isFinite(cooldownMs) && cooldownMs > 0 ? cooldownMs : 60000; - const existing = unavailable.get(key); - const existingRemainingMs = - existing && Date.now() - existing.unavailableSince < existing.cooldownMs - ? existing.cooldownMs - (Date.now() - existing.unavailableSince) - : 0; - const effectiveCooldownMs = Math.max(safeCooldownMs, existingRemainingMs); - - unavailable.set(key, { - provider, - model, - unavailableSince: now, - cooldownMs: effectiveCooldownMs, - reason: reason || "unknown", - }); -} - -/** - * Marca provider/model como problemรกtico com cooldown adaptativo. - * Mantรฉm retrocompatibilidade: nรฃo altera o comportamento de setModelUnavailable, - * apenas oferece uma estratรฉgia mais agressiva para falhas recorrentes. - * - * @param {string} provider - * @param {string} model - * @param {{ status?: number, baseCooldownMs?: number, reason?: string, profile?: ProviderProfile | null }} [options] - * @returns {{ cooldownMs: number, failureCount: number, quarantined: boolean, threshold: number, resetAfterMs: number }} - */ -export function markModelAsProblematic(provider, model, options = {}) { - const key = makeKey(provider, model); - const now = Date.now(); - const status = Number.isFinite(options.status) ? Number(options.status) : null; - const profile = options.profile || null; - const explicitBaseCooldownMs = - Number.isFinite(options.baseCooldownMs) && Number(options.baseCooldownMs) > 0 - ? Number(options.baseCooldownMs) - : 0; - const statusBaseCooldown = profile - ? getProfileStatusCooldown(status, profile) - : getLegacyStatusCooldown(status); - const baseCooldownMs = Math.max(explicitBaseCooldownMs, statusBaseCooldown); - - const prev = failureState.get(key); - const resetAfterMs = getFailureWindowMs(profile); - const withinFailureWindow = prev && now - prev.lastFailureAt <= prev.resetAfterMs; - const failureCount = withinFailureWindow ? prev.failureCount + 1 : 1; - failureState.set(key, { failureCount, lastFailureAt: now, resetAfterMs }); - - const threshold = getFailureThreshold(profile); - const cooldownMs = getScaledCooldown(baseCooldownMs, failureCount, profile); - const quarantined = failureCount >= threshold; - - if (quarantined) { - setModelUnavailable(provider, model, cooldownMs, options.reason || "problematic_model"); - } - - return { - cooldownMs, - failureCount, - quarantined, - threshold, - resetAfterMs, - }; -} - -/** - * Clear unavailability for a model (e.g. after manual reset). - * - * @param {string} provider - * @param {string} model - * @returns {boolean} true if entry existed and was removed - */ -export function clearModelUnavailability(provider, model) { - const key = makeKey(provider, model); - failureState.delete(key); - return unavailable.delete(key); -} - -/** - * Get a report of all currently unavailable models. - * - * @returns {Array<{ provider: string, model: string, reason: string, remainingMs: number, unavailableSince: string }>} - */ -export function getAvailabilityReport() { - const now = Date.now(); - const report = []; - - for (const [key, entry] of unavailable.entries()) { - const elapsed = now - entry.unavailableSince; - if (elapsed >= entry.cooldownMs) { - unavailable.delete(key); - continue; - } - - report.push({ - provider: entry.provider, - model: entry.model, - reason: entry.reason || "unknown", - remainingMs: entry.cooldownMs - elapsed, - unavailableSince: new Date(entry.unavailableSince).toISOString(), - }); - } - - return report; -} - -/** - * Get total count of unavailable models. - * @returns {number} - */ -export function getUnavailableCount() { - // Prune expired entries first - getAvailabilityReport(); - return unavailable.size; -} - -/** - * Reset all availability states (for testing or admin). - */ -export function resetAllAvailability() { - unavailable.clear(); - failureState.clear(); -} diff --git a/src/lib/combos/intelligentRouting.ts b/src/lib/combos/intelligentRouting.ts index 1f2a285a6a..7f42f5a0cb 100644 --- a/src/lib/combos/intelligentRouting.ts +++ b/src/lib/combos/intelligentRouting.ts @@ -31,13 +31,6 @@ export type IntelligentProviderScore = { factors: IntelligentRoutingWeights; }; -export type IntelligentExclusionEntry = { - provider: string; - excludedAt: string; - cooldownMs: number; - reason: string; -}; - export const DEFAULT_INTELLIGENT_WEIGHTS: IntelligentRoutingWeights = { quota: 0.2, health: 0.25, @@ -161,50 +154,3 @@ export function buildIntelligentProviderScores(combo: { factors: weights, })); } - -export function extractIntelligentHealthState(health: unknown): { - incidentMode: boolean; - exclusions: IntelligentExclusionEntry[]; -} { - const healthRecord = isRecord(health) ? health : {}; - const providerHealth = isRecord(healthRecord.providerHealth) ? healthRecord.providerHealth : {}; - const providerBreakers = Object.entries(providerHealth).map(([provider, status]) => { - const statusRecord = isRecord(status) ? status : {}; - return { - provider, - state: typeof statusRecord.state === "string" ? statusRecord.state : "CLOSED", - lastFailure: typeof statusRecord.lastFailure === "string" ? statusRecord.lastFailure : null, - }; - }); - const breakersFromArray = Array.isArray(healthRecord.circuitBreakers) - ? healthRecord.circuitBreakers - .map((entry) => { - const breaker = isRecord(entry) ? entry : {}; - const provider = - typeof breaker.provider === "string" - ? breaker.provider - : typeof breaker.name === "string" - ? breaker.name - : "unknown"; - return { - provider, - state: typeof breaker.state === "string" ? breaker.state : "CLOSED", - lastFailure: typeof breaker.lastFailure === "string" ? breaker.lastFailure : null, - }; - }) - .filter((entry) => typeof entry.provider === "string") - : []; - - const breakers = breakersFromArray.length > 0 ? breakersFromArray : providerBreakers; - const openBreakers = breakers.filter((breaker) => breaker.state === "OPEN"); - - return { - incidentMode: openBreakers.length / Math.max(breakers.length, 1) > 0.5, - exclusions: openBreakers.map((breaker) => ({ - provider: breaker.provider, - excludedAt: breaker.lastFailure || new Date().toISOString(), - cooldownMs: 5 * 60 * 1000, - reason: "Circuit breaker OPEN", - })), - }; -} diff --git a/src/lib/dataPaths.js b/src/lib/dataPaths.js deleted file mode 100644 index 3595429936..0000000000 --- a/src/lib/dataPaths.js +++ /dev/null @@ -1,70 +0,0 @@ -"use strict"; -var __importDefault = (this && this.__importDefault) || function (mod) { - return (mod && mod.__esModule) ? mod : { "default": mod }; -}; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.APP_NAME = void 0; -exports.getLegacyDotDataDir = getLegacyDotDataDir; -exports.getDefaultDataDir = getDefaultDataDir; -exports.resolveDataDir = resolveDataDir; -exports.isSamePath = isSamePath; -const path_1 = __importDefault(require("path")); -const os_1 = __importDefault(require("os")); -exports.APP_NAME = "omniroute"; -function fallbackHomeDir() { - const envHome = process.env.HOME || process.env.USERPROFILE; - if (typeof envHome === "string" && envHome.trim().length > 0) { - return path_1.default.resolve(envHome); - } - return os_1.default.tmpdir(); -} -function safeHomeDir() { - try { - return os_1.default.homedir(); - } - catch { - return fallbackHomeDir(); - } -} -function normalizeConfiguredPath(dir) { - if (typeof dir !== "string") - return null; - const trimmed = dir.trim(); - if (!trimmed) - return null; - return path_1.default.resolve(trimmed); -} -function getLegacyDotDataDir() { - return path_1.default.join(safeHomeDir(), `.${exports.APP_NAME}`); -} -function getDefaultDataDir() { - const homeDir = safeHomeDir(); - if (process.platform === "win32") { - const appData = process.env.APPDATA || path_1.default.join(homeDir, "AppData", "Roaming"); - return path_1.default.join(appData, exports.APP_NAME); - } - // Support XDG on Linux/macOS when explicitly configured. - const xdgConfigHome = normalizeConfiguredPath(process.env.XDG_CONFIG_HOME); - if (xdgConfigHome) { - return path_1.default.join(xdgConfigHome, exports.APP_NAME); - } - return getLegacyDotDataDir(); -} -function resolveDataDir({ isCloud = false } = {}) { - if (isCloud) - return "/tmp"; - const configured = normalizeConfiguredPath(process.env.DATA_DIR); - if (configured) - return configured; - return getDefaultDataDir(); -} -function isSamePath(a, b) { - if (!a || !b) - return false; - const normalizedA = path_1.default.resolve(a); - const normalizedB = path_1.default.resolve(b); - if (process.platform === "win32") { - return normalizedA.toLowerCase() === normalizedB.toLowerCase(); - } - return normalizedA === normalizedB; -} diff --git a/src/lib/monitoring/observability.ts b/src/lib/monitoring/observability.ts index 8feb1cdcc7..0088fd5192 100644 --- a/src/lib/monitoring/observability.ts +++ b/src/lib/monitoring/observability.ts @@ -4,7 +4,8 @@ interface CircuitBreakerStatus { name: string; state: string; failureCount?: number; - lastFailureTime?: string | null; + lastFailureTime?: number | string | null; + retryAfterMs?: number; } interface SessionSnapshot { @@ -155,13 +156,31 @@ export function buildHealthPayload({ platform: process.platform, }; + const providerBreakers = circuitBreakers + .filter((cb) => !cb.name.startsWith("test-") && !cb.name.startsWith("test_")) + .map((cb) => { + const lastFailure = + typeof cb.lastFailureTime === "number" && Number.isFinite(cb.lastFailureTime) + ? new Date(cb.lastFailureTime).toISOString() + : typeof cb.lastFailureTime === "string" + ? cb.lastFailureTime + : null; + return { + provider: cb.name, + state: cb.state, + failureCount: cb.failureCount || 0, + lastFailure, + retryAfterMs: cb.retryAfterMs || 0, + }; + }); + const providerHealth: Record = {}; - for (const cb of circuitBreakers) { - if (cb.name.startsWith("test-") || cb.name.startsWith("test_")) continue; - providerHealth[cb.name] = { - state: cb.state, - failures: cb.failureCount || 0, - lastFailure: cb.lastFailureTime || null, + for (const breaker of providerBreakers) { + providerHealth[breaker.provider] = { + state: breaker.state, + failures: breaker.failureCount, + lastFailure: breaker.lastFailure, + retryAfterMs: breaker.retryAfterMs, }; } @@ -197,6 +216,7 @@ export function buildHealthPayload({ ...breakerCounts, total: breakerCounts.open + breakerCounts.halfOpen + breakerCounts.closed, }, + providerBreakers, providerHealth, providerSummary: { catalogCount, diff --git a/src/lib/resilience/settings.ts b/src/lib/resilience/settings.ts new file mode 100644 index 0000000000..f6f65e8faf --- /dev/null +++ b/src/lib/resilience/settings.ts @@ -0,0 +1,419 @@ +import { DEFAULT_API_LIMITS, PROVIDER_PROFILES } from "@omniroute/open-sse/config/constants"; + +type JsonRecord = Record; +type AuthCategory = "oauth" | "apikey"; + +export interface RequestQueueSettings { + autoEnableApiKeyProviders: boolean; + requestsPerMinute: number; + minTimeBetweenRequestsMs: number; + concurrentRequests: number; + maxWaitMs: number; +} + +export interface ConnectionCooldownProfileSettings { + baseCooldownMs: number; + useUpstreamRetryHints: boolean; + maxBackoffSteps: number; +} + +export interface ProviderBreakerProfileSettings { + failureThreshold: number; + resetTimeoutMs: number; +} + +export interface WaitForCooldownSettings { + enabled: boolean; + maxRetries: number; + maxRetryWaitSec: number; + maxRetryWaitMs: number; +} + +export interface ResilienceSettings { + requestQueue: RequestQueueSettings; + connectionCooldown: Record; + providerBreaker: Record; + waitForCooldown: WaitForCooldownSettings; +} + +export interface ResilienceSettingsPatch { + requestQueue?: Partial; + connectionCooldown?: Partial>>; + providerBreaker?: Partial>>; + waitForCooldown?: Partial; +} + +function asRecord(value: unknown): JsonRecord { + return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {}; +} + +function toInteger( + value: unknown, + fallback: number, + options: { min?: number; max?: number } = {} +): number { + const min = options.min ?? 0; + const max = options.max ?? Number.MAX_SAFE_INTEGER; + const parsed = + typeof value === "number" + ? value + : typeof value === "string" && value.trim().length > 0 + ? Number(value) + : Number.NaN; + + if (!Number.isFinite(parsed)) { + return fallback; + } + + return Math.min(max, Math.max(min, Math.trunc(parsed))); +} + +function toBoolean(value: unknown, fallback: boolean): boolean { + return typeof value === "boolean" ? value : fallback; +} + +export const DEFAULT_REQUEST_QUEUE_MAX_WAIT_MS = (() => { + const parsed = Number(process.env.RATE_LIMIT_MAX_WAIT_MS || "120000"); + return Number.isFinite(parsed) && parsed > 0 ? Math.trunc(parsed) : 120000; +})(); + +export const DEFAULT_RESILIENCE_SETTINGS: ResilienceSettings = { + requestQueue: { + autoEnableApiKeyProviders: true, + requestsPerMinute: DEFAULT_API_LIMITS.requestsPerMinute, + minTimeBetweenRequestsMs: DEFAULT_API_LIMITS.minTimeBetweenRequests, + concurrentRequests: DEFAULT_API_LIMITS.concurrentRequests, + maxWaitMs: DEFAULT_REQUEST_QUEUE_MAX_WAIT_MS, + }, + connectionCooldown: { + oauth: { + baseCooldownMs: PROVIDER_PROFILES.oauth.transientCooldown, + useUpstreamRetryHints: PROVIDER_PROFILES.oauth.rateLimitCooldown === 0, + maxBackoffSteps: PROVIDER_PROFILES.oauth.maxBackoffLevel, + }, + apikey: { + baseCooldownMs: PROVIDER_PROFILES.apikey.transientCooldown, + useUpstreamRetryHints: PROVIDER_PROFILES.apikey.rateLimitCooldown === 0, + maxBackoffSteps: PROVIDER_PROFILES.apikey.maxBackoffLevel, + }, + }, + providerBreaker: { + oauth: { + failureThreshold: PROVIDER_PROFILES.oauth.circuitBreakerThreshold, + resetTimeoutMs: PROVIDER_PROFILES.oauth.circuitBreakerReset, + }, + apikey: { + failureThreshold: PROVIDER_PROFILES.apikey.circuitBreakerThreshold, + resetTimeoutMs: PROVIDER_PROFILES.apikey.circuitBreakerReset, + }, + }, + waitForCooldown: { + enabled: true, + maxRetries: 3, + maxRetryWaitSec: 30, + maxRetryWaitMs: 30000, + }, +}; + +function normalizeRequestQueueSettings( + next: unknown, + fallback: RequestQueueSettings +): RequestQueueSettings { + const record = asRecord(next); + const requestsPerMinute = toInteger(record.requestsPerMinute, fallback.requestsPerMinute, { + min: 1, + max: 1_000_000, + }); + const minTimeBetweenRequestsMs = toInteger( + record.minTimeBetweenRequestsMs, + fallback.minTimeBetweenRequestsMs, + { min: 0, max: 60 * 60 * 1000 } + ); + const concurrentRequests = toInteger(record.concurrentRequests, fallback.concurrentRequests, { + min: 1, + max: 10_000, + }); + const maxWaitMs = toInteger(record.maxWaitMs, fallback.maxWaitMs, { + min: 1, + max: 24 * 60 * 60 * 1000, + }); + + return { + autoEnableApiKeyProviders: toBoolean( + record.autoEnableApiKeyProviders, + fallback.autoEnableApiKeyProviders + ), + requestsPerMinute, + minTimeBetweenRequestsMs, + concurrentRequests, + maxWaitMs, + }; +} + +function normalizeConnectionCooldownProfile( + next: unknown, + fallback: ConnectionCooldownProfileSettings +): ConnectionCooldownProfileSettings { + const record = asRecord(next); + return { + baseCooldownMs: toInteger(record.baseCooldownMs, fallback.baseCooldownMs, { + min: 0, + max: 24 * 60 * 60 * 1000, + }), + useUpstreamRetryHints: toBoolean(record.useUpstreamRetryHints, fallback.useUpstreamRetryHints), + maxBackoffSteps: toInteger(record.maxBackoffSteps, fallback.maxBackoffSteps, { + min: 0, + max: 32, + }), + }; +} + +function normalizeLegacyConnectionCooldownProfile( + next: unknown, + fallback: ConnectionCooldownProfileSettings +): ConnectionCooldownProfileSettings { + const record = asRecord(next); + const transientCooldown = toInteger(record.transientCooldown, fallback.baseCooldownMs, { + min: 0, + max: 24 * 60 * 60 * 1000, + }); + const legacyRateLimitCooldown = toInteger(record.rateLimitCooldown, transientCooldown, { + min: 0, + max: 24 * 60 * 60 * 1000, + }); + const useUpstreamRetryHints = + typeof record.rateLimitCooldown === "number" + ? record.rateLimitCooldown === 0 + : fallback.useUpstreamRetryHints; + + return { + baseCooldownMs: useUpstreamRetryHints + ? transientCooldown + : Math.max(transientCooldown, legacyRateLimitCooldown), + useUpstreamRetryHints, + maxBackoffSteps: toInteger(record.maxBackoffLevel, fallback.maxBackoffSteps, { + min: 0, + max: 32, + }), + }; +} + +function normalizeProviderBreakerProfile( + next: unknown, + fallback: ProviderBreakerProfileSettings +): ProviderBreakerProfileSettings { + const record = asRecord(next); + return { + failureThreshold: toInteger(record.failureThreshold, fallback.failureThreshold, { + min: 1, + max: 1000, + }), + resetTimeoutMs: toInteger(record.resetTimeoutMs, fallback.resetTimeoutMs, { + min: 1000, + max: 24 * 60 * 60 * 1000, + }), + }; +} + +function normalizeWaitForCooldownSettings( + next: unknown, + fallback: WaitForCooldownSettings +): WaitForCooldownSettings { + const record = asRecord(next); + const maxRetryWaitSec = toInteger(record.maxRetryWaitSec, fallback.maxRetryWaitSec, { + min: 0, + max: 300, + }); + const maxRetries = toInteger(record.maxRetries, fallback.maxRetries, { min: 0, max: 10 }); + const enabled = + toBoolean(record.enabled, fallback.enabled) && maxRetries > 0 && maxRetryWaitSec > 0; + + return { + enabled, + maxRetries, + maxRetryWaitSec, + maxRetryWaitMs: maxRetryWaitSec * 1000, + }; +} + +function buildLegacyFallback(settings: JsonRecord): ResilienceSettings { + const profiles = asRecord(settings.providerProfiles); + const defaults = asRecord(settings.rateLimitDefaults); + + const oauthLegacy = asRecord(profiles.oauth); + const apikeyLegacy = asRecord(profiles.apikey); + + const waitMaxRetrySec = toInteger( + settings.maxRetryIntervalSec, + DEFAULT_RESILIENCE_SETTINGS.waitForCooldown.maxRetryWaitSec, + { min: 0, max: 300 } + ); + const waitMaxRetries = toInteger( + settings.requestRetry, + DEFAULT_RESILIENCE_SETTINGS.waitForCooldown.maxRetries, + { min: 0, max: 10 } + ); + + return { + requestQueue: { + autoEnableApiKeyProviders: DEFAULT_RESILIENCE_SETTINGS.requestQueue.autoEnableApiKeyProviders, + requestsPerMinute: toInteger( + defaults.requestsPerMinute, + DEFAULT_RESILIENCE_SETTINGS.requestQueue.requestsPerMinute, + { min: 1, max: 1_000_000 } + ), + minTimeBetweenRequestsMs: toInteger( + defaults.minTimeBetweenRequests, + DEFAULT_RESILIENCE_SETTINGS.requestQueue.minTimeBetweenRequestsMs, + { min: 0, max: 60 * 60 * 1000 } + ), + concurrentRequests: toInteger( + defaults.concurrentRequests, + DEFAULT_RESILIENCE_SETTINGS.requestQueue.concurrentRequests, + { min: 1, max: 10_000 } + ), + maxWaitMs: DEFAULT_RESILIENCE_SETTINGS.requestQueue.maxWaitMs, + }, + connectionCooldown: { + oauth: normalizeLegacyConnectionCooldownProfile( + oauthLegacy, + DEFAULT_RESILIENCE_SETTINGS.connectionCooldown.oauth + ), + apikey: normalizeLegacyConnectionCooldownProfile( + apikeyLegacy, + DEFAULT_RESILIENCE_SETTINGS.connectionCooldown.apikey + ), + }, + providerBreaker: { + oauth: { + failureThreshold: toInteger( + oauthLegacy.circuitBreakerThreshold, + DEFAULT_RESILIENCE_SETTINGS.providerBreaker.oauth.failureThreshold, + { min: 1, max: 1000 } + ), + resetTimeoutMs: toInteger( + oauthLegacy.circuitBreakerReset, + DEFAULT_RESILIENCE_SETTINGS.providerBreaker.oauth.resetTimeoutMs, + { min: 1000, max: 24 * 60 * 60 * 1000 } + ), + }, + apikey: { + failureThreshold: toInteger( + apikeyLegacy.circuitBreakerThreshold, + DEFAULT_RESILIENCE_SETTINGS.providerBreaker.apikey.failureThreshold, + { min: 1, max: 1000 } + ), + resetTimeoutMs: toInteger( + apikeyLegacy.circuitBreakerReset, + DEFAULT_RESILIENCE_SETTINGS.providerBreaker.apikey.resetTimeoutMs, + { min: 1000, max: 24 * 60 * 60 * 1000 } + ), + }, + }, + waitForCooldown: { + enabled: waitMaxRetries > 0 && waitMaxRetrySec > 0, + maxRetries: waitMaxRetries, + maxRetryWaitSec: waitMaxRetrySec, + maxRetryWaitMs: waitMaxRetrySec * 1000, + }, + }; +} + +export function resolveResilienceSettings( + settings: Record | null | undefined +): ResilienceSettings { + const record = asRecord(settings); + const current = asRecord(record.resilienceSettings); + const fallback = buildLegacyFallback(record); + + return { + requestQueue: normalizeRequestQueueSettings(current.requestQueue, fallback.requestQueue), + connectionCooldown: { + oauth: normalizeConnectionCooldownProfile( + asRecord(current.connectionCooldown).oauth, + fallback.connectionCooldown.oauth + ), + apikey: normalizeConnectionCooldownProfile( + asRecord(current.connectionCooldown).apikey, + fallback.connectionCooldown.apikey + ), + }, + providerBreaker: { + oauth: normalizeProviderBreakerProfile( + asRecord(current.providerBreaker).oauth, + fallback.providerBreaker.oauth + ), + apikey: normalizeProviderBreakerProfile( + asRecord(current.providerBreaker).apikey, + fallback.providerBreaker.apikey + ), + }, + waitForCooldown: normalizeWaitForCooldownSettings( + current.waitForCooldown, + fallback.waitForCooldown + ), + }; +} + +export function mergeResilienceSettings( + current: ResilienceSettings, + updates: ResilienceSettingsPatch +): ResilienceSettings { + return { + requestQueue: normalizeRequestQueueSettings(updates.requestQueue, current.requestQueue), + connectionCooldown: { + oauth: normalizeConnectionCooldownProfile( + updates.connectionCooldown?.oauth, + current.connectionCooldown.oauth + ), + apikey: normalizeConnectionCooldownProfile( + updates.connectionCooldown?.apikey, + current.connectionCooldown.apikey + ), + }, + providerBreaker: { + oauth: normalizeProviderBreakerProfile( + updates.providerBreaker?.oauth, + current.providerBreaker.oauth + ), + apikey: normalizeProviderBreakerProfile( + updates.providerBreaker?.apikey, + current.providerBreaker.apikey + ), + }, + waitForCooldown: normalizeWaitForCooldownSettings( + updates.waitForCooldown, + current.waitForCooldown + ), + }; +} + +export function buildLegacyResilienceCompat(settings: ResilienceSettings) { + return { + profiles: { + oauth: { + transientCooldown: settings.connectionCooldown.oauth.baseCooldownMs, + rateLimitCooldown: settings.connectionCooldown.oauth.useUpstreamRetryHints + ? 0 + : settings.connectionCooldown.oauth.baseCooldownMs, + maxBackoffLevel: settings.connectionCooldown.oauth.maxBackoffSteps, + circuitBreakerThreshold: settings.providerBreaker.oauth.failureThreshold, + circuitBreakerReset: settings.providerBreaker.oauth.resetTimeoutMs, + }, + apikey: { + transientCooldown: settings.connectionCooldown.apikey.baseCooldownMs, + rateLimitCooldown: settings.connectionCooldown.apikey.useUpstreamRetryHints + ? 0 + : settings.connectionCooldown.apikey.baseCooldownMs, + maxBackoffLevel: settings.connectionCooldown.apikey.maxBackoffSteps, + circuitBreakerThreshold: settings.providerBreaker.apikey.failureThreshold, + circuitBreakerReset: settings.providerBreaker.apikey.resetTimeoutMs, + }, + }, + defaults: { + requestsPerMinute: settings.requestQueue.requestsPerMinute, + minTimeBetweenRequests: settings.requestQueue.minTimeBetweenRequestsMs, + concurrentRequests: settings.requestQueue.concurrentRequests, + }, + }; +} diff --git a/src/shared/utils/circuitBreaker.ts b/src/shared/utils/circuitBreaker.ts index 3f9e0e332a..3700bba31c 100644 --- a/src/shared/utils/circuitBreaker.ts +++ b/src/shared/utils/circuitBreaker.ts @@ -180,6 +180,7 @@ export class CircuitBreaker { state: this.state, failureCount: this.failureCount, lastFailureTime: this.lastFailureTime, + retryAfterMs: this.getRetryAfterMs(), }; } diff --git a/src/shared/validation/schemas.ts b/src/shared/validation/schemas.ts index ae120f3da7..fc6d83101e 100644 --- a/src/shared/validation/schemas.ts +++ b/src/shared/validation/schemas.ts @@ -616,11 +616,6 @@ export const updateModelAliasSchema = z.object({ alias: z.string().trim().min(1, "Alias is required").max(200), }); -export const clearModelAvailabilitySchema = z.object({ - provider: z.string().trim().min(1, "provider is required").max(120), - model: modelIdSchema, -}); - /** Align with `sanitizeUpstreamHeadersMap` โ€” allow non-ASCII names; reject Host / hop-by-hop / whitespace / ":". */ const upstreamHeaderNameSchema = z .string() @@ -709,7 +704,7 @@ export const toggleRateLimitSchema = z.object({ enabled: z.boolean(), }); -const resilienceProfileSchema = z.object({ +const legacyResilienceProfileSchema = z.object({ transientCooldown: z.number().min(0), rateLimitCooldown: z.number().min(0), maxBackoffLevel: z.number().int().min(0), @@ -717,30 +712,87 @@ const resilienceProfileSchema = z.object({ circuitBreakerReset: z.number().min(0), }); -const resilienceDefaultsSchema = z +const legacyResilienceDefaultsSchema = z .object({ requestsPerMinute: z.number().int().min(1).optional(), - minTimeBetweenRequests: z.number().int().min(1).optional(), + minTimeBetweenRequests: z.number().int().min(0).optional(), concurrentRequests: z.number().int().min(1).optional(), }) .strict(); +const requestQueueSettingsSchema = z + .object({ + autoEnableApiKeyProviders: z.boolean().optional(), + requestsPerMinute: z.number().int().min(1).optional(), + minTimeBetweenRequestsMs: z.number().int().min(0).optional(), + concurrentRequests: z.number().int().min(1).optional(), + maxWaitMs: z.number().int().min(1).optional(), + }) + .strict(); + +const connectionCooldownProfileSchema = z + .object({ + baseCooldownMs: z.number().int().min(0).optional(), + useUpstreamRetryHints: z.boolean().optional(), + maxBackoffSteps: z.number().int().min(0).optional(), + }) + .strict(); + +const providerBreakerProfileSchema = z + .object({ + failureThreshold: z.number().int().min(1).optional(), + resetTimeoutMs: z.number().int().min(1000).optional(), + }) + .strict(); + +const waitForCooldownSettingsSchema = z + .object({ + enabled: z.boolean().optional(), + maxRetries: z.number().int().min(0).max(10).optional(), + maxRetryWaitSec: z.number().int().min(0).max(300).optional(), + }) + .strict(); + export const updateResilienceSchema = z .object({ - profiles: z + requestQueue: requestQueueSettingsSchema.optional(), + connectionCooldown: z .object({ - oauth: resilienceProfileSchema.optional(), - apikey: resilienceProfileSchema.optional(), + oauth: connectionCooldownProfileSchema.optional(), + apikey: connectionCooldownProfileSchema.optional(), }) .strict() .optional(), - defaults: resilienceDefaultsSchema.optional(), + providerBreaker: z + .object({ + oauth: providerBreakerProfileSchema.optional(), + apikey: providerBreakerProfileSchema.optional(), + }) + .strict() + .optional(), + waitForCooldown: waitForCooldownSettingsSchema.optional(), + profiles: z + .object({ + oauth: legacyResilienceProfileSchema.optional(), + apikey: legacyResilienceProfileSchema.optional(), + }) + .strict() + .optional(), + defaults: legacyResilienceDefaultsSchema.optional(), }) + .strict() .superRefine((value, ctx) => { - if (!value.profiles && !value.defaults) { + if ( + !value.requestQueue && + !value.connectionCooldown && + !value.providerBreaker && + !value.waitForCooldown && + !value.profiles && + !value.defaults + ) { ctx.addIssue({ code: z.ZodIssueCode.custom, - message: "Must provide profiles or defaults", + message: "Must provide resilience settings to update", path: [], }); } diff --git a/src/sse/handlers/chat.ts b/src/sse/handlers/chat.ts index 70f546be1b..b66d1ea534 100644 --- a/src/sse/handlers/chat.ts +++ b/src/sse/handlers/chat.ts @@ -42,11 +42,6 @@ import { // Pipeline integration โ€” wired modules import { getCircuitBreaker } from "../../shared/utils/circuitBreaker"; -import { - isModelAvailable, - markModelAsProblematic, - clearModelUnavailability, -} from "../../domain/modelAvailability"; import { markAccountExhaustedFrom429 } from "../../domain/quotaCache"; import { RequestTelemetry, recordTelemetry } from "../../shared/utils/requestTelemetry"; import { generateRequestId } from "../../shared/utils/requestId"; @@ -107,6 +102,8 @@ function intersectAllowedConnectionIds(primary: unknown, secondary: unknown): st return first || second || null; } +const PROVIDER_BREAKER_FAILURE_STATUSES = new Set([408, 500, 502, 503, 504]); + /** * Handle chat completion request * Supports: OpenAI, Claude, Gemini, OpenAI Responses API formats @@ -324,14 +321,6 @@ export async function handleChat(request: any, clientRawRequest: any = null) { return false; } - // Fixed-account combo steps must bypass the provider/model cooldown gate here. - // A previous account failure can quarantine the model globally, but the next - // step may intentionally pin a different connection for the same model. - if (!hasForcedConnection && !isModelAvailable(provider, resolvedModel)) { - log.debug("AVAILABILITY", `${provider}/${modelInfo.model} in cooldown, skipping`); - return false; - } - const creds = await getProviderCredentialsWithQuotaPreflight( provider, null, @@ -524,7 +513,7 @@ async function handleSingleModelChat( ? "fixed combo step connection" : undefined; - // 2. Pipeline gates (availability + circuit breaker) + // 2. Pipeline gates (availability + provider circuit breaker) const providerProfile = await getRuntimeProviderProfile(provider); const gate = await checkPipelineGates(provider, model, { ignoreCircuitBreaker: forceLiveComboTest || hasForcedConnection, @@ -535,8 +524,8 @@ async function handleSingleModelChat( if (gate) return gate; const breaker = getCircuitBreaker(provider, { - failureThreshold: providerProfile.circuitBreakerThreshold, - resetTimeout: providerProfile.circuitBreakerReset, + failureThreshold: providerProfile.failureThreshold, + resetTimeout: providerProfile.resetTimeoutMs, onStateChange: (name: string, from: string, to: string) => log.info("CIRCUIT", `${name}: ${from} โ†’ ${to}`), }); @@ -549,9 +538,10 @@ async function handleSingleModelChat( const retrySettings = disableCooldownAwareRetry ? { ...baseRetrySettings, - requestRetry: 0, - maxRetryIntervalSec: 0, - maxRetryIntervalMs: 0, + enabled: false, + maxRetries: 0, + maxRetryWaitSec: 0, + maxRetryWaitMs: 0, } : baseRetrySettings; const requestSignal = request?.signal ?? null; @@ -597,26 +587,6 @@ async function handleSingleModelChat( ); if (!credentials || "allRateLimited" in credentials) { - if ([408, 429, 500, 502, 503, 504].includes(Number(lastStatus))) { - const quarantine = markModelAsProblematic(provider, model, { - status: Number(lastStatus), - baseCooldownMs: lastCooldownMs, - reason: `HTTP ${lastStatus}`, - profile: providerProfile, - }); - if (quarantine.quarantined) { - log.info( - "AVAILABILITY", - `${provider}/${model} marked unavailable โ€” all accounts exhausted (HTTP ${lastStatus}, cooldown ${Math.ceil(quarantine.cooldownMs / 1000)}s, failureCount ${quarantine.failureCount}/${quarantine.threshold})` - ); - } else { - log.info( - "AVAILABILITY", - `${provider}/${model} recorded exhaustion failure ${quarantine.failureCount}/${quarantine.threshold} (HTTP ${lastStatus}, cooldown basis ${Math.ceil(quarantine.cooldownMs / 1000)}s)` - ); - } - } - if (credentials?.allRateLimited) { const retryDecision = getCooldownAwareRetryDecision({ retryAfter: credentials.retryAfter, @@ -628,7 +598,7 @@ async function handleSingleModelChat( const waitSec = Math.max(Math.ceil(retryDecision.waitMs / 1000), 0); log.info( "COOLDOWN_RETRY", - `${provider}/${model} all accounts cooling down (${retryDecision.retryAfterHuman || `retry in ${waitSec}s`}) โ€” waiting ${waitSec}s before retry ${requestRetryAttempt + 1}/${retrySettings.requestRetry}` + `${provider}/${model} all connections cooling down (${retryDecision.retryAfterHuman || `retry in ${waitSec}s`}) โ€” waiting ${waitSec}s before retry ${requestRetryAttempt + 1}/${retrySettings.maxRetries}` ); const completed = await waitForCooldownAwareRetry(retryDecision.waitMs, requestSignal); @@ -643,12 +613,21 @@ async function handleSingleModelChat( requestRetryAttempt += 1; log.info( "COOLDOWN_RETRY", - `${provider}/${model} cooldown elapsed โ€” restarting request attempt ${requestRetryAttempt}/${retrySettings.requestRetry}` + `${provider}/${model} cooldown elapsed โ€” restarting request attempt ${requestRetryAttempt}/${retrySettings.maxRetries}` ); continue requestAttemptLoop; } } + const breakerFailureStatus = Number(lastStatus ?? credentials?.lastErrorCode); + if ( + !forceLiveComboTest && + credentials?.allRateLimited && + PROVIDER_BREAKER_FAILURE_STATUSES.has(breakerFailureStatus) + ) { + breaker._onFailure(); + } + return handleNoCredentials( credentials, excludedConnectionIds.size > 0 ? Array.from(excludedConnectionIds)[0] : null, @@ -717,11 +696,9 @@ async function handleSingleModelChat( const proxyInfo = await safeResolveProxy(credentials.connectionId); const proxyStartTime = Date.now(); - // 4. Execute chat via core (with circuit breaker + optional TLS) + // 4. Execute chat via core after breaker gate checks (with optional TLS tracking) if (telemetry) telemetry.startPhase("connect"); const { result, tlsFingerprintUsed } = await executeChatWithBreaker({ - bypassCircuitBreaker: forceLiveComboTest, - breaker, body: requestBody, provider, model, @@ -765,7 +742,9 @@ async function handleSingleModelChat( if (result.success) { clearModelLock(provider, credentials.connectionId, model); - clearModelUnavailability(provider, model); + if (!forceLiveComboTest) { + breaker._onSuccess(); + } if (injectedHandoff && runtimeOptions.sessionId && comboName) { deleteHandoff(runtimeOptions.sessionId, comboName); } @@ -873,6 +852,10 @@ async function handleSingleModelChat( continue; } + if (!forceLiveComboTest && PROVIDER_BREAKER_FAILURE_STATUSES.has(Number(result.status))) { + breaker._onFailure(); + } + return result.response; } } diff --git a/src/sse/handlers/chatHelpers.ts b/src/sse/handlers/chatHelpers.ts index ef5dd9213e..8099889ed3 100644 --- a/src/sse/handlers/chatHelpers.ts +++ b/src/sse/handlers/chatHelpers.ts @@ -14,6 +14,7 @@ import { handleChatCore } from "@omniroute/open-sse/handlers/chatCore.ts"; import { errorResponse, modelCooldownResponse, + providerCircuitOpenResponse, unavailableResponse, } from "@omniroute/open-sse/utils/error.ts"; import { HTTP_STATUS } from "@omniroute/open-sse/config/constants.ts"; @@ -23,14 +24,10 @@ import { isTlsFingerprintActive, } from "@omniroute/open-sse/utils/proxyFetch.ts"; import { resolveProxyForConnection } from "@/lib/localDb"; -import { getCircuitBreaker, CircuitBreakerOpenError } from "../../shared/utils/circuitBreaker"; -import { getModelCooldownInfo, isModelAvailable } from "../../domain/modelAvailability"; +import { getCircuitBreaker } from "../../shared/utils/circuitBreaker"; import { logProxyEvent } from "../../lib/proxyLogger"; import { logTranslationEvent } from "../../lib/translatorEvents"; -import { - getRuntimeProviderProfile, - clearProviderFailure, -} from "@omniroute/open-sse/services/accountFallback.ts"; +import { getRuntimeProviderProfile } from "@omniroute/open-sse/services/accountFallback.ts"; export async function resolveModelOrError(modelStr: string, body: any, endpointPath: string = "") { const modelInfo = await getModelInfo(modelStr); @@ -79,30 +76,16 @@ export async function checkPipelineGates( providerProfile?: { circuitBreakerThreshold?: number; circuitBreakerReset?: number; + failureThreshold?: number; + resetTimeoutMs?: number; } | null; } = {} ) { const bypassReason = options.bypassReason || "pipeline override"; - const modelAvailable = isModelAvailable(provider, model); - if (!modelAvailable && options.ignoreModelCooldown) { - log.info("AVAILABILITY", `${provider}/${model} cooldown bypassed (${bypassReason})`); - } else if (!modelAvailable) { - const cooldownInfo = getModelCooldownInfo(provider, model); - const retryAfterSec = cooldownInfo - ? Math.max(Math.ceil(cooldownInfo.remainingMs / 1000), 1) - : 1; - log.warn("AVAILABILITY", `${provider}/${model} is in cooldown, rejecting request`); - return unavailableResponse( - HTTP_STATUS.SERVICE_UNAVAILABLE, - `Model ${provider}/${model} is temporarily unavailable (cooldown)`, - retryAfterSec - ); - } - const providerProfile = options.providerProfile ?? (await getRuntimeProviderProfile(provider)); const breaker = getCircuitBreaker(provider, { - failureThreshold: providerProfile.circuitBreakerThreshold, - resetTimeout: providerProfile.circuitBreakerReset, + failureThreshold: providerProfile.failureThreshold ?? providerProfile.circuitBreakerThreshold, + resetTimeout: providerProfile.resetTimeoutMs ?? providerProfile.circuitBreakerReset, onStateChange: (name: string, from: string, to: string) => log.info("CIRCUIT", `${name}: ${from} โ†’ ${to}`), }); @@ -112,19 +95,13 @@ export async function checkPipelineGates( const retryAfterMs = breaker.getRetryAfterMs(); const retryAfterSec = Math.max(Math.ceil(retryAfterMs / 1000), 1); log.warn("CIRCUIT", `Circuit breaker OPEN for ${provider}, rejecting request`); - return unavailableResponse( - HTTP_STATUS.SERVICE_UNAVAILABLE, - `Provider ${provider} circuit breaker is open`, - retryAfterSec - ); + return providerCircuitOpenResponse(provider, retryAfterSec); } return null; } export async function executeChatWithBreaker({ - bypassCircuitBreaker, - breaker, body, provider, model, @@ -173,46 +150,18 @@ export async function executeChatWithBreaker({ }, onRequestSuccess: async () => { await clearAccountError(credentials.connectionId, credentials); - // Clear provider-level failure state on successful request - clearProviderFailure(provider); }, }) ); - if (bypassCircuitBreaker) { - if (!proxyInfo?.proxy && isTlsFingerprintActive()) { - const tracked = await runWithTlsTracking(chatFn); - return { result: tracked.result, tlsFingerprintUsed: tracked.tlsFingerprintUsed }; - } - - const result = await chatFn(); - return { result, tlsFingerprintUsed: false }; - } - if (!proxyInfo?.proxy && isTlsFingerprintActive()) { - const tracked = await breaker.execute(async () => runWithTlsTracking(chatFn)); + const tracked = await runWithTlsTracking(chatFn); return { result: tracked.result, tlsFingerprintUsed: tracked.tlsFingerprintUsed }; } - const result = await breaker.execute(chatFn); + const result = await chatFn(); return { result, tlsFingerprintUsed: false }; } catch (cbErr: any) { - if (cbErr instanceof CircuitBreakerOpenError) { - log.warn("CIRCUIT", `${provider} circuit open during retry: ${cbErr.message}`); - return { - result: { - success: false, - response: unavailableResponse( - HTTP_STATUS.SERVICE_UNAVAILABLE, - `Provider ${provider} circuit breaker is open`, - Math.ceil(cbErr.retryAfterMs / 1000) - ), - status: HTTP_STATUS.SERVICE_UNAVAILABLE, - }, - tlsFingerprintUsed: false, - }; - } - if (cbErr?.code === "PROXY_UNREACHABLE" || /proxy unreachable/i.test(cbErr?.message || "")) { const detail = cbErr?.message || "Proxy unreachable"; log.warn("PROXY", detail); diff --git a/src/sse/services/auth.ts b/src/sse/services/auth.ts index ec601a3810..59e64366ad 100644 --- a/src/sse/services/auth.ts +++ b/src/sse/services/auth.ts @@ -19,12 +19,14 @@ import { hasPerModelQuota, getRuntimeProviderProfile, recordModelLockoutFailure, - isProviderInCooldown, - getProviderCooldownRemainingMs, } from "@omniroute/open-sse/services/accountFallback.ts"; import { isLocalProvider } from "@omniroute/open-sse/config/providerRegistry.ts"; import { COOLDOWN_MS } from "@omniroute/open-sse/config/constants.ts"; import { preflightQuota } from "@omniroute/open-sse/services/quotaPreflight.ts"; +import { + classifyProviderError, + PROVIDER_ERROR_TYPES, +} from "@omniroute/open-sse/services/errorClassifier.ts"; import { getCodexModelScope } from "@omniroute/open-sse/executors/codex.ts"; import { getProviderAlias, resolveProviderId } from "@/shared/constants/providers"; import { isModelExcludedByConnection } from "@/domain/connectionModelRules"; @@ -253,6 +255,31 @@ function isTerminalConnectionStatus(connection: ProviderConnectionView): boolean return status === "credits_exhausted" || status === "banned" || status === "expired"; } +function resolveTerminalConnectionStatus( + status: number, + result: { permanent?: boolean; creditsExhausted?: boolean }, + providerErrorType: string | null = null +): string | null { + if (result.creditsExhausted || status === 402) return "credits_exhausted"; + if ( + providerErrorType === PROVIDER_ERROR_TYPES.PROJECT_ROUTE_ERROR || + providerErrorType === PROVIDER_ERROR_TYPES.OAUTH_INVALID_TOKEN + ) { + return null; + } + if (result.permanent || providerErrorType === PROVIDER_ERROR_TYPES.FORBIDDEN || status === 403) { + return "banned"; + } + if ( + providerErrorType === PROVIDER_ERROR_TYPES.ACCOUNT_DEACTIVATED || + providerErrorType === PROVIDER_ERROR_TYPES.UNAUTHORIZED || + status === 401 + ) { + return "expired"; + } + return null; +} + export function resolveQuotaLimitPolicy( provider: string, providerSpecificData: JsonRecord @@ -684,22 +711,6 @@ export async function getProviderCredentials( return true; }); - // Check if the entire provider is in cooldown (too many transient failures) - if (isProviderInCooldown(provider)) { - const cooldownRemaining = getProviderCooldownRemainingMs(provider); - log.warn( - "AUTH", - `${provider} | provider in cooldown for ${Math.ceil((cooldownRemaining || 0) / 1000)}s (${availableConnections.length} connections bypassed)` - ); - return { - allRateLimited: true, - retryAfter: new Date(Date.now() + (cooldownRemaining || 0)).toISOString(), - retryAfterHuman: formatRetryAfter( - new Date(Date.now() + (cooldownRemaining || 0)).toISOString() - ), - }; - } - log.debug( "AUTH", `${provider} | available: ${availableConnections.length}/${connections.length}` @@ -1206,6 +1217,15 @@ export async function markAccountUnavailable( const effectiveProviderProfile = providerProfile || (provider ? await getRuntimeProviderProfile(provider) : null); + const fallbackResult = checkFallbackError( + status, + errorText, + backoffLevel, + model, + provider, + null, + effectiveProviderProfile + ); // Read passthroughModels from connection config (user-configured per-model quota) const connProviderSpecificData = (conn?.providerSpecificData as Record) || {}; @@ -1216,18 +1236,20 @@ export async function markAccountUnavailable( const isPerModelQuotaProvider = hasPerModelQuota(provider, model, connectionPassthroughModels); if (isPerModelQuotaProvider && provider && model && (status === 404 || status === 429)) { const reason = status === 404 ? "not_found" : "rate_limited"; - const fallbackCooldown = - status === 404 - ? (effectiveProviderProfile?.transientCooldown ?? COOLDOWN_MS.notFoundLocal) - : 0; const lockout = recordModelLockoutFailure( provider, connectionId, model, reason, status, - fallbackCooldown, - effectiveProviderProfile + status === 404 + ? (effectiveProviderProfile?.baseCooldownMs ?? COOLDOWN_MS.notFoundLocal) + : (fallbackResult.baseCooldownMs ?? effectiveProviderProfile?.baseCooldownMs ?? 0), + effectiveProviderProfile, + { + exactCooldownMs: + fallbackResult.usedUpstreamRetryHint === true ? fallbackResult.cooldownMs : null, + } ); // Update last error for observability (without changing terminal status) updateProviderConnection(connectionId, { @@ -1242,18 +1264,12 @@ export async function markAccountUnavailable( ); return { shouldFallback: true, cooldownMs: lockout.cooldownMs }; } - - const result = checkFallbackError( - status, - errorText, - backoffLevel, - model, - provider, - null, - effectiveProviderProfile - ); - const { shouldFallback, cooldownMs, newBackoffLevel, reason } = result; + const result = fallbackResult; + const { shouldFallback, cooldownMs: rawCooldownMs, newBackoffLevel, reason } = result; if (!shouldFallback) return { shouldFallback: false, cooldownMs: 0 }; + const providerErrorType = classifyProviderError(status, errorText); + const terminalStatus = resolveTerminalConnectionStatus(status, result, providerErrorType); + const cooldownMs = terminalStatus ? 0 : rawCooldownMs; // โ”€โ”€ 404 model-only lockout: connection stays active โ”€โ”€ // For local providers (detected by URL), a 404 means the specific model @@ -1286,7 +1302,6 @@ export async function markAccountUnavailable( return { shouldFallback: true, cooldownMs: lockout.cooldownMs }; } - const rateLimitedUntil = getUnavailableUntil(cooldownMs); const errorMsg = typeof errorText === "string" ? errorText.slice(0, 100) : "Provider error"; // T09: Codex per-scope lockout (do not block the whole account globally). @@ -1294,7 +1309,7 @@ export async function markAccountUnavailable( const scope = getCodexModelScope(model); const existingScopeMap = asRecord(conn.providerSpecificData.codexScopeRateLimitedUntil); const persistedScopeUntil = getCodexScopeRateLimitedUntil(conn.providerSpecificData, model); - const scopeRateLimitedUntil = persistedScopeUntil || rateLimitedUntil; + const scopeRateLimitedUntil = persistedScopeUntil || getUnavailableUntil(cooldownMs); const scopeCooldownMs = Math.max(new Date(scopeRateLimitedUntil).getTime() - Date.now(), 0); await updateProviderConnection(connectionId, { @@ -1323,14 +1338,27 @@ export async function markAccountUnavailable( return { shouldFallback: true, cooldownMs: scopeCooldownMs }; } - await updateProviderConnection(connectionId, { - rateLimitedUntil, - testStatus: "unavailable", + const baseUpdate = { lastError: errorMsg, + lastErrorType: providerErrorType, errorCode: status, lastErrorAt: new Date().toISOString(), backoffLevel: newBackoffLevel ?? backoffLevel, - }); + }; + + if (cooldownMs > 0) { + await updateProviderConnection(connectionId, { + ...baseUpdate, + rateLimitedUntil: getUnavailableUntil(cooldownMs), + testStatus: "unavailable", + }); + } else { + await updateProviderConnection(connectionId, { + ...baseUpdate, + rateLimitedUntil: null, + ...(terminalStatus ? { testStatus: terminalStatus } : {}), + }); + } // T-AUTODISABLE: If auto-disable setting is enabled and error is permanent/terminal, // mark account as inactive so it is never retried again. @@ -1354,11 +1382,6 @@ export async function markAccountUnavailable( } } - // Per-model lockout: lock the specific model if known - if (provider && model && cooldownMs > 0) { - lockModel(provider, connectionId, model, reason || "unknown", cooldownMs); - } - if (provider && status && errorMsg) { console.error(`โŒ ${provider} [${status}]: ${errorMsg}`); } diff --git a/src/sse/services/cooldownAwareRetry.ts b/src/sse/services/cooldownAwareRetry.ts index 99f1ef5f4a..14a95c605d 100644 --- a/src/sse/services/cooldownAwareRetry.ts +++ b/src/sse/services/cooldownAwareRetry.ts @@ -1,14 +1,14 @@ import { formatRetryAfter } from "@omniroute/open-sse/services/accountFallback.ts"; +import { resolveResilienceSettings } from "@/lib/resilience/settings"; -const DEFAULT_REQUEST_RETRY = 3; -const DEFAULT_MAX_RETRY_INTERVAL_SEC = 30; const MAX_REQUEST_RETRY = 10; const MAX_RETRY_INTERVAL_SEC = 300; export interface CooldownAwareRetrySettings { - requestRetry: number; - maxRetryIntervalSec: number; - maxRetryIntervalMs: number; + enabled: boolean; + maxRetries: number; + maxRetryWaitSec: number; + maxRetryWaitMs: number; } function normalizeInteger( @@ -35,20 +35,26 @@ function normalizeInteger( export function resolveCooldownAwareRetrySettings( settings: Record | null | undefined ): CooldownAwareRetrySettings { - const requestRetry = normalizeInteger(settings?.requestRetry, DEFAULT_REQUEST_RETRY, { + const waitForCooldown = resolveResilienceSettings(settings).waitForCooldown; + const maxRetries = normalizeInteger(waitForCooldown.maxRetries, waitForCooldown.maxRetries, { min: 0, max: MAX_REQUEST_RETRY, }); - const maxRetryIntervalSec = normalizeInteger( - settings?.maxRetryIntervalSec, - DEFAULT_MAX_RETRY_INTERVAL_SEC, - { min: 0, max: MAX_RETRY_INTERVAL_SEC } + const maxRetryWaitSec = normalizeInteger( + waitForCooldown.maxRetryWaitSec, + waitForCooldown.maxRetryWaitSec, + { + min: 0, + max: MAX_RETRY_INTERVAL_SEC, + } ); + const enabled = Boolean(waitForCooldown.enabled) && maxRetries > 0 && maxRetryWaitSec > 0; return { - requestRetry, - maxRetryIntervalSec, - maxRetryIntervalMs: maxRetryIntervalSec * 1000, + enabled, + maxRetries, + maxRetryWaitSec, + maxRetryWaitMs: maxRetryWaitSec * 1000, }; } @@ -90,9 +96,10 @@ export function getCooldownAwareRetryDecision({ } { const closest = computeClosestRetryAfter(retryAfter); if ( - settings.requestRetry <= 0 || - settings.maxRetryIntervalMs <= 0 || - attempt >= settings.requestRetry || + !settings.enabled || + settings.maxRetries <= 0 || + settings.maxRetryWaitMs <= 0 || + attempt >= settings.maxRetries || closest.waitMs === null ) { return { @@ -103,7 +110,7 @@ export function getCooldownAwareRetryDecision({ }; } - if (closest.waitMs > settings.maxRetryIntervalMs) { + if (closest.waitMs > settings.maxRetryWaitMs) { return { shouldRetry: false, retryAfter: closest.retryAfter, diff --git a/src/types/settings.ts b/src/types/settings.ts index 9f4f32b38e..273b5bf941 100644 --- a/src/types/settings.ts +++ b/src/types/settings.ts @@ -1,4 +1,5 @@ import type { HideableSidebarItemId } from "@/shared/constants/sidebarVisibility"; +import type { ResilienceSettings } from "@/lib/resilience/settings"; /** * Application settings stored in SQLite key-value pairs. @@ -20,15 +21,13 @@ export interface Settings { jwtSecret?: string; hideHealthCheckLogs?: boolean; hiddenSidebarItems?: HideableSidebarItemId[]; + resilienceSettings?: ResilienceSettings; } export interface ComboDefaults { strategy: "priority" | "weighted" | "round-robin" | "context-relay"; maxRetries: number; retryDelayMs: number; - timeoutMs: number; - healthCheckEnabled: boolean; - healthCheckTimeoutMs: number; maxComboDepth: number; trackMetrics: boolean; concurrencyPerModel?: number; diff --git a/tests/e2e/resilience-plan-alignment.spec.ts b/tests/e2e/resilience-plan-alignment.spec.ts new file mode 100644 index 0000000000..817ebcf5bc --- /dev/null +++ b/tests/e2e/resilience-plan-alignment.spec.ts @@ -0,0 +1,374 @@ +import { expect, test, type Page } from "@playwright/test"; +import { gotoDashboardRoute } from "./helpers/dashboardAuth"; + +const resilienceSettings = { + requestQueue: { + autoEnableApiKeyProviders: true, + requestsPerMinute: 100, + minTimeBetweenRequestsMs: 200, + concurrentRequests: 10, + maxWaitMs: 120000, + }, + connectionCooldown: { + oauth: { + baseCooldownMs: 60000, + useUpstreamRetryHints: false, + maxBackoffSteps: 8, + }, + apikey: { + baseCooldownMs: 3000, + useUpstreamRetryHints: true, + maxBackoffSteps: 5, + }, + }, + providerBreaker: { + oauth: { + failureThreshold: 3, + resetTimeoutMs: 60000, + }, + apikey: { + failureThreshold: 5, + resetTimeoutMs: 30000, + }, + }, + waitForCooldown: { + enabled: true, + maxRetries: 3, + maxRetryWaitSec: 30, + }, +}; + +async function mockResilienceSettings(page: Page) { + await page.route("**/api/resilience", async (route) => { + const method = route.request().method(); + if (method === "GET") { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify(resilienceSettings), + }); + return; + } + + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ + ok: true, + ...resilienceSettings, + }), + }); + }); +} + +async function mockHealthPageApis(page: Page) { + const now = new Date().toISOString(); + + await page.route("**/api/monitoring/health", async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ + status: "error", + system: { + uptime: 3723, + version: "3.7.0", + nodeVersion: "22.12.0", + memoryUsage: { + rss: 64 * 1024 * 1024, + heapUsed: 24 * 1024 * 1024, + heapTotal: 48 * 1024 * 1024, + }, + }, + providerHealth: { + openai: { + state: "OPEN", + failures: 3, + retryAfterMs: 15000, + lastFailure: now, + }, + gemini: { + state: "HALF_OPEN", + failures: 1, + retryAfterMs: 5000, + lastFailure: now, + }, + groq: { + state: "CLOSED", + failures: 0, + retryAfterMs: 0, + lastFailure: null, + }, + }, + providerSummary: { + configuredCount: 3, + activeCount: 2, + monitoredCount: 3, + }, + rateLimitStatus: {}, + lockouts: {}, + sessions: { + activeCount: 0, + stickyBoundCount: 0, + byApiKey: {}, + top: [], + }, + quotaMonitor: { + active: 0, + alerting: 0, + exhausted: 0, + errors: 0, + byProvider: {}, + monitors: [], + }, + }), + }); + }); + + await page.route("**/api/telemetry/summary", async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ p50: 120, p95: 240, p99: 450, totalRequests: 18 }), + }); + }); + + await page.route("**/api/cache/stats", async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ size: 3, maxSize: 100, hitRate: 50, hits: 2, misses: 2 }), + }); + }); + + await page.route("**/api/rate-limits", async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ + cacheStats: { + defaultCount: 0, + tool: { entries: 0, patterns: 0 }, + family: { entries: 0, patterns: 0 }, + session: { entries: 0, patterns: 0 }, + }, + }), + }); + }); + + await page.route("**/api/health/degradation", async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ + summary: { full: 0, reduced: 0, minimal: 0, default: 0 }, + features: [], + }), + }); + }); + + await page.route("**/api/v1/db/health", async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ + isHealthy: true, + issues: [], + repairedCount: 0, + backupCreated: false, + }), + }); + }); +} + +async function mockProvidersPageApis(page: Page) { + await page.route("**/api/providers", async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ + connections: [ + { + id: "conn-openai-main", + provider: "openai", + authType: "apikey", + name: "OpenAI Main", + testStatus: "active", + }, + { + id: "conn-gemini-main", + provider: "gemini", + authType: "apikey", + name: "Gemini Main", + testStatus: "active", + }, + ], + }), + }); + }); + + await page.route("**/api/provider-nodes", async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ nodes: [], ccCompatibleProviderEnabled: false }), + }); + }); + + await page.route("**/api/providers/expiration", async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({}), + }); + }); + + await page.route("**/api/system/env/repair", async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ available: false, missingCount: 0 }), + }); + }); +} + +async function mockIntelligentCombosPageApis(page: Page) { + await page.route("**/api/combos", async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ + combos: [ + { + id: "combo-auto", + name: "combo-auto", + models: ["openai/gpt-4o-mini", "gemini/gemini-2.5-pro"], + strategy: "auto", + config: { + candidatePool: ["openai", "gemini", "anthropic"], + modePack: "ship-fast", + explorationRate: 0.15, + }, + isActive: true, + }, + ], + }), + }); + }); + + await page.route("**/api/combos/metrics", async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ metrics: {} }), + }); + }); + + await page.route("**/api/providers", async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ + connections: [ + { id: "conn-openai", provider: "openai", name: "OpenAI", testStatus: "active" }, + { id: "conn-gemini", provider: "gemini", name: "Gemini", testStatus: "active" }, + { + id: "conn-anthropic", + provider: "anthropic", + name: "Anthropic", + testStatus: "active", + }, + ], + }), + }); + }); + + await page.route("**/api/provider-nodes", async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ nodes: [] }), + }); + }); + + await page.route("**/api/settings/proxy", async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ combos: {} }), + }); + }); +} + +test.describe("Resilience Plan Alignment", () => { + test("resilience settings page only shows the plan-aligned cooldown fields", async ({ page }) => { + await mockResilienceSettings(page); + + await gotoDashboardRoute(page, "/dashboard/settings?tab=resilience"); + await expect(page.getByText("Connection Cooldown")).toBeVisible({ timeout: 15000 }); + await expect(page.getByText("Base cooldown", { exact: true }).first()).toBeVisible(); + await expect(page.getByText("Use upstream retry hints", { exact: true }).first()).toBeVisible(); + await expect(page.getByText("Max backoff steps", { exact: true }).first()).toBeVisible(); + await expect(page.getByText(/Rate-limit fallback/i)).toHaveCount(0); + }); + + test("health page renders provider breaker runtime state for multiple providers", async ({ + page, + }) => { + await mockHealthPageApis(page); + + await gotoDashboardRoute(page, "/dashboard/health"); + await expect(page.getByText("Provider Health")).toBeVisible({ timeout: 15000 }); + await expect(page.getByText("OpenAI")).toBeVisible(); + await expect(page.getByText("Gemini")).toBeVisible(); + await expect(page.getByText("Groq")).toBeVisible(); + await expect(page.getByText("Recovering", { exact: true }).first()).toBeVisible(); + await expect(page.getByText("Down", { exact: true }).first()).toBeVisible(); + }); + + test("providers page no longer requests legacy model availability data", async ({ page }) => { + let availabilityRequests = 0; + + await mockProvidersPageApis(page); + await page.route("**/api/models/availability", async (route) => { + availabilityRequests += 1; + await route.fulfill({ + status: 410, + contentType: "application/json", + body: JSON.stringify({ error: "removed" }), + }); + }); + + await gotoDashboardRoute(page, "/dashboard/providers"); + await page.waitForLoadState("networkidle"); + + expect(availabilityRequests).toBe(0); + await expect(page.getByText("OpenAI").first()).toBeVisible(); + await expect(page.getByText(/Model Availability/i)).toHaveCount(0); + }); + + test("intelligent combo panel stays config-only and does not fetch breaker runtime state", async ({ + page, + }) => { + let monitoringHealthRequests = 0; + + await mockIntelligentCombosPageApis(page); + await page.route("**/api/monitoring/health", async (route) => { + monitoringHealthRequests += 1; + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ providerHealth: {} }), + }); + }); + + await gotoDashboardRoute(page, "/dashboard/combos?filter=intelligent"); + await page.waitForLoadState("networkidle"); + + expect(monitoringHealthRequests).toBe(0); + await expect(page.getByText("Intelligent Routing Dashboard")).toBeVisible({ timeout: 15000 }); + await expect(page.getByText("Routing Inputs", { exact: true })).toBeVisible(); + await expect(page.getByText(/Excluded Providers/i)).toHaveCount(0); + await expect(page.getByText(/Incident Mode/i)).toHaveCount(0); + }); +}); diff --git a/tests/integration/_chatPipelineHarness.ts b/tests/integration/_chatPipelineHarness.ts index d5a7cdbad1..57b2d745ed 100644 --- a/tests/integration/_chatPipelineHarness.ts +++ b/tests/integration/_chatPipelineHarness.ts @@ -33,8 +33,6 @@ export async function createChatPipelineHarness(prefix) { const { initTranslators } = await import("../../open-sse/translator/index.ts"); const { clearInflight } = await import("../../open-sse/services/requestDedup.ts"); const { BaseExecutor } = await import("../../open-sse/executors/base.ts"); - const { resetAllAvailability, setModelUnavailable } = - await import("../../src/domain/modelAvailability.ts"); const { resetAllCircuitBreakers } = await import("../../src/shared/utils/circuitBreaker.ts"); const originalFetch = globalThis.fetch; @@ -209,7 +207,6 @@ export async function createChatPipelineHarness(prefix) { clearInflight(); idempotencyLayerModule.clearIdempotency(); semanticCacheModule.clearCache(); - resetAllAvailability(); resetAllCircuitBreakers(); apiKeysDb.resetApiKeyState(); readCacheDb.invalidateDbCache(); @@ -229,7 +226,6 @@ export async function createChatPipelineHarness(prefix) { idempotencyLayerModule.clearIdempotency(); semanticCacheModule.clearCache(); clearSkillState(); - resetAllAvailability(); resetAllCircuitBreakers(); core.resetDbInstance(); fs.rmSync(testDataDir, { recursive: true, force: true }); @@ -292,7 +288,6 @@ export async function createChatPipelineHarness(prefix) { semanticCacheModule, seedApiKey, seedConnection, - setModelUnavailable, settingsDb, skillByIdRouteModule, skillExecutor, diff --git a/tests/integration/chat-pipeline.test.ts b/tests/integration/chat-pipeline.test.ts index 10f5d366f4..ce6f4641b3 100644 --- a/tests/integration/chat-pipeline.test.ts +++ b/tests/integration/chat-pipeline.test.ts @@ -23,9 +23,8 @@ const { handleChat } = await import("../../src/sse/handlers/chat.ts"); const { initTranslators } = await import("../../open-sse/translator/index.ts"); const { clearInflight } = await import("../../open-sse/services/requestDedup.ts"); const { BaseExecutor } = await import("../../open-sse/executors/base.ts"); -const { resetAllAvailability, setModelUnavailable } = - await import("../../src/domain/modelAvailability.ts"); -const { resetAllCircuitBreakers } = await import("../../src/shared/utils/circuitBreaker.ts"); +const { getCircuitBreaker, resetAllCircuitBreakers } = + await import("../../src/shared/utils/circuitBreaker.ts"); const { clearProviderFailure } = await import("../../open-sse/services/accountFallback.ts"); const originalFetch = globalThis.fetch; @@ -295,7 +294,6 @@ async function resetStorage() { globalThis.fetch = originalFetch; process.env.REQUIRE_API_KEY = "false"; clearInflight(); - resetAllAvailability(); resetAllCircuitBreakers(); apiKeysDb.resetApiKeyState(); readCacheDb.invalidateDbCache(); @@ -442,7 +440,6 @@ test.after(async () => { BaseExecutor.RETRY_CONFIG.delayMs = originalRetryDelayMs; globalThis.fetch = originalFetch; clearInflight(); - resetAllAvailability(); resetAllCircuitBreakers(); core.resetDbInstance(); fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); @@ -909,26 +906,6 @@ test("chat pipeline returns current no-credentials contract when no provider con assert.match(json.error.message, /No credentials for provider: openai/); }); -test("chat pipeline returns 503 when the requested model is temporarily unavailable", async () => { - await seedConnection("openai", { apiKey: "sk-openai-unavailable" }); - setModelUnavailable("openai", "gpt-4o-mini", 60000, "test cooldown"); - - const response = await handleChat( - buildRequest({ - body: { - model: "openai/gpt-4o-mini", - stream: false, - messages: [{ role: "user", content: "Provider unavailable" }], - }, - }) - ); - - const json = await response.json(); - assert.equal(response.status, 503); - assert.ok(Number(response.headers.get("Retry-After")) >= 1); - assert.match(json.error.message, /temporarily unavailable/i); -}); - test("chat pipeline surfaces upstream 500 responses as structured errors", async () => { await seedConnection("openai", { apiKey: "sk-openai-500" }); @@ -993,6 +970,46 @@ test("chat pipeline returns 429 with Retry-After when the upstream rate-limits t assert.match(json.error.message, /\[openai\/gpt-4o-mini\]/); }); +test("chat pipeline keeps provider breaker closed for repeated connection-scoped 429s", async () => { + await seedConnection("openai", { apiKey: "sk-openai-429-breaker" }); + await settingsDb.updateSettings({ + requestRetry: 0, + maxRetryIntervalSec: 0, + }); + + globalThis.fetch = async () => + new Response( + JSON.stringify({ + error: { + message: "Rate limit exceeded. Your quota will reset after 30s.", + }, + }), + { + status: 429, + headers: { "Content-Type": "application/json" }, + } + ); + + for (let i = 0; i < 3; i += 1) { + const response = await handleChat( + buildRequest({ + body: { + model: "openai/gpt-4o-mini", + stream: false, + messages: [{ role: "user", content: `Trigger 429 #${i + 1}` }], + }, + }) + ); + assert.equal(response.status, 429); + } + + const breaker = getCircuitBreaker("openai"); + const status = breaker.getStatus(); + + assert.equal(status.state, "CLOSED"); + assert.equal(status.failureCount, 0); +}); + test("chat pipeline maps upstream timeouts to 504 responses", async () => { await seedConnection("openai", { apiKey: "sk-openai-timeout" }); diff --git a/tests/integration/chatcore-compression-integration.test.ts b/tests/integration/chatcore-compression-integration.test.ts index 13adcd92e6..2ab34d8fa0 100644 --- a/tests/integration/chatcore-compression-integration.test.ts +++ b/tests/integration/chatcore-compression-integration.test.ts @@ -15,14 +15,12 @@ const readCacheDb = await import("../../src/lib/db/readCache.ts"); const combosDb = await import("../../src/lib/db/combos.ts"); const { handleChatCore } = await import("../../open-sse/handlers/chatCore.ts"); const { estimateTokens, getTokenLimit } = await import("../../open-sse/services/contextManager.ts"); -const { resetAllAvailability } = await import("../../src/domain/modelAvailability.ts"); const { resetAllCircuitBreakers } = await import("../../src/shared/utils/circuitBreaker.ts"); const originalFetch = globalThis.fetch; async function resetStorage() { globalThis.fetch = originalFetch; - resetAllAvailability(); resetAllCircuitBreakers(); readCacheDb.invalidateDbCache(); await new Promise((resolve) => setTimeout(resolve, 20)); diff --git a/tests/integration/integration-wiring.test.ts b/tests/integration/integration-wiring.test.ts index a9e99a6809..4ecd2da607 100644 --- a/tests/integration/integration-wiring.test.ts +++ b/tests/integration/integration-wiring.test.ts @@ -84,8 +84,8 @@ describe("Pipeline Wiring โ€” sse chat handler", () => { assert.match(src, /getCircuitBreaker|CircuitBreakerOpenError/); }); - it("should import model availability integration", () => { - assert.match(src, /isModelAvailable|setModelUnavailable/); + it("should use credential preflight instead of global model quarantine gates", () => { + assert.match(src, /getProviderCredentialsWithQuotaPreflight/); }); it("should import request telemetry integration", () => { @@ -129,7 +129,6 @@ describe("Pipeline Wiring โ€” middleware proxy", () => { describe("API Routes โ€” existence check", () => { const routes = [ "src/app/api/cache/stats/route.ts", - "src/app/api/models/availability/route.ts", "src/app/api/telemetry/summary/route.ts", "src/app/api/usage/budget/route.ts", "src/app/api/usage/quota/route.ts", @@ -152,10 +151,6 @@ describe("API Routes โ€” export HTTP methods", () => { assertRouteMethods("src/app/api/cache/stats/route.ts", ["GET", "DELETE"]); }); - it("/api/models/availability should export GET and POST", () => { - assertRouteMethods("src/app/api/models/availability/route.ts", ["GET", "POST"]); - }); - it("/api/telemetry/summary should export GET", () => { assertRouteMethods("src/app/api/telemetry/summary/route.ts", ["GET"]); }); diff --git a/tests/integration/proxy-pipeline.test.ts b/tests/integration/proxy-pipeline.test.ts index 847c27cb4a..42540df355 100644 --- a/tests/integration/proxy-pipeline.test.ts +++ b/tests/integration/proxy-pipeline.test.ts @@ -88,25 +88,25 @@ describe("Chat Pipeline โ€” combo fallback support", () => { assert.match(src, /handleSingleModel.*handleSingleModelChat/s); }); - it("should check model availability before attempting combo models", () => { - assert.match(src, /isModelAvailable/); + it("should preflight provider credentials before attempting combo models", () => { + assert.match(src, /getProviderCredentialsWithQuotaPreflight/); }); }); describe("Chat Pipeline โ€” circuit breaker integration", () => { const helpersSrc = readSrc("sse/handlers/chatHelpers.ts"); - it("should import CircuitBreakerOpenError", () => { + it("should import providerCircuitOpenResponse", () => { assert.ok(helpersSrc, "chatHelpers.ts should exist"); - assert.match(helpersSrc, /CircuitBreakerOpenError/); + assert.match(helpersSrc, /providerCircuitOpenResponse/); }); - it("should handle CircuitBreakerOpenError with retry-after", () => { + it("should handle circuit-open responses with retry-after", () => { assert.match(helpersSrc, /retryAfterMs/); }); - it("should reject requests when circuit is open", () => { - assert.match(helpersSrc, /circuit breaker is open/i); + it("should reject requests when circuit is open via structured provider breaker response", () => { + assert.match(helpersSrc, /providerCircuitOpenResponse\(provider,\s*retryAfterSec\)/); }); }); diff --git a/tests/integration/resilience-http-e2e.test.ts b/tests/integration/resilience-http-e2e.test.ts new file mode 100644 index 0000000000..f6ea094ab2 --- /dev/null +++ b/tests/integration/resilience-http-e2e.test.ts @@ -0,0 +1,746 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import fsp from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import http from "node:http"; +import net from "node:net"; +import { spawn } from "node:child_process"; +import { fileURLToPath } from "node:url"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-resilience-http-e2e-")); +const DASHBOARD_PORT = await getFreePort(); +const REPO_ROOT = fileURLToPath(new URL("../..", import.meta.url)); + +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.DISABLE_SQLITE_AUTO_BACKUP = "true"; +process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "resilience-http-e2e-secret-123456"; +process.env.REQUIRE_API_KEY = "false"; + +const core = await import("../../src/lib/db/core.ts"); +const providersDb = await import("../../src/lib/db/providers.ts"); +const combosDb = await import("../../src/lib/db/combos.ts"); +const settingsDb = await import("../../src/lib/db/settings.ts"); + +function getFreePort() { + return new Promise((resolve, reject) => { + const server = net.createServer(); + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + if (!address || typeof address === "string") { + server.close(); + reject(new Error("Failed to allocate a free port")); + return; + } + const { port } = address; + server.close((closeError) => { + if (closeError) reject(closeError); + else resolve(port); + }); + }); + }); +} + +function sleep(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function buildCompletion(content: string) { + return { + status: 200, + body: { + id: `chatcmpl_${Math.random().toString(16).slice(2, 8)}`, + object: "chat.completion", + choices: [{ index: 0, message: { role: "assistant", content }, finish_reason: "stop" }], + usage: { prompt_tokens: 4, completion_tokens: 2, total_tokens: 6 }, + }, + }; +} + +function buildError(status: number, message: string, headers: Record = {}) { + return { + status, + headers, + body: { error: { message } }, + }; +} + +type PlannedResponse = { + status: number; + body: Record; + headers?: Record; + delayMs?: number; +}; + +type TokenBehavior = { + defaultResponse: PlannedResponse; + queue: PlannedResponse[]; + hits: number; + startedAt: number[]; + bodies: Array>; +}; + +function createFakeOpenAiRelay() { + const behaviors = new Map(); + let server: http.Server | null = null; + let baseUrl = ""; + + const handleRequest = async (req: http.IncomingMessage, res: http.ServerResponse) => { + const chunks: Buffer[] = []; + req.on("data", (chunk) => chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk))); + req.on("end", async () => { + const authHeader = String(req.headers.authorization || ""); + const token = authHeader.replace(/^Bearer\s+/i, "").trim(); + const rawBody = Buffer.concat(chunks).toString("utf8"); + const parsedBody = rawBody ? JSON.parse(rawBody) : {}; + + if (req.method === "GET" && req.url?.startsWith("/v1/models")) { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ data: [{ id: "test-model", object: "model" }] })); + return; + } + + if (req.method !== "POST" || !req.url?.startsWith("/v1/chat/completions")) { + res.writeHead(404, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ error: { message: `Unhandled path: ${req.method} ${req.url}` } })); + return; + } + + const behavior = behaviors.get(token); + if (!behavior) { + res.writeHead(401, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ error: { message: `Unknown token: ${token || "missing"}` } })); + return; + } + + behavior.hits += 1; + behavior.startedAt.push(Date.now()); + behavior.bodies.push(parsedBody as Record); + + const planned = behavior.queue.shift() || behavior.defaultResponse; + if (planned.delayMs && planned.delayMs > 0) { + await sleep(planned.delayMs); + } + + const headers = { "Content-Type": "application/json", ...(planned.headers || {}) }; + res.writeHead(planned.status, headers); + res.end(JSON.stringify(planned.body)); + }); + }; + + return { + async start() { + const port = await getFreePort(); + await new Promise((resolve, reject) => { + server = http.createServer((req, res) => { + void handleRequest(req, res); + }); + server.once("error", reject); + server.listen(port, "127.0.0.1", () => resolve()); + }); + baseUrl = `http://127.0.0.1:${port}/v1`; + return baseUrl; + }, + getBaseUrl() { + if (!baseUrl) throw new Error("Fake relay has not started yet"); + return baseUrl; + }, + configureToken( + token: string, + config: { defaultResponse: PlannedResponse; queue?: PlannedResponse[] } + ) { + behaviors.set(token, { + defaultResponse: config.defaultResponse, + queue: [...(config.queue || [])], + hits: 0, + startedAt: [], + bodies: [], + }); + }, + getState(token: string) { + const state = behaviors.get(token); + if (!state) throw new Error(`Unknown token state for ${token}`); + return state; + }, + resetState(token: string, queue?: PlannedResponse[]) { + const state = behaviors.get(token); + if (!state) throw new Error(`Unknown token state for ${token}`); + state.hits = 0; + state.startedAt = []; + state.bodies = []; + state.queue = [...(queue || [])]; + }, + async stop() { + if (!server) return; + await new Promise((resolve) => server?.close(() => resolve())); + server = null; + }, + }; +} + +function createServerProcess(dataDir: string, port: number) { + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + const child = spawn(process.execPath, ["scripts/run-next.mjs", "dev"], { + cwd: REPO_ROOT, + env: { + ...process.env, + DATA_DIR: dataDir, + PORT: String(port), + DASHBOARD_PORT: String(port), + API_PORT: String(port), + HOST: "127.0.0.1", + REQUIRE_API_KEY: "false", + API_KEY_SECRET: process.env.API_KEY_SECRET || "resilience-http-e2e-secret-123456", + DISABLE_SQLITE_AUTO_BACKUP: "true", + INITIAL_PASSWORD: "", + NEXT_TELEMETRY_DISABLED: "1", + OMNIROUTE_DISABLE_BACKGROUND_SERVICES: "true", + OMNIROUTE_DISABLE_TOKEN_HEALTHCHECK: "true", + OMNIROUTE_DISABLE_LOCAL_HEALTHCHECK: "true", + OMNIROUTE_HIDE_HEALTHCHECK_LOGS: "true", + }, + stdio: ["ignore", "pipe", "pipe"], + }); + + child.stdout.on("data", (chunk) => { + const lines = String(chunk).split(/\r?\n/).filter(Boolean); + stdoutLines.push(...lines); + if (stdoutLines.length > 200) stdoutLines.splice(0, stdoutLines.length - 200); + }); + child.stderr.on("data", (chunk) => { + const lines = String(chunk).split(/\r?\n/).filter(Boolean); + stderrLines.push(...lines); + if (stderrLines.length > 200) stderrLines.splice(0, stderrLines.length - 200); + }); + + return { + child, + stdoutLines, + stderrLines, + baseUrl: `http://127.0.0.1:${port}`, + }; +} + +async function waitForServer( + baseUrl: string, + logs: { stdoutLines: string[]; stderrLines: string[] } +) { + const startedAt = Date.now(); + let lastError = ""; + while (Date.now() - startedAt < 120_000) { + try { + const response = await fetch(`${baseUrl}/api/monitoring/health`, { + signal: AbortSignal.timeout(5_000), + }); + if (response.ok) return; + lastError = `HTTP ${response.status}`; + } catch (error) { + lastError = error instanceof Error ? error.message : String(error); + } + await sleep(500); + } + + throw new Error( + [ + `Timed out waiting for OmniRoute to start: ${lastError}`, + "--- stdout ---", + ...logs.stdoutLines.slice(-40), + "--- stderr ---", + ...logs.stderrLines.slice(-40), + ].join("\n") + ); +} + +async function stopProcess(child: ReturnType) { + if (child.killed) return; + child.kill("SIGTERM"); + const exited = await Promise.race([ + new Promise((resolve) => child.once("exit", () => resolve(true))), + sleep(5_000).then(() => false), + ]); + if (!exited && !child.killed) { + child.kill("SIGKILL"); + await new Promise((resolve) => child.once("exit", () => resolve())); + } +} + +async function seedCompatibleProvider(prefix: string, apiKey: string, baseUrl: string) { + const providerId = `openai-compatible-chat-e2e-${prefix}`; + await providersDb.createProviderNode({ + id: providerId, + type: "openai-compatible", + name: `E2E ${prefix}`, + prefix, + apiType: "chat", + baseUrl, + }); + await providersDb.createProviderConnection({ + provider: providerId, + authType: "apikey", + name: `conn-${prefix}`, + apiKey, + isActive: true, + testStatus: "active", + providerSpecificData: { + baseUrl, + apiType: "chat", + }, + }); + return { providerId, model: `${prefix}/test-model`, apiKey }; +} + +function buildResilienceConfig(overrides: Record = {}) { + const base = { + requestQueue: { + autoEnableApiKeyProviders: true, + requestsPerMinute: 120, + minTimeBetweenRequestsMs: 0, + concurrentRequests: 4, + maxWaitMs: 2_000, + }, + connectionCooldown: { + oauth: { + baseCooldownMs: 500, + useUpstreamRetryHints: true, + maxBackoffSteps: 3, + }, + apikey: { + baseCooldownMs: 300, + useUpstreamRetryHints: false, + maxBackoffSteps: 0, + }, + }, + providerBreaker: { + oauth: { + failureThreshold: 3, + resetTimeoutMs: 2_000, + }, + apikey: { + failureThreshold: 2, + resetTimeoutMs: 1_500, + }, + }, + waitForCooldown: { + enabled: false, + maxRetries: 0, + maxRetryWaitSec: 0, + }, + }; + + return { + ...base, + ...overrides, + requestQueue: { + ...base.requestQueue, + ...((overrides.requestQueue as Record) || {}), + }, + connectionCooldown: { + oauth: { + ...base.connectionCooldown.oauth, + ...(((overrides.connectionCooldown as Record)?.oauth as Record< + string, + unknown + >) || {}), + }, + apikey: { + ...base.connectionCooldown.apikey, + ...(((overrides.connectionCooldown as Record)?.apikey as Record< + string, + unknown + >) || {}), + }, + }, + providerBreaker: { + oauth: { + ...base.providerBreaker.oauth, + ...(((overrides.providerBreaker as Record)?.oauth as Record< + string, + unknown + >) || {}), + }, + apikey: { + ...base.providerBreaker.apikey, + ...(((overrides.providerBreaker as Record)?.apikey as Record< + string, + unknown + >) || {}), + }, + }, + waitForCooldown: { + ...base.waitForCooldown, + ...((overrides.waitForCooldown as Record) || {}), + }, + }; +} + +async function patchResilience(baseUrl: string, config: Record) { + const response = await fetch(`${baseUrl}/api/resilience`, { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(config), + signal: AbortSignal.timeout(10_000), + }); + const payload = await response.json(); + assert.equal(response.status, 200, JSON.stringify(payload)); + return payload; +} + +async function getJson(url: string) { + const response = await fetch(url, { signal: AbortSignal.timeout(10_000) }); + const json = await response.json(); + return { response, json }; +} + +async function postChat(baseUrl: string, model: string, content: string) { + const response = await fetch(`${baseUrl}/api/v1/chat/completions`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + model, + stream: false, + messages: [{ role: "user", content }], + }), + signal: AbortSignal.timeout(20_000), + }); + const text = await response.text(); + const json = text ? JSON.parse(text) : {}; + return { response, json }; +} + +const relay = createFakeOpenAiRelay(); +let app: + | { + child: ReturnType; + stdoutLines: string[]; + stderrLines: string[]; + baseUrl: string; + } + | undefined; + +const TOKENS = { + p1: "sk-e2e-p1", + p2: "sk-e2e-p2", + p3: "sk-e2e-p3", + p4: "sk-e2e-p4", + p5: "sk-e2e-p5", + p6: "sk-e2e-p6", + p7: "sk-e2e-p7", + p8: "sk-e2e-p8", +}; + +test.before(async () => { + const fakeBaseUrl = await relay.start(); + + relay.configureToken(TOKENS.p1, { + defaultResponse: buildCompletion("primary healthy again"), + queue: [buildError(503, "primary transient failure")], + }); + relay.configureToken(TOKENS.p2, { + defaultResponse: buildCompletion("secondary stable"), + }); + relay.configureToken(TOKENS.p3, { + defaultResponse: buildCompletion("wait-for-cooldown via upstream hint"), + queue: [buildError(429, "rate limited, retry after 1 second", { "Retry-After": "1" })], + }); + relay.configureToken(TOKENS.p4, { + defaultResponse: buildCompletion("ignored upstream retry hint"), + queue: [buildError(429, "rate limited, retry after 5 seconds", { "Retry-After": "5" })], + }); + relay.configureToken(TOKENS.p5, { + defaultResponse: buildCompletion("breaker target recovered"), + queue: [buildError(503, "breaker failure #1"), buildError(503, "breaker failure #2")], + }); + relay.configureToken(TOKENS.p6, { + defaultResponse: buildCompletion("round robin A"), + }); + relay.configureToken(TOKENS.p7, { + defaultResponse: buildCompletion("round robin B"), + }); + relay.configureToken(TOKENS.p8, { + defaultResponse: { + ...buildCompletion("queued connection request"), + delayMs: 250, + }, + }); + + await seedCompatibleProvider("p1", TOKENS.p1, fakeBaseUrl); + await seedCompatibleProvider("p2", TOKENS.p2, fakeBaseUrl); + await seedCompatibleProvider("p3", TOKENS.p3, fakeBaseUrl); + await seedCompatibleProvider("p4", TOKENS.p4, fakeBaseUrl); + await seedCompatibleProvider("p5", TOKENS.p5, fakeBaseUrl); + await seedCompatibleProvider("p6", TOKENS.p6, fakeBaseUrl); + await seedCompatibleProvider("p7", TOKENS.p7, fakeBaseUrl); + await seedCompatibleProvider("p8", TOKENS.p8, fakeBaseUrl); + + await combosDb.createCombo({ + name: "res-priority-fallback", + strategy: "priority", + config: { maxRetries: 0, retryDelayMs: 0 }, + models: ["p1/test-model", "p2/test-model"], + }); + await combosDb.createCombo({ + name: "res-breaker-combo", + strategy: "priority", + config: { maxRetries: 0, retryDelayMs: 0 }, + models: ["p5/test-model", "p2/test-model"], + }); + await combosDb.createCombo({ + name: "res-rr", + strategy: "round-robin", + config: { maxRetries: 0, retryDelayMs: 0, concurrencyPerModel: 1, queueTimeoutMs: 800 }, + models: ["p6/test-model", "p7/test-model"], + }); + + await settingsDb.updateSettings({ + resilienceSettings: buildResilienceConfig(), + requestRetry: 0, + maxRetryIntervalSec: 0, + requireLogin: false, + setupComplete: true, + }); + + core.closeDbInstance(); + + app = createServerProcess(TEST_DATA_DIR, DASHBOARD_PORT); + await waitForServer(app.baseUrl, app); + + await patchResilience(app.baseUrl, buildResilienceConfig()); + + const warmup = await postChat(app.baseUrl, "p2/test-model", "warm up chat route"); + assert.equal(warmup.response.status, 200, JSON.stringify(warmup.json)); + relay.resetState(TOKENS.p2); +}); + +test.after(async () => { + if (app) { + await stopProcess(app.child); + } + await relay.stop(); + core.closeDbInstance(); + await fsp.rm(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +test("resilience API only exposes configuration, not runtime breaker state", async () => { + assert.ok(app); + const { response, json } = await getJson(`${app.baseUrl}/api/resilience`); + + assert.equal(response.status, 200); + assert.deepEqual(Object.keys(json).sort(), [ + "connectionCooldown", + "legacy", + "providerBreaker", + "requestQueue", + "waitForCooldown", + ]); + assert.equal("providerBreakers" in json, false); + assert.equal("runtime" in json, false); +}); + +test("request queue serializes concurrent requests on the same connection", async () => { + assert.ok(app); + await patchResilience( + app.baseUrl, + buildResilienceConfig({ + requestQueue: { + concurrentRequests: 1, + maxWaitMs: 1_500, + }, + }) + ); + relay.resetState(TOKENS.p8); + + const startedAt = Date.now(); + const [first, second] = await Promise.all([ + postChat(app.baseUrl, "p8/test-model", "queue-one"), + postChat(app.baseUrl, "p8/test-model", "queue-two"), + ]); + const elapsed = Date.now() - startedAt; + const state = relay.getState(TOKENS.p8); + + assert.equal(first.response.status, 200, JSON.stringify(first.json)); + assert.equal(second.response.status, 200, JSON.stringify(second.json)); + assert.equal(state.hits, 2); + assert.ok(elapsed >= 450, `expected queued elapsed >= 450ms, got ${elapsed}ms`); + assert.ok( + state.startedAt[1] - state.startedAt[0] >= 180, + `expected second request to be delayed by queue, got ${state.startedAt[1] - state.startedAt[0]}ms` + ); +}); + +test("priority combo falls back on 503 and skips the cooled-down primary on the next request", async () => { + assert.ok(app); + await patchResilience(app.baseUrl, buildResilienceConfig()); + relay.resetState(TOKENS.p1, [buildError(503, "primary transient failure")]); + relay.resetState(TOKENS.p2); + + const first = await postChat(app.baseUrl, "res-priority-fallback", "priority fallback request"); + assert.equal(first.response.status, 200, JSON.stringify(first.json)); + assert.equal(first.json.choices[0].message.content, "secondary stable"); + assert.equal(relay.getState(TOKENS.p1).hits, 1); + assert.equal(relay.getState(TOKENS.p2).hits, 1); + + const second = await postChat(app.baseUrl, "res-priority-fallback", "priority fallback again"); + assert.equal(second.response.status, 200, JSON.stringify(second.json)); + assert.equal(second.json.choices[0].message.content, "secondary stable"); + assert.equal(relay.getState(TOKENS.p1).hits, 1); + assert.equal(relay.getState(TOKENS.p2).hits, 2); +}); + +test("wait-for-cooldown honors upstream Retry-After when enabled", async () => { + assert.ok(app); + await patchResilience( + app.baseUrl, + buildResilienceConfig({ + connectionCooldown: { + apikey: { + useUpstreamRetryHints: true, + baseCooldownMs: 200, + }, + }, + waitForCooldown: { + enabled: true, + maxRetries: 1, + maxRetryWaitSec: 2, + }, + }) + ); + relay.resetState(TOKENS.p3); + const warmup = await postChat(app.baseUrl, "p3/test-model", "warm provider-specific route"); + assert.equal(warmup.response.status, 200, JSON.stringify(warmup.json)); + relay.resetState(TOKENS.p3, [ + buildError(429, "rate limited, retry after 1 second", { "Retry-After": "1" }), + ]); + + const startedAt = Date.now(); + const result = await postChat(app.baseUrl, "p3/test-model", "wait for cooldown via upstream"); + const elapsed = Date.now() - startedAt; + + assert.equal(result.response.status, 200, JSON.stringify(result.json)); + assert.equal(result.json.choices[0].message.content, "wait-for-cooldown via upstream hint"); + assert.equal(relay.getState(TOKENS.p3).hits, 2); + assert.ok(elapsed >= 800, `expected upstream wait >= 800ms, got ${elapsed}ms`); +}); + +test("connection cooldown can ignore upstream Retry-After and use the configured local cooldown", async () => { + assert.ok(app); + await patchResilience( + app.baseUrl, + buildResilienceConfig({ + connectionCooldown: { + apikey: { + useUpstreamRetryHints: false, + baseCooldownMs: 200, + }, + }, + waitForCooldown: { + enabled: true, + maxRetries: 1, + maxRetryWaitSec: 2, + }, + }) + ); + relay.resetState(TOKENS.p4); + const warmup = await postChat(app.baseUrl, "p4/test-model", "warm provider-specific route"); + assert.equal(warmup.response.status, 200, JSON.stringify(warmup.json)); + relay.resetState(TOKENS.p4, [ + buildError(429, "rate limited, retry after 30 seconds", { "Retry-After": "30" }), + ]); + + const startedAt = Date.now(); + const result = await postChat(app.baseUrl, "p4/test-model", "ignore upstream retry-after"); + const elapsed = Date.now() - startedAt; + + assert.equal(result.response.status, 200, JSON.stringify(result.json)); + assert.equal(result.json.choices[0].message.content, "ignored upstream retry hint"); + assert.equal(relay.getState(TOKENS.p4).hits, 2); + assert.ok( + elapsed < 5_000, + `expected ignored upstream hint to avoid a 30s wait, got ${elapsed}ms` + ); +}); + +test("provider circuit breaker opens after repeated final failures and Health reports it", async () => { + assert.ok(app); + await patchResilience( + app.baseUrl, + buildResilienceConfig({ + waitForCooldown: { + enabled: false, + maxRetries: 0, + maxRetryWaitSec: 0, + }, + providerBreaker: { + apikey: { + failureThreshold: 2, + resetTimeoutMs: 1_500, + }, + }, + }) + ); + relay.resetState(TOKENS.p5, [ + buildError(503, "breaker failure #1"), + buildError(503, "breaker failure #2"), + ]); + + const first = await postChat(app.baseUrl, "p5/test-model", "breaker first failure"); + const second = await postChat(app.baseUrl, "p5/test-model", "breaker second failure"); + const third = await postChat(app.baseUrl, "p5/test-model", "breaker should now be open"); + + assert.equal(first.response.status, 503); + assert.equal(second.response.status, 503); + assert.equal(third.response.status, 503); + assert.match(String(second.json.error?.message || ""), /reset after/i); + assert.match(String(third.json.error?.message || ""), /circuit breaker is open/i); + assert.equal(relay.getState(TOKENS.p5).hits, 1); + + const health = await getJson(`${app.baseUrl}/api/monitoring/health`); + assert.equal(health.response.status, 200); + const breakerEntry = (health.json.providerBreakers || []).find( + (entry: Record) => entry.provider === "openai-compatible-chat-e2e-p5" + ); + assert.ok(breakerEntry, "expected provider breaker entry for p5"); + assert.equal(breakerEntry.state, "OPEN"); + assert.ok(Number(breakerEntry.failureCount) >= 2); + assert.ok(Number(breakerEntry.retryAfterMs) > 0); + assert.ok( + (health.json.providerBreakers || []).every( + (entry: Record) => !String(entry.provider || "").includes(":") + ) + ); +}); + +test("combo respects the global provider breaker and falls through without a combo-local breaker", async () => { + assert.ok(app); + relay.resetState(TOKENS.p2); + + const result = await postChat( + app.baseUrl, + "res-breaker-combo", + "combo should skip broken provider" + ); + + assert.equal(result.response.status, 200, JSON.stringify(result.json)); + assert.equal(result.json.choices[0].message.content, "secondary stable"); + assert.equal(relay.getState(TOKENS.p5).hits, 1); + assert.equal(relay.getState(TOKENS.p2).hits, 1); +}); + +test("round-robin combo still alternates healthy providers after combo breaker removal", async () => { + assert.ok(app); + await patchResilience(app.baseUrl, buildResilienceConfig()); + relay.resetState(TOKENS.p6); + relay.resetState(TOKENS.p7); + + const first = await postChat(app.baseUrl, "res-rr", "round robin one"); + const second = await postChat(app.baseUrl, "res-rr", "round robin two"); + + assert.equal(first.response.status, 200, JSON.stringify(first.json)); + assert.equal(second.response.status, 200, JSON.stringify(second.json)); + assert.equal(first.json.choices[0].message.content, "round robin A"); + assert.equal(second.json.choices[0].message.content, "round robin B"); + assert.equal(relay.getState(TOKENS.p6).hits, 1); + assert.equal(relay.getState(TOKENS.p7).hits, 1); +}); diff --git a/tests/integration/security-hardening.test.ts b/tests/integration/security-hardening.test.ts index df01b34c2f..cb6060e247 100644 --- a/tests/integration/security-hardening.test.ts +++ b/tests/integration/security-hardening.test.ts @@ -232,7 +232,6 @@ test("T06 route payload validation uses validateBody in critical endpoints", () "src/app/api/policies/route.ts", "src/app/api/fallback/chains/route.ts", "src/app/api/models/route.ts", - "src/app/api/models/availability/route.ts", "src/app/api/provider-models/route.ts", "src/app/api/pricing/route.ts", "src/app/api/rate-limits/route.ts", diff --git a/tests/unit/account-fallback-service.test.ts b/tests/unit/account-fallback-service.test.ts index 912aa9836e..2095057770 100644 --- a/tests/unit/account-fallback-service.test.ts +++ b/tests/unit/account-fallback-service.test.ts @@ -207,8 +207,37 @@ test("lockModelIfPerModelQuota only locks supported providers and real models", }); test("getProviderProfile differentiates oauth and api-key providers", () => { - assert.deepEqual(getProviderProfile("claude"), PROVIDER_PROFILES.oauth); - assert.deepEqual(getProviderProfile("openai"), PROVIDER_PROFILES.apikey); + const oauthProfile = getProviderProfile("claude"); + assert.equal(oauthProfile.transientCooldown, PROVIDER_PROFILES.oauth.transientCooldown); + assert.equal( + oauthProfile.rateLimitCooldown, + oauthProfile.useUpstreamRetryHints ? 0 : oauthProfile.baseCooldownMs + ); + assert.equal(oauthProfile.maxBackoffLevel, PROVIDER_PROFILES.oauth.maxBackoffLevel); + assert.equal( + oauthProfile.circuitBreakerThreshold, + PROVIDER_PROFILES.oauth.circuitBreakerThreshold + ); + assert.equal(oauthProfile.circuitBreakerReset, PROVIDER_PROFILES.oauth.circuitBreakerReset); + assert.equal(oauthProfile.baseCooldownMs, PROVIDER_PROFILES.oauth.transientCooldown); + assert.equal(oauthProfile.failureThreshold, PROVIDER_PROFILES.oauth.circuitBreakerThreshold); + assert.equal(oauthProfile.resetTimeoutMs, PROVIDER_PROFILES.oauth.circuitBreakerReset); + + const apiKeyProfile = getProviderProfile("openai"); + assert.equal(apiKeyProfile.transientCooldown, PROVIDER_PROFILES.apikey.transientCooldown); + assert.equal( + apiKeyProfile.rateLimitCooldown, + apiKeyProfile.useUpstreamRetryHints ? 0 : apiKeyProfile.baseCooldownMs + ); + assert.equal(apiKeyProfile.maxBackoffLevel, PROVIDER_PROFILES.apikey.maxBackoffLevel); + assert.equal( + apiKeyProfile.circuitBreakerThreshold, + PROVIDER_PROFILES.apikey.circuitBreakerThreshold + ); + assert.equal(apiKeyProfile.circuitBreakerReset, PROVIDER_PROFILES.apikey.circuitBreakerReset); + assert.equal(apiKeyProfile.baseCooldownMs, PROVIDER_PROFILES.apikey.transientCooldown); + assert.equal(apiKeyProfile.failureThreshold, PROVIDER_PROFILES.apikey.circuitBreakerThreshold); + assert.equal(apiKeyProfile.resetTimeoutMs, PROVIDER_PROFILES.apikey.circuitBreakerReset); }); test("shouldMarkAccountExhaustedFrom429 skips connection poisoning for compatible providers", () => { @@ -229,11 +258,11 @@ test("recordModelLockoutFailure uses provider profile cooldowns, backoff, and re const compatibleProvider = "openai-compatible-custom-node"; const compatibleModel = "custom-model-a"; const profile = { - transientCooldown: 250, - rateLimitCooldown: 125, - maxBackoffLevel: 2, - circuitBreakerThreshold: 60, - circuitBreakerReset: 500, + baseCooldownMs: 125, + useUpstreamRetryHints: false, + maxBackoffSteps: 2, + failureThreshold: 60, + resetTimeoutMs: 500, }; const first = recordModelLockoutFailure( @@ -298,8 +327,8 @@ test("recordModelLockoutFailure uses provider profile cooldowns, backoff, and re }); // Provider-level failure circuit breaker tests -test("isProviderFailureCode correctly identifies transient error codes", () => { - assert.equal(isProviderFailureCode(429), true); +test("isProviderFailureCode correctly identifies provider-wide transient error codes", () => { + assert.equal(isProviderFailureCode(429), false); assert.equal(isProviderFailureCode(408), true); assert.equal(isProviderFailureCode(500), true); assert.equal(isProviderFailureCode(502), true); @@ -360,62 +389,16 @@ test("recordProviderFailure tracks failures and triggers cooldown after threshol } }); -test("recordProviderFailure resets counter after failure window expires", () => { - const originalNow = Date.now; - let now = 1_700_000_000_000; - Date.now = () => now; +test("checkFallbackError no longer mutates provider breaker state on per-connection failures", () => { + const provider = "test-provider-check"; + clearProviderFailure(provider); - try { - const provider = "test-provider-window"; - clearProviderFailure(provider); - - // Record 3 failures - for (let i = 0; i < 3; i++) { - recordProviderFailure(provider); - now += 1000; - } - assert.equal(isProviderInCooldown(provider), false); - - // Wait for failure window to expire (20 minutes + 1 second) - now += 20 * 60 * 1000 + 1000; - - // Next failure should reset counter, not trigger cooldown - recordProviderFailure(provider); - assert.equal(isProviderInCooldown(provider), false); - - // Need 4 more failures to trigger cooldown - for (let i = 0; i < 4; i++) { - recordProviderFailure(provider); - now += 1000; - } - assert.equal(isProviderInCooldown(provider), true); - } finally { - Date.now = originalNow; - clearProviderFailure("test-provider-window"); + for (let i = 0; i < 5; i++) { + checkFallbackError(429, "rate limited", 0, null, provider); } -}); -test("checkFallbackError records provider failure for transient errors", () => { - const originalNow = Date.now; - let now = 1_700_000_000_000; - Date.now = () => now; - - try { - const provider = "test-provider-check"; - clearProviderFailure(provider); - - // Simulate 5 transient errors through checkFallbackError - for (let i = 0; i < 5; i++) { - checkFallbackError(429, "rate limited", 0, null, provider); - now += 1000; - } - - // Provider should now be in cooldown - assert.equal(isProviderInCooldown(provider), true); - } finally { - Date.now = originalNow; - clearProviderFailure("test-provider-check"); - } + assert.equal(isProviderInCooldown(provider), false); + clearProviderFailure(provider); }); test("checkFallbackError does not record provider failure for non-transient errors", () => { diff --git a/tests/unit/auth-terminal-status.test.ts b/tests/unit/auth-terminal-status.test.ts index a62c4ba788..e3cad8eea0 100644 --- a/tests/unit/auth-terminal-status.test.ts +++ b/tests/unit/auth-terminal-status.test.ts @@ -109,3 +109,129 @@ test("markAccountUnavailable does not overwrite terminal status", async () => { const after = await providersDb.getProviderConnectionById(conn.id); assert.equal(after.testStatus, "credits_exhausted"); }); + +test("markAccountUnavailable marks 401 connections as expired without adding cooldown", async () => { + await resetStorage(); + + const conn = await providersDb.createProviderConnection({ + provider: "openai", + authType: "apikey", + apiKey: "sk-expired", + isActive: true, + testStatus: "active", + }); + + const result = await auth.markAccountUnavailable( + conn.id, + 401, + "unauthorized", + "openai", + "gpt-4.1" + ); + const after = await providersDb.getProviderConnectionById(conn.id); + + assert.equal(result.shouldFallback, true); + assert.equal(result.cooldownMs, 0); + assert.equal(after.testStatus, "expired"); + assert.ok(!after.rateLimitedUntil); +}); + +test("markAccountUnavailable marks 402 connections as credits_exhausted without adding cooldown", async () => { + await resetStorage(); + + const conn = await providersDb.createProviderConnection({ + provider: "openai", + authType: "apikey", + apiKey: "sk-credits", + isActive: true, + testStatus: "active", + }); + + const result = await auth.markAccountUnavailable( + conn.id, + 402, + "payment required", + "openai", + "gpt-4.1" + ); + const after = await providersDb.getProviderConnectionById(conn.id); + + assert.equal(result.shouldFallback, true); + assert.equal(result.cooldownMs, 0); + assert.equal(after.testStatus, "credits_exhausted"); + assert.ok(!after.rateLimitedUntil); +}); + +test("markAccountUnavailable marks 403 connections as banned without adding cooldown", async () => { + await resetStorage(); + + const conn = await providersDb.createProviderConnection({ + provider: "openai", + authType: "apikey", + apiKey: "sk-banned", + isActive: true, + testStatus: "active", + }); + + const result = await auth.markAccountUnavailable(conn.id, 403, "forbidden", "openai", "gpt-4.1"); + const after = await providersDb.getProviderConnectionById(conn.id); + + assert.equal(result.shouldFallback, true); + assert.equal(result.cooldownMs, 0); + assert.equal(after.testStatus, "banned"); + assert.ok(!after.rateLimitedUntil); +}); + +test("markAccountUnavailable keeps project-route 403 errors non-terminal", async () => { + await resetStorage(); + + const conn = await providersDb.createProviderConnection({ + provider: "openai", + authType: "apikey", + apiKey: "sk-project-route", + isActive: true, + testStatus: "active", + }); + + const result = await auth.markAccountUnavailable( + conn.id, + 403, + "The service has not been used in project", + "openai", + "gpt-4.1" + ); + const after = await providersDb.getProviderConnectionById(conn.id); + + assert.equal(result.shouldFallback, true); + assert.equal(result.cooldownMs, 0); + assert.equal(after.testStatus, "active"); + assert.equal(after.lastErrorType, "project_route_error"); + assert.ok(!after.rateLimitedUntil); +}); + +test("markAccountUnavailable keeps oauth-invalid 401 errors non-terminal", async () => { + await resetStorage(); + + const conn = await providersDb.createProviderConnection({ + provider: "openai", + authType: "apikey", + apiKey: "sk-oauth-invalid", + isActive: true, + testStatus: "active", + }); + + const result = await auth.markAccountUnavailable( + conn.id, + 401, + "Invalid authentication credentials provided", + "openai", + "gpt-4.1" + ); + const after = await providersDb.getProviderConnectionById(conn.id); + + assert.equal(result.shouldFallback, true); + assert.equal(result.cooldownMs, 0); + assert.equal(after.testStatus, "active"); + assert.equal(after.lastErrorType, "oauth_invalid_token"); + assert.ok(!after.rateLimitedUntil); +}); diff --git a/tests/unit/autocombo-unification.test.ts b/tests/unit/autocombo-unification.test.ts index a622651fdc..4ff04fa2dc 100644 --- a/tests/unit/autocombo-unification.test.ts +++ b/tests/unit/autocombo-unification.test.ts @@ -128,29 +128,40 @@ test("sidebar visibility excludes the removed auto-combo item", async () => { assert.deepEqual(sidebarVisibility.normalizeHiddenSidebarItems(["auto-combo", "home"]), ["home"]); }); -test("intelligent routing helpers normalize config and health state", () => { +test("intelligent routing helpers normalize config and build provider scores", () => { const normalizedConfig = intelligentRouting.normalizeIntelligentRoutingConfig({ candidatePool: ["openai", "anthropic"], explorationRate: "0.25", + modePack: "", + routerStrategy: "", weights: { quota: 0.4 }, }); assert.deepEqual(normalizedConfig.candidatePool, ["openai", "anthropic"]); assert.equal(normalizedConfig.explorationRate, 0.25); + assert.equal(normalizedConfig.modePack, "ship-fast"); + assert.equal(normalizedConfig.routerStrategy, "rules"); assert.equal(normalizedConfig.weights.quota, 0.4); assert.equal( normalizedConfig.weights.health, intelligentRouting.DEFAULT_INTELLIGENT_WEIGHTS.health ); - const healthState = intelligentRouting.extractIntelligentHealthState({ - circuitBreakers: [ - { provider: "openai", state: "OPEN", lastFailure: "2026-04-12T12:00:00Z" }, - { provider: "anthropic", state: "OPEN", lastFailure: "2026-04-12T12:01:00Z" }, - { provider: "google", state: "CLOSED" }, - ], + const providerScores = intelligentRouting.buildIntelligentProviderScores({ + config: normalizedConfig, }); - assert.equal(healthState.incidentMode, true); - assert.equal(healthState.exclusions.length, 2); + assert.equal(providerScores.length, 2); + assert.deepEqual( + providerScores.map((entry) => ({ + provider: entry.provider, + model: entry.model, + score: entry.score, + quotaWeight: entry.factors.quota, + })), + [ + { provider: "openai", model: "auto", score: 0.5, quotaWeight: 0.4 }, + { provider: "anthropic", model: "auto", score: 0.5, quotaWeight: 0.4 }, + ] + ); }); diff --git a/tests/unit/batch-a-domain.test.ts b/tests/unit/batch-a-domain.test.ts index 5efb40bf48..fd7e1237b7 100644 --- a/tests/unit/batch-a-domain.test.ts +++ b/tests/unit/batch-a-domain.test.ts @@ -1,166 +1,13 @@ /** * Batch A โ€” Domain Layer + Infrastructure Tests * - * Tests for: modelAvailability, costRules, fallbackPolicy, + * Tests for: costRules, fallbackPolicy, * errorCodes, requestId, fetchTimeout */ import { describe, it, before, after } from "node:test"; import assert from "node:assert/strict"; -// โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ T-19: Model Availability โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - -import { - isModelAvailable, - setModelUnavailable, - markModelAsProblematic, - clearModelUnavailability, - getAvailabilityReport, - getUnavailableCount, - resetAllAvailability, -} from "../../src/domain/modelAvailability.ts"; - -describe("modelAvailability", () => { - before(() => resetAllAvailability()); - after(() => resetAllAvailability()); - - it("should report model as available by default", () => { - assert.equal(isModelAvailable("openai", "gpt-4o"), true); - }); - - it("should mark model as unavailable", () => { - setModelUnavailable("openai", "gpt-4o", 60000, "rate limited"); - assert.equal(isModelAvailable("openai", "gpt-4o"), false); - }); - - it("should report unavailable models", () => { - const report = getAvailabilityReport(); - assert.equal(report.length, 1); - assert.equal(report[0].provider, "openai"); - assert.equal(report[0].model, "gpt-4o"); - assert.equal(report[0].reason, "rate limited"); - assert.ok(report[0].remainingMs > 0); - }); - - it("should count unavailable models", () => { - assert.equal(getUnavailableCount(), 1); - }); - - it("should clear model unavailability", () => { - clearModelUnavailability("openai", "gpt-4o"); - assert.equal(isModelAvailable("openai", "gpt-4o"), true); - assert.equal(getUnavailableCount(), 0); - }); - - it("should auto-expire after cooldown", () => { - setModelUnavailable("anthropic", "claude-sonnet-4-20250514", 1, "test"); - // Wait 2ms for expiry - const start = Date.now(); - while (Date.now() - start < 5) {} // spin wait - assert.equal(isModelAvailable("anthropic", "claude-sonnet-4-20250514"), true); - }); - - it("should apply adaptive quarantine for repeated problematic failures", () => { - const first = markModelAsProblematic("nvidia", "z-ai/glm4.7", { - status: 502, - baseCooldownMs: 3000, - reason: "HTTP 502", - }); - const second = markModelAsProblematic("nvidia", "z-ai/glm4.7", { - status: 502, - baseCooldownMs: 3000, - reason: "HTTP 502", - }); - - assert.equal(first.failureCount, 1); - assert.equal(first.cooldownMs, 120000); - assert.equal(second.failureCount, 2); - assert.equal(second.cooldownMs, 240000); - - const report = getAvailabilityReport(); - const entry = report.find((r) => r.provider === "nvidia" && r.model === "z-ai/glm4.7"); - assert.ok(entry, "nvidia model should be quarantined"); - assert.ok(entry.remainingMs >= 200000, "cooldown should be preserved/escalated"); - }); - - it("should reset failure history after model recovery", () => { - clearModelUnavailability("nvidia", "z-ai/glm4.7"); - const afterReset = markModelAsProblematic("nvidia", "z-ai/glm4.7", { - status: 502, - baseCooldownMs: 3000, - reason: "HTTP 502", - }); - - assert.equal(afterReset.failureCount, 1); - assert.equal(afterReset.cooldownMs, 120000); - }); - - it("should use provider profile threshold, cooldowns, and reset window for global quarantine", () => { - const originalNow = Date.now; - let now = 10_000; - Date.now = () => now; - - try { - const profile = { - transientCooldown: 200, - rateLimitCooldown: 100, - maxBackoffLevel: 2, - circuitBreakerThreshold: 3, - circuitBreakerReset: 500, - }; - - const first = markModelAsProblematic("openai", "gpt-4o-mini", { - status: 429, - reason: "HTTP 429", - profile, - }); - const second = markModelAsProblematic("openai", "gpt-4o-mini", { - status: 429, - reason: "HTTP 429", - profile, - }); - - assert.equal(first.failureCount, 1); - assert.equal(first.cooldownMs, 100); - assert.equal(first.quarantined, false); - assert.equal(isModelAvailable("openai", "gpt-4o-mini"), true); - - assert.equal(second.failureCount, 2); - assert.equal(second.cooldownMs, 200); - assert.equal(second.quarantined, false); - assert.equal(isModelAvailable("openai", "gpt-4o-mini"), true); - - now += 10; - const third = markModelAsProblematic("openai", "gpt-4o-mini", { - status: 429, - reason: "HTTP 429", - profile, - }); - - assert.equal(third.failureCount, 3); - assert.equal(third.cooldownMs, 400); - assert.equal(third.quarantined, true); - assert.equal(isModelAvailable("openai", "gpt-4o-mini"), false); - - now += 600; - assert.equal(isModelAvailable("openai", "gpt-4o-mini"), true); - - const afterWindow = markModelAsProblematic("openai", "gpt-4o-mini", { - status: 429, - reason: "HTTP 429", - profile, - }); - - assert.equal(afterWindow.failureCount, 1); - assert.equal(afterWindow.cooldownMs, 100); - assert.equal(afterWindow.quarantined, false); - } finally { - Date.now = originalNow; - clearModelUnavailability("openai", "gpt-4o-mini"); - } - }); -}); - // โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ T-19: Cost Rules โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ import { diff --git a/tests/unit/chat-combo-live-test.test.ts b/tests/unit/chat-combo-live-test.test.ts index 107d601ccd..31c99f7517 100644 --- a/tests/unit/chat-combo-live-test.test.ts +++ b/tests/unit/chat-combo-live-test.test.ts @@ -12,8 +12,6 @@ const providersDb = await import("../../src/lib/db/providers.ts"); const chatRoute = await import("../../src/app/api/v1/chat/completions/route.ts"); const { generateSignature, invalidateBySignature, setCachedResponse } = await import("../../src/lib/semanticCache.ts"); -const { clearModelUnavailability, resetAllAvailability, setModelUnavailable } = - await import("../../src/domain/modelAvailability.ts"); const { getCircuitBreaker, resetAllCircuitBreakers, STATE } = await import("../../src/shared/utils/circuitBreaker.ts"); @@ -28,7 +26,6 @@ async function resetStorage() { core.resetDbInstance(); fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); - resetAllAvailability(); resetAllCircuitBreakers(); } @@ -79,27 +76,20 @@ test.beforeEach(async () => { test.afterEach(async () => { await flushBackgroundWork(); globalThis.fetch = originalFetch; - resetAllAvailability(); resetAllCircuitBreakers(); }); test.after(async () => { await flushBackgroundWork(); globalThis.fetch = originalFetch; - resetAllAvailability(); resetAllCircuitBreakers(); core.resetDbInstance(); fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); }); -test("combo live test bypasses local cooldown and breaker state to perform a real upstream request", async () => { +test("combo live test bypasses connection cooldown and breaker state to perform a real upstream request", async () => { const created = await seedSuppressedConnection(); - setModelUnavailable("openai", "gpt-4o-mini", 60_000, "test cooldown"); - const breaker = getCircuitBreaker("openai"); - breaker.state = STATE.OPEN; - breaker.lastFailureTime = Date.now(); - const fetchCalls = []; globalThis.fetch = async (url, init = {}) => { fetchCalls.push({ url: String(url), init }); @@ -120,7 +110,10 @@ test("combo live test bypasses local cooldown and breaker state to perform a rea assert.equal(blockedByCooldown.status, 503); assert.equal(fetchCalls.length, 0); - clearModelUnavailability("openai", "gpt-4o-mini"); + const breaker = getCircuitBreaker("openai"); + breaker.state = STATE.OPEN; + breaker.lastFailureTime = Date.now(); + breaker.resetTimeout = 60_000; const blockedByBreaker = await chatRoute.POST(makeRequest()); assert.equal(blockedByBreaker.status, 503); diff --git a/tests/unit/chat-helpers.test.ts b/tests/unit/chat-helpers.test.ts index 64726e297d..102ccfd84c 100644 --- a/tests/unit/chat-helpers.test.ts +++ b/tests/unit/chat-helpers.test.ts @@ -18,13 +18,10 @@ const { safeLogEvents, withSessionHeader, } = await import("../../src/sse/handlers/chatHelpers.ts"); -const { getModelCooldownInfo, setModelUnavailable, resetAllAvailability } = - await import("../../src/domain/modelAvailability.ts"); -const { getCircuitBreaker, resetAllCircuitBreakers, CircuitBreakerOpenError, STATE } = +const { getCircuitBreaker, resetAllCircuitBreakers, STATE } = await import("../../src/shared/utils/circuitBreaker.ts"); async function resetStorage() { - resetAllAvailability(); resetAllCircuitBreakers(); core.resetDbInstance(); fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); @@ -92,22 +89,6 @@ test("resolveModelOrError rejects malformed model strings", async () => { assert.match(json.error.message, /Invalid model format/i); }); -test("checkPipelineGates blocks models in cooldown", async () => { - setModelUnavailable("openai", "gpt-4o-mini", 12_000, "cooldown"); - - const response = await checkPipelineGates("openai", "gpt-4o-mini"); - const json = await response.json(); - const cooldownInfo = getModelCooldownInfo("openai", "gpt-4o-mini"); - const retryAfter = Number(response.headers.get("Retry-After")); - - assert.equal(response.status, 503); - assert.ok(cooldownInfo); - assert.ok(retryAfter >= 1); - assert.ok(retryAfter <= 12); - assert.ok(retryAfter >= Math.ceil((cooldownInfo?.remainingMs || 0) / 1000) - 1); - assert.match(json.error.message, /temporarily unavailable/i); -}); - test("checkPipelineGates blocks providers with an open circuit breaker", async () => { const breaker = getCircuitBreaker("openai"); breaker.state = STATE.OPEN; @@ -116,8 +97,8 @@ test("checkPipelineGates blocks providers with an open circuit breaker", async ( const response = await checkPipelineGates("openai", "gpt-4o-mini", { providerProfile: { - circuitBreakerThreshold: 5, - circuitBreakerReset: 5_000, + failureThreshold: 5, + resetTimeoutMs: 5_000, }, }); const json = await response.json(); @@ -127,6 +108,8 @@ test("checkPipelineGates blocks providers with an open circuit breaker", async ( assert.ok(retryAfter >= 4); assert.ok(retryAfter <= 5); assert.match(json.error.message, /circuit breaker is open/i); + assert.equal(json.error.code, "provider_circuit_open"); + assert.equal(response.headers.get("X-OmniRoute-Provider-Breaker"), "open"); }); test("checkPipelineGates reapplies runtime breaker settings to existing breakers", async () => { @@ -139,8 +122,8 @@ test("checkPipelineGates reapplies runtime breaker settings to existing breakers const response = await checkPipelineGates("openai", "gpt-4o-mini", { providerProfile: { - circuitBreakerThreshold: 60, - circuitBreakerReset: 5_000, + failureThreshold: 60, + resetTimeoutMs: 5_000, }, }); @@ -233,60 +216,44 @@ test("safeResolveProxy returns the direct route when no proxy config is present" }); }); -test("executeChatWithBreaker converts circuit-open and proxy-fast-fail errors", async () => { - const credentials = { connectionId: "conn_helper" }; - const openResult = await executeChatWithBreaker({ - bypassCircuitBreaker: false, - breaker: { - execute: async () => { - throw new CircuitBreakerOpenError("already open", "openai", 5_000); - }, - }, - body: { model: "openai/gpt-4o-mini" }, - provider: "openai", - model: "gpt-4o-mini", - refreshedCredentials: credentials, - proxyInfo: null, - log: console, - clientRawRequest: null, - credentials, - apiKeyInfo: null, - userAgent: "", - comboName: null, - comboStrategy: null, - isCombo: false, - extendedContext: false, - }); +test("executeChatWithBreaker converts proxy fast-fail errors", async () => { + const originalFetch = globalThis.fetch; + globalThis.fetch = async () => { + const error = new Error("Proxy unreachable"); + (error as Error & { code?: string }).code = "PROXY_UNREACHABLE"; + throw error; + }; - const proxyResult = await executeChatWithBreaker({ - bypassCircuitBreaker: false, - breaker: { - execute: async () => { - const error = new Error("Proxy unreachable"); - error.code = "PROXY_UNREACHABLE"; - throw error; - }, - }, - body: { model: "openai/gpt-4o-mini" }, - provider: "openai", - model: "gpt-4o-mini", - refreshedCredentials: credentials, - proxyInfo: null, - log: console, - clientRawRequest: null, - credentials, - apiKeyInfo: null, - userAgent: "", - comboName: null, - comboStrategy: null, - isCombo: false, - extendedContext: false, - }); + try { + const credentials = { + connectionId: "conn_helper", + apiKey: "sk-openai-helper", + providerSpecificData: {}, + }; + const proxyResult = await executeChatWithBreaker({ + body: { model: "openai/gpt-4o-mini" }, + provider: "openai", + model: "gpt-4o-mini", + refreshedCredentials: credentials, + proxyInfo: null, + log: console, + clientRawRequest: null, + credentials, + apiKeyInfo: null, + userAgent: "", + comboName: null, + comboStrategy: null, + isCombo: false, + extendedContext: false, + comboStepId: null, + comboExecutionKey: null, + }); - assert.equal(openResult.result.status, 503); - assert.equal(openResult.result.response.status, 503); - assert.equal(proxyResult.result.status, 503); - assert.equal(proxyResult.result.error, "Proxy unreachable"); + assert.equal(proxyResult.result.status, 502); + assert.match(String(proxyResult.result.error || ""), /Proxy unreachable/); + } finally { + globalThis.fetch = originalFetch; + } }); test("safeLogEvents tolerates success and timeout payloads", () => { diff --git a/tests/unit/chat-route-coverage.test.ts b/tests/unit/chat-route-coverage.test.ts index 2c7b4ca71d..f38085ff94 100644 --- a/tests/unit/chat-route-coverage.test.ts +++ b/tests/unit/chat-route-coverage.test.ts @@ -14,13 +14,11 @@ const { resetStorage, seedApiKey, seedConnection, - setModelUnavailable, settingsDb, toPlainHeaders, } = harness; const { getCircuitBreaker, STATE } = await import("../../src/shared/utils/circuitBreaker.ts"); -const { clearModelUnavailability } = await import("../../src/domain/modelAvailability.ts"); const { clearProviderFailure } = await import("../../open-sse/services/accountFallback.ts"); const { getDefaultTaskModelMap, resetTaskRoutingStats, setTaskRoutingConfig } = await import("../../open-sse/services/taskAwareRouter.ts"); @@ -345,9 +343,11 @@ test("handleChat returns 400 when no provider credentials exist", async () => { assert.match(json.error.message, /No credentials for provider: openai/); }); -test("handleChat returns 503 for cooled-down models and open circuit breakers", async () => { - await seedConnection("openai", { apiKey: "sk-openai-breaker" }); - setModelUnavailable("openai", "gpt-4o-mini", 60_000, "test cooldown"); +test("handleChat returns 503 for cooled-down connections and 503 for open circuit breakers", async () => { + await seedConnection("openai", { + apiKey: "sk-openai-breaker", + rateLimitedUntil: new Date(Date.now() + 60_000).toISOString(), + }); const cooldownResponse = await handleChat( buildRequest({ @@ -360,15 +360,13 @@ test("handleChat returns 503 for cooled-down models and open circuit breakers", ); const cooldownJson = await cooldownResponse.json(); assert.equal(cooldownResponse.status, 503); - assert.match(cooldownJson.error.message, /temporarily unavailable/i); - - clearModelUnavailability("openai", "gpt-4o-mini"); - const freshBreaker = getCircuitBreaker("openai"); - freshBreaker.reset(); + assert.ok(Number(cooldownResponse.headers.get("Retry-After")) >= 1); + assert.match(cooldownJson.error.message, /\[openai\/gpt-4o-mini\]/i); const breaker = getCircuitBreaker("openai"); breaker.state = STATE.OPEN; breaker.lastFailureTime = Date.now(); + breaker.resetTimeout = 60_000; const breakerBlocked = await handleChat( buildRequest({ @@ -382,6 +380,8 @@ test("handleChat returns 503 for cooled-down models and open circuit breakers", const breakerJson = await breakerBlocked.json(); assert.equal(breakerBlocked.status, 503); + assert.equal(breakerBlocked.headers.get("X-OmniRoute-Provider-Breaker"), "open"); + assert.equal(breakerJson.error.code, "provider_circuit_open"); assert.match(breakerJson.error.message, /circuit breaker is open/i); }); diff --git a/tests/unit/combo-circuit-breaker.test.ts b/tests/unit/combo-circuit-breaker.test.ts deleted file mode 100644 index 5b06a3a848..0000000000 --- a/tests/unit/combo-circuit-breaker.test.ts +++ /dev/null @@ -1,230 +0,0 @@ -import test from "node:test"; -import assert from "node:assert/strict"; - -// Reset circuit breaker registry between tests -const { CircuitBreaker, getCircuitBreaker, getAllCircuitBreakerStatuses, STATE } = - await import("../../src/shared/utils/circuitBreaker.ts"); - -const { handleComboChat, getComboFromData } = await import("../../open-sse/services/combo.ts"); -const { normalizeComboStep } = await import("../../src/lib/combos/steps.ts"); - -const { PROVIDER_PROFILES } = await import("../../open-sse/config/constants.ts"); - -// โ”€โ”€โ”€ Helpers โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - -/** Create a mock logger */ -function mockLog() { - const entries = []; - return { - info: (tag, msg) => entries.push({ level: "info", tag, msg }), - warn: (tag, msg) => entries.push({ level: "warn", tag, msg }), - error: (tag, msg) => entries.push({ level: "error", tag, msg }), - entries, - }; -} - -/** Create a handleSingleModel that returns given status codes in sequence */ -function mockHandler(statusSequence) { - let callIndex = 0; - return async (body, modelStr) => { - const status = statusSequence[callIndex] ?? statusSequence[statusSequence.length - 1] ?? 200; - callIndex++; - if (status === 200) { - return new Response(JSON.stringify({ ok: true }), { status: 200 }); - } - return new Response(JSON.stringify({ error: { message: `Error ${status}` } }), { - status, - statusText: `Error ${status}`, - }); - }; -} - -function getComboTargetBreakerKey(combo, index = 0) { - const step = normalizeComboStep(combo.models[index], { - comboName: combo.name, - index, - }); - return `combo:${combo.name}:${step.id}`; -} - -// โ”€โ”€โ”€ Circuit Breaker Integration Tests โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ -// NOTE: combo.ts uses the full model string (e.g. "combo:groq/llama-3.3-70b") -// as the circuit breaker key, not just the provider prefix. - -test("handleComboChat: circuit breaker opens after repeated 502 errors", async () => { - const combo = { - name: "test-combo", - models: [{ model: "groq/llama-3.3-70b", weight: 0 }], - strategy: "priority", - config: { maxRetries: 0 }, - }; - const breakerKey = getComboTargetBreakerKey(combo); - const breaker = getCircuitBreaker(breakerKey, { - failureThreshold: PROVIDER_PROFILES.apikey.circuitBreakerThreshold, - resetTimeout: 60000, - }); - breaker.reset(); - - const log = mockLog(); - - // Send requests that all fail with 502 until the API-key profile threshold is reached. - for (let i = 0; i < PROVIDER_PROFILES.apikey.circuitBreakerThreshold; i++) { - await handleComboChat({ - body: {}, - combo, - handleSingleModel: mockHandler([502]), - isModelAvailable: () => true, - log, - settings: null, - allCombos: null, - }); - } - - // Breaker should now be OPEN - const status = breaker.getStatus(); - console.log("=== BREAKER STATUS AFTER THRESHOLD CALLS ===", status); - assert.equal( - status.state, - STATE.OPEN, - "Breaker should be OPEN after the API-key failure threshold is reached" - ); - assert.equal( - status.failureCount, - PROVIDER_PROFILES.apikey.circuitBreakerThreshold, - "Failure count should match the API-key profile threshold" - ); -}); - -test("handleComboChat: skips models with open circuit breaker", async () => { - // Set up: groq breaker is OPEN, fireworks breaker is CLOSED - const combo = { - name: "test-skip-combo", - models: [ - { model: "groq/llama-3.3-70b", weight: 0 }, - { model: "fireworks/deepseek-v3p1", weight: 0 }, - ], - strategy: "priority", - }; - const groqBreakerKey = getComboTargetBreakerKey(combo, 0); - const groqBreaker = getCircuitBreaker(groqBreakerKey, { - failureThreshold: 3, - resetTimeout: 60000, - }); - groqBreaker.reset(); - // Force open the breaker - groqBreaker._onFailure(); - groqBreaker._onFailure(); - groqBreaker._onFailure(); - assert.equal(groqBreaker.getStatus().state, STATE.OPEN); - - const fireworksBreakerKey = getComboTargetBreakerKey(combo, 1); - const fireworksBreaker = getCircuitBreaker(fireworksBreakerKey, { - failureThreshold: 5, - resetTimeout: 30000, - }); - fireworksBreaker.reset(); - - const log = mockLog(); - - const result = await handleComboChat({ - body: {}, - combo, - handleSingleModel: mockHandler([200]), // fireworks will succeed - isModelAvailable: () => true, - log, - settings: null, - allCombos: null, - }); - - assert.equal(result.ok, true, "Should succeed via fireworks fallback"); - - // Check logs show groq was skipped - const skipLog = log.entries.find( - (e) => e.msg.includes("circuit breaker OPEN") && e.msg.includes("groq") - ); - assert.ok(skipLog, "Should log that groq was skipped due to breaker"); -}); - -test("handleComboChat: returns 503 when all breakers are open", async () => { - const combo = { - name: "test-all-open", - models: [ - { model: "groq/llama-3.3-70b", weight: 0 }, - { model: "fireworks/deepseek-v3p1", weight: 0 }, - ], - strategy: "priority", - }; - - // Open both breakers explicitly before invoking the combo. - const groqBreaker = getCircuitBreaker(getComboTargetBreakerKey(combo, 0), { - failureThreshold: 3, - resetTimeout: 60000, - }); - groqBreaker.reset(); - groqBreaker._onFailure(); - groqBreaker._onFailure(); - groqBreaker._onFailure(); - assert.equal(groqBreaker.getStatus().state, STATE.OPEN); - - const fireworksBreaker = getCircuitBreaker(getComboTargetBreakerKey(combo, 1), { - failureThreshold: 5, - resetTimeout: 30000, - }); - fireworksBreaker.reset(); - for (let i = 0; i < 5; i++) fireworksBreaker._onFailure(); - assert.equal(fireworksBreaker.getStatus().state, STATE.OPEN); - - const log = mockLog(); - - const result = await handleComboChat({ - body: {}, - combo, - handleSingleModel: mockHandler([200]), // Won't be called - isModelAvailable: () => true, - log, - settings: null, - allCombos: null, - }); - - assert.equal(result.status, 503, "Should return 503"); - const body = await result.json(); - assert.ok(body.error.message.includes("circuit breakers open"), "Should mention breakers"); -}); - -test("handleComboChat: 429 errors also trigger circuit breaker", async () => { - const combo = { - name: "test-429", - models: [{ model: "cerebras/llama-3.3-70b", weight: 0 }], - strategy: "priority", - }; - const breakerKey = getComboTargetBreakerKey(combo); - const breaker = getCircuitBreaker(breakerKey, { - failureThreshold: 5, - resetTimeout: 30000, - }); - breaker.reset(); - - const log = mockLog(); - - // 5 x 429 should open breaker - for (let i = 0; i < 5; i++) { - await handleComboChat({ - body: {}, - combo, - handleSingleModel: mockHandler([429]), - isModelAvailable: () => true, - log, - settings: null, - allCombos: null, - }); - } - - assert.equal(breaker.getStatus().state, STATE.OPEN, "429s should open breaker"); -}); - -test("circuit breaker uses provider profile thresholds", () => { - // OAuth providers (e.g. claude) should have lower threshold - assert.equal(PROVIDER_PROFILES.oauth.circuitBreakerThreshold, 3); - // API providers (e.g. groq) should have higher threshold - assert.equal(PROVIDER_PROFILES.apikey.circuitBreakerThreshold, 5); -}); diff --git a/tests/unit/combo-config.test.ts b/tests/unit/combo-config.test.ts index 014ff4e44a..0c39a25993 100644 --- a/tests/unit/combo-config.test.ts +++ b/tests/unit/combo-config.test.ts @@ -13,7 +13,9 @@ test("getDefaultComboConfig returns a fresh copy of the defaults", () => { assert.notEqual(first, second); assert.equal(first.strategy, "priority"); assert.equal(first.maxRetries, 1); - assert.equal(first.timeoutMs, 600000); + assert.equal(first.retryDelayMs, 2000); + assert.ok(!("timeoutMs" in first)); + assert.ok(!("healthCheckEnabled" in first)); assert.equal(first.handoffThreshold, 0.85); assert.equal(first.maxMessagesForSummary, 30); assert.deepEqual(first.handoffProviders, ["codex"]); @@ -27,7 +29,6 @@ test("resolveComboConfig applies the full cascade from defaults to combo overrid { config: { maxRetries: 4, - timeoutMs: 45000, }, }, { @@ -47,12 +48,12 @@ test("resolveComboConfig applies the full cascade from defaults to combo overrid assert.equal(result.strategy, "round-robin"); assert.equal(result.retryDelayMs, 500); - assert.equal(result.timeoutMs, 45000); assert.equal(result.maxRetries, 4); - assert.equal(result.healthCheckEnabled, true); + assert.ok(!("timeoutMs" in result)); + assert.ok(!("healthCheckEnabled" in result)); }); -test("resolveComboConfig ignores null and undefined overrides", () => { +test("resolveComboConfig ignores null, undefined, and legacy resilience overrides", () => { const result = resolveComboConfig( { config: { @@ -75,7 +76,7 @@ test("resolveComboConfig ignores null and undefined overrides", () => { "openai" ); - assert.equal(result.timeoutMs, 600000); + assert.ok(!("timeoutMs" in result)); assert.equal(result.queueTimeoutMs, 15000); assert.equal(result.concurrencyPerModel, 9); assert.equal(result.trackMetrics, false); diff --git a/tests/unit/combo-context-relay.test.ts b/tests/unit/combo-context-relay.test.ts index 7675f05848..b3aa09f78d 100644 --- a/tests/unit/combo-context-relay.test.ts +++ b/tests/unit/combo-context-relay.test.ts @@ -13,12 +13,10 @@ const handoffDb = await import("../../src/lib/db/contextHandoffs.ts"); const { registerCodexConnection } = await import("../../open-sse/services/codexQuotaFetcher.ts"); const { clearSessions, touchSession } = await import("../../open-sse/services/sessionManager.ts"); const { resetAllComboMetrics } = await import("../../open-sse/services/comboMetrics.ts"); -const { resetAllCircuitBreakers, getCircuitBreaker } = - await import("../../src/shared/utils/circuitBreaker.ts"); +const { resetAllCircuitBreakers } = await import("../../src/shared/utils/circuitBreaker.ts"); const { resetAll: resetAllSemaphores } = await import("../../open-sse/services/rateLimitSemaphore.ts"); const { _resetAllDecks } = await import("../../src/shared/utils/shuffleDeck.ts"); -const { normalizeComboStep } = await import("../../src/lib/combos/steps.ts"); const originalFetch = globalThis.fetch; @@ -39,6 +37,24 @@ function okResponse(body = { choices: [{ message: { content: "ok" } }] }) { }); } +function providerBreakerOpenResponse() { + return new Response( + JSON.stringify({ + error: { + message: "Provider circuit breaker is open", + code: "provider_circuit_open", + }, + }), + { + status: 503, + headers: { + "content-type": "application/json", + "x-omniroute-provider-breaker": "open", + }, + } + ); +} + function buildQuotaResponse(usedPercent, resetAfterSeconds = 3600) { return new Response( JSON.stringify({ @@ -161,23 +177,13 @@ test("handleComboChat context-relay skips unavailable models and falls through t assert.deepEqual(calls, ["openai/gpt-4o-mini"]); }); -test("handleComboChat context-relay skips models with an open circuit breaker", async () => { +test("handleComboChat context-relay skips targets that report an open provider circuit breaker", async () => { const combo = { name: "relay-breaker", strategy: "context-relay", models: ["codex/gpt-5.4", "openai/gpt-4o-mini"], config: { maxRetries: 0 }, }; - const firstStep = normalizeComboStep(combo.models[0], { - comboName: combo.name, - index: 0, - }); - const breaker = getCircuitBreaker(`combo:${combo.name}:${firstStep.id}`, { - failureThreshold: 1, - resetTimeout: 60000, - }); - breaker._onFailure(); - const log = createLog(); const calls = []; @@ -188,6 +194,9 @@ test("handleComboChat context-relay skips models with an open circuit breaker", combo, handleSingleModel: async (_body, modelStr) => { calls.push(modelStr); + if (modelStr === "codex/gpt-5.4") { + return providerBreakerOpenResponse(); + } return okResponse(); }, isModelAvailable: async () => true, @@ -197,8 +206,10 @@ test("handleComboChat context-relay skips models with an open circuit breaker", }); assert.equal(result.ok, true); - assert.deepEqual(calls, ["openai/gpt-4o-mini"]); - assert.ok(log.entries.some((entry) => entry.msg.includes("circuit breaker OPEN"))); + assert.deepEqual(calls, ["codex/gpt-5.4", "openai/gpt-4o-mini"]); + assert.ok( + log.entries.some((entry) => String(entry.msg).includes("provider circuit breaker OPEN")) + ); }); test("handleComboChat context-relay persists a handoff when codex quota reaches the warning threshold", async () => { diff --git a/tests/unit/combo-routing-engine.test.ts b/tests/unit/combo-routing-engine.test.ts index 823dfbc3bb..c8dd07e2ac 100644 --- a/tests/unit/combo-routing-engine.test.ts +++ b/tests/unit/combo-routing-engine.test.ts @@ -24,8 +24,7 @@ const { saveModelsDevCapabilities, clearModelsDevCapabilities } = await import("../../src/lib/modelsDevSync.ts"); const { getComboMetrics, recordComboRequest, resetAllComboMetrics } = await import("../../open-sse/services/comboMetrics.ts"); -const { getCircuitBreaker, resetAllCircuitBreakers } = - await import("../../src/shared/utils/circuitBreaker.ts"); +const { resetAllCircuitBreakers } = await import("../../src/shared/utils/circuitBreaker.ts"); const { acquire: acquireSemaphore, resetAll: resetAllSemaphores } = await import("../../open-sse/services/rateLimitSemaphore.ts"); const { _resetAllDecks } = await import("../../src/shared/utils/shuffleDeck.ts"); @@ -54,6 +53,24 @@ function errorResponse(status: number, message: string = `Error ${status}`) { }); } +function providerBreakerOpenResponse() { + return new Response( + JSON.stringify({ + error: { + message: "Provider circuit breaker is open", + code: "provider_circuit_open", + }, + }), + { + status: 503, + headers: { + "content-type": "application/json", + "x-omniroute-provider-breaker": "open", + }, + } + ); +} + function streamResponse(chunks: any[]) { return new Response(chunks.join(""), { status: 200, @@ -83,7 +100,7 @@ function capabilityEntry(limitContext: any) { }; } -function getComboTargetBreakerKey(comboName: string, index: number, stepInput: any) { +function getComboTargetExecutionKey(comboName: string, index: number, stepInput: any) { const step = normalizeComboStep(stepInput, { comboName, index }); if (!step) throw new Error(`Failed to normalize combo step for ${comboName}#${index}`); return `combo:${comboName}:${step.id}`; @@ -1038,7 +1055,7 @@ test("handleComboChat round-robin returns 404 when no models are configured", as test("handleComboChat round-robin falls through semaphore timeouts and malformed success payloads", async () => { const release = await acquireSemaphore( - getComboTargetBreakerKey("rr-timeout-fallback", 0, "model-a"), + getComboTargetExecutionKey("rr-timeout-fallback", 0, "model-a"), { maxConcurrency: 1, timeoutMs: 100, @@ -1376,38 +1393,36 @@ test("handleComboChat returns a 503 when every model is unavailable before execu assert.equal(payload.error.code, "ALL_ACCOUNTS_INACTIVE"); }); -test("handleComboChat returns the circuit-breaker unavailable response when all breakers are open", async () => { - for (const [index, modelStr] of ["openai/model-a", "openai/model-b"].entries()) { - const breaker = getCircuitBreaker( - getComboTargetBreakerKey("all-breakers-open", index, modelStr), - { - failureThreshold: 1, - resetTimeout: 60000, - } - ); - breaker._onFailure(); - } - +test("handleComboChat falls through targets that return provider circuit breaker open responses", async () => { + const calls = []; + const log = createLog(); const result = await handleComboChat({ body: {}, combo: { - name: "all-breakers-open", + name: "provider-breaker-open", strategy: "priority", models: ["openai/model-a", "openai/model-b"], }, - handleSingleModel: async () => { - throw new Error("handleSingleModel should not run when all breakers are open"); + handleSingleModel: async (_body, modelStr) => { + calls.push(modelStr); + if (modelStr === "openai/model-a") { + return providerBreakerOpenResponse(); + } + return okResponse(); }, isModelAvailable: async () => true, - log: createLog(), + log, settings: null, relayOptions: null as any, allCombos: null, relayOptions: null, }); - assert.equal(result.status, 503); - assert.match((await result.json()).error.message, /circuit breakers open/); + assert.equal(result.ok, true); + assert.deepEqual(calls, ["openai/model-a", "openai/model-b"]); + assert.ok( + log.entries.some((entry) => String(entry.msg).includes("provider circuit breaker OPEN")) + ); }); test("handleComboChat auto strategy honors LKGP after filtering to tool-capable models", async () => { @@ -1799,39 +1814,37 @@ test("handleComboChat round-robin resolves nested combos and returns inactive wh assert.equal(payload.error.code, "ALL_ACCOUNTS_INACTIVE"); }); -test("handleComboChat round-robin returns circuit-breaker unavailable when every model is open", async () => { - for (const [index, modelStr] of ["openai/model-a", "openai/model-b"].entries()) { - const breaker = getCircuitBreaker( - getComboTargetBreakerKey("rr-breakers-open", index, modelStr), - { - failureThreshold: 1, - resetTimeout: 60000, - } - ); - breaker._onFailure(); - } - +test("handleComboChat round-robin skips targets that return provider circuit breaker open responses", async () => { + const calls = []; + const log = createLog(); const result = await handleComboChat({ body: {}, combo: { - name: "rr-breakers-open", + name: "rr-provider-breaker-open", strategy: "round-robin", models: ["openai/model-a", "openai/model-b"], config: { maxRetries: 0 }, }, - handleSingleModel: async () => { - throw new Error("round-robin should not execute when all breakers are open"); + handleSingleModel: async (_body, modelStr) => { + calls.push(modelStr); + if (modelStr === "openai/model-a") { + return providerBreakerOpenResponse(); + } + return okResponse(); }, isModelAvailable: async () => true, - log: createLog(), + log, settings: null, relayOptions: null as any, allCombos: null, relayOptions: null, }); - assert.equal(result.status, 503); - assert.match((await result.json()).error.message, /circuit breakers open/); + assert.equal(result.ok, true); + assert.deepEqual(calls, ["openai/model-a", "openai/model-b"]); + assert.ok( + log.entries.some((entry) => String(entry.msg).includes("provider circuit breaker OPEN")) + ); }); test("handleComboChat round-robin retries a transient failure on the same model before succeeding", async () => { diff --git a/tests/unit/domain-branch-hardening.test.ts b/tests/unit/domain-branch-hardening.test.ts index 63d8e387d7..8f6071dbee 100644 --- a/tests/unit/domain-branch-hardening.test.ts +++ b/tests/unit/domain-branch-hardening.test.ts @@ -11,7 +11,6 @@ const core = await import("../../src/lib/db/core.ts"); const costRules = await import("../../src/domain/costRules.ts"); const fallbackPolicy = await import("../../src/domain/fallbackPolicy.ts"); const lockoutPolicy = await import("../../src/domain/lockoutPolicy.ts"); -const modelAvailability = await import("../../src/domain/modelAvailability.ts"); const providerExpiration = await import("../../src/domain/providerExpiration.ts"); const quotaCache = await import("../../src/domain/quotaCache.ts"); const comboResolver = await import("../../src/domain/comboResolver.ts"); @@ -28,7 +27,6 @@ function isoFromNow(offsetMs) { async function resetStorage() { costRules.resetCostData(); fallbackPolicy.resetAllFallbacks(); - modelAvailability.resetAllAvailability(); providerExpiration.resetExpirations(); quotaCache.stopBackgroundRefresh(); core.resetDbInstance(); @@ -63,7 +61,6 @@ test.after(async () => { quotaCache.stopBackgroundRefresh(); costRules.resetCostData(); fallbackPolicy.resetAllFallbacks(); - modelAvailability.resetAllAvailability(); providerExpiration.resetExpirations(); core.resetDbInstance(); fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); @@ -192,30 +189,6 @@ test("resolveComboModel also covers implicit defaults and missing optional field assert.deepEqual(comboResolver.getComboFallbacks({}, 0), []); }); -test("modelAvailability tracks missing, active and expired cooldowns", () => { - let now = 1_000; - Date.now = () => now; - - assert.equal(modelAvailability.isModelAvailable("openai", "gpt-4o"), true); - - modelAvailability.setModelUnavailable("openai", "gpt-4o", 100, undefined); - modelAvailability.setModelUnavailable("anthropic", "claude-sonnet", 500, "capacity"); - - assert.equal(modelAvailability.isModelAvailable("openai", "gpt-4o"), false); - assert.equal(modelAvailability.getUnavailableCount(), 2); - - const report = modelAvailability.getAvailabilityReport(); - assert.equal(report.length, 2); - assert.equal(report[0].reason, "unknown"); - assert.equal(report[1].reason, "capacity"); - - now += 150; - assert.equal(modelAvailability.isModelAvailable("openai", "gpt-4o"), true); - assert.equal(modelAvailability.clearModelUnavailability("openai", "gpt-4o"), false); - assert.equal(modelAvailability.clearModelUnavailability("anthropic", "claude-sonnet"), true); - assert.equal(modelAvailability.getUnavailableCount(), 0); -}); - test("providerExpiration derives status, sorting, summary and header-based expiration hints", () => { const expired = providerExpiration.setExpiration( "conn-expired", diff --git a/tests/unit/error-classification.test.ts b/tests/unit/error-classification.test.ts index 9fb24d7181..0f6be74c08 100644 --- a/tests/unit/error-classification.test.ts +++ b/tests/unit/error-classification.test.ts @@ -6,7 +6,7 @@ const { checkFallbackError, getProviderProfile, parseRetryFromErrorText } = const { getProviderCategory } = await import("../../open-sse/config/providerRegistry.ts"); -const { COOLDOWN_MS, PROVIDER_PROFILES, RateLimitReason } = +const { BACKOFF_CONFIG, COOLDOWN_MS, PROVIDER_PROFILES, RateLimitReason } = await import("../../open-sse/config/constants.ts"); // โ”€โ”€โ”€ Provider Category Tests โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ @@ -41,12 +41,36 @@ test("getProviderCategory: unknown provider defaults to 'apikey'", () => { test("getProviderProfile: OAuth provider returns oauth profile", () => { const profile = getProviderProfile("claude"); - assert.deepEqual(profile, PROVIDER_PROFILES.oauth); + assert.equal(profile.baseCooldownMs, PROVIDER_PROFILES.oauth.transientCooldown); + assert.equal(profile.useUpstreamRetryHints, false); + assert.equal(profile.maxBackoffSteps, PROVIDER_PROFILES.oauth.maxBackoffLevel); + assert.equal(profile.failureThreshold, PROVIDER_PROFILES.oauth.circuitBreakerThreshold); + assert.equal(profile.resetTimeoutMs, PROVIDER_PROFILES.oauth.circuitBreakerReset); + assert.equal(profile.transientCooldown, PROVIDER_PROFILES.oauth.transientCooldown); + assert.equal( + profile.rateLimitCooldown, + profile.useUpstreamRetryHints ? 0 : profile.baseCooldownMs + ); + assert.equal(profile.maxBackoffLevel, PROVIDER_PROFILES.oauth.maxBackoffLevel); + assert.equal(profile.circuitBreakerThreshold, PROVIDER_PROFILES.oauth.circuitBreakerThreshold); + assert.equal(profile.circuitBreakerReset, PROVIDER_PROFILES.oauth.circuitBreakerReset); }); test("getProviderProfile: API provider returns apikey profile", () => { const profile = getProviderProfile("groq"); - assert.deepEqual(profile, PROVIDER_PROFILES.apikey); + assert.equal(profile.baseCooldownMs, PROVIDER_PROFILES.apikey.transientCooldown); + assert.equal(profile.useUpstreamRetryHints, true); + assert.equal(profile.maxBackoffSteps, PROVIDER_PROFILES.apikey.maxBackoffLevel); + assert.equal(profile.failureThreshold, PROVIDER_PROFILES.apikey.circuitBreakerThreshold); + assert.equal(profile.resetTimeoutMs, PROVIDER_PROFILES.apikey.circuitBreakerReset); + assert.equal(profile.transientCooldown, PROVIDER_PROFILES.apikey.transientCooldown); + assert.equal( + profile.rateLimitCooldown, + profile.useUpstreamRetryHints ? 0 : profile.baseCooldownMs + ); + assert.equal(profile.maxBackoffLevel, PROVIDER_PROFILES.apikey.maxBackoffLevel); + assert.equal(profile.circuitBreakerThreshold, PROVIDER_PROFILES.apikey.circuitBreakerThreshold); + assert.equal(profile.circuitBreakerReset, PROVIDER_PROFILES.apikey.circuitBreakerReset); }); test("getProviderProfile: profiles have different thresholds", () => { @@ -64,7 +88,7 @@ test("getProviderProfile: profiles have different thresholds", () => { // โ”€โ”€โ”€ Exponential Backoff for Transient Errors โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ -test("502 transient: exponential backoff 5s โ†’ 10s โ†’ 20s โ†’ 40s โ†’ 60s (capped)", () => { +test("502 transient: exponential backoff doubles until the configured max backoff step", () => { const cooldowns = []; for (let level = 0; level < 6; level++) { const result = checkFallbackError(502, "", level, null, null); @@ -73,14 +97,14 @@ test("502 transient: exponential backoff 5s โ†’ 10s โ†’ 20s โ†’ 40s โ†’ 60s (cap assert.equal(result.newBackoffLevel, level + 1); assert.equal(result.reason, RateLimitReason.SERVER_ERROR); } - // Without provider: uses COOLDOWN_MS.transientInitial (5s) as base - assert.equal(cooldowns[0], COOLDOWN_MS.transientInitial); // 5s - assert.equal(cooldowns[1], COOLDOWN_MS.transientInitial * 2); // 10s - assert.equal(cooldowns[2], COOLDOWN_MS.transientInitial * 4); // 20s - assert.equal(cooldowns[3], COOLDOWN_MS.transientInitial * 8); // 40s - // Level 4: 5s * 16 = 80s โ†’ capped at 60s - assert.equal(cooldowns[4], COOLDOWN_MS.transientMax); // 60s - assert.equal(cooldowns[5], COOLDOWN_MS.transientMax); // 60s (stays capped) + assert.deepEqual(cooldowns, [ + COOLDOWN_MS.transientInitial, + COOLDOWN_MS.transientInitial * 2, + COOLDOWN_MS.transientInitial * 4, + COOLDOWN_MS.transientInitial * 8, + COOLDOWN_MS.transientInitial * 16, + COOLDOWN_MS.transientInitial * 32, + ]); }); test("502 with OAuth provider: uses oauth profile transientCooldown", () => { @@ -116,11 +140,13 @@ test("429 rate limit: still uses quota-based exponential backoff", () => { assert.equal(result.reason, RateLimitReason.RATE_LIMIT_EXCEEDED); }); -test("401 auth error: still uses flat cooldown, no backoff", () => { +test("401 auth error: returns terminal auth semantics without connection cooldown", () => { const result = checkFallbackError(401, "", 0, null, "groq"); assert.equal(result.shouldFallback, true); - assert.equal(result.cooldownMs, COOLDOWN_MS.unauthorized); + assert.equal(result.cooldownMs, 0); + assert.equal(result.baseCooldownMs, 0); assert.equal(result.newBackoffLevel, undefined); + assert.equal(result.reason, RateLimitReason.AUTH_ERROR); }); test("400 bad request: still returns shouldFallback false", () => { @@ -162,7 +188,7 @@ test("parseRetryFromErrorText: parses will reset after variant", () => { // โ”€โ”€โ”€ T06: Keyword Matching for Long Cooldowns โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ -test("quota will reset keyword triggers long cooldown from body", () => { +test("quota reset text is ignored when upstream retry hints are disabled", () => { const result = checkFallbackError( 429, "Your quota will reset after 27h41m36s", @@ -172,19 +198,31 @@ test("quota will reset keyword triggers long cooldown from body", () => { null ); assert.equal(result.shouldFallback, true); - assert.ok(result.cooldownMs > 60_000, "cooldownMs should be > 60s"); - assert.equal(result.newBackoffLevel, 0, "backoffLevel should reset to 0"); + assert.equal(result.cooldownMs, PROVIDER_PROFILES.oauth.transientCooldown); + assert.equal(result.newBackoffLevel, 1); + assert.equal(result.usedUpstreamRetryHint, false); }); -test("exhausted your capacity keyword triggers long cooldown", () => { +test("quota reset text is honored when upstream retry hints are enabled", () => { const result = checkFallbackError( 429, "You have exhausted your capacity. Your quota will reset after 2h", 0, null, - "antigravity", + "groq", null ); assert.equal(result.shouldFallback, true); - assert.ok(result.cooldownMs > 60_000); + assert.equal(result.cooldownMs, 2 * 60 * 60 * 1000); + assert.equal(result.newBackoffLevel, 0); + assert.equal(result.usedUpstreamRetryHint, true); +}); + +test("high transient backoff levels clamp to the configured maxBackoffSteps", () => { + const result = checkFallbackError(502, "", BACKOFF_CONFIG.maxLevel + 5, null, null); + assert.equal(result.newBackoffLevel, BACKOFF_CONFIG.maxLevel); + assert.equal( + result.cooldownMs, + COOLDOWN_MS.transientInitial * Math.pow(2, BACKOFF_CONFIG.maxLevel) + ); }); diff --git a/tests/unit/rate-limit-manager.test.ts b/tests/unit/rate-limit-manager.test.ts index cb1aa96b6a..e61d523103 100644 --- a/tests/unit/rate-limit-manager.test.ts +++ b/tests/unit/rate-limit-manager.test.ts @@ -157,9 +157,14 @@ test("rate limit manager parses retry hints from response bodies and locks model "gpt-4o" ); - const lockout = accountFallback.getModelLockoutInfo("openai", "conn-body", "gpt-4o"); - assert.equal(lockout.reason, "rate_limit_exceeded"); - assert.ok(lockout.remainingMs > 0); + assert.equal(accountFallback.getModelLockoutInfo("openai", "conn-body", "gpt-4o"), null); + const limiterState = await rateLimitManager.__getLimiterStateForTests( + "openai", + "conn-body", + "gpt-4o" + ); + assert.equal(limiterState?.key, "openai:conn-body"); + assert.equal(rateLimitManager.getRateLimitStatus("openai", "conn-body").active, true); rateLimitManager.updateFromResponseBody( "openai", diff --git a/tests/unit/sse-auth.test.ts b/tests/unit/sse-auth.test.ts index 98b31f54c8..04d558a7c2 100644 --- a/tests/unit/sse-auth.test.ts +++ b/tests/unit/sse-auth.test.ts @@ -14,6 +14,7 @@ const settingsDb = await import("../../src/lib/db/settings.ts"); const apiKeysDb = await import("../../src/lib/db/apiKeys.ts"); const auth = await import("../../src/sse/services/auth.ts"); const quotaCache = await import("../../src/domain/quotaCache.ts"); +const { COOLDOWN_MS } = await import("../../open-sse/config/constants.ts"); async function resetStorage() { core.resetDbInstance(); @@ -760,7 +761,7 @@ test("getProviderCredentials exposes copilotToken when present in providerSpecif assert.equal(selected.copilotToken, "copilot-token-value"); }); -test("markAccountUnavailable uses configured cooldowns for local 404 model lockouts", async () => { +test("markAccountUnavailable keeps local 404 failures model-scoped with the local not-found cooldown", async () => { await settingsDb.updateSettings({ providerProfiles: { apikey: { @@ -790,7 +791,7 @@ test("markAccountUnavailable uses configured cooldowns for local 404 model locko const updated = await providersDb.getProviderConnectionById(connection.id); assert.equal(result.shouldFallback, true); - assert.equal(result.cooldownMs, 250); + assert.equal(result.cooldownMs, COOLDOWN_MS.notFoundLocal); assert.equal(updated.testStatus, "active"); assert.equal(updated.rateLimitedUntil, undefined); assert.equal(updated.lastErrorType, "not_found"); @@ -843,7 +844,7 @@ test("markAccountUnavailable applies a model-only lockout for compatible provide assert.equal(Number(updated.errorCode), 429); }); -test("markAccountUnavailable honors configured api-key rate-limit cooldowns", async () => { +test("markAccountUnavailable uses the unified configured api-key connection cooldown", async () => { await settingsDb.updateSettings({ providerProfiles: { apikey: { @@ -869,7 +870,7 @@ test("markAccountUnavailable honors configured api-key rate-limit cooldowns", as ); assert.equal(result.shouldFallback, true); - assert.equal(result.cooldownMs, 125); + assert.equal(result.cooldownMs, 200); }); test("markAccountUnavailable stores Codex scope-specific cooldowns without a global rate limit", async () => { @@ -1021,7 +1022,7 @@ test("markAccountUnavailable auto-disables permanently banned accounts when the assert.equal(result.shouldFallback, true); assert.equal(updated.isActive, false); - assert.equal(updated.testStatus, "unavailable"); + assert.equal(updated.testStatus, "banned"); }); test("markAccountUnavailable leaves permanently banned accounts active when auto-disable is disabled", async () => { @@ -1041,7 +1042,7 @@ test("markAccountUnavailable leaves permanently banned accounts active when auto assert.equal(result.shouldFallback, true); assert.equal(updated.isActive, true); - assert.equal(updated.testStatus, "unavailable"); + assert.equal(updated.testStatus, "banned"); }); test("markAccountUnavailable swallows auto-disable persistence errors", async () => { @@ -1085,7 +1086,7 @@ test("markAccountUnavailable swallows auto-disable persistence errors", async () assert.equal(result.shouldFallback, true); assert.equal(updated.isActive, true); - assert.equal(updated.testStatus, "unavailable"); + assert.equal(updated.testStatus, "banned"); } finally { db.prepare = originalPrepare; } diff --git a/tests/unit/thundering-herd.test.ts b/tests/unit/thundering-herd.test.ts index aa1c28b5c2..e3af4f81f1 100644 --- a/tests/unit/thundering-herd.test.ts +++ b/tests/unit/thundering-herd.test.ts @@ -9,7 +9,7 @@ import assert from "node:assert/strict"; const { checkFallbackError, getProviderProfile } = await import("../../open-sse/services/accountFallback.ts"); -const { PROVIDER_PROFILES, DEFAULT_API_LIMITS, COOLDOWN_MS, RateLimitReason } = +const { BACKOFF_CONFIG, PROVIDER_PROFILES, DEFAULT_API_LIMITS, COOLDOWN_MS, RateLimitReason } = await import("../../open-sse/config/constants.ts"); const { getProviderCategory } = await import("../../open-sse/config/providerRegistry.ts"); @@ -36,10 +36,13 @@ test("API profile has shorter transient cooldown", () => { // โ”€โ”€โ”€ Backoff Ceiling Tests (prevents infinite growth) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ -test("Exponential backoff is capped at transientMax for high backoff levels", () => { - // Level 20 โ†’ 5s * 2^20 = 5.2M ms, but capped at 60s +test("Exponential backoff clamps to the configured maxBackoffLevel", () => { const result = checkFallbackError(502, "", 20, null, null); - assert.equal(result.cooldownMs, COOLDOWN_MS.transientMax); + assert.equal(result.newBackoffLevel, BACKOFF_CONFIG.maxLevel); + assert.equal( + result.cooldownMs, + COOLDOWN_MS.transientInitial * Math.pow(2, BACKOFF_CONFIG.maxLevel) + ); }); test("API provider backoff level caps at profile maxBackoffLevel", () => { From 1d3d99bb9efe12e43a96e6acb545475eee1d6db4 Mon Sep 17 00:00:00 2001 From: diegosouzapw Date: Tue, 21 Apr 2026 04:36:16 -0300 Subject: [PATCH 8/8] fix(combo): resolve cross-provider thinking 400 and http clipboard (#1444) Integrated into release/v3.7.0 --- open-sse/translator/helpers/claudeHelper.ts | 8 +++++++- src/shared/utils/clipboard.ts | 11 ++++------- tests/unit/translator-helper-branches.test.ts | 3 ++- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/open-sse/translator/helpers/claudeHelper.ts b/open-sse/translator/helpers/claudeHelper.ts index 2936b60240..bf4e589d0f 100644 --- a/open-sse/translator/helpers/claudeHelper.ts +++ b/open-sse/translator/helpers/claudeHelper.ts @@ -247,10 +247,16 @@ export function prepareClaudeRequest( let hasToolUse = false; let hasThinking = false; - // Always replace signature for all thinking blocks + // Convert thinking blocks to redacted_thinking and replace signature. + // When requests cross provider boundaries (e.g., combo fallback), the + // original thinking signature is invalid for the new provider, causing + // "Invalid signature in thinking block" 400 errors. redacted_thinking + // blocks are accepted without signature validation. for (const block of content) { if (block.type === "thinking" || block.type === "redacted_thinking") { + block.type = "redacted_thinking"; block.signature = DEFAULT_THINKING_CLAUDE_SIGNATURE; + delete block.thinking; hasThinking = true; } if (block.type === "tool_use") hasToolUse = true; diff --git a/src/shared/utils/clipboard.ts b/src/shared/utils/clipboard.ts index ee6d014dfb..50f6671dc9 100644 --- a/src/shared/utils/clipboard.ts +++ b/src/shared/utils/clipboard.ts @@ -11,13 +11,10 @@ * @returns true if copy succeeded, false otherwise */ export async function copyToClipboard(text: string): Promise { - // Method 1: Clipboard API (requires HTTPS / secure context) - if ( - typeof navigator !== "undefined" && - navigator.clipboard && - typeof window !== "undefined" && - window.isSecureContext - ) { + // Method 1: Clipboard API + // Works on HTTPS, localhost (treated as secure context), and some browsers + // even on HTTP. Try unconditionally โ€” the catch handles failures. + if (typeof navigator !== "undefined" && navigator.clipboard) { try { await navigator.clipboard.writeText(text); return true; diff --git a/tests/unit/translator-helper-branches.test.ts b/tests/unit/translator-helper-branches.test.ts index e31c393352..33a59a0302 100644 --- a/tests/unit/translator-helper-branches.test.ts +++ b/tests/unit/translator-helper-branches.test.ts @@ -306,9 +306,10 @@ test("claudeHelper validates content, ordering and request preparation branches" assert.equal(prepared.messages[4].content[0].type, "tool_result"); assert.deepEqual( prepared.messages[5].content.map((block) => block.type), - ["thinking", "text"] + ["redacted_thinking", "text"] ); assert.ok(prepared.messages[5].content[0].signature); + assert.equal(prepared.messages[5].content[0].thinking, undefined); assert.equal(prepared.tools.length, 2); assert.equal(prepared.tools[0].cache_control, undefined); assert.deepEqual(prepared.tools[1].cache_control, { type: "ephemeral", ttl: "1h" });