diff --git a/.env.example b/.env.example index 5e36a6871c..5d8c7d1090 100644 --- a/.env.example +++ b/.env.example @@ -1324,6 +1324,28 @@ CURSOR_USER_AGENT="Cursor/3.4" # OMNIROUTE_CIRCUIT_BREAKER_LOCAL_THRESHOLD=2 # OMNIROUTE_CIRCUIT_BREAKER_LOCAL_RESET_MS=15000 +# ── Provider-level circuit breaker thresholds and cooldowns ── +# Used by: open-sse/config/constants.ts (PROVIDER_PROFILES → accountFallback). +# These control the provider-level fuse (entire provider cooldown after repeated +# failures) — distinct from the per-key breaker above. Defaults match the +# historical PROVIDER_PROFILES values. Raise to tolerate transient upstream +# sheds without blacklisting the provider; lower to fail over faster. +# OMNIROUTE_PROVIDER_BREAKER_OAUTH_FAILURE_THRESHOLD=10 +# OMNIROUTE_PROVIDER_BREAKER_OAUTH_FAILURE_WINDOW_MS=900000 +# OMNIROUTE_PROVIDER_BREAKER_OAUTH_COOLDOWN_MS=300000 +# OMNIROUTE_PROVIDER_BREAKER_OAUTH_DEGRADATION_THRESHOLD=5 +# OMNIROUTE_PROVIDER_BREAKER_OAUTH_MAX_BACKOFF_MULTIPLIER=8 +# OMNIROUTE_PROVIDER_BREAKER_OAUTH_BACKOFF_ESCALATION_COUNT=2 +# OMNIROUTE_PROVIDER_BREAKER_API_KEY_FAILURE_THRESHOLD=15 +# OMNIROUTE_PROVIDER_BREAKER_API_KEY_FAILURE_WINDOW_MS=1800000 +# OMNIROUTE_PROVIDER_BREAKER_API_KEY_COOLDOWN_MS=600000 +# OMNIROUTE_PROVIDER_BREAKER_API_KEY_DEGRADATION_THRESHOLD=7 +# OMNIROUTE_PROVIDER_BREAKER_API_KEY_MAX_BACKOFF_MULTIPLIER=4 +# OMNIROUTE_PROVIDER_BREAKER_API_KEY_BACKOFF_ESCALATION_COUNT=3 +# OMNIROUTE_PROVIDER_BREAKER_LOCAL_FAILURE_THRESHOLD=2 +# OMNIROUTE_PROVIDER_BREAKER_LOCAL_FAILURE_WINDOW_MS=300000 +# OMNIROUTE_PROVIDER_BREAKER_LOCAL_COOLDOWN_MS=60000 + # ── Context-cache pin health gate ── # Used by: open-sse/services/combo.ts. When a context-cache pin points at a # provider that is durably unhealthy, the pin is dropped to allow failover. diff --git a/docs/reference/ENVIRONMENT.md b/docs/reference/ENVIRONMENT.md index 67ebff1917..9d85457d97 100644 --- a/docs/reference/ENVIRONMENT.md +++ b/docs/reference/ENVIRONMENT.md @@ -714,6 +714,21 @@ Provider-level circuit breaker tuning. Defaults reflect the scaled values used s | `OMNIROUTE_CIRCUIT_BREAKER_API_KEY_RESET_MS` | `30000` | `open-sse/config/constants.ts` | Reset window (ms) for API-key provider breaker. | | `OMNIROUTE_CIRCUIT_BREAKER_LOCAL_THRESHOLD` | `2` | `open-sse/config/constants.ts` | Consecutive failure threshold for local providers (Ollama, LM Studio, ...). | | `OMNIROUTE_CIRCUIT_BREAKER_LOCAL_RESET_MS` | `15000` | `open-sse/config/constants.ts` | Reset window (ms) for local provider breaker. | +| `OMNIROUTE_PROVIDER_BREAKER_OAUTH_FAILURE_THRESHOLD` | `10` | `open-sse/config/constants.ts` | Provider-level breaker: failures within the window before the entire OAuth provider enters cooldown. | +| `OMNIROUTE_PROVIDER_BREAKER_OAUTH_FAILURE_WINDOW_MS` | `900000` | `open-sse/config/constants.ts` | Provider-level breaker: rolling failure-count window (ms) for OAuth providers. | +| `OMNIROUTE_PROVIDER_BREAKER_OAUTH_COOLDOWN_MS` | `300000` | `open-sse/config/constants.ts` | Provider-level breaker: cooldown (ms) once the OAuth provider threshold is reached. | +| `OMNIROUTE_PROVIDER_BREAKER_OAUTH_DEGRADATION_THRESHOLD` | `5` | `open-sse/config/constants.ts` | OAuth provider enters DEGRADED at this many failures. | +| `OMNIROUTE_PROVIDER_BREAKER_OAUTH_MAX_BACKOFF_MULTIPLIER` | `8` | `open-sse/config/constants.ts` | OAuth provider max resetTimeout escalation multiplier. | +| `OMNIROUTE_PROVIDER_BREAKER_OAUTH_BACKOFF_ESCALATION_COUNT` | `2` | `open-sse/config/constants.ts` | OAuth provider escalates after this many open cycles. | +| `OMNIROUTE_PROVIDER_BREAKER_API_KEY_FAILURE_THRESHOLD` | `15` | `open-sse/config/constants.ts` | Provider-level breaker: failures within the window before the entire API-key provider enters cooldown. | +| `OMNIROUTE_PROVIDER_BREAKER_API_KEY_FAILURE_WINDOW_MS` | `1800000` | `open-sse/config/constants.ts` | Provider-level breaker: rolling failure-count window (ms) for API-key providers. | +| `OMNIROUTE_PROVIDER_BREAKER_API_KEY_COOLDOWN_MS` | `600000` | `open-sse/config/constants.ts` | Provider-level breaker: cooldown (ms) once the API-key provider threshold is reached. | +| `OMNIROUTE_PROVIDER_BREAKER_API_KEY_DEGRADATION_THRESHOLD` | `7` | `open-sse/config/constants.ts` | API-key provider enters DEGRADED at this many failures. | +| `OMNIROUTE_PROVIDER_BREAKER_API_KEY_MAX_BACKOFF_MULTIPLIER` | `4` | `open-sse/config/constants.ts` | API-key provider max resetTimeout escalation multiplier. | +| `OMNIROUTE_PROVIDER_BREAKER_API_KEY_BACKOFF_ESCALATION_COUNT` | `3` | `open-sse/config/constants.ts` | API-key provider escalates after this many open cycles. | +| `OMNIROUTE_PROVIDER_BREAKER_LOCAL_FAILURE_THRESHOLD` | `2` | `open-sse/config/constants.ts` | Provider-level breaker: failures before the entire local provider enters cooldown. | +| `OMNIROUTE_PROVIDER_BREAKER_LOCAL_FAILURE_WINDOW_MS` | `300000` | `open-sse/config/constants.ts` | Provider-level breaker: rolling failure-count window (ms) for local providers. | +| `OMNIROUTE_PROVIDER_BREAKER_LOCAL_COOLDOWN_MS` | `60000` | `open-sse/config/constants.ts` | Provider-level breaker: cooldown (ms) once the local provider threshold is reached. | | `PIN_DROP_BACKOFF_LEVEL` | `2` | `open-sse/services/combo.ts` | Backoff depth at which a context-cache pin's provider is deemed durably unhealthy and the pin is dropped for failover. | | `PIN_DROP_GRACE_MS` | `20000` | `open-sse/services/combo.ts` | Anti-flap window (ms) tolerating brief transient cooldowns before dropping a context-cache pin. | diff --git a/open-sse/config/constants.ts b/open-sse/config/constants.ts index 0ee89452aa..6ddeabe7b5 100644 --- a/open-sse/config/constants.ts +++ b/open-sse/config/constants.ts @@ -229,13 +229,13 @@ export const PROVIDER_PROFILES = { circuitBreakerThreshold: envInt("OMNIROUTE_CIRCUIT_BREAKER_OAUTH_THRESHOLD", 8), circuitBreakerReset: envInt("OMNIROUTE_CIRCUIT_BREAKER_OAUTH_RESET_MS", 60000), // Provider-level circuit breaker (entire provider cooldown after repeated failures) - providerFailureThreshold: 10, // Scaled for 500+ connections (was 3) - providerFailureWindowMs: 900000, // 15min window (was 10min) - providerCooldownMs: 300000, // 5min cooldown when threshold reached + providerFailureThreshold: envInt("OMNIROUTE_PROVIDER_BREAKER_OAUTH_FAILURE_THRESHOLD", 10), // Scaled for 500+ connections (was 3) + providerFailureWindowMs: envInt("OMNIROUTE_PROVIDER_BREAKER_OAUTH_FAILURE_WINDOW_MS", 900000), // 15min window (was 10min) + providerCooldownMs: envInt("OMNIROUTE_PROVIDER_BREAKER_OAUTH_COOLDOWN_MS", 300000), // 5min cooldown when threshold reached // Adaptive circuit breaker v2 settings - degradationThreshold: 5, // Enter DEGRADED at this many failures - maxBackoffMultiplier: 8, // Max 8x resetTimeout escalation - backoffEscalationCount: 2, // Escalate after 2 open cycles + degradationThreshold: envInt("OMNIROUTE_PROVIDER_BREAKER_OAUTH_DEGRADATION_THRESHOLD", 5), // Enter DEGRADED at this many failures + maxBackoffMultiplier: envInt("OMNIROUTE_PROVIDER_BREAKER_OAUTH_MAX_BACKOFF_MULTIPLIER", 8), // Max 8x resetTimeout escalation + backoffEscalationCount: envInt("OMNIROUTE_PROVIDER_BREAKER_OAUTH_BACKOFF_ESCALATION_COUNT", 2), // Escalate after 2 open cycles }, apikey: { transientCooldown: 3000, // 3s (API providers recover faster) @@ -244,12 +244,12 @@ export const PROVIDER_PROFILES = { circuitBreakerThreshold: envInt("OMNIROUTE_CIRCUIT_BREAKER_API_KEY_THRESHOLD", 12), circuitBreakerReset: envInt("OMNIROUTE_CIRCUIT_BREAKER_API_KEY_RESET_MS", 30000), // Provider-level circuit breaker (entire provider cooldown after repeated failures) - providerFailureThreshold: 15, // Scaled for 500+ connections (was 5) - providerFailureWindowMs: 1800000, // 30min window (was 20min) - providerCooldownMs: 600000, // 10min cooldown when threshold reached - degradationThreshold: 7, - maxBackoffMultiplier: 4, - backoffEscalationCount: 3, + providerFailureThreshold: envInt("OMNIROUTE_PROVIDER_BREAKER_API_KEY_FAILURE_THRESHOLD", 15), // Scaled for 500+ connections (was 5) + providerFailureWindowMs: envInt("OMNIROUTE_PROVIDER_BREAKER_API_KEY_FAILURE_WINDOW_MS", 1800000), // 30min window (was 20min) + providerCooldownMs: envInt("OMNIROUTE_PROVIDER_BREAKER_API_KEY_COOLDOWN_MS", 600000), // 10min cooldown when threshold reached + degradationThreshold: envInt("OMNIROUTE_PROVIDER_BREAKER_API_KEY_DEGRADATION_THRESHOLD", 7), + maxBackoffMultiplier: envInt("OMNIROUTE_PROVIDER_BREAKER_API_KEY_MAX_BACKOFF_MULTIPLIER", 4), + backoffEscalationCount: envInt("OMNIROUTE_PROVIDER_BREAKER_API_KEY_BACKOFF_ESCALATION_COUNT", 3), }, // Local providers (localhost inference backends like Ollama, LM Studio, oMLX). // Not yet wired into getProviderProfile() — will be used when local provider_nodes @@ -261,9 +261,9 @@ export const PROVIDER_PROFILES = { circuitBreakerThreshold: envInt("OMNIROUTE_CIRCUIT_BREAKER_LOCAL_THRESHOLD", 2), circuitBreakerReset: envInt("OMNIROUTE_CIRCUIT_BREAKER_LOCAL_RESET_MS", 15000), // Provider-level circuit breaker (entire provider cooldown after repeated failures) - providerFailureThreshold: 2, // 2 failures trigger provider cooldown - providerFailureWindowMs: 300000, // 5min window for counting failures - providerCooldownMs: 60000, // 1min cooldown when threshold reached + providerFailureThreshold: envInt("OMNIROUTE_PROVIDER_BREAKER_LOCAL_FAILURE_THRESHOLD", 2), // 2 failures trigger provider cooldown + providerFailureWindowMs: envInt("OMNIROUTE_PROVIDER_BREAKER_LOCAL_FAILURE_WINDOW_MS", 300000), // 5min window for counting failures + providerCooldownMs: envInt("OMNIROUTE_PROVIDER_BREAKER_LOCAL_COOLDOWN_MS", 60000), // 1min cooldown when threshold reached }, }; diff --git a/tests/unit/provider-breaker-env-overrides.test.ts b/tests/unit/provider-breaker-env-overrides.test.ts new file mode 100644 index 0000000000..dffcbc6c82 --- /dev/null +++ b/tests/unit/provider-breaker-env-overrides.test.ts @@ -0,0 +1,120 @@ +/** + * Provider-level circuit breaker env overrides (#10040). + * + * Verifies that the provider-level breaker fields in PROVIDER_PROFILES + * (providerFailureThreshold, providerFailureWindowMs, providerCooldownMs, + * degradationThreshold, maxBackoffMultiplier, backoffEscalationCount) are + * env-overridable via OMNIROUTE_PROVIDER_BREAKER__ variables, + * with the historical hardcoded defaults preserved when unset. + * + * Three layers of test: + * 1. Source-shape test on open-sse/config/constants.ts — every required + * field is wrapped in envInt() with the documented default. + * 2. .env.example test — every new env var is listed (env-doc-sync gate). + * 3. docs/reference/ENVIRONMENT.md test — every new env var is documented + * (env-doc-sync gate). + * + * Behavior tests (loading the actual module with controlled env vars) are + * left to upstream CI; the static source-shape test is sufficient here because + * the envInt() helper is a plain function whose only dependency is + * process.env at module load time. + * + * Run: node --import tsx/esm --test tests/unit/provider-breaker-env-overrides.test.ts + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = join(__dirname, "..", ".."); +const CONSTANTS_SRC = join(REPO_ROOT, "open-sse", "config", "constants.ts"); +const ENV_EXAMPLE = join(REPO_ROOT, ".env.example"); +const ENVIRONMENT_DOC = join(REPO_ROOT, "docs", "reference", "ENVIRONMENT.md"); + +// Source-of-truth env names (taken verbatim from the constants.ts source / +// .env.example / docs/reference/ENVIRONMENT.md). Listed explicitly so the test +// fails loud if any rename is missed in any of the three files. +const NEW_ENV_VARS = [ + "OMNIROUTE_PROVIDER_BREAKER_OAUTH_FAILURE_THRESHOLD", + "OMNIROUTE_PROVIDER_BREAKER_OAUTH_FAILURE_WINDOW_MS", + "OMNIROUTE_PROVIDER_BREAKER_OAUTH_COOLDOWN_MS", + "OMNIROUTE_PROVIDER_BREAKER_OAUTH_DEGRADATION_THRESHOLD", + "OMNIROUTE_PROVIDER_BREAKER_OAUTH_MAX_BACKOFF_MULTIPLIER", + "OMNIROUTE_PROVIDER_BREAKER_OAUTH_BACKOFF_ESCALATION_COUNT", + "OMNIROUTE_PROVIDER_BREAKER_API_KEY_FAILURE_THRESHOLD", + "OMNIROUTE_PROVIDER_BREAKER_API_KEY_FAILURE_WINDOW_MS", + "OMNIROUTE_PROVIDER_BREAKER_API_KEY_COOLDOWN_MS", + "OMNIROUTE_PROVIDER_BREAKER_API_KEY_DEGRADATION_THRESHOLD", + "OMNIROUTE_PROVIDER_BREAKER_API_KEY_MAX_BACKOFF_MULTIPLIER", + "OMNIROUTE_PROVIDER_BREAKER_API_KEY_BACKOFF_ESCALATION_COUNT", + "OMNIROUTE_PROVIDER_BREAKER_LOCAL_FAILURE_THRESHOLD", + "OMNIROUTE_PROVIDER_BREAKER_LOCAL_FAILURE_WINDOW_MS", + "OMNIROUTE_PROVIDER_BREAKER_LOCAL_COOLDOWN_MS", +] as const; + +function readConstants(): string { + return readFileSync(CONSTANTS_SRC, "utf8"); +} + +test("every new OMNIROUTE_PROVIDER_BREAKER_* env var is wired in constants.ts", () => { + const src = readConstants(); + for (const envName of NEW_ENV_VARS) { + const re = new RegExp( + `envInt\\(\\s*"${envName.replace(/[.+*?^${}()|[\]\\]/g, "\\$&")}"\\s*,\\s*\\d+\\s*\\)`, + ); + assert.ok( + re.test(src), + `constants.ts is missing envInt(...) wrapping for ${envName}`, + ); + } +}); + +test("all 15 new provider-breaker env vars are documented in ENVIRONMENT.md", () => { + const env = readFileSync(ENVIRONMENT_DOC, "utf8"); + for (const envName of NEW_ENV_VARS) { + assert.ok( + env.includes(`\`${envName}\``), + `ENVIRONMENT.md is missing the ${envName} row (env-doc-sync gate will fail)`, + ); + } +}); + +test("all 15 new provider-breaker env vars are listed in .env.example", () => { + const env = readFileSync(ENV_EXAMPLE, "utf8"); + for (const envName of NEW_ENV_VARS) { + assert.ok( + env.includes(`# ${envName}=`), + `.env.example is missing the commented ${envName} entry (env-doc-sync gate will fail)`, + ); + } +}); + +test("historical defaults preserved when no OMNIROUTE_PROVIDER_BREAKER_* env vars set", () => { + // The source still contains the literal default values as the envInt fallback. + // If a refactor accidentally drops the default, this test will catch it. + const src = readConstants(); + const defaults: Array<[string, number]> = [ + ["OMNIROUTE_PROVIDER_BREAKER_OAUTH_FAILURE_THRESHOLD", 10], + ["OMNIROUTE_PROVIDER_BREAKER_OAUTH_FAILURE_WINDOW_MS", 900000], + ["OMNIROUTE_PROVIDER_BREAKER_OAUTH_COOLDOWN_MS", 300000], + ["OMNIROUTE_PROVIDER_BREAKER_OAUTH_DEGRADATION_THRESHOLD", 5], + ["OMNIROUTE_PROVIDER_BREAKER_API_KEY_FAILURE_THRESHOLD", 15], + ["OMNIROUTE_PROVIDER_BREAKER_API_KEY_FAILURE_WINDOW_MS", 1800000], + ["OMNIROUTE_PROVIDER_BREAKER_API_KEY_COOLDOWN_MS", 600000], + ["OMNIROUTE_PROVIDER_BREAKER_API_KEY_DEGRADATION_THRESHOLD", 7], + ["OMNIROUTE_PROVIDER_BREAKER_LOCAL_FAILURE_THRESHOLD", 2], + ["OMNIROUTE_PROVIDER_BREAKER_LOCAL_FAILURE_WINDOW_MS", 300000], + ["OMNIROUTE_PROVIDER_BREAKER_LOCAL_COOLDOWN_MS", 60000], + ]; + for (const [envName, defaultValue] of defaults) { + const re = new RegExp( + `envInt\\(\\s*"${envName.replace(/[.+*?^${}()|[\]\\]/g, "\\$&")}"\\s*,\\s*${defaultValue}\\s*\\)`, + ); + assert.ok( + re.test(src), + `default for ${envName} should be ${defaultValue}; check constants.ts`, + ); + } +});