From c32be09e6a79049ab3e7b68c32fd72f51028aefb Mon Sep 17 00:00:00 2001 From: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> Date: Tue, 1 Sep 2026 00:19:09 -0300 Subject: [PATCH] feat(sse): wire the PROVIDER_PROFILES window gate into the global provider cooldown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit providerFailureThreshold / providerFailureWindowMs / providerCooldownMs shipped in PROVIDER_PROFILES with no runtime consumer (2026-08-31 docs audit, P0.1). Provider-level entries in providerCooldownTracker now honor them: the whole provider only counts as cooling after providerFailureThreshold failures inside providerFailureWindowMs, then cools for providerCooldownMs. Connection-level entries keep the pre-existing exponential backoff, and the layer stays opt-in (PROVIDER_COOLDOWN_ENABLED, default off) — default behavior is unchanged. TDD: tests/unit/provider-cooldown-window-gate.test.ts written first (4 red on the old behavior), then the wiring; legacy tracker suite aligned to the new contract (23/23 green). Docs: AGENTS.md breaker section + RESILIENCE_GUIDE opt-in layer subsection; executors soft-drift refresh (104 -> 106). --- AGENTS.md | 12 +- docs/architecture/ARCHITECTURE.md | 2 +- docs/architecture/CODEBASE_DOCUMENTATION.md | 4 +- docs/architecture/RESILIENCE_GUIDE.md | 19 +++ open-sse/services/providerCooldownTracker.ts | 72 ++++++++++- .../provider-cooldown-window-gate.test.ts | 121 ++++++++++++++++++ .../services/providerCooldownTracker.test.ts | 9 ++ 7 files changed, 230 insertions(+), 9 deletions(-) create mode 100644 tests/unit/provider-cooldown-window-gate.test.ts diff --git a/AGENTS.md b/AGENTS.md index 7d449324fd..7b0030b56f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -131,10 +131,14 @@ breaker runs on `circuitBreakerThreshold` / `circuitBreakerReset`: | API key | `7` | `12` | `30s` | | Local | (derived) | `2` | `15s` | -`PROVIDER_PROFILES` also defines `providerFailureThreshold` (10/15/2) and `providerCooldownMs` -(5min/10min/1min); those fields are loaded into the profile but have **no runtime consumer -today** — do not tune or document them as the live breaker. Every default is overridable -through the `OMNIROUTE_PROVIDER_BREAKER_*` and `OMNIROUTE_CIRCUIT_BREAKER_*` env vars; the +`PROVIDER_PROFILES` also defines `providerFailureThreshold` (10/15/2), +`providerFailureWindowMs` (15/30/5 min) and `providerCooldownMs` (5/10/1 min): these power the +**window gate of the opt-in global Provider Cooldown** (`PROVIDER_COOLDOWN_ENABLED`, default +off) — a provider-level entry in `open-sse/services/providerCooldownTracker.ts` only counts as +cooling after `providerFailureThreshold` failures inside `providerFailureWindowMs`, and then +cools for `providerCooldownMs`. They are NOT the live breaker's thresholds — do not tune them +expecting breaker behavior. Every default is overridable through the +`OMNIROUTE_PROVIDER_BREAKER_*` and `OMNIROUTE_CIRCUIT_BREAKER_*` env vars; the runtime-accurate reference table lives in `docs/architecture/RESILIENCE_GUIDE.md`. Only provider-level failure statuses should trip the provider breaker: diff --git a/docs/architecture/ARCHITECTURE.md b/docs/architecture/ARCHITECTURE.md index ad5a72cec3..209074cb45 100644 --- a/docs/architecture/ARCHITECTURE.md +++ b/docs/architecture/ARCHITECTURE.md @@ -17,7 +17,7 @@ It provides a single OpenAI-compatible endpoint (`/v1/*`) and routes traffic acr Core capabilities: -- OpenAI-compatible API surface for CLI/tools (352 providers, 104 executors) +- OpenAI-compatible API surface for CLI/tools (352 providers, 106 executors) - Request/response translation across provider formats - Model combo fallback (multi-model sequence) - Structured combo steps (`provider + model + connection`) with runtime ordering by `compositeTiers` diff --git a/docs/architecture/CODEBASE_DOCUMENTATION.md b/docs/architecture/CODEBASE_DOCUMENTATION.md index bc4d224c23..6f21abf187 100644 --- a/docs/architecture/CODEBASE_DOCUMENTATION.md +++ b/docs/architecture/CODEBASE_DOCUMENTATION.md @@ -450,7 +450,7 @@ open-sse/ ├── types.d.ts ├── config/ Provider registries, header profiles, identity, … ├── handlers/ Request handlers (chat, embeddings, audio, image, …) -├── executors/ 104 provider-specific HTTP executors +├── executors/ 106 provider-specific HTTP executors ├── translator/ Format conversion (OpenAI ↔ Claude ↔ Gemini ↔ Cursor ↔ Kiro) ├── transformer/ Responses API ↔ Chat Completions stream transformer ├── services/ 80+ service modules (combos, fallback, quotas, identity, …) @@ -480,7 +480,7 @@ open-sse/ ### 4.2 `open-sse/executors/` -104 provider executors, each extending `BaseExecutor` (`base.ts`): +106 provider executors, each extending `BaseExecutor` (`base.ts`): `antigravity`, `azure-openai`, `blackbox-web`, `cliproxyapi`, `chatgpt-web-codex`, `cloudflare-ai`, `codex`, `commandCode`, `cursor`, `default`, `devin-cli`, diff --git a/docs/architecture/RESILIENCE_GUIDE.md b/docs/architecture/RESILIENCE_GUIDE.md index d2744a0e81..1d81a2a1f6 100644 --- a/docs/architecture/RESILIENCE_GUIDE.md +++ b/docs/architecture/RESILIENCE_GUIDE.md @@ -50,6 +50,25 @@ OmniRoute has three distinct but related resilience mechanisms. Each has a diffe --- +### Opt-in global Provider Cooldown (window gate) + +A fourth, **opt-in** layer (`PROVIDER_COOLDOWN_ENABLED`, default **off**) keeps a +cross-request memory of failing providers in +`open-sse/services/providerCooldownTracker.ts`, consulted by combo target +resolution so consecutive combo requests stop re-walking a provider that just +failed. Provider-level entries honor the `PROVIDER_PROFILES` window gate: + +| Profile | trips after (`providerFailureThreshold`) | inside (`providerFailureWindowMs`) | cools for (`providerCooldownMs`) | +| ------- | ---------------------------------------: | ---------------------------------: | -------------------------------: | +| OAuth | `10` | `15min` | `5min` | +| API key | `15` | `30min` | `10min` | + +Below the threshold the provider is **not** considered cooling; a success clears +the window. Connection-level entries (`provider:connectionId`) keep the +exponential `minRetryCooldownMs → maxRetryCooldownMs` backoff instead. Overrides: +`OMNIROUTE_PROVIDER_BREAKER_{OAUTH,API_KEY}_{FAILURE_THRESHOLD,FAILURE_WINDOW_MS,COOLDOWN_MS}`. +Regression guard: `tests/unit/provider-cooldown-window-gate.test.ts`. + ## 2. Connection Cooldown **Scope:** single provider connection/account/key. diff --git a/open-sse/services/providerCooldownTracker.ts b/open-sse/services/providerCooldownTracker.ts index 13fbd17af5..a67d2b2815 100644 --- a/open-sse/services/providerCooldownTracker.ts +++ b/open-sse/services/providerCooldownTracker.ts @@ -11,6 +11,8 @@ import { DEFAULT_RESILIENCE_SETTINGS, type ResilienceSettings, } from "../../src/lib/resilience/settings"; +import { PROVIDER_PROFILES } from "../config/constants.ts"; +import { getProviderCategory } from "../config/providerRegistry.ts"; interface CooldownEntry { /** Timestamp of last recorded failure (ms since epoch) */ @@ -19,6 +21,47 @@ interface CooldownEntry { failureCount: number; /** How long this entry must be retained for cleanup purposes */ retentionMs: number; + /** + * Provider-level entries only: timestamps of recent failures, pruned to the + * profile's `providerFailureWindowMs`. Powers the PROVIDER_PROFILES window + * gate (`providerFailureThreshold` failures inside the window trip a + * `providerCooldownMs` cooldown for the whole provider). + */ + failureTimestamps?: number[]; +} + +// ── PROVIDER_PROFILES window gate (whole-provider scope) ───────────────────── +// `providerFailureThreshold` / `providerFailureWindowMs` / `providerCooldownMs` +// shipped in PROVIDER_PROFILES with no runtime consumer (2026-08-31 docs +// audit, P0.1). Provider-level entries (no connectionId) now honor them: the +// provider only counts as cooling after `providerFailureThreshold` failures +// inside `providerFailureWindowMs`, and then cools for `providerCooldownMs`. +// Connection-level entries keep the pre-existing exponential backoff. +function providerWindowProfile(provider: string) { + const category = getProviderCategory(provider); + const profile = PROVIDER_PROFILES[category] ?? PROVIDER_PROFILES.apikey; + return { + failureThreshold: profile.providerFailureThreshold, + failureWindowMs: profile.providerFailureWindowMs, + cooldownMs: profile.providerCooldownMs, + }; +} + +function pruneWindow(timestamps: number[], windowMs: number, now: number): number[] { + const cutoff = now - windowMs; + const pruned = timestamps.filter((t) => t >= cutoff); + // Memory bound: the gate only ever needs `failureThreshold` recent samples; + // keep a small multiple so bursts cannot grow the array unbounded. + return pruned.length > 200 ? pruned.slice(-200) : pruned; +} + +function providerWindowCooldownMs(provider: string, entry: CooldownEntry, now: number): number { + const { failureThreshold, failureWindowMs, cooldownMs } = providerWindowProfile(provider); + const inWindow = pruneWindow(entry.failureTimestamps ?? [], failureWindowMs, now); + if (inWindow.length < failureThreshold) return 0; + const elapsed = now - entry.lastFailureAt; + const remaining = cooldownMs - elapsed; + return remaining > 0 ? remaining : 0; } // Global cooldown state: keyed by "provider:connectionId" or "provider" @@ -90,8 +133,21 @@ export function recordProviderCooldown( existing.lastFailureAt = now; existing.failureCount++; existing.retentionMs = Math.max(existing.retentionMs, retentionMs); + if (!connectionId) { + const { failureWindowMs } = providerWindowProfile(provider); + existing.failureTimestamps = pruneWindow( + [...(existing.failureTimestamps ?? []), now], + failureWindowMs, + now + ); + } } else { - cooldownMap.set(key, { lastFailureAt: now, failureCount: 1, retentionMs }); + cooldownMap.set(key, { + lastFailureAt: now, + failureCount: 1, + retentionMs, + ...(connectionId ? {} : { failureTimestamps: [now] }), + }); } startCleanupIfNeeded(); @@ -119,6 +175,11 @@ export function isProviderInCooldown( if (entry.failureCount === 0) return false; const now = Date.now(); + + if (!connectionId) { + return providerWindowCooldownMs(provider, entry, now) > 0; + } + const elapsed = now - entry.lastFailureAt; const minCooldownMs = @@ -151,6 +212,12 @@ export function getRemainingCooldownMs( if (!entry) return 0; const now = Date.now(); + + if (!connectionId) { + if (entry.failureCount === 0) return 0; + return providerWindowCooldownMs(provider, entry, now); + } + const elapsed = now - entry.lastFailureAt; const minCooldownMs = @@ -183,8 +250,9 @@ export function recordProviderSuccess(provider: string, connectionId: string | u const key = cooldownKey(provider, connectionId); const entry = cooldownMap.get(key); if (entry) { - // Reset failure count but keep the entry + // Reset failure count and the provider-level failure window, keep the entry entry.failureCount = 0; + entry.failureTimestamps = []; } } diff --git a/tests/unit/provider-cooldown-window-gate.test.ts b/tests/unit/provider-cooldown-window-gate.test.ts new file mode 100644 index 0000000000..e42a295545 --- /dev/null +++ b/tests/unit/provider-cooldown-window-gate.test.ts @@ -0,0 +1,121 @@ +import { test, beforeEach, mock } from "node:test"; +import assert from "node:assert"; +import { + recordProviderCooldown, + isProviderInCooldown, + getRemainingCooldownMs, + recordProviderSuccess, + clearCooldownState, +} from "../../open-sse/services/providerCooldownTracker.ts"; +import { PROVIDER_PROFILES } from "../../open-sse/config/constants.ts"; +import { DEFAULT_RESILIENCE_SETTINGS } from "../../src/lib/resilience/settings.ts"; + +// Provider-level entries (no connectionId) must honor the PROVIDER_PROFILES +// window gate: `providerFailureThreshold` failures inside +// `providerFailureWindowMs` put the whole provider in a `providerCooldownMs` +// cooldown — below the threshold the provider must NOT be considered cooling. +// These fields shipped in PROVIDER_PROFILES with no runtime consumer (2026-08-31 +// docs audit, P0.1); this suite is the regression guard for wiring them in. +// Connection-level entries keep the pre-existing exponential-backoff behavior. + +const settings = DEFAULT_RESILIENCE_SETTINGS; +// "openai" resolves to the apikey category in the provider registry. +const APIKEY = PROVIDER_PROFILES.apikey; + +beforeEach(() => { + clearCooldownState(); +}); + +test("provider-level: below providerFailureThreshold the provider is NOT in cooldown", () => { + for (let i = 0; i < APIKEY.providerFailureThreshold - 1; i++) { + recordProviderCooldown("openai", undefined, settings); + } + assert.equal( + isProviderInCooldown("openai", undefined, settings), + false, + `expected no provider-level cooldown below the ${APIKEY.providerFailureThreshold}-failure threshold` + ); + assert.equal(getRemainingCooldownMs("openai", undefined, settings), 0); +}); + +test("provider-level: reaching providerFailureThreshold trips a providerCooldownMs cooldown", () => { + for (let i = 0; i < APIKEY.providerFailureThreshold; i++) { + recordProviderCooldown("openai", undefined, settings); + } + assert.equal(isProviderInCooldown("openai", undefined, settings), true); + const remaining = getRemainingCooldownMs("openai", undefined, settings); + assert.ok( + remaining > 0 && remaining <= APIKEY.providerCooldownMs, + `remaining ${remaining}ms should be within (0, providerCooldownMs=${APIKEY.providerCooldownMs}]` + ); + assert.ok( + remaining > APIKEY.providerCooldownMs - 5_000, + `a freshly tripped cooldown should last ~providerCooldownMs (got ${remaining}ms)` + ); +}); + +test("provider-level: failures outside providerFailureWindowMs do not count toward the threshold", () => { + mock.timers.enable({ apis: ["Date"], now: 1_000_000 }); + try { + // threshold-1 failures, then jump past the window before the next one + for (let i = 0; i < APIKEY.providerFailureThreshold - 1; i++) { + recordProviderCooldown("openai", undefined, settings); + } + mock.timers.setTime(1_000_000 + APIKEY.providerFailureWindowMs + 60_000); + recordProviderCooldown("openai", undefined, settings); + assert.equal( + isProviderInCooldown("openai", undefined, settings), + false, + "stale failures beyond the window must not trip the provider cooldown" + ); + } finally { + mock.timers.reset(); + } +}); + +test("provider-level: the cooldown expires providerCooldownMs after the tripping failure", () => { + mock.timers.enable({ apis: ["Date"], now: 2_000_000 }); + try { + for (let i = 0; i < APIKEY.providerFailureThreshold; i++) { + recordProviderCooldown("openai", undefined, settings); + } + assert.equal(isProviderInCooldown("openai", undefined, settings), true); + mock.timers.setTime(2_000_000 + APIKEY.providerCooldownMs + 1_000); + assert.equal( + isProviderInCooldown("openai", undefined, settings), + false, + "provider cooldown must expire after providerCooldownMs" + ); + } finally { + mock.timers.reset(); + } +}); + +test("provider-level: recordProviderSuccess clears the failure window", () => { + for (let i = 0; i < APIKEY.providerFailureThreshold; i++) { + recordProviderCooldown("openai", undefined, settings); + } + assert.equal(isProviderInCooldown("openai", undefined, settings), true); + recordProviderSuccess("openai", undefined); + assert.equal(isProviderInCooldown("openai", undefined, settings), false); + recordProviderCooldown("openai", undefined, settings); + assert.equal( + isProviderInCooldown("openai", undefined, settings), + false, + "one failure after a success must not re-trip the threshold gate" + ); +}); + +test("connection-level entries keep the pre-existing backoff behavior (no window gate)", () => { + recordProviderCooldown("openai", "conn-1", settings); + assert.equal( + isProviderInCooldown("openai", "conn-1", settings), + true, + "a single connection-level failure still starts the legacy min-cooldown backoff" + ); + const remaining = getRemainingCooldownMs("openai", "conn-1", settings); + assert.ok( + remaining > 0 && remaining <= settings.providerCooldown.minRetryCooldownMs, + `connection-level cooldown should follow minRetryCooldownMs (got ${remaining}ms)` + ); +}); diff --git a/tests/unit/services/providerCooldownTracker.test.ts b/tests/unit/services/providerCooldownTracker.test.ts index 8d3f88f35f..e79cf39d24 100644 --- a/tests/unit/services/providerCooldownTracker.test.ts +++ b/tests/unit/services/providerCooldownTracker.test.ts @@ -9,6 +9,7 @@ import { getCooldownEntryCount, cleanupExpiredCooldownEntries, } from "../../../open-sse/services/providerCooldownTracker.ts"; +import { PROVIDER_PROFILES } from "../../../open-sse/config/constants.ts"; import { resolveResilienceSettings, DEFAULT_RESILIENCE_SETTINGS, @@ -183,8 +184,16 @@ test("different connections have independent cooldowns", () => { test("provider-only key works without connectionId", () => { const settings = makeSettings(); + // Provider-level entries honor the PROVIDER_PROFILES window gate: a single + // failure no longer cools the whole provider (2026-08-31 audit, P0.1 wiring). recordProviderCooldown("openai", undefined, settings); + assert.equal(isProviderInCooldown("openai", undefined, settings), false); + // Reaching the profile threshold trips the whole-provider cooldown, still + // independent from any connection-level key. + for (let i = 1; i < PROVIDER_PROFILES.apikey.providerFailureThreshold; i++) { + recordProviderCooldown("openai", undefined, settings); + } assert.ok(isProviderInCooldown("openai", undefined, settings)); assert.equal(isProviderInCooldown("openai", "conn-1", settings), false); });