diff --git a/.env.example b/.env.example index d06f164c34..bcff1ff98b 100644 --- a/.env.example +++ b/.env.example @@ -2101,3 +2101,24 @@ QUOTA_STORE_DRIVER=sqlite # sqlite | redis # BIFROST_API_KEY= # BIFROST_STREAMING_ENABLED=true # BIFROST_TIMEOUT_MS=30000 + +# ───────────────────────────────────────────────────────────────────────────── +# Account rotation config (operator-managed; consumed by open-sse/services/rotationConfig.ts) +# Lets a supervising front-end mirror its rotation rules onto the backend's account-fallback +# engine. All optional; defaults preserve the historical behavior. +# ───────────────────────────────────────────────────────────────────────────── +# OMNIROUTE_ROTATION_ENABLED=true +# OMNIROUTE_ROTATION_RATE_LIMIT_RESET_SECONDS=0 +# OMNIROUTE_ROTATION_DISABLE_TAG_WITHOUT_RESET=true +# OMNIROUTE_ROTATE_ON_429=true +# OMNIROUTE_ROTATE_429_THRESHOLD=1 +# OMNIROUTE_ROTATE_429_WINDOW_SECONDS=120 +# OMNIROUTE_ROTATE_ON_500=true +# OMNIROUTE_ROTATE_500_THRESHOLD=1 +# OMNIROUTE_ROTATE_500_WINDOW_SECONDS=120 +# OMNIROUTE_ROTATE_ON_502=true +# OMNIROUTE_ROTATE_502_THRESHOLD=1 +# OMNIROUTE_ROTATE_502_WINDOW_SECONDS=120 +# OMNIROUTE_ROTATE_ON_400=false +# OMNIROUTE_ROTATE_400_THRESHOLD=1 +# OMNIROUTE_ROTATE_400_WINDOW_SECONDS=120 diff --git a/changelog.d/features/6763-operator-configurable-account-rotation.md b/changelog.d/features/6763-operator-configurable-account-rotation.md new file mode 100644 index 0000000000..71ff263a3a --- /dev/null +++ b/changelog.d/features/6763-operator-configurable-account-rotation.md @@ -0,0 +1 @@ +- **feat(resilience):** operator-configurable account rotation policy — a new `rotationConfig` layer lets operators tune how connections rotate on failure, wired into `accountFallback` (#6763 — thanks @artickc). diff --git a/docs/reference/ENVIRONMENT.md b/docs/reference/ENVIRONMENT.md index f4ba64fdbf..927cb93973 100644 --- a/docs/reference/ENVIRONMENT.md +++ b/docs/reference/ENVIRONMENT.md @@ -1093,6 +1093,21 @@ Provider quota endpoints, network tunnels (Tailscale, Ngrok, MITM debug proxy), | `QDRANT_EMBEDDING_MODEL` | `text-embedding-3-small` | _(opt-in cluster profile)_ | Default embedding model name recorded in the Qdrant collection metadata. Actual embeddings are generated by whatever provider the `embeddingModel` field in OmniRoute's settings points to. | | `QDRANT_VECTOR_SIZE` | `1536` | _(opt-in cluster profile)_ | Embedding vector dimension. Must match the model you embed with (text-embedding-3-small → 1536; ada-002 → 1536; nomic-embed-text → 768). | | `QDRANT_HNSW_EF_CONSTRUCT` | `128` | _(opt-in cluster profile)_ | HNSW index construction-time accuracy. Higher = slower build, faster search. | +| `OMNIROUTE_ROTATION_ENABLED` | `true` | `open-sse/services/rotationConfig.ts` | Master switch for operator-configurable account rotation. When `false`, none of the `OMNIROUTE_ROTATE_*` classes below trigger account fallback (the master-off state also blocks the default-enabled 429/500/502 classes). Lets a supervising front-end (e.g. the VibeProxy desktop app) mirror its own rotation rules onto the backend's account-fallback engine. | +| `OMNIROUTE_ROTATION_RATE_LIMIT_RESET_SECONDS` | `0` | `open-sse/services/rotationConfig.ts` | Cooldown (seconds) applied to a rate-limited account when the upstream gives no explicit reset hint. `0` = use the engine default cooldown instead of a fixed override. | +| `OMNIROUTE_ROTATION_DISABLE_TAG_WITHOUT_RESET` | `true` | `open-sse/services/rotationConfig.ts` | Mirror of the front-end "don't tag as rate-limited without a reset time" preference. | +| `OMNIROUTE_ROTATE_ON_429` | `true` | `open-sse/services/rotationConfig.ts` | Per-status fallback enable for `429` errors. When `false` (and `OMNIROUTE_ROTATION_ENABLED=true`), a `429` no longer triggers account rotation and is returned to the client instead. | +| `OMNIROUTE_ROTATE_429_THRESHOLD` | `1` | `open-sse/services/rotationConfig.ts` | Number of `429` errors within `OMNIROUTE_ROTATE_429_WINDOW_SECONDS` required before the account is rotated. `1` (default) rotates immediately, preserving historical behavior. | +| `OMNIROUTE_ROTATE_429_WINDOW_SECONDS` | `120` | `open-sse/services/rotationConfig.ts` | Sliding window (seconds) over which `429` errors are counted toward `OMNIROUTE_ROTATE_429_THRESHOLD`. | +| `OMNIROUTE_ROTATE_ON_500` | `true` | `open-sse/services/rotationConfig.ts` | Per-status fallback enable for `5xx` server errors (excluding `502`, which has its own class). When `false`, these errors no longer trigger account rotation. | +| `OMNIROUTE_ROTATE_500_THRESHOLD` | `1` | `open-sse/services/rotationConfig.ts` | Number of `5xx` errors within `OMNIROUTE_ROTATE_500_WINDOW_SECONDS` required before the account is rotated. `1` (default) rotates immediately. | +| `OMNIROUTE_ROTATE_500_WINDOW_SECONDS` | `120` | `open-sse/services/rotationConfig.ts` | Sliding window (seconds) over which `5xx` errors are counted toward `OMNIROUTE_ROTATE_500_THRESHOLD`. | +| `OMNIROUTE_ROTATE_ON_502` | `true` | `open-sse/services/rotationConfig.ts` | Per-status fallback enable for `502` (bad gateway) errors. When `false`, `502`s no longer trigger account rotation. | +| `OMNIROUTE_ROTATE_502_THRESHOLD` | `1` | `open-sse/services/rotationConfig.ts` | Number of `502` errors within `OMNIROUTE_ROTATE_502_WINDOW_SECONDS` required before the account is rotated. `1` (default) rotates immediately. | +| `OMNIROUTE_ROTATE_502_WINDOW_SECONDS` | `120` | `open-sse/services/rotationConfig.ts` | Sliding window (seconds) over which `502` errors are counted toward `OMNIROUTE_ROTATE_502_THRESHOLD`. | +| `OMNIROUTE_ROTATE_ON_400` | `false` | `open-sse/services/rotationConfig.ts` | Opt-in (default OFF): when `true`, a plain `400` (bad request) also triggers account rotation. This is additive only — it never blocks the engine's existing behavior where a `400` carrying rate-limit/quota text still falls over regardless of this flag. | +| `OMNIROUTE_ROTATE_400_THRESHOLD` | `1` | `open-sse/services/rotationConfig.ts` | Number of `400` errors within `OMNIROUTE_ROTATE_400_WINDOW_SECONDS` required before the account is rotated (only consulted when `OMNIROUTE_ROTATE_ON_400=true`). | +| `OMNIROUTE_ROTATE_400_WINDOW_SECONDS` | `120` | `open-sse/services/rotationConfig.ts` | Sliding window (seconds) over which `400` errors are counted toward `OMNIROUTE_ROTATE_400_THRESHOLD`. | --- diff --git a/open-sse/config/errorConfig.ts b/open-sse/config/errorConfig.ts index e685fe60a8..f5ac29fde3 100644 --- a/open-sse/config/errorConfig.ts +++ b/open-sse/config/errorConfig.ts @@ -199,3 +199,41 @@ export function matchErrorRuleByStatus(statusCode: number): ErrorRule | null { export function findMatchingErrorRule(statusCode: number, message: unknown): ErrorRule | null { return matchErrorRuleByText(message) || matchErrorRuleByStatus(statusCode); } + +export interface ServiceSupervisorCooldown { + shouldFallback: true; + cooldownMs: number; + baseCooldownMs: number; + newBackoffLevel: 0; + reason: string; + skipProviderBreaker: true; +} + +/** + * G-02: detect embedded service supervisor failures (X-Omni-Fallback-Hint: connection_cooldown). + * These are NOT upstream AI provider failures — they are local supervisor state changes. Returns + * a short 5s connection-cooldown decision (no provider circuit-breaker trip), or null when the + * status/header don't match. + */ +export function serviceSupervisorCooldown( + status: number, + headers: Headers | Record | null +): ServiceSupervisorCooldown | null { + if (status !== 503 || !headers) return null; + const hintValue = + typeof (headers as Headers).get === "function" + ? (headers as Headers).get("x-omni-fallback-hint") + : (headers as Record)["x-omni-fallback-hint"] || + (headers as Record)["X-Omni-Fallback-Hint"]; + if (typeof hintValue !== "string" || hintValue.toLowerCase() !== "connection_cooldown") { + return null; + } + return { + shouldFallback: true, + cooldownMs: 5_000, + baseCooldownMs: 5_000, + newBackoffLevel: 0, + reason: "service_not_running", + skipProviderBreaker: true, + }; +} diff --git a/open-sse/services/accountFallback.ts b/open-sse/services/accountFallback.ts index 92e7fb65b9..5a1e1888de 100644 --- a/open-sse/services/accountFallback.ts +++ b/open-sse/services/accountFallback.ts @@ -11,8 +11,10 @@ import { findMatchingErrorRule, matchErrorRuleByText, matchErrorRuleByStatus, + serviceSupervisorCooldown, } from "../config/errorConfig.ts"; import { getProviderErrorRuleMatch } from "../config/providerErrorRules.ts"; +import * as rot from "./rotationConfig.ts"; import { getPassthroughProviders, getProviderCategory } from "../config/providerRegistry.ts"; import { DEFAULT_RESILIENCE_SETTINGS, @@ -1267,7 +1269,8 @@ export function checkFallbackError( provider: string | null = null, headers: Headers | Record | null = null, profileOverride: ProviderProfile | null = null, - structuredError?: { code?: string | null; type?: string | null } | null + structuredError?: { code?: string | null; type?: string | null } | null, + rotation?: { account?: unknown } | null ): { shouldFallback: boolean; cooldownMs: number; @@ -1287,27 +1290,10 @@ export function checkFallbackError( * caller can persist an explicit reset window instead of the engine's scaled cooldown. */ configuredCooldownMs?: number; } { - // G-02: detect embedded service supervisor failures (X-Omni-Fallback-Hint: connection_cooldown). - // These are NOT upstream AI provider failures — they are local supervisor state changes. - // Apply a short 5s connection cooldown without tripping the provider circuit breaker. - if (status === 503 && headers) { - const hintValue = - typeof (headers as Headers).get === "function" - ? (headers as Headers).get("x-omni-fallback-hint") - : (headers as Record)["x-omni-fallback-hint"] || - (headers as Record)["X-Omni-Fallback-Hint"]; - if (typeof hintValue === "string" && hintValue.toLowerCase() === "connection_cooldown") { - return { - shouldFallback: true, - cooldownMs: 5_000, - baseCooldownMs: 5_000, - newBackoffLevel: 0, - reason: "service_not_running", - skipProviderBreaker: true, - }; - } - } - + const svc = serviceSupervisorCooldown(status, headers); + if (svc) return svc; + const rg = rot.gateFor(status, rotation?.account); + if (rg) return rg; const errorStr = (errorText || "").toString(); const profile = profileOverride ?? (provider ? getProviderProfile(provider) : null); const maxBackoffSteps = profile?.maxBackoffSteps ?? BACKOFF_CONFIG.maxLevel; @@ -1396,6 +1382,8 @@ export function checkFallbackError( }; } + const ro = rot.overrideFor(reason, rotation?.account); + if (ro) return ro; const scaled = getScaledBaseCooldown(reason, backoffLevel); return { shouldFallback: true, @@ -1761,13 +1749,15 @@ export function resetAccountState( export function applyErrorState( account: T, status: number, - errorText: string | null, - provider: string | null = null + errText: string | null, + prov: string | null = null ): T | AccountState { if (!account) return account; - const backoffLevel = account.backoffLevel || 0; - const fallbackDecision = checkFallbackError(status, errorText, backoffLevel, null, provider); + const lvl = account.backoffLevel || 0; + const fallbackDecision = checkFallbackError(status, errText, lvl, null, prov, null, null, null, { + account, + }); const { cooldownMs, reason } = fallbackDecision; const newBackoffLevel = "newBackoffLevel" in fallbackDecision ? fallbackDecision.newBackoffLevel : undefined; @@ -1789,8 +1779,8 @@ export function applyErrorState( const nextState: T | AccountState = { ...account, rateLimitedUntil: effectiveCooldownMs > 0 ? getUnavailableUntil(effectiveCooldownMs) : null, - backoffLevel: newBackoffLevel ?? backoffLevel, - lastError: { status, message: errorText, timestamp: new Date().toISOString(), reason }, + backoffLevel: newBackoffLevel ?? lvl, + lastError: { status, message: errText, timestamp: new Date().toISOString(), reason }, status: "error", }; diff --git a/open-sse/services/rotationConfig.ts b/open-sse/services/rotationConfig.ts new file mode 100644 index 0000000000..42c47a92e9 --- /dev/null +++ b/open-sse/services/rotationConfig.ts @@ -0,0 +1,390 @@ +/** + * Runtime rotation configuration. + * + * OmniRoute's account-fallback engine (accountFallback.ts) historically rotated accounts using + * only hardcoded constants (COOLDOWN_MS / BACKOFF_CONFIG / ERROR_RULES): every retryable error + * cooled the account down immediately, on a fixed exponential backoff, with no operator control. + * + * A front-end/orchestrator (e.g. the VibeProxy desktop app) that manages the SAME set of accounts + * needs the backend to rotate according to the operator's own rules. This module exposes those + * rules as a runtime config, sourced from environment variables (so a supervising process can set + * them per launch) with an optional per-connection override (read from a connection's + * `providerSpecificData.rotationOverrides`). + * + * Config surface (all optional — defaults preserve the pre-existing engine behavior): + * - master enable OMNIROUTE_ROTATION_ENABLED (default true) + * - rate-limit reset/cooldown seconds OMNIROUTE_ROTATION_RATE_LIMIT_RESET_SECONDS (0 => engine default) + * - per-status fallback enable OMNIROUTE_ROTATE_ON_{429,500,502,400} (429/500/502 default true, 400 default false) + * - per-status threshold (errors in window) OMNIROUTE_ROTATE_{status}_THRESHOLD (default 1 => immediate, current behavior) + * - per-status window seconds OMNIROUTE_ROTATE_{status}_WINDOW_SECONDS (default 120) + * + * Everything here is a pure function or a small in-memory sliding-window counter — no DB / IO on + * the hot path — so it is cheap to consult per request and trivially unit-testable. + */ + +import { RateLimitReason } from "../config/constants.ts"; +import { COOLDOWN_MS } from "../config/errorConfig.ts"; + +export interface RotationErrorClassConfig { + enabled: boolean; + /** Number of errors of this class (within the window) required before an account is rotated. */ + threshold: number; + /** Sliding window (ms) over which errors are counted. */ + windowMs: number; +} + +export interface RotationConfig { + /** Master switch. When false, none of the configurable error classes trigger account fallback. */ + enabled: boolean; + /** Cooldown (ms) applied to a rate-limited account when the upstream gives no explicit hint. 0 => engine default. */ + rateLimitResetMs: number; + /** Mirror of the front-end "don't tag as rate-limited without a reset time" preference. */ + disableTagWithoutReset: boolean; + rateLimit429: RotationErrorClassConfig; + serverError500: RotationErrorClassConfig; + badGateway502: RotationErrorClassConfig; + badRequest400: RotationErrorClassConfig; +} + +const GLOBAL_KEY = "__omniroute_rotation_config__"; +const DEFAULT_WINDOW_MS = 120_000; + +function envBool(name: string, dflt: boolean): boolean { + const raw = process.env[name]; + if (raw === undefined || raw === null || raw === "") return dflt; + const v = raw.trim().toLowerCase(); + if (v === "true" || v === "1" || v === "yes" || v === "on") return true; + if (v === "false" || v === "0" || v === "no" || v === "off") return false; + return dflt; +} + +function envInt(name: string, dflt: number, min = 0): number { + const raw = process.env[name]; + if (raw === undefined || raw === null || raw === "") return dflt; + const n = Number.parseInt(raw.trim(), 10); + if (!Number.isFinite(n)) return dflt; + return Math.max(min, n); +} + +function buildClass( + enableEnv: string, + thresholdEnv: string, + windowEnv: string, + enableDefault: boolean +): RotationErrorClassConfig { + return { + enabled: envBool(enableEnv, enableDefault), + threshold: envInt(thresholdEnv, 1, 1), + windowMs: envInt(windowEnv, DEFAULT_WINDOW_MS / 1000, 1) * 1000, + }; +} + +function buildFromEnv(): RotationConfig { + return { + enabled: envBool("OMNIROUTE_ROTATION_ENABLED", true), + rateLimitResetMs: envInt("OMNIROUTE_ROTATION_RATE_LIMIT_RESET_SECONDS", 0, 0) * 1000, + disableTagWithoutReset: envBool("OMNIROUTE_ROTATION_DISABLE_TAG_WITHOUT_RESET", true), + rateLimit429: buildClass( + "OMNIROUTE_ROTATE_ON_429", + "OMNIROUTE_ROTATE_429_THRESHOLD", + "OMNIROUTE_ROTATE_429_WINDOW_SECONDS", + true + ), + serverError500: buildClass( + "OMNIROUTE_ROTATE_ON_500", + "OMNIROUTE_ROTATE_500_THRESHOLD", + "OMNIROUTE_ROTATE_500_WINDOW_SECONDS", + true + ), + badGateway502: buildClass( + "OMNIROUTE_ROTATE_ON_502", + "OMNIROUTE_ROTATE_502_THRESHOLD", + "OMNIROUTE_ROTATE_502_WINDOW_SECONDS", + true + ), + badRequest400: buildClass( + "OMNIROUTE_ROTATE_ON_400", + "OMNIROUTE_ROTATE_400_THRESHOLD", + "OMNIROUTE_ROTATE_400_WINDOW_SECONDS", + false + ), + }; +} + +/** + * The global (env-derived) rotation config, parsed once and cached on `globalThis` so the Next.js + * app-route module graph and the startup graph share one instance (same pattern as the other + * runtime-config singletons in this codebase). + */ +export function getGlobalRotationConfig(): RotationConfig { + const g = globalThis as Record; + let cfg = g[GLOBAL_KEY] as RotationConfig | undefined; + if (!cfg) { + cfg = buildFromEnv(); + g[GLOBAL_KEY] = cfg; + } + return cfg; +} + +/** Test/reset hook: clears the cached global config so the next read re-parses env. */ +export function resetGlobalRotationConfigForTest(): void { + const g = globalThis as Record; + delete g[GLOBAL_KEY]; + clearRotationErrorCounters(); +} + +function coerceBool(v: unknown, dflt: boolean): boolean { + if (typeof v === "boolean") return v; + if (typeof v === "string") return envBoolFromString(v, dflt); + return dflt; +} + +function envBoolFromString(v: string, dflt: boolean): boolean { + const s = v.trim().toLowerCase(); + if (s === "true" || s === "1" || s === "yes" || s === "on") return true; + if (s === "false" || s === "0" || s === "no" || s === "off") return false; + return dflt; +} + +function coerceInt(v: unknown, dflt: number, min = 0): number { + const n = typeof v === "number" ? v : typeof v === "string" ? Number.parseInt(v, 10) : NaN; + if (!Number.isFinite(n)) return dflt; + return Math.max(min, Math.floor(n)); +} + +/** + * Merges a connection's per-connection overrides (from + * `providerSpecificData.rotationOverrides`) over the global env config. Any absent override key + * inherits the global value. Returns the global config unchanged when there are no overrides. + */ +export function resolveRotationConfig(overrides?: Record | null): RotationConfig { + const base = getGlobalRotationConfig(); + if (!overrides || typeof overrides !== "object") return base; + + const cls = ( + src: RotationErrorClassConfig, + enableKey: string, + thrKey: string, + winKey: string + ): RotationErrorClassConfig => ({ + enabled: enableKey in overrides ? coerceBool(overrides[enableKey], src.enabled) : src.enabled, + threshold: thrKey in overrides ? coerceInt(overrides[thrKey], src.threshold, 1) : src.threshold, + windowMs: + winKey in overrides ? coerceInt(overrides[winKey], src.windowMs / 1000, 1) * 1000 : src.windowMs, + }); + + return { + enabled: base.enabled, + rateLimitResetMs: + "rateLimitResetSeconds" in overrides + ? coerceInt(overrides.rateLimitResetSeconds, base.rateLimitResetMs / 1000, 0) * 1000 + : base.rateLimitResetMs, + disableTagWithoutReset: base.disableTagWithoutReset, + rateLimit429: cls(base.rateLimit429, "rotateOn429", "error429Threshold", "error429WindowSeconds"), + serverError500: cls(base.serverError500, "rotateOn500", "error500Threshold", "error500WindowSeconds"), + badGateway502: cls(base.badGateway502, "rotateOn502", "error502Threshold", "error502WindowSeconds"), + badRequest400: cls(base.badRequest400, "rotateOn400", "error400Threshold", "error400WindowSeconds"), + }; +} + +/** Maps an HTTP status to its configured error class (or null for statuses this config doesn't gate). */ +export function classForStatus(status: number, cfg: RotationConfig): RotationErrorClassConfig | null { + if (status === 429) return cfg.rateLimit429; + if (status === 502) return cfg.badGateway502; + if (status >= 500 && status < 600) return cfg.serverError500; + if (status === 400) return cfg.badRequest400; + return null; // 401/402/403/404/… are not gated by this config +} + +/** + * True when the operator config should BLOCK account fallback for this status. + * + * This is RESTRICTIVE and applies only to the default-enabled classes (429 / 502 / other 5xx): + * when the operator disables one, fallback for it is blocked (the error returns to the client + * instead of rotating). 400 is NEVER restrictively blocked here — it is handled additively by + * {@link shouldForceFallbackFor400}, so the engine's existing 400 behavior (a 400 carrying + * rate-limit/quota text still falls over; a plain malformed 400 does not) is fully preserved. + * Statuses this config does not gate (401/403/404/…) are never blocked. + */ +export function isFallbackBlockedForStatus(status: number, cfg: RotationConfig): boolean { + const c = classForStatus(status, cfg); + if (c === null) return false; // ungated statuses: engine default + if (c === cfg.badRequest400) return false; // 400 is additive, never restrictively blocked + if (!cfg.enabled) return true; // master off blocks the gated 429/500/502 classes + return !c.enabled; // per-class disable +} + +/** True when the operator opted IN to rotating on a 400 (bad request) — off by default. */ +export function shouldForceFallbackFor400(status: number, cfg: RotationConfig): boolean { + return status === 400 && cfg.enabled && cfg.badRequest400.enabled; +} + +/** Rate-limit cooldown override (ms) or null to use the engine default. */ +export function rateLimitCooldownOverrideMs(cfg: RotationConfig): number | null { + return cfg.rateLimitResetMs > 0 ? cfg.rateLimitResetMs : null; +} + +// ── Sliding-window per-key error counter (for threshold-based fallback) ────────────────────── + +const COUNTER_KEY = "__omniroute_rotation_counters__"; + +function counters(): Map { + const g = globalThis as Record; + let m = g[COUNTER_KEY] as Map | undefined; + if (!m) { + m = new Map(); + g[COUNTER_KEY] = m; + } + return m; +} + +export function clearRotationErrorCounters(): void { + counters().clear(); +} + +/** + * Records an error for (key, status) and returns true when the number of errors within the class + * window reaches the configured threshold (i.e. the account should now be rotated). When the + * threshold is 1 (default) this returns true on the first error — preserving the engine's + * historical "rotate immediately" behavior. `nowMs` is injectable for tests. + */ +export function recordErrorAndCheckThreshold( + key: string, + status: number, + cfg: RotationConfig, + nowMs: number = Date.now() +): boolean { + const cls = classForStatus(status, cfg); + if (cls === null) return true; // not gated => defer to engine (treat as immediate) + if (cls.threshold <= 1) return true; // immediate rotation (historical behavior) + + const bucketKey = `${key}::${status}`; + const list = counters().get(bucketKey) ?? []; + const windowStart = nowMs - cls.windowMs; + const pruned = list.filter((ts) => ts >= windowStart); + pruned.push(nowMs); + counters().set(bucketKey, pruned); + + if (pruned.length >= cls.threshold) { + counters().delete(bucketKey); // reset after reaching the threshold + return true; + } + return false; +} + +// ── accountFallback.ts integration helpers ────────────────────────────────────────────────── +// These encapsulate the "runtime rotation config" glue that `checkFallbackError` / +// `applyErrorState` (open-sse/services/accountFallback.ts, a size-frozen file) consult before +// falling back to their own hardcoded heuristics. Keeping the glue here (rather than inline in +// accountFallback.ts) keeps that file's line budget stable as this config surface grows. + +export interface RotationGateDecision { + shouldFallback: boolean; + cooldownMs: number; + baseCooldownMs?: number; + newBackoffLevel?: number; + reason?: string; +} + +/** + * Evaluates the runtime rotation config gate for a given status BEFORE the engine's own error + * classification runs. Returns a decision that should short-circuit `checkFallbackError` + * (block fallback, hold pending threshold/window, or force-fallback an opted-in 400), or `null` + * when the engine should proceed with its normal heuristics. + */ +export function evaluateRotationGate( + status: number, + rotationCfg: RotationConfig, + rotationKey?: string | null +): RotationGateDecision | null { + if (isFallbackBlockedForStatus(status, rotationCfg)) { + return { shouldFallback: false, cooldownMs: 0, reason: RateLimitReason.UNKNOWN }; + } + if ( + rotationKey && + classForStatus(status, rotationCfg) && + !recordErrorAndCheckThreshold(rotationKey, status, rotationCfg) + ) { + return { shouldFallback: false, cooldownMs: 0, reason: RateLimitReason.UNKNOWN }; + } + if (shouldForceFallbackFor400(status, rotationCfg)) { + const overrideMs = rateLimitCooldownOverrideMs(rotationCfg); + const cooldownMs = overrideMs ?? COOLDOWN_MS.rateLimit; + return { + shouldFallback: true, + cooldownMs, + baseCooldownMs: cooldownMs, + newBackoffLevel: 0, + reason: RateLimitReason.RATE_LIMIT_EXCEEDED, + }; + } + return null; +} + +export interface RotationRateLimitFallback { + shouldFallback: true; + cooldownMs: number; + baseCooldownMs: number; + newBackoffLevel: 0; + usedUpstreamRetryHint: false; + reason: string; +} + +/** + * Operator-configured rate-limit cooldown override (no upstream retry hint available). Applies + * only to the rate-limit reason so 5xx / capacity errors keep their scaled exponential backoff. + * Returns `null` when the reason isn't rate-limit or no override is configured, in which case + * the caller should fall through to its own scaled-backoff calculation. + */ +export function rotationRateLimitFallback( + reason: string, + rotationCfg: RotationConfig +): RotationRateLimitFallback | null { + if (reason !== RateLimitReason.RATE_LIMIT_EXCEEDED) return null; + const overrideMs = rateLimitCooldownOverrideMs(rotationCfg); + if (overrideMs === null) return null; + return { + shouldFallback: true, + cooldownMs: overrideMs, + baseCooldownMs: overrideMs, + newBackoffLevel: 0, + usedUpstreamRetryHint: false, + reason, + }; +} + +/** Combines extractRotationContext + resolveRotationConfig + evaluateRotationGate for an account. */ +export function gateFor(status: number, account?: unknown): RotationGateDecision | null { + const { rotationOverrides, rotationKey } = extractRotationContext(account); + return evaluateRotationGate(status, resolveRotationConfig(rotationOverrides), rotationKey); +} + +/** Combines extractRotationContext + resolveRotationConfig + rotationRateLimitFallback for an account. */ +export function overrideFor(reason: string, account?: unknown): RotationRateLimitFallback | null { + const { rotationOverrides } = extractRotationContext(account); + return rotationRateLimitFallback(reason, resolveRotationConfig(rotationOverrides)); +} + +/** + * Extracts a connection's per-connection rotation overrides and rotation key from its account + * state (`providerSpecificData.rotationOverrides` and `id`). Both are optional — absent => + * global env config / count-immediately. `account` is typed `unknown` here because callers pass + * a generic `AccountState`-shaped value; this only does structural checks, no behavior change. + */ +export function extractRotationContext(account: unknown): { + rotationOverrides: Record | null; + rotationKey: string | null; +} { + const rec = account && typeof account === "object" ? (account as Record) : null; + const psd = rec ? rec["providerSpecificData"] : undefined; + const rotationOverrides = + psd && + typeof psd === "object" && + (psd as Record).rotationOverrides && + typeof (psd as Record).rotationOverrides === "object" + ? ((psd as Record).rotationOverrides as Record) + : null; + const id = rec ? rec["id"] : undefined; + const rotationKey = typeof id === "string" && id.length > 0 ? id : null; + return { rotationOverrides, rotationKey }; +} diff --git a/stryker.conf.json b/stryker.conf.json index 949fb9b5ed..d443617d97 100644 --- a/stryker.conf.json +++ b/stryker.conf.json @@ -215,6 +215,7 @@ "tests/unit/rate-limit-manager.test.ts", "tests/unit/rate-limit-queue-timeout-lockout.test.ts", "tests/unit/responses-handler.test.ts", + "tests/unit/rotation-config-omniroute.test.ts", "tests/unit/route-explainability.test.ts", "tests/unit/route-guard-middleware-local-only.test.ts", "tests/unit/route-guard-plugins-local-only.test.ts", diff --git a/tests/unit/rotation-config-omniroute.test.ts b/tests/unit/rotation-config-omniroute.test.ts new file mode 100644 index 0000000000..0c172b758d --- /dev/null +++ b/tests/unit/rotation-config-omniroute.test.ts @@ -0,0 +1,196 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +// Runtime rotation config (operator-managed, e.g. VibeProxy): env-driven per-status fallback +// enable, rate-limit cooldown override, per-connection overrides, and a sliding-window +// threshold counter — with defaults that preserve the engine's historical behavior. + +const rc = await import("../../open-sse/services/rotationConfig.ts"); +const { checkFallbackError } = await import("../../open-sse/services/accountFallback.ts"); + +const ROTATION_ENV_KEYS = [ + "OMNIROUTE_ROTATION_ENABLED", + "OMNIROUTE_ROTATION_RATE_LIMIT_RESET_SECONDS", + "OMNIROUTE_ROTATION_DISABLE_TAG_WITHOUT_RESET", + "OMNIROUTE_ROTATE_ON_429", + "OMNIROUTE_ROTATE_429_THRESHOLD", + "OMNIROUTE_ROTATE_429_WINDOW_SECONDS", + "OMNIROUTE_ROTATE_ON_500", + "OMNIROUTE_ROTATE_500_THRESHOLD", + "OMNIROUTE_ROTATE_500_WINDOW_SECONDS", + "OMNIROUTE_ROTATE_ON_502", + "OMNIROUTE_ROTATE_502_THRESHOLD", + "OMNIROUTE_ROTATE_502_WINDOW_SECONDS", + "OMNIROUTE_ROTATE_ON_400", + "OMNIROUTE_ROTATE_400_THRESHOLD", + "OMNIROUTE_ROTATE_400_WINDOW_SECONDS", +]; + +function clearEnv() { + for (const k of ROTATION_ENV_KEYS) delete process.env[k]; + rc.resetGlobalRotationConfigForTest(); +} + +test("defaults preserve historical behavior (no env set)", () => { + clearEnv(); + const cfg = rc.getGlobalRotationConfig(); + assert.equal(cfg.enabled, true); + assert.equal(cfg.rateLimit429.enabled, true); + assert.equal(cfg.serverError500.enabled, true); + assert.equal(cfg.badGateway502.enabled, true); + assert.equal(cfg.badRequest400.enabled, false); // 400 is opt-in + assert.equal(cfg.rateLimit429.threshold, 1); // immediate + assert.equal(rc.rateLimitCooldownOverrideMs(cfg), null); // no override => engine default + clearEnv(); +}); + +test("env parse: disable 502, enable 400, set reset seconds + thresholds", () => { + clearEnv(); + process.env.OMNIROUTE_ROTATE_ON_502 = "false"; + process.env.OMNIROUTE_ROTATE_ON_400 = "true"; + process.env.OMNIROUTE_ROTATION_RATE_LIMIT_RESET_SECONDS = "10"; + process.env.OMNIROUTE_ROTATE_429_THRESHOLD = "6"; + process.env.OMNIROUTE_ROTATE_429_WINDOW_SECONDS = "120"; + rc.resetGlobalRotationConfigForTest(); + + const cfg = rc.getGlobalRotationConfig(); + assert.equal(cfg.badGateway502.enabled, false); + assert.equal(cfg.badRequest400.enabled, true); + assert.equal(rc.rateLimitCooldownOverrideMs(cfg), 10_000); + assert.equal(cfg.rateLimit429.threshold, 6); + assert.equal(cfg.rateLimit429.windowMs, 120_000); + clearEnv(); +}); + +test("isFallbackBlockedForStatus: restrictive for 429/500/502, never for 400/ungated", () => { + clearEnv(); + process.env.OMNIROUTE_ROTATE_ON_502 = "false"; + rc.resetGlobalRotationConfigForTest(); + const cfg = rc.getGlobalRotationConfig(); + + assert.equal(rc.isFallbackBlockedForStatus(502, cfg), true); // disabled => blocked + assert.equal(rc.isFallbackBlockedForStatus(429, cfg), false); // enabled default + assert.equal(rc.isFallbackBlockedForStatus(500, cfg), false); + assert.equal(rc.isFallbackBlockedForStatus(400, cfg), false); // additive, never blocked + assert.equal(rc.isFallbackBlockedForStatus(401, cfg), false); // ungated + clearEnv(); +}); + +test("shouldForceFallbackFor400 only when opted in", () => { + clearEnv(); + let cfg = rc.getGlobalRotationConfig(); + assert.equal(rc.shouldForceFallbackFor400(400, cfg), false); // default off + + process.env.OMNIROUTE_ROTATE_ON_400 = "true"; + rc.resetGlobalRotationConfigForTest(); + cfg = rc.getGlobalRotationConfig(); + assert.equal(rc.shouldForceFallbackFor400(400, cfg), true); + assert.equal(rc.shouldForceFallbackFor400(429, cfg), false); // not a 400 + clearEnv(); +}); + +test("resolveRotationConfig merges per-connection overrides over global", () => { + clearEnv(); + process.env.OMNIROUTE_ROTATE_429_THRESHOLD = "6"; + rc.resetGlobalRotationConfigForTest(); + + const merged = rc.resolveRotationConfig({ + rotateOn429: false, + error429Threshold: 3, + rateLimitResetSeconds: 20, + }); + assert.equal(merged.rateLimit429.enabled, false); // overridden + assert.equal(merged.rateLimit429.threshold, 3); // overridden (not the global 6) + assert.equal(merged.rateLimitResetMs, 20_000); // overridden + assert.equal(merged.serverError500.enabled, true); // inherited + clearEnv(); +}); + +test("recordErrorAndCheckThreshold: sliding window reaches threshold then resets", () => { + clearEnv(); + process.env.OMNIROUTE_ROTATE_429_THRESHOLD = "3"; + process.env.OMNIROUTE_ROTATE_429_WINDOW_SECONDS = "60"; + rc.resetGlobalRotationConfigForTest(); + const cfg = rc.getGlobalRotationConfig(); + + const t0 = 1_000_000; + assert.equal(rc.recordErrorAndCheckThreshold("conn-a", 429, cfg, t0), false); // 1 + assert.equal(rc.recordErrorAndCheckThreshold("conn-a", 429, cfg, t0 + 1000), false); // 2 + assert.equal(rc.recordErrorAndCheckThreshold("conn-a", 429, cfg, t0 + 2000), true); // 3 => rotate + // Counter reset after threshold — next error starts a fresh window. + assert.equal(rc.recordErrorAndCheckThreshold("conn-a", 429, cfg, t0 + 3000), false); + clearEnv(); +}); + +test("recordErrorAndCheckThreshold: errors outside the window do not accumulate", () => { + clearEnv(); + process.env.OMNIROUTE_ROTATE_500_THRESHOLD = "3"; + process.env.OMNIROUTE_ROTATE_500_WINDOW_SECONDS = "60"; + rc.resetGlobalRotationConfigForTest(); + const cfg = rc.getGlobalRotationConfig(); + + const t0 = 2_000_000; + assert.equal(rc.recordErrorAndCheckThreshold("conn-b", 500, cfg, t0), false); // 1 + // 2 minutes later — the first error has fallen out of the 60s window. + assert.equal(rc.recordErrorAndCheckThreshold("conn-b", 500, cfg, t0 + 120_000), false); // still 1 + clearEnv(); +}); + +test("recordErrorAndCheckThreshold: threshold 1 (default) => immediate rotate", () => { + clearEnv(); + const cfg = rc.getGlobalRotationConfig(); + assert.equal(rc.recordErrorAndCheckThreshold("conn-c", 429, cfg, 1), true); + clearEnv(); +}); + +test("integration: OMNIROUTE_ROTATE_ON_502=false blocks 502 fallback in checkFallbackError", () => { + clearEnv(); + process.env.OMNIROUTE_ROTATE_ON_502 = "false"; + rc.resetGlobalRotationConfigForTest(); + + const blocked = checkFallbackError(502, "bad gateway", 0, null, "openai"); + assert.equal(blocked.shouldFallback, false); + clearEnv(); + + // With default config, a 502 still falls over (historical behavior). + const dflt = checkFallbackError(502, "bad gateway", 0, null, "openai"); + assert.equal(dflt.shouldFallback, true); + clearEnv(); +}); + +test("integration: 400 with rate-limit text still falls over under default config (#4976 preserved)", () => { + clearEnv(); + const res = checkFallbackError( + 400, + "Detected high-frequency non-compliant requests from you.", + 0, + null, + "mimocode" + ); + assert.equal(res.shouldFallback, true); + clearEnv(); +}); + +test("integration: plain 400 does not fall over by default, but does when opted in", () => { + clearEnv(); + const dflt = checkFallbackError(400, "Invalid JSON: unexpected token", 0, null, "openai"); + assert.equal(dflt.shouldFallback, false); + clearEnv(); + + process.env.OMNIROUTE_ROTATE_ON_400 = "true"; + rc.resetGlobalRotationConfigForTest(); + const opted = checkFallbackError(400, "Invalid JSON: unexpected token", 0, null, "openai"); + assert.equal(opted.shouldFallback, true); + clearEnv(); +}); + +test("integration: rate-limit cooldown override applies to a 429 with no upstream hint", () => { + clearEnv(); + process.env.OMNIROUTE_ROTATION_RATE_LIMIT_RESET_SECONDS = "10"; + rc.resetGlobalRotationConfigForTest(); + + const res = checkFallbackError(429, "rate limit", 0, null, "openai"); + assert.equal(res.shouldFallback, true); + assert.equal(res.cooldownMs, 10_000); + clearEnv(); +});