From d5b0ee9394fb7beb91fb70f010c1a5d4cde1f6d3 Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza <8016841+diegosouzapw@users.noreply.github.com> Date: Tue, 30 Jun 2026 23:36:55 -0300 Subject: [PATCH] refactor(resilience): split settings.ts into types + normalize leaves (#5745) Decompose the (fully pure) resilience settings module into two sibling leaves: - src/lib/resilience/settings/types.ts: the settings shape (11 public interfaces + JsonRecord/AuthCategory), zero imports. - src/lib/resilience/settings/normalize.ts: the coercers (asRecord/toInteger/ toBoolean/feature-flag resolvers) + the 11 per-section normalize* functions. settings.ts keeps DEFAULT_RESILIENCE_SETTINGS, DEFAULT_REQUEST_QUEUE_MAX_WAIT_MS, buildLegacyFallback, and the public orchestrators (resolveResilienceSettings, mergeResilienceSettings, buildLegacyResilienceCompat); it imports the coercers/normalizers for internal use and re-exports the 11 settings interfaces, so the public API is unchanged. Host 840->363 LOC; leaves 182 + 359 LOC (< 800 cap); host was frozen-satisfied (841), so this is debt reduction. 472 moved lines are byte-identical; no cycles (leaves never import the host). New split-guard test characterizes the coercers/normalizers and the host resolve/merge/compat orchestration. --- src/lib/resilience/settings.ts | 539 +----------------- src/lib/resilience/settings/normalize.ts | 359 ++++++++++++ src/lib/resilience/settings/types.ts | 182 ++++++ ...esilience-settings-normalize-split.test.ts | 124 ++++ 4 files changed, 696 insertions(+), 508 deletions(-) create mode 100644 src/lib/resilience/settings/normalize.ts create mode 100644 src/lib/resilience/settings/types.ts create mode 100644 tests/unit/resilience-settings-normalize-split.test.ts diff --git a/src/lib/resilience/settings.ts b/src/lib/resilience/settings.ts index 834a827967..ea1f2b8418 100644 --- a/src/lib/resilience/settings.ts +++ b/src/lib/resilience/settings.ts @@ -1,241 +1,37 @@ import { DEFAULT_API_LIMITS, PROVIDER_PROFILES } from "@omniroute/open-sse/config/constants"; -import { resolveFeatureFlag } from "@/shared/utils/featureFlags"; -type JsonRecord = Record; -type AuthCategory = "oauth" | "apikey"; +import type { JsonRecord, ResilienceSettings, ResilienceSettingsPatch } from "./settings/types"; +import { + asRecord, + toInteger, + resolveStreamRecoveryDefaults, + normalizeLegacyConnectionCooldownProfile, + normalizeRequestQueueSettings, + normalizeConnectionCooldownProfile, + normalizeProviderBreakerProfile, + normalizeWaitForCooldownSettings, + normalizeComboCooldownWaitSettings, + normalizeQuotaShareConcurrencyLimitSettings, + normalizeProviderCooldownSettings, + normalizeQuotaPreflightSettings, + normalizeStreamRecoverySettings, +} from "./settings/normalize"; -export interface RequestQueueSettings { - autoEnableApiKeyProviders: boolean; - requestsPerMinute: number; - minTimeBetweenRequestsMs: number; - concurrentRequests: number; - maxWaitMs: number; -} - -export interface ConnectionCooldownProfileSettings { - baseCooldownMs: number; - useUpstreamRetryHints: boolean; - /** - * Issue #2100 follow-up: opt-in toggle for upstream 429 hint trust at the - * circuit-breaker cooldown layer (independent of `useUpstreamRetryHints` - * which controls retry scheduling). - * - * Stored shape is intentionally optional / `boolean | undefined`: when - * unset, the per-provider default from `providerHints.ts` applies. - * Normalize/merge MUST preserve `undefined` — do not coerce via - * `toBoolean(value, fallback)`. - */ - useUpstream429BreakerHints?: boolean; - maxBackoffSteps: number; -} - -export interface ProviderBreakerProfileSettings { - failureThreshold: number; - degradationThreshold: number; - resetTimeoutMs: number; -} - -export interface WaitForCooldownSettings { - enabled: boolean; - maxRetries: number; - maxRetryWaitSec: number; - maxRetryWaitMs: number; -} - -/** - * Quota-share combo cooldown-aware retry (Variante A). A quota-share (`qtSd/…`) - * combo that would crystallize a 429 `model_cooldown` for a SHORT transient - * cooldown waits it out and re-dispatches instead. Guards (gating + the - * `quota_exhausted`/auth/not-found exclusions) live in - * open-sse/services/combo/comboCooldownRetry.ts; `maxWaitMs`/`maxAttempts`/ - * `budgetMs` bound a single wait, the retry cycles, and the total wait time. - */ -export interface ComboCooldownWaitSettings { - enabled: boolean; - maxWaitMs: number; - maxAttempts: number; - budgetMs: number; -} - -/** - * Per-connection concurrency limit for quota-share (`qtSd/…`) combos (FASE 2.1). - * The quota-share gating in selectQuotaShareTarget is fail-open and cannot - * hard-limit a single-connection pool, so concurrent requests to one - * subscription account can still flood it (→ 429 + cooldown). When a connection - * declares a positive `max_concurrent` ceiling, this layer serializes concurrent - * requests to that account through a per-connection semaphore (excess requests - * wait in the queue instead of flooding). Kill-switch only: the cap itself comes - * from each connection's `max_concurrent`. Wiring lives in - * open-sse/services/combo/quotaShareConcurrency.ts. - */ -export interface QuotaShareConcurrencyLimitSettings { - enabled: boolean; -} - -export interface ProviderCooldownSettings { - /** - * Minimum cooldown (ms) before a failed provider/connection can be retried. - * This prevents subsequent requests from immediately re-walking failing providers. - * Scaled exponentially with failure count: minRetryCooldownMs * 2^(failures-1). - * Default: 5000 (5 seconds). - */ - minRetryCooldownMs: number; - /** - * Maximum cooldown (ms) before a failed provider/connection is retried regardless. - * Hard cap to prevent providers from being skipped indefinitely. - * Default: 300000 (5 minutes). - */ - maxRetryCooldownMs: number; - /** - * Enable/disable global provider cooldown tracking. - * When disabled, only per-request cooldown state is used. - * Default: true. - */ - enabled: boolean; -} - -export interface QuotaPreflightSettings { - /** - * Master switch for the auto-routing quota cutoff (buildAutoCandidates). When - * disabled (default), candidates are NOT dropped for low quota before scoring — - * the soft quota penalty + connection cooldown still apply, so behavior is - * unchanged. Opt-in because the hard cutoff interacts with the auto-routing - * scorer and must be validated per deployment. Default: false. - */ - enabled: boolean; - /** - * Global minimum-remaining cutoff (percent, 0-100). A connection is skipped - * when its remaining quota drops to this value or below. Matches the - * dashboard's quota bars (which show REMAINING %, not used %), so the - * number means the same thing in both places. Default: 2 (stop at 2% - * remaining = 98% used). - */ - defaultThresholdPercent: number; - /** - * Global warn threshold (percent, 0-100 remaining %). Fires when remaining - * quota drops to this value or below. Must be HIGHER than the cutoff so - * warnings appear before the block point. Default: 20 (warn at 20% - * remaining = 80% used). - */ - warnThresholdPercent: number; - /** - * Per-(provider, window) defaults for providers that expose multiple quota - * windows (e.g. Codex's session + weekly). Values are minimum-remaining % - * cutoffs. Resolution order, low-to-high precedence: - * defaultThresholdPercent - * → providerWindowDefaults[provider][window] - * → connection.quotaWindowThresholds[window] - */ - providerWindowDefaults: Record>; -} - -export interface StreamRecoverySettings { - /** - * Opt-in transparent recovery of truncated upstream streams (free-claude-code port). - * When enabled, the opening SSE window is briefly held (see STREAM_RECOVERY in - * open-sse/config/constants.ts) so an early cutoff can be retried before any byte - * reaches the client. OFF by default because holding the window adds up to - * STREAM_RECOVERY.HOLDBACK_MS of time-to-first-token latency on every stream. - * Default seeds from the STREAM_RECOVERY_ENABLED feature flag / env var. - */ - enabled: boolean; - /** - * Opt-in mid-stream continuation (Fase 4.4): when an upstream stream truncates AFTER - * bytes already reached the client, re-request with the partial text as an assistant - * prefill and stitch the missing suffix (plain-text OpenAI-compatible streams only; - * never with a tool call in flight). OFF by default because the recovered tail arrives - * as one burst rather than token-by-token. Default seeds from the - * STREAM_RECOVERY_MIDSTREAM_ENABLED feature flag / env var. - */ - continueMidStream: boolean; -} - -export interface ResilienceSettings { - requestQueue: RequestQueueSettings; - connectionCooldown: Record; - providerBreaker: Record; - waitForCooldown: WaitForCooldownSettings; - comboCooldownWait: ComboCooldownWaitSettings; - quotaShareConcurrencyLimit: QuotaShareConcurrencyLimitSettings; - providerCooldown: ProviderCooldownSettings; - quotaPreflight: QuotaPreflightSettings; - streamRecovery: StreamRecoverySettings; -} - -export interface ResilienceSettingsPatch { - requestQueue?: Partial; - connectionCooldown?: Partial>>; - providerBreaker?: Partial>>; - waitForCooldown?: Partial; - comboCooldownWait?: Partial; - quotaShareConcurrencyLimit?: Partial; - providerCooldown?: Partial; - quotaPreflight?: Partial; - streamRecovery?: Partial; -} - -function asRecord(value: unknown): JsonRecord { - return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {}; -} - -function toInteger( - value: unknown, - fallback: number, - options: { min?: number; max?: number } = {} -): number { - const min = options.min ?? 0; - const max = options.max ?? Number.MAX_SAFE_INTEGER; - const parsed = - typeof value === "number" - ? value - : typeof value === "string" && value.trim().length > 0 - ? Number(value) - : Number.NaN; - - if (!Number.isFinite(parsed)) { - return fallback; - } - - return Math.min(max, Math.max(min, Math.trunc(parsed))); -} - -function toBoolean(value: unknown, fallback: boolean): boolean { - return typeof value === "boolean" ? value : fallback; -} - -function parseFeatureFlagBoolean(value: string, fallback: boolean): boolean { - const normalized = value.trim().toLowerCase(); - if (normalized === "true" || normalized === "1" || normalized === "yes" || normalized === "on") { - return true; - } - if (normalized === "false" || normalized === "0" || normalized === "no" || normalized === "off") { - return false; - } - return fallback; -} - -function resolveBooleanFeatureFlag(key: string, fallback: boolean): boolean { - try { - return parseFeatureFlagBoolean(resolveFeatureFlag(key), fallback); - } catch (error) { - const envValue = process.env[key]; - if (typeof envValue === "string" && envValue.trim() !== "") { - return parseFeatureFlagBoolean(envValue, fallback); - } - console.error( - `[resilience] Failed to resolve ${key}, falling back to ${String(fallback)}:`, - error instanceof Error ? error.message : error - ); - return fallback; - } -} - -function resolveStreamRecoveryDefaults(): StreamRecoverySettings { - return { - enabled: resolveBooleanFeatureFlag("STREAM_RECOVERY_ENABLED", false), - continueMidStream: resolveBooleanFeatureFlag("STREAM_RECOVERY_MIDSTREAM_ENABLED", false), - }; -} +// Re-export the settings shape (moved to ./settings/types) so this module's +// public API is unchanged. +export type { + RequestQueueSettings, + ConnectionCooldownProfileSettings, + ProviderBreakerProfileSettings, + WaitForCooldownSettings, + ComboCooldownWaitSettings, + QuotaShareConcurrencyLimitSettings, + ProviderCooldownSettings, + QuotaPreflightSettings, + StreamRecoverySettings, + ResilienceSettings, + ResilienceSettingsPatch, +} from "./settings/types"; export const DEFAULT_REQUEST_QUEUE_MAX_WAIT_MS = (() => { const parsed = Number(process.env.RATE_LIMIT_MAX_WAIT_MS || "120000"); @@ -338,279 +134,6 @@ export const DEFAULT_RESILIENCE_SETTINGS: ResilienceSettings = { }, }; -function normalizeRequestQueueSettings( - next: unknown, - fallback: RequestQueueSettings -): RequestQueueSettings { - const record = asRecord(next); - const requestsPerMinute = toInteger(record.requestsPerMinute, fallback.requestsPerMinute, { - min: 1, - max: 1_000_000, - }); - const minTimeBetweenRequestsMs = toInteger( - record.minTimeBetweenRequestsMs, - fallback.minTimeBetweenRequestsMs, - { min: 0, max: 60 * 60 * 1000 } - ); - const concurrentRequests = toInteger(record.concurrentRequests, fallback.concurrentRequests, { - min: 1, - max: 10_000, - }); - const maxWaitMs = toInteger(record.maxWaitMs, fallback.maxWaitMs, { - min: 1, - max: 24 * 60 * 60 * 1000, - }); - - return { - autoEnableApiKeyProviders: toBoolean( - record.autoEnableApiKeyProviders, - fallback.autoEnableApiKeyProviders - ), - requestsPerMinute, - minTimeBetweenRequestsMs, - concurrentRequests, - maxWaitMs, - }; -} - -function normalizeConnectionCooldownProfile( - next: unknown, - fallback: ConnectionCooldownProfileSettings -): ConnectionCooldownProfileSettings { - const record = asRecord(next); - // useUpstream429BreakerHints uses a 3-state input contract: - // - boolean → user override, store as-is - // - null → explicit unset sentinel, drop key so the per-provider - // default in `providerHints.ts` resolves at runtime - // - omitted → leave existing fallback value unchanged (partial-merge) - // Never coerce via `toBoolean(value, fallback)` because that would - // collapse the unset state. - const hasHintsKey = Object.prototype.hasOwnProperty.call(record, "useUpstream429BreakerHints"); - const rawHints = record.useUpstream429BreakerHints; - let useUpstream429BreakerHints: boolean | undefined; - if (!hasHintsKey) { - useUpstream429BreakerHints = fallback.useUpstream429BreakerHints; - } else if (rawHints === null) { - useUpstream429BreakerHints = undefined; - } else if (typeof rawHints === "boolean") { - useUpstream429BreakerHints = rawHints; - } else { - useUpstream429BreakerHints = fallback.useUpstream429BreakerHints; - } - const out: ConnectionCooldownProfileSettings = { - baseCooldownMs: toInteger(record.baseCooldownMs, fallback.baseCooldownMs, { - min: 0, - max: 24 * 60 * 60 * 1000, - }), - useUpstreamRetryHints: toBoolean(record.useUpstreamRetryHints, fallback.useUpstreamRetryHints), - maxBackoffSteps: toInteger(record.maxBackoffSteps, fallback.maxBackoffSteps, { - min: 0, - max: 32, - }), - }; - // Only attach the key when defined — preserves omission across round-trips. - if (useUpstream429BreakerHints !== undefined) { - out.useUpstream429BreakerHints = useUpstream429BreakerHints; - } - return out; -} - -function normalizeLegacyConnectionCooldownProfile( - next: unknown, - fallback: ConnectionCooldownProfileSettings -): ConnectionCooldownProfileSettings { - const record = asRecord(next); - const transientCooldown = toInteger(record.transientCooldown, fallback.baseCooldownMs, { - min: 0, - max: 24 * 60 * 60 * 1000, - }); - const legacyRateLimitCooldown = toInteger(record.rateLimitCooldown, transientCooldown, { - min: 0, - max: 24 * 60 * 60 * 1000, - }); - const useUpstreamRetryHints = - typeof record.rateLimitCooldown === "number" - ? record.rateLimitCooldown === 0 - : fallback.useUpstreamRetryHints; - - return { - baseCooldownMs: useUpstreamRetryHints - ? transientCooldown - : Math.max(transientCooldown, legacyRateLimitCooldown), - useUpstreamRetryHints, - maxBackoffSteps: toInteger(record.maxBackoffLevel, fallback.maxBackoffSteps, { - min: 0, - max: 32, - }), - }; -} - -function normalizeProviderBreakerProfile( - next: unknown, - fallback: ProviderBreakerProfileSettings -): ProviderBreakerProfileSettings { - const record = asRecord(next); - const failureThreshold = toInteger(record.failureThreshold, fallback.failureThreshold, { - min: 1, - max: 1000, - }); - const degradationThreshold = Math.min( - toInteger(record.degradationThreshold, fallback.degradationThreshold, { - min: 1, - max: 1000, - }), - failureThreshold <= 1 ? 1 : failureThreshold - 1 - ); - - return { - failureThreshold, - degradationThreshold, - resetTimeoutMs: toInteger(record.resetTimeoutMs, fallback.resetTimeoutMs, { - min: 1000, - max: 24 * 60 * 60 * 1000, - }), - }; -} - -function normalizeProviderWindowDefaults( - next: unknown, - fallback: Record> -): Record> { - // Accept either an explicit object or fall back. Drop providers/windows - // whose values are not a valid 0-100 integer so a malformed setting can't - // accidentally disable cutoffs entirely. - const rawProviders = asRecord(next ?? fallback); - const out: Record> = {}; - for (const [provider, windows] of Object.entries(rawProviders)) { - if (!provider || typeof windows !== "object" || windows === null) continue; - const windowMap: Record = {}; - for (const [windowName, percent] of Object.entries(windows as Record)) { - if (!windowName) continue; - const parsed = - typeof percent === "number" - ? percent - : typeof percent === "string" && percent.trim() !== "" - ? Number(percent) - : NaN; - if (Number.isFinite(parsed)) { - const clamped = Math.min(100, Math.max(0, Math.trunc(parsed))); - windowMap[windowName] = clamped; - } - } - if (Object.keys(windowMap).length > 0) { - out[provider] = windowMap; - } - } - return out; -} - -function normalizeQuotaPreflightSettings( - next: unknown, - fallback: QuotaPreflightSettings -): QuotaPreflightSettings { - const record = asRecord(next); - // Remaining-% semantics: cutoff is the lowest acceptable remaining %, warn - // is the higher "you're getting close" remaining %. So warn MUST be greater - // than cutoff — otherwise the warn log would only fire after the request - // is already blocked. - const defaultThresholdPercent = toInteger( - record.defaultThresholdPercent, - fallback.defaultThresholdPercent, - { min: 0, max: 99 } - ); - const warnRaw = toInteger(record.warnThresholdPercent, fallback.warnThresholdPercent, { - min: 0, - max: 100, - }); - const warnThresholdPercent = - warnRaw <= defaultThresholdPercent ? Math.min(100, defaultThresholdPercent + 1) : warnRaw; - const providerWindowDefaults = normalizeProviderWindowDefaults( - record.providerWindowDefaults, - fallback.providerWindowDefaults - ); - const enabled = typeof record.enabled === "boolean" ? record.enabled : fallback.enabled; - return { enabled, defaultThresholdPercent, warnThresholdPercent, providerWindowDefaults }; -} - -function normalizeWaitForCooldownSettings( - next: unknown, - fallback: WaitForCooldownSettings -): WaitForCooldownSettings { - const record = asRecord(next); - const maxRetryWaitSec = toInteger(record.maxRetryWaitSec, fallback.maxRetryWaitSec, { - min: 0, - max: 300, - }); - const maxRetries = toInteger(record.maxRetries, fallback.maxRetries, { min: 0, max: 10 }); - const enabled = - toBoolean(record.enabled, fallback.enabled) && maxRetries > 0 && maxRetryWaitSec > 0; - - return { - enabled, - maxRetries, - maxRetryWaitSec, - maxRetryWaitMs: maxRetryWaitSec * 1000, - }; -} - -function normalizeComboCooldownWaitSettings( - next: unknown, - fallback: ComboCooldownWaitSettings -): ComboCooldownWaitSettings { - const record = asRecord(next); - // Hard ceiling of 30s on a single wait — this layer only ever exists for - // SHORT transient cooldowns; anything longer should fall through to the - // existing 429 crystallization (and the cross-request cooldown layers). - const maxWaitMs = toInteger(record.maxWaitMs, fallback.maxWaitMs, { min: 0, max: 30000 }); - const maxAttempts = toInteger(record.maxAttempts, fallback.maxAttempts, { min: 0, max: 10 }); - // Budget can never be smaller than a single wait, otherwise no wait could - // ever fire; floor it at maxWaitMs. - const budgetMs = toInteger(record.budgetMs, fallback.budgetMs, { - min: maxWaitMs, - max: 5 * 60 * 1000, - }); - const enabled = toBoolean(record.enabled, fallback.enabled) && maxWaitMs > 0 && maxAttempts > 0; - - return { enabled, maxWaitMs, maxAttempts, budgetMs }; -} - -function normalizeQuotaShareConcurrencyLimitSettings( - next: unknown, - fallback: QuotaShareConcurrencyLimitSettings -): QuotaShareConcurrencyLimitSettings { - const record = asRecord(next); - return { enabled: toBoolean(record.enabled, fallback.enabled) }; -} - -function normalizeProviderCooldownSettings( - next: unknown, - fallback: ProviderCooldownSettings -): ProviderCooldownSettings { - const record = asRecord(next); - const enabled = toBoolean(record.enabled, fallback.enabled); - const minRetryCooldownMs = toInteger(record.minRetryCooldownMs, fallback.minRetryCooldownMs, { - min: 0, - max: 60 * 60 * 1000, - }); - const maxRetryCooldownMs = toInteger(record.maxRetryCooldownMs, fallback.maxRetryCooldownMs, { - min: minRetryCooldownMs, - max: 24 * 60 * 60 * 1000, - }); - - return { enabled, minRetryCooldownMs, maxRetryCooldownMs }; -} - -function normalizeStreamRecoverySettings( - next: unknown, - fallback: StreamRecoverySettings -): StreamRecoverySettings { - const record = asRecord(next); - return { - enabled: toBoolean(record.enabled, fallback.enabled), - continueMidStream: toBoolean(record.continueMidStream, fallback.continueMidStream), - }; -} - function buildLegacyFallback(settings: JsonRecord): ResilienceSettings { const profiles = asRecord(settings.providerProfiles); const defaults = asRecord(settings.rateLimitDefaults); diff --git a/src/lib/resilience/settings/normalize.ts b/src/lib/resilience/settings/normalize.ts new file mode 100644 index 0000000000..9325873439 --- /dev/null +++ b/src/lib/resilience/settings/normalize.ts @@ -0,0 +1,359 @@ +/** + * resilience/settings/normalize — coercion + per-section normalizers (pure). + * + * Extracted verbatim from resilience/settings.ts. Pure functions: no DB, no + * module state. Depends only on the feature-flag resolver and the settings + * types. The host imports these for its resolve/merge/legacy orchestration. + * + * @module lib/resilience/settings/normalize + */ + +import { resolveFeatureFlag } from "@/shared/utils/featureFlags"; +import type { + JsonRecord, + RequestQueueSettings, + ConnectionCooldownProfileSettings, + ProviderBreakerProfileSettings, + WaitForCooldownSettings, + ComboCooldownWaitSettings, + QuotaShareConcurrencyLimitSettings, + ProviderCooldownSettings, + QuotaPreflightSettings, + StreamRecoverySettings, +} from "./types"; + +export function asRecord(value: unknown): JsonRecord { + return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {}; +} + +export function toInteger( + value: unknown, + fallback: number, + options: { min?: number; max?: number } = {} +): number { + const min = options.min ?? 0; + const max = options.max ?? Number.MAX_SAFE_INTEGER; + const parsed = + typeof value === "number" + ? value + : typeof value === "string" && value.trim().length > 0 + ? Number(value) + : Number.NaN; + + if (!Number.isFinite(parsed)) { + return fallback; + } + + return Math.min(max, Math.max(min, Math.trunc(parsed))); +} + +export function toBoolean(value: unknown, fallback: boolean): boolean { + return typeof value === "boolean" ? value : fallback; +} + +export function parseFeatureFlagBoolean(value: string, fallback: boolean): boolean { + const normalized = value.trim().toLowerCase(); + if (normalized === "true" || normalized === "1" || normalized === "yes" || normalized === "on") { + return true; + } + if (normalized === "false" || normalized === "0" || normalized === "no" || normalized === "off") { + return false; + } + return fallback; +} + +export function resolveBooleanFeatureFlag(key: string, fallback: boolean): boolean { + try { + return parseFeatureFlagBoolean(resolveFeatureFlag(key), fallback); + } catch (error) { + const envValue = process.env[key]; + if (typeof envValue === "string" && envValue.trim() !== "") { + return parseFeatureFlagBoolean(envValue, fallback); + } + console.error( + `[resilience] Failed to resolve ${key}, falling back to ${String(fallback)}:`, + error instanceof Error ? error.message : error + ); + return fallback; + } +} + +export function resolveStreamRecoveryDefaults(): StreamRecoverySettings { + return { + enabled: resolveBooleanFeatureFlag("STREAM_RECOVERY_ENABLED", false), + continueMidStream: resolveBooleanFeatureFlag("STREAM_RECOVERY_MIDSTREAM_ENABLED", false), + }; +} + +export function normalizeRequestQueueSettings( + next: unknown, + fallback: RequestQueueSettings +): RequestQueueSettings { + const record = asRecord(next); + const requestsPerMinute = toInteger(record.requestsPerMinute, fallback.requestsPerMinute, { + min: 1, + max: 1_000_000, + }); + const minTimeBetweenRequestsMs = toInteger( + record.minTimeBetweenRequestsMs, + fallback.minTimeBetweenRequestsMs, + { min: 0, max: 60 * 60 * 1000 } + ); + const concurrentRequests = toInteger(record.concurrentRequests, fallback.concurrentRequests, { + min: 1, + max: 10_000, + }); + const maxWaitMs = toInteger(record.maxWaitMs, fallback.maxWaitMs, { + min: 1, + max: 24 * 60 * 60 * 1000, + }); + + return { + autoEnableApiKeyProviders: toBoolean( + record.autoEnableApiKeyProviders, + fallback.autoEnableApiKeyProviders + ), + requestsPerMinute, + minTimeBetweenRequestsMs, + concurrentRequests, + maxWaitMs, + }; +} + +export function normalizeConnectionCooldownProfile( + next: unknown, + fallback: ConnectionCooldownProfileSettings +): ConnectionCooldownProfileSettings { + const record = asRecord(next); + // useUpstream429BreakerHints uses a 3-state input contract: + // - boolean → user override, store as-is + // - null → explicit unset sentinel, drop key so the per-provider + // default in `providerHints.ts` resolves at runtime + // - omitted → leave existing fallback value unchanged (partial-merge) + // Never coerce via `toBoolean(value, fallback)` because that would + // collapse the unset state. + const hasHintsKey = Object.prototype.hasOwnProperty.call(record, "useUpstream429BreakerHints"); + const rawHints = record.useUpstream429BreakerHints; + let useUpstream429BreakerHints: boolean | undefined; + if (!hasHintsKey) { + useUpstream429BreakerHints = fallback.useUpstream429BreakerHints; + } else if (rawHints === null) { + useUpstream429BreakerHints = undefined; + } else if (typeof rawHints === "boolean") { + useUpstream429BreakerHints = rawHints; + } else { + useUpstream429BreakerHints = fallback.useUpstream429BreakerHints; + } + const out: ConnectionCooldownProfileSettings = { + baseCooldownMs: toInteger(record.baseCooldownMs, fallback.baseCooldownMs, { + min: 0, + max: 24 * 60 * 60 * 1000, + }), + useUpstreamRetryHints: toBoolean(record.useUpstreamRetryHints, fallback.useUpstreamRetryHints), + maxBackoffSteps: toInteger(record.maxBackoffSteps, fallback.maxBackoffSteps, { + min: 0, + max: 32, + }), + }; + // Only attach the key when defined — preserves omission across round-trips. + if (useUpstream429BreakerHints !== undefined) { + out.useUpstream429BreakerHints = useUpstream429BreakerHints; + } + return out; +} + +export function normalizeLegacyConnectionCooldownProfile( + next: unknown, + fallback: ConnectionCooldownProfileSettings +): ConnectionCooldownProfileSettings { + const record = asRecord(next); + const transientCooldown = toInteger(record.transientCooldown, fallback.baseCooldownMs, { + min: 0, + max: 24 * 60 * 60 * 1000, + }); + const legacyRateLimitCooldown = toInteger(record.rateLimitCooldown, transientCooldown, { + min: 0, + max: 24 * 60 * 60 * 1000, + }); + const useUpstreamRetryHints = + typeof record.rateLimitCooldown === "number" + ? record.rateLimitCooldown === 0 + : fallback.useUpstreamRetryHints; + + return { + baseCooldownMs: useUpstreamRetryHints + ? transientCooldown + : Math.max(transientCooldown, legacyRateLimitCooldown), + useUpstreamRetryHints, + maxBackoffSteps: toInteger(record.maxBackoffLevel, fallback.maxBackoffSteps, { + min: 0, + max: 32, + }), + }; +} + +export function normalizeProviderBreakerProfile( + next: unknown, + fallback: ProviderBreakerProfileSettings +): ProviderBreakerProfileSettings { + const record = asRecord(next); + const failureThreshold = toInteger(record.failureThreshold, fallback.failureThreshold, { + min: 1, + max: 1000, + }); + const degradationThreshold = Math.min( + toInteger(record.degradationThreshold, fallback.degradationThreshold, { + min: 1, + max: 1000, + }), + failureThreshold <= 1 ? 1 : failureThreshold - 1 + ); + + return { + failureThreshold, + degradationThreshold, + resetTimeoutMs: toInteger(record.resetTimeoutMs, fallback.resetTimeoutMs, { + min: 1000, + max: 24 * 60 * 60 * 1000, + }), + }; +} + +export function normalizeProviderWindowDefaults( + next: unknown, + fallback: Record> +): Record> { + // Accept either an explicit object or fall back. Drop providers/windows + // whose values are not a valid 0-100 integer so a malformed setting can't + // accidentally disable cutoffs entirely. + const rawProviders = asRecord(next ?? fallback); + const out: Record> = {}; + for (const [provider, windows] of Object.entries(rawProviders)) { + if (!provider || typeof windows !== "object" || windows === null) continue; + const windowMap: Record = {}; + for (const [windowName, percent] of Object.entries(windows as Record)) { + if (!windowName) continue; + const parsed = + typeof percent === "number" + ? percent + : typeof percent === "string" && percent.trim() !== "" + ? Number(percent) + : NaN; + if (Number.isFinite(parsed)) { + const clamped = Math.min(100, Math.max(0, Math.trunc(parsed))); + windowMap[windowName] = clamped; + } + } + if (Object.keys(windowMap).length > 0) { + out[provider] = windowMap; + } + } + return out; +} + +export function normalizeQuotaPreflightSettings( + next: unknown, + fallback: QuotaPreflightSettings +): QuotaPreflightSettings { + const record = asRecord(next); + // Remaining-% semantics: cutoff is the lowest acceptable remaining %, warn + // is the higher "you're getting close" remaining %. So warn MUST be greater + // than cutoff — otherwise the warn log would only fire after the request + // is already blocked. + const defaultThresholdPercent = toInteger( + record.defaultThresholdPercent, + fallback.defaultThresholdPercent, + { min: 0, max: 99 } + ); + const warnRaw = toInteger(record.warnThresholdPercent, fallback.warnThresholdPercent, { + min: 0, + max: 100, + }); + const warnThresholdPercent = + warnRaw <= defaultThresholdPercent ? Math.min(100, defaultThresholdPercent + 1) : warnRaw; + const providerWindowDefaults = normalizeProviderWindowDefaults( + record.providerWindowDefaults, + fallback.providerWindowDefaults + ); + const enabled = typeof record.enabled === "boolean" ? record.enabled : fallback.enabled; + return { enabled, defaultThresholdPercent, warnThresholdPercent, providerWindowDefaults }; +} + +export function normalizeWaitForCooldownSettings( + next: unknown, + fallback: WaitForCooldownSettings +): WaitForCooldownSettings { + const record = asRecord(next); + const maxRetryWaitSec = toInteger(record.maxRetryWaitSec, fallback.maxRetryWaitSec, { + min: 0, + max: 300, + }); + const maxRetries = toInteger(record.maxRetries, fallback.maxRetries, { min: 0, max: 10 }); + const enabled = + toBoolean(record.enabled, fallback.enabled) && maxRetries > 0 && maxRetryWaitSec > 0; + + return { + enabled, + maxRetries, + maxRetryWaitSec, + maxRetryWaitMs: maxRetryWaitSec * 1000, + }; +} + +export function normalizeComboCooldownWaitSettings( + next: unknown, + fallback: ComboCooldownWaitSettings +): ComboCooldownWaitSettings { + const record = asRecord(next); + // Hard ceiling of 30s on a single wait — this layer only ever exists for + // SHORT transient cooldowns; anything longer should fall through to the + // existing 429 crystallization (and the cross-request cooldown layers). + const maxWaitMs = toInteger(record.maxWaitMs, fallback.maxWaitMs, { min: 0, max: 30000 }); + const maxAttempts = toInteger(record.maxAttempts, fallback.maxAttempts, { min: 0, max: 10 }); + // Budget can never be smaller than a single wait, otherwise no wait could + // ever fire; floor it at maxWaitMs. + const budgetMs = toInteger(record.budgetMs, fallback.budgetMs, { + min: maxWaitMs, + max: 5 * 60 * 1000, + }); + const enabled = toBoolean(record.enabled, fallback.enabled) && maxWaitMs > 0 && maxAttempts > 0; + + return { enabled, maxWaitMs, maxAttempts, budgetMs }; +} + +export function normalizeQuotaShareConcurrencyLimitSettings( + next: unknown, + fallback: QuotaShareConcurrencyLimitSettings +): QuotaShareConcurrencyLimitSettings { + const record = asRecord(next); + return { enabled: toBoolean(record.enabled, fallback.enabled) }; +} + +export function normalizeProviderCooldownSettings( + next: unknown, + fallback: ProviderCooldownSettings +): ProviderCooldownSettings { + const record = asRecord(next); + const enabled = toBoolean(record.enabled, fallback.enabled); + const minRetryCooldownMs = toInteger(record.minRetryCooldownMs, fallback.minRetryCooldownMs, { + min: 0, + max: 60 * 60 * 1000, + }); + const maxRetryCooldownMs = toInteger(record.maxRetryCooldownMs, fallback.maxRetryCooldownMs, { + min: minRetryCooldownMs, + max: 24 * 60 * 60 * 1000, + }); + + return { enabled, minRetryCooldownMs, maxRetryCooldownMs }; +} + +export function normalizeStreamRecoverySettings( + next: unknown, + fallback: StreamRecoverySettings +): StreamRecoverySettings { + const record = asRecord(next); + return { + enabled: toBoolean(record.enabled, fallback.enabled), + continueMidStream: toBoolean(record.continueMidStream, fallback.continueMidStream), + }; +} diff --git a/src/lib/resilience/settings/types.ts b/src/lib/resilience/settings/types.ts new file mode 100644 index 0000000000..395f8dd698 --- /dev/null +++ b/src/lib/resilience/settings/types.ts @@ -0,0 +1,182 @@ +/** + * resilience/settings/types — resilience settings shape (pure types). + * + * Extracted verbatim from resilience/settings.ts. Zero imports, type-only. + * The host re-exports the public interfaces so its API is unchanged; the + * normalize layer imports these to type its coercion. + * + * @module lib/resilience/settings/types + */ + +export type JsonRecord = Record; +export type AuthCategory = "oauth" | "apikey"; + +export interface RequestQueueSettings { + autoEnableApiKeyProviders: boolean; + requestsPerMinute: number; + minTimeBetweenRequestsMs: number; + concurrentRequests: number; + maxWaitMs: number; +} + +export interface ConnectionCooldownProfileSettings { + baseCooldownMs: number; + useUpstreamRetryHints: boolean; + /** + * Issue #2100 follow-up: opt-in toggle for upstream 429 hint trust at the + * circuit-breaker cooldown layer (independent of `useUpstreamRetryHints` + * which controls retry scheduling). + * + * Stored shape is intentionally optional / `boolean | undefined`: when + * unset, the per-provider default from `providerHints.ts` applies. + * Normalize/merge MUST preserve `undefined` — do not coerce via + * `toBoolean(value, fallback)`. + */ + useUpstream429BreakerHints?: boolean; + maxBackoffSteps: number; +} + +export interface ProviderBreakerProfileSettings { + failureThreshold: number; + degradationThreshold: number; + resetTimeoutMs: number; +} + +export interface WaitForCooldownSettings { + enabled: boolean; + maxRetries: number; + maxRetryWaitSec: number; + maxRetryWaitMs: number; +} + +/** + * Quota-share combo cooldown-aware retry (Variante A). A quota-share (`qtSd/…`) + * combo that would crystallize a 429 `model_cooldown` for a SHORT transient + * cooldown waits it out and re-dispatches instead. Guards (gating + the + * `quota_exhausted`/auth/not-found exclusions) live in + * open-sse/services/combo/comboCooldownRetry.ts; `maxWaitMs`/`maxAttempts`/ + * `budgetMs` bound a single wait, the retry cycles, and the total wait time. + */ +export interface ComboCooldownWaitSettings { + enabled: boolean; + maxWaitMs: number; + maxAttempts: number; + budgetMs: number; +} + +/** + * Per-connection concurrency limit for quota-share (`qtSd/…`) combos (FASE 2.1). + * The quota-share gating in selectQuotaShareTarget is fail-open and cannot + * hard-limit a single-connection pool, so concurrent requests to one + * subscription account can still flood it (→ 429 + cooldown). When a connection + * declares a positive `max_concurrent` ceiling, this layer serializes concurrent + * requests to that account through a per-connection semaphore (excess requests + * wait in the queue instead of flooding). Kill-switch only: the cap itself comes + * from each connection's `max_concurrent`. Wiring lives in + * open-sse/services/combo/quotaShareConcurrency.ts. + */ +export interface QuotaShareConcurrencyLimitSettings { + enabled: boolean; +} + +export interface ProviderCooldownSettings { + /** + * Minimum cooldown (ms) before a failed provider/connection can be retried. + * This prevents subsequent requests from immediately re-walking failing providers. + * Scaled exponentially with failure count: minRetryCooldownMs * 2^(failures-1). + * Default: 5000 (5 seconds). + */ + minRetryCooldownMs: number; + /** + * Maximum cooldown (ms) before a failed provider/connection is retried regardless. + * Hard cap to prevent providers from being skipped indefinitely. + * Default: 300000 (5 minutes). + */ + maxRetryCooldownMs: number; + /** + * Enable/disable global provider cooldown tracking. + * When disabled, only per-request cooldown state is used. + * Default: true. + */ + enabled: boolean; +} + +export interface QuotaPreflightSettings { + /** + * Master switch for the auto-routing quota cutoff (buildAutoCandidates). When + * disabled (default), candidates are NOT dropped for low quota before scoring — + * the soft quota penalty + connection cooldown still apply, so behavior is + * unchanged. Opt-in because the hard cutoff interacts with the auto-routing + * scorer and must be validated per deployment. Default: false. + */ + enabled: boolean; + /** + * Global minimum-remaining cutoff (percent, 0-100). A connection is skipped + * when its remaining quota drops to this value or below. Matches the + * dashboard's quota bars (which show REMAINING %, not used %), so the + * number means the same thing in both places. Default: 2 (stop at 2% + * remaining = 98% used). + */ + defaultThresholdPercent: number; + /** + * Global warn threshold (percent, 0-100 remaining %). Fires when remaining + * quota drops to this value or below. Must be HIGHER than the cutoff so + * warnings appear before the block point. Default: 20 (warn at 20% + * remaining = 80% used). + */ + warnThresholdPercent: number; + /** + * Per-(provider, window) defaults for providers that expose multiple quota + * windows (e.g. Codex's session + weekly). Values are minimum-remaining % + * cutoffs. Resolution order, low-to-high precedence: + * defaultThresholdPercent + * → providerWindowDefaults[provider][window] + * → connection.quotaWindowThresholds[window] + */ + providerWindowDefaults: Record>; +} + +export interface StreamRecoverySettings { + /** + * Opt-in transparent recovery of truncated upstream streams (free-claude-code port). + * When enabled, the opening SSE window is briefly held (see STREAM_RECOVERY in + * open-sse/config/constants.ts) so an early cutoff can be retried before any byte + * reaches the client. OFF by default because holding the window adds up to + * STREAM_RECOVERY.HOLDBACK_MS of time-to-first-token latency on every stream. + * Default seeds from the STREAM_RECOVERY_ENABLED feature flag / env var. + */ + enabled: boolean; + /** + * Opt-in mid-stream continuation (Fase 4.4): when an upstream stream truncates AFTER + * bytes already reached the client, re-request with the partial text as an assistant + * prefill and stitch the missing suffix (plain-text OpenAI-compatible streams only; + * never with a tool call in flight). OFF by default because the recovered tail arrives + * as one burst rather than token-by-token. Default seeds from the + * STREAM_RECOVERY_MIDSTREAM_ENABLED feature flag / env var. + */ + continueMidStream: boolean; +} + +export interface ResilienceSettings { + requestQueue: RequestQueueSettings; + connectionCooldown: Record; + providerBreaker: Record; + waitForCooldown: WaitForCooldownSettings; + comboCooldownWait: ComboCooldownWaitSettings; + quotaShareConcurrencyLimit: QuotaShareConcurrencyLimitSettings; + providerCooldown: ProviderCooldownSettings; + quotaPreflight: QuotaPreflightSettings; + streamRecovery: StreamRecoverySettings; +} + +export interface ResilienceSettingsPatch { + requestQueue?: Partial; + connectionCooldown?: Partial>>; + providerBreaker?: Partial>>; + waitForCooldown?: Partial; + comboCooldownWait?: Partial; + quotaShareConcurrencyLimit?: Partial; + providerCooldown?: Partial; + quotaPreflight?: Partial; + streamRecovery?: Partial; +} diff --git a/tests/unit/resilience-settings-normalize-split.test.ts b/tests/unit/resilience-settings-normalize-split.test.ts new file mode 100644 index 0000000000..5e1ab767c8 --- /dev/null +++ b/tests/unit/resilience-settings-normalize-split.test.ts @@ -0,0 +1,124 @@ +/** + * Split-guard — resilience/settings ↔ settings/types + settings/normalize + * + * Guards the decomposition of the (fully pure) resilience settings module into + * two leaves: settings/types.ts (the shape) and settings/normalize.ts (coercion + * + per-section normalizers). Characterizes the coercers/normalizers and proves + * the host still exposes DEFAULT_RESILIENCE_SETTINGS + resolve/merge/compat with + * the re-exported types. DB-free — the whole layer is pure. + */ + +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; + +import { + asRecord, + toInteger, + toBoolean, + normalizeProviderBreakerProfile, + normalizeWaitForCooldownSettings, +} from "../../src/lib/resilience/settings/normalize.ts"; + +import { + DEFAULT_RESILIENCE_SETTINGS, + resolveResilienceSettings, + mergeResilienceSettings, + buildLegacyResilienceCompat, +} from "../../src/lib/resilience/settings.ts"; +import type { ResilienceSettings } from "../../src/lib/resilience/settings.ts"; + +describe("resilience/settings normalize split-guard", () => { + it("asRecord returns {} for non-objects and passes objects through", () => { + assert.deepEqual(asRecord(null), {}); + assert.deepEqual(asRecord(42), {}); + assert.deepEqual(asRecord([1, 2]), {}); + assert.deepEqual(asRecord({ a: 1 }), { a: 1 }); + }); + + it("toInteger clamps to [min,max], truncates, and falls back on NaN", () => { + assert.equal(toInteger(5.9, 0), 5); + assert.equal(toInteger("7", 0), 7); + assert.equal(toInteger("nope", 3), 3); + assert.equal(toInteger(-10, 0, { min: 1, max: 100 }), 1); + assert.equal(toInteger(9999, 0, { min: 1, max: 100 }), 100); + }); + + it("toBoolean only accepts real booleans", () => { + assert.equal(toBoolean(true, false), true); + assert.equal(toBoolean("true", false), false); + assert.equal(toBoolean(undefined, true), true); + }); + + it("normalizeProviderBreakerProfile keeps degradation below failure threshold", () => { + const out = normalizeProviderBreakerProfile( + { failureThreshold: 3, degradationThreshold: 10, resetTimeoutMs: 60000 }, + { failureThreshold: 5, degradationThreshold: 4, resetTimeoutMs: 30000 } + ); + assert.equal(out.failureThreshold, 3); + assert.equal(out.degradationThreshold, 2); // clamped to failureThreshold - 1 + assert.equal(out.resetTimeoutMs, 60000); + }); + + it("normalizeWaitForCooldownSettings derives ms and disables on zero retries/wait", () => { + const on = normalizeWaitForCooldownSettings( + { enabled: true, maxRetries: 2, maxRetryWaitSec: 30 }, + { enabled: true, maxRetries: 3, maxRetryWaitSec: 30, maxRetryWaitMs: 30000 } + ); + assert.equal(on.enabled, true); + assert.equal(on.maxRetryWaitMs, 30000); + + const off = normalizeWaitForCooldownSettings( + { enabled: true, maxRetries: 0, maxRetryWaitSec: 30 }, + { enabled: true, maxRetries: 3, maxRetryWaitSec: 30, maxRetryWaitMs: 30000 } + ); + assert.equal(off.enabled, false); // maxRetries 0 forces disabled + }); + + it("host exposes DEFAULT_RESILIENCE_SETTINGS with the full section set", () => { + const keys = Object.keys(DEFAULT_RESILIENCE_SETTINGS).sort(); + assert.deepEqual(keys, [ + "comboCooldownWait", + "connectionCooldown", + "providerBreaker", + "providerCooldown", + "quotaPreflight", + "quotaShareConcurrencyLimit", + "requestQueue", + "streamRecovery", + "waitForCooldown", + ]); + }); + + it("host resolveResilienceSettings(null) yields normalized defaults", () => { + const resolved: ResilienceSettings = resolveResilienceSettings(null); + assert.equal(typeof resolved.requestQueue.requestsPerMinute, "number"); + assert.equal(typeof resolved.providerBreaker.oauth.failureThreshold, "number"); + // degradation is always kept below failure by the normalizer. + assert.ok( + resolved.providerBreaker.oauth.degradationThreshold < + resolved.providerBreaker.oauth.failureThreshold || + resolved.providerBreaker.oauth.failureThreshold <= 1 + ); + }); + + it("host mergeResilienceSettings applies a partial patch", () => { + const base = resolveResilienceSettings(null); + const merged = mergeResilienceSettings(base, { + requestQueue: { concurrentRequests: 7 }, + }); + assert.equal(merged.requestQueue.concurrentRequests, 7); + assert.equal(merged.requestQueue.requestsPerMinute, base.requestQueue.requestsPerMinute); + }); + + it("host buildLegacyResilienceCompat round-trips connection cooldown into profiles", () => { + const compat = buildLegacyResilienceCompat(DEFAULT_RESILIENCE_SETTINGS); + assert.equal( + compat.profiles.oauth.transientCooldown, + DEFAULT_RESILIENCE_SETTINGS.connectionCooldown.oauth.baseCooldownMs + ); + assert.equal( + compat.defaults.requestsPerMinute, + DEFAULT_RESILIENCE_SETTINGS.requestQueue.requestsPerMinute + ); + }); +});