refactor(resilience): split settings.ts into types + normalize leaves (#5745)

Decompose the (fully pure) resilience settings module into two sibling leaves:
- src/lib/resilience/settings/types.ts: the settings shape (11 public
  interfaces + JsonRecord/AuthCategory), zero imports.
- src/lib/resilience/settings/normalize.ts: the coercers (asRecord/toInteger/
  toBoolean/feature-flag resolvers) + the 11 per-section normalize* functions.

settings.ts keeps DEFAULT_RESILIENCE_SETTINGS, DEFAULT_REQUEST_QUEUE_MAX_WAIT_MS,
buildLegacyFallback, and the public orchestrators (resolveResilienceSettings,
mergeResilienceSettings, buildLegacyResilienceCompat); it imports the
coercers/normalizers for internal use and re-exports the 11 settings interfaces,
so the public API is unchanged. Host 840->363 LOC; leaves 182 + 359 LOC
(< 800 cap); host was frozen-satisfied (841), so this is debt reduction.

472 moved lines are byte-identical; no cycles (leaves never import the host).
New split-guard test characterizes the coercers/normalizers and the host
resolve/merge/compat orchestration.
This commit is contained in:
Diego Rodrigues de Sa e Souza
2026-06-30 23:36:55 -03:00
committed by GitHub
parent 027e0e57d8
commit d5b0ee9394
4 changed files with 696 additions and 508 deletions

View File

@@ -1,241 +1,37 @@
import { DEFAULT_API_LIMITS, PROVIDER_PROFILES } from "@omniroute/open-sse/config/constants";
import { resolveFeatureFlag } from "@/shared/utils/featureFlags";
type JsonRecord = Record<string, unknown>;
type AuthCategory = "oauth" | "apikey";
import type { JsonRecord, ResilienceSettings, ResilienceSettingsPatch } from "./settings/types";
import {
asRecord,
toInteger,
resolveStreamRecoveryDefaults,
normalizeLegacyConnectionCooldownProfile,
normalizeRequestQueueSettings,
normalizeConnectionCooldownProfile,
normalizeProviderBreakerProfile,
normalizeWaitForCooldownSettings,
normalizeComboCooldownWaitSettings,
normalizeQuotaShareConcurrencyLimitSettings,
normalizeProviderCooldownSettings,
normalizeQuotaPreflightSettings,
normalizeStreamRecoverySettings,
} from "./settings/normalize";
export interface RequestQueueSettings {
autoEnableApiKeyProviders: boolean;
requestsPerMinute: number;
minTimeBetweenRequestsMs: number;
concurrentRequests: number;
maxWaitMs: number;
}
export interface ConnectionCooldownProfileSettings {
baseCooldownMs: number;
useUpstreamRetryHints: boolean;
/**
* Issue #2100 follow-up: opt-in toggle for upstream 429 hint trust at the
* circuit-breaker cooldown layer (independent of `useUpstreamRetryHints`
* which controls retry scheduling).
*
* Stored shape is intentionally optional / `boolean | undefined`: when
* unset, the per-provider default from `providerHints.ts` applies.
* Normalize/merge MUST preserve `undefined` — do not coerce via
* `toBoolean(value, fallback)`.
*/
useUpstream429BreakerHints?: boolean;
maxBackoffSteps: number;
}
export interface ProviderBreakerProfileSettings {
failureThreshold: number;
degradationThreshold: number;
resetTimeoutMs: number;
}
export interface WaitForCooldownSettings {
enabled: boolean;
maxRetries: number;
maxRetryWaitSec: number;
maxRetryWaitMs: number;
}
/**
* Quota-share combo cooldown-aware retry (Variante A). A quota-share (`qtSd/…`)
* combo that would crystallize a 429 `model_cooldown` for a SHORT transient
* cooldown waits it out and re-dispatches instead. Guards (gating + the
* `quota_exhausted`/auth/not-found exclusions) live in
* open-sse/services/combo/comboCooldownRetry.ts; `maxWaitMs`/`maxAttempts`/
* `budgetMs` bound a single wait, the retry cycles, and the total wait time.
*/
export interface ComboCooldownWaitSettings {
enabled: boolean;
maxWaitMs: number;
maxAttempts: number;
budgetMs: number;
}
/**
* Per-connection concurrency limit for quota-share (`qtSd/…`) combos (FASE 2.1).
* The quota-share gating in selectQuotaShareTarget is fail-open and cannot
* hard-limit a single-connection pool, so concurrent requests to one
* subscription account can still flood it (→ 429 + cooldown). When a connection
* declares a positive `max_concurrent` ceiling, this layer serializes concurrent
* requests to that account through a per-connection semaphore (excess requests
* wait in the queue instead of flooding). Kill-switch only: the cap itself comes
* from each connection's `max_concurrent`. Wiring lives in
* open-sse/services/combo/quotaShareConcurrency.ts.
*/
export interface QuotaShareConcurrencyLimitSettings {
enabled: boolean;
}
export interface ProviderCooldownSettings {
/**
* Minimum cooldown (ms) before a failed provider/connection can be retried.
* This prevents subsequent requests from immediately re-walking failing providers.
* Scaled exponentially with failure count: minRetryCooldownMs * 2^(failures-1).
* Default: 5000 (5 seconds).
*/
minRetryCooldownMs: number;
/**
* Maximum cooldown (ms) before a failed provider/connection is retried regardless.
* Hard cap to prevent providers from being skipped indefinitely.
* Default: 300000 (5 minutes).
*/
maxRetryCooldownMs: number;
/**
* Enable/disable global provider cooldown tracking.
* When disabled, only per-request cooldown state is used.
* Default: true.
*/
enabled: boolean;
}
export interface QuotaPreflightSettings {
/**
* Master switch for the auto-routing quota cutoff (buildAutoCandidates). When
* disabled (default), candidates are NOT dropped for low quota before scoring —
* the soft quota penalty + connection cooldown still apply, so behavior is
* unchanged. Opt-in because the hard cutoff interacts with the auto-routing
* scorer and must be validated per deployment. Default: false.
*/
enabled: boolean;
/**
* Global minimum-remaining cutoff (percent, 0-100). A connection is skipped
* when its remaining quota drops to this value or below. Matches the
* dashboard's quota bars (which show REMAINING %, not used %), so the
* number means the same thing in both places. Default: 2 (stop at 2%
* remaining = 98% used).
*/
defaultThresholdPercent: number;
/**
* Global warn threshold (percent, 0-100 remaining %). Fires when remaining
* quota drops to this value or below. Must be HIGHER than the cutoff so
* warnings appear before the block point. Default: 20 (warn at 20%
* remaining = 80% used).
*/
warnThresholdPercent: number;
/**
* Per-(provider, window) defaults for providers that expose multiple quota
* windows (e.g. Codex's session + weekly). Values are minimum-remaining %
* cutoffs. Resolution order, low-to-high precedence:
* defaultThresholdPercent
* → providerWindowDefaults[provider][window]
* → connection.quotaWindowThresholds[window]
*/
providerWindowDefaults: Record<string, Record<string, number>>;
}
export interface StreamRecoverySettings {
/**
* Opt-in transparent recovery of truncated upstream streams (free-claude-code port).
* When enabled, the opening SSE window is briefly held (see STREAM_RECOVERY in
* open-sse/config/constants.ts) so an early cutoff can be retried before any byte
* reaches the client. OFF by default because holding the window adds up to
* STREAM_RECOVERY.HOLDBACK_MS of time-to-first-token latency on every stream.
* Default seeds from the STREAM_RECOVERY_ENABLED feature flag / env var.
*/
enabled: boolean;
/**
* Opt-in mid-stream continuation (Fase 4.4): when an upstream stream truncates AFTER
* bytes already reached the client, re-request with the partial text as an assistant
* prefill and stitch the missing suffix (plain-text OpenAI-compatible streams only;
* never with a tool call in flight). OFF by default because the recovered tail arrives
* as one burst rather than token-by-token. Default seeds from the
* STREAM_RECOVERY_MIDSTREAM_ENABLED feature flag / env var.
*/
continueMidStream: boolean;
}
export interface ResilienceSettings {
requestQueue: RequestQueueSettings;
connectionCooldown: Record<AuthCategory, ConnectionCooldownProfileSettings>;
providerBreaker: Record<AuthCategory, ProviderBreakerProfileSettings>;
waitForCooldown: WaitForCooldownSettings;
comboCooldownWait: ComboCooldownWaitSettings;
quotaShareConcurrencyLimit: QuotaShareConcurrencyLimitSettings;
providerCooldown: ProviderCooldownSettings;
quotaPreflight: QuotaPreflightSettings;
streamRecovery: StreamRecoverySettings;
}
export interface ResilienceSettingsPatch {
requestQueue?: Partial<RequestQueueSettings>;
connectionCooldown?: Partial<Record<AuthCategory, Partial<ConnectionCooldownProfileSettings>>>;
providerBreaker?: Partial<Record<AuthCategory, Partial<ProviderBreakerProfileSettings>>>;
waitForCooldown?: Partial<WaitForCooldownSettings>;
comboCooldownWait?: Partial<ComboCooldownWaitSettings>;
quotaShareConcurrencyLimit?: Partial<QuotaShareConcurrencyLimitSettings>;
providerCooldown?: Partial<ProviderCooldownSettings>;
quotaPreflight?: Partial<QuotaPreflightSettings>;
streamRecovery?: Partial<StreamRecoverySettings>;
}
function asRecord(value: unknown): JsonRecord {
return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {};
}
function toInteger(
value: unknown,
fallback: number,
options: { min?: number; max?: number } = {}
): number {
const min = options.min ?? 0;
const max = options.max ?? Number.MAX_SAFE_INTEGER;
const parsed =
typeof value === "number"
? value
: typeof value === "string" && value.trim().length > 0
? Number(value)
: Number.NaN;
if (!Number.isFinite(parsed)) {
return fallback;
}
return Math.min(max, Math.max(min, Math.trunc(parsed)));
}
function toBoolean(value: unknown, fallback: boolean): boolean {
return typeof value === "boolean" ? value : fallback;
}
function parseFeatureFlagBoolean(value: string, fallback: boolean): boolean {
const normalized = value.trim().toLowerCase();
if (normalized === "true" || normalized === "1" || normalized === "yes" || normalized === "on") {
return true;
}
if (normalized === "false" || normalized === "0" || normalized === "no" || normalized === "off") {
return false;
}
return fallback;
}
function resolveBooleanFeatureFlag(key: string, fallback: boolean): boolean {
try {
return parseFeatureFlagBoolean(resolveFeatureFlag(key), fallback);
} catch (error) {
const envValue = process.env[key];
if (typeof envValue === "string" && envValue.trim() !== "") {
return parseFeatureFlagBoolean(envValue, fallback);
}
console.error(
`[resilience] Failed to resolve ${key}, falling back to ${String(fallback)}:`,
error instanceof Error ? error.message : error
);
return fallback;
}
}
function resolveStreamRecoveryDefaults(): StreamRecoverySettings {
return {
enabled: resolveBooleanFeatureFlag("STREAM_RECOVERY_ENABLED", false),
continueMidStream: resolveBooleanFeatureFlag("STREAM_RECOVERY_MIDSTREAM_ENABLED", false),
};
}
// Re-export the settings shape (moved to ./settings/types) so this module's
// public API is unchanged.
export type {
RequestQueueSettings,
ConnectionCooldownProfileSettings,
ProviderBreakerProfileSettings,
WaitForCooldownSettings,
ComboCooldownWaitSettings,
QuotaShareConcurrencyLimitSettings,
ProviderCooldownSettings,
QuotaPreflightSettings,
StreamRecoverySettings,
ResilienceSettings,
ResilienceSettingsPatch,
} from "./settings/types";
export const DEFAULT_REQUEST_QUEUE_MAX_WAIT_MS = (() => {
const parsed = Number(process.env.RATE_LIMIT_MAX_WAIT_MS || "120000");
@@ -338,279 +134,6 @@ export const DEFAULT_RESILIENCE_SETTINGS: ResilienceSettings = {
},
};
function normalizeRequestQueueSettings(
next: unknown,
fallback: RequestQueueSettings
): RequestQueueSettings {
const record = asRecord(next);
const requestsPerMinute = toInteger(record.requestsPerMinute, fallback.requestsPerMinute, {
min: 1,
max: 1_000_000,
});
const minTimeBetweenRequestsMs = toInteger(
record.minTimeBetweenRequestsMs,
fallback.minTimeBetweenRequestsMs,
{ min: 0, max: 60 * 60 * 1000 }
);
const concurrentRequests = toInteger(record.concurrentRequests, fallback.concurrentRequests, {
min: 1,
max: 10_000,
});
const maxWaitMs = toInteger(record.maxWaitMs, fallback.maxWaitMs, {
min: 1,
max: 24 * 60 * 60 * 1000,
});
return {
autoEnableApiKeyProviders: toBoolean(
record.autoEnableApiKeyProviders,
fallback.autoEnableApiKeyProviders
),
requestsPerMinute,
minTimeBetweenRequestsMs,
concurrentRequests,
maxWaitMs,
};
}
function normalizeConnectionCooldownProfile(
next: unknown,
fallback: ConnectionCooldownProfileSettings
): ConnectionCooldownProfileSettings {
const record = asRecord(next);
// useUpstream429BreakerHints uses a 3-state input contract:
// - boolean → user override, store as-is
// - null → explicit unset sentinel, drop key so the per-provider
// default in `providerHints.ts` resolves at runtime
// - omitted → leave existing fallback value unchanged (partial-merge)
// Never coerce via `toBoolean(value, fallback)` because that would
// collapse the unset state.
const hasHintsKey = Object.prototype.hasOwnProperty.call(record, "useUpstream429BreakerHints");
const rawHints = record.useUpstream429BreakerHints;
let useUpstream429BreakerHints: boolean | undefined;
if (!hasHintsKey) {
useUpstream429BreakerHints = fallback.useUpstream429BreakerHints;
} else if (rawHints === null) {
useUpstream429BreakerHints = undefined;
} else if (typeof rawHints === "boolean") {
useUpstream429BreakerHints = rawHints;
} else {
useUpstream429BreakerHints = fallback.useUpstream429BreakerHints;
}
const out: ConnectionCooldownProfileSettings = {
baseCooldownMs: toInteger(record.baseCooldownMs, fallback.baseCooldownMs, {
min: 0,
max: 24 * 60 * 60 * 1000,
}),
useUpstreamRetryHints: toBoolean(record.useUpstreamRetryHints, fallback.useUpstreamRetryHints),
maxBackoffSteps: toInteger(record.maxBackoffSteps, fallback.maxBackoffSteps, {
min: 0,
max: 32,
}),
};
// Only attach the key when defined — preserves omission across round-trips.
if (useUpstream429BreakerHints !== undefined) {
out.useUpstream429BreakerHints = useUpstream429BreakerHints;
}
return out;
}
function normalizeLegacyConnectionCooldownProfile(
next: unknown,
fallback: ConnectionCooldownProfileSettings
): ConnectionCooldownProfileSettings {
const record = asRecord(next);
const transientCooldown = toInteger(record.transientCooldown, fallback.baseCooldownMs, {
min: 0,
max: 24 * 60 * 60 * 1000,
});
const legacyRateLimitCooldown = toInteger(record.rateLimitCooldown, transientCooldown, {
min: 0,
max: 24 * 60 * 60 * 1000,
});
const useUpstreamRetryHints =
typeof record.rateLimitCooldown === "number"
? record.rateLimitCooldown === 0
: fallback.useUpstreamRetryHints;
return {
baseCooldownMs: useUpstreamRetryHints
? transientCooldown
: Math.max(transientCooldown, legacyRateLimitCooldown),
useUpstreamRetryHints,
maxBackoffSteps: toInteger(record.maxBackoffLevel, fallback.maxBackoffSteps, {
min: 0,
max: 32,
}),
};
}
function normalizeProviderBreakerProfile(
next: unknown,
fallback: ProviderBreakerProfileSettings
): ProviderBreakerProfileSettings {
const record = asRecord(next);
const failureThreshold = toInteger(record.failureThreshold, fallback.failureThreshold, {
min: 1,
max: 1000,
});
const degradationThreshold = Math.min(
toInteger(record.degradationThreshold, fallback.degradationThreshold, {
min: 1,
max: 1000,
}),
failureThreshold <= 1 ? 1 : failureThreshold - 1
);
return {
failureThreshold,
degradationThreshold,
resetTimeoutMs: toInteger(record.resetTimeoutMs, fallback.resetTimeoutMs, {
min: 1000,
max: 24 * 60 * 60 * 1000,
}),
};
}
function normalizeProviderWindowDefaults(
next: unknown,
fallback: Record<string, Record<string, number>>
): Record<string, Record<string, number>> {
// Accept either an explicit object or fall back. Drop providers/windows
// whose values are not a valid 0-100 integer so a malformed setting can't
// accidentally disable cutoffs entirely.
const rawProviders = asRecord(next ?? fallback);
const out: Record<string, Record<string, number>> = {};
for (const [provider, windows] of Object.entries(rawProviders)) {
if (!provider || typeof windows !== "object" || windows === null) continue;
const windowMap: Record<string, number> = {};
for (const [windowName, percent] of Object.entries(windows as Record<string, unknown>)) {
if (!windowName) continue;
const parsed =
typeof percent === "number"
? percent
: typeof percent === "string" && percent.trim() !== ""
? Number(percent)
: NaN;
if (Number.isFinite(parsed)) {
const clamped = Math.min(100, Math.max(0, Math.trunc(parsed)));
windowMap[windowName] = clamped;
}
}
if (Object.keys(windowMap).length > 0) {
out[provider] = windowMap;
}
}
return out;
}
function normalizeQuotaPreflightSettings(
next: unknown,
fallback: QuotaPreflightSettings
): QuotaPreflightSettings {
const record = asRecord(next);
// Remaining-% semantics: cutoff is the lowest acceptable remaining %, warn
// is the higher "you're getting close" remaining %. So warn MUST be greater
// than cutoff — otherwise the warn log would only fire after the request
// is already blocked.
const defaultThresholdPercent = toInteger(
record.defaultThresholdPercent,
fallback.defaultThresholdPercent,
{ min: 0, max: 99 }
);
const warnRaw = toInteger(record.warnThresholdPercent, fallback.warnThresholdPercent, {
min: 0,
max: 100,
});
const warnThresholdPercent =
warnRaw <= defaultThresholdPercent ? Math.min(100, defaultThresholdPercent + 1) : warnRaw;
const providerWindowDefaults = normalizeProviderWindowDefaults(
record.providerWindowDefaults,
fallback.providerWindowDefaults
);
const enabled = typeof record.enabled === "boolean" ? record.enabled : fallback.enabled;
return { enabled, defaultThresholdPercent, warnThresholdPercent, providerWindowDefaults };
}
function normalizeWaitForCooldownSettings(
next: unknown,
fallback: WaitForCooldownSettings
): WaitForCooldownSettings {
const record = asRecord(next);
const maxRetryWaitSec = toInteger(record.maxRetryWaitSec, fallback.maxRetryWaitSec, {
min: 0,
max: 300,
});
const maxRetries = toInteger(record.maxRetries, fallback.maxRetries, { min: 0, max: 10 });
const enabled =
toBoolean(record.enabled, fallback.enabled) && maxRetries > 0 && maxRetryWaitSec > 0;
return {
enabled,
maxRetries,
maxRetryWaitSec,
maxRetryWaitMs: maxRetryWaitSec * 1000,
};
}
function normalizeComboCooldownWaitSettings(
next: unknown,
fallback: ComboCooldownWaitSettings
): ComboCooldownWaitSettings {
const record = asRecord(next);
// Hard ceiling of 30s on a single wait — this layer only ever exists for
// SHORT transient cooldowns; anything longer should fall through to the
// existing 429 crystallization (and the cross-request cooldown layers).
const maxWaitMs = toInteger(record.maxWaitMs, fallback.maxWaitMs, { min: 0, max: 30000 });
const maxAttempts = toInteger(record.maxAttempts, fallback.maxAttempts, { min: 0, max: 10 });
// Budget can never be smaller than a single wait, otherwise no wait could
// ever fire; floor it at maxWaitMs.
const budgetMs = toInteger(record.budgetMs, fallback.budgetMs, {
min: maxWaitMs,
max: 5 * 60 * 1000,
});
const enabled = toBoolean(record.enabled, fallback.enabled) && maxWaitMs > 0 && maxAttempts > 0;
return { enabled, maxWaitMs, maxAttempts, budgetMs };
}
function normalizeQuotaShareConcurrencyLimitSettings(
next: unknown,
fallback: QuotaShareConcurrencyLimitSettings
): QuotaShareConcurrencyLimitSettings {
const record = asRecord(next);
return { enabled: toBoolean(record.enabled, fallback.enabled) };
}
function normalizeProviderCooldownSettings(
next: unknown,
fallback: ProviderCooldownSettings
): ProviderCooldownSettings {
const record = asRecord(next);
const enabled = toBoolean(record.enabled, fallback.enabled);
const minRetryCooldownMs = toInteger(record.minRetryCooldownMs, fallback.minRetryCooldownMs, {
min: 0,
max: 60 * 60 * 1000,
});
const maxRetryCooldownMs = toInteger(record.maxRetryCooldownMs, fallback.maxRetryCooldownMs, {
min: minRetryCooldownMs,
max: 24 * 60 * 60 * 1000,
});
return { enabled, minRetryCooldownMs, maxRetryCooldownMs };
}
function normalizeStreamRecoverySettings(
next: unknown,
fallback: StreamRecoverySettings
): StreamRecoverySettings {
const record = asRecord(next);
return {
enabled: toBoolean(record.enabled, fallback.enabled),
continueMidStream: toBoolean(record.continueMidStream, fallback.continueMidStream),
};
}
function buildLegacyFallback(settings: JsonRecord): ResilienceSettings {
const profiles = asRecord(settings.providerProfiles);
const defaults = asRecord(settings.rateLimitDefaults);

View File

@@ -0,0 +1,359 @@
/**
* resilience/settings/normalize — coercion + per-section normalizers (pure).
*
* Extracted verbatim from resilience/settings.ts. Pure functions: no DB, no
* module state. Depends only on the feature-flag resolver and the settings
* types. The host imports these for its resolve/merge/legacy orchestration.
*
* @module lib/resilience/settings/normalize
*/
import { resolveFeatureFlag } from "@/shared/utils/featureFlags";
import type {
JsonRecord,
RequestQueueSettings,
ConnectionCooldownProfileSettings,
ProviderBreakerProfileSettings,
WaitForCooldownSettings,
ComboCooldownWaitSettings,
QuotaShareConcurrencyLimitSettings,
ProviderCooldownSettings,
QuotaPreflightSettings,
StreamRecoverySettings,
} from "./types";
export function asRecord(value: unknown): JsonRecord {
return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {};
}
export function toInteger(
value: unknown,
fallback: number,
options: { min?: number; max?: number } = {}
): number {
const min = options.min ?? 0;
const max = options.max ?? Number.MAX_SAFE_INTEGER;
const parsed =
typeof value === "number"
? value
: typeof value === "string" && value.trim().length > 0
? Number(value)
: Number.NaN;
if (!Number.isFinite(parsed)) {
return fallback;
}
return Math.min(max, Math.max(min, Math.trunc(parsed)));
}
export function toBoolean(value: unknown, fallback: boolean): boolean {
return typeof value === "boolean" ? value : fallback;
}
export function parseFeatureFlagBoolean(value: string, fallback: boolean): boolean {
const normalized = value.trim().toLowerCase();
if (normalized === "true" || normalized === "1" || normalized === "yes" || normalized === "on") {
return true;
}
if (normalized === "false" || normalized === "0" || normalized === "no" || normalized === "off") {
return false;
}
return fallback;
}
export function resolveBooleanFeatureFlag(key: string, fallback: boolean): boolean {
try {
return parseFeatureFlagBoolean(resolveFeatureFlag(key), fallback);
} catch (error) {
const envValue = process.env[key];
if (typeof envValue === "string" && envValue.trim() !== "") {
return parseFeatureFlagBoolean(envValue, fallback);
}
console.error(
`[resilience] Failed to resolve ${key}, falling back to ${String(fallback)}:`,
error instanceof Error ? error.message : error
);
return fallback;
}
}
export function resolveStreamRecoveryDefaults(): StreamRecoverySettings {
return {
enabled: resolveBooleanFeatureFlag("STREAM_RECOVERY_ENABLED", false),
continueMidStream: resolveBooleanFeatureFlag("STREAM_RECOVERY_MIDSTREAM_ENABLED", false),
};
}
export function normalizeRequestQueueSettings(
next: unknown,
fallback: RequestQueueSettings
): RequestQueueSettings {
const record = asRecord(next);
const requestsPerMinute = toInteger(record.requestsPerMinute, fallback.requestsPerMinute, {
min: 1,
max: 1_000_000,
});
const minTimeBetweenRequestsMs = toInteger(
record.minTimeBetweenRequestsMs,
fallback.minTimeBetweenRequestsMs,
{ min: 0, max: 60 * 60 * 1000 }
);
const concurrentRequests = toInteger(record.concurrentRequests, fallback.concurrentRequests, {
min: 1,
max: 10_000,
});
const maxWaitMs = toInteger(record.maxWaitMs, fallback.maxWaitMs, {
min: 1,
max: 24 * 60 * 60 * 1000,
});
return {
autoEnableApiKeyProviders: toBoolean(
record.autoEnableApiKeyProviders,
fallback.autoEnableApiKeyProviders
),
requestsPerMinute,
minTimeBetweenRequestsMs,
concurrentRequests,
maxWaitMs,
};
}
export function normalizeConnectionCooldownProfile(
next: unknown,
fallback: ConnectionCooldownProfileSettings
): ConnectionCooldownProfileSettings {
const record = asRecord(next);
// useUpstream429BreakerHints uses a 3-state input contract:
// - boolean → user override, store as-is
// - null → explicit unset sentinel, drop key so the per-provider
// default in `providerHints.ts` resolves at runtime
// - omitted → leave existing fallback value unchanged (partial-merge)
// Never coerce via `toBoolean(value, fallback)` because that would
// collapse the unset state.
const hasHintsKey = Object.prototype.hasOwnProperty.call(record, "useUpstream429BreakerHints");
const rawHints = record.useUpstream429BreakerHints;
let useUpstream429BreakerHints: boolean | undefined;
if (!hasHintsKey) {
useUpstream429BreakerHints = fallback.useUpstream429BreakerHints;
} else if (rawHints === null) {
useUpstream429BreakerHints = undefined;
} else if (typeof rawHints === "boolean") {
useUpstream429BreakerHints = rawHints;
} else {
useUpstream429BreakerHints = fallback.useUpstream429BreakerHints;
}
const out: ConnectionCooldownProfileSettings = {
baseCooldownMs: toInteger(record.baseCooldownMs, fallback.baseCooldownMs, {
min: 0,
max: 24 * 60 * 60 * 1000,
}),
useUpstreamRetryHints: toBoolean(record.useUpstreamRetryHints, fallback.useUpstreamRetryHints),
maxBackoffSteps: toInteger(record.maxBackoffSteps, fallback.maxBackoffSteps, {
min: 0,
max: 32,
}),
};
// Only attach the key when defined — preserves omission across round-trips.
if (useUpstream429BreakerHints !== undefined) {
out.useUpstream429BreakerHints = useUpstream429BreakerHints;
}
return out;
}
export function normalizeLegacyConnectionCooldownProfile(
next: unknown,
fallback: ConnectionCooldownProfileSettings
): ConnectionCooldownProfileSettings {
const record = asRecord(next);
const transientCooldown = toInteger(record.transientCooldown, fallback.baseCooldownMs, {
min: 0,
max: 24 * 60 * 60 * 1000,
});
const legacyRateLimitCooldown = toInteger(record.rateLimitCooldown, transientCooldown, {
min: 0,
max: 24 * 60 * 60 * 1000,
});
const useUpstreamRetryHints =
typeof record.rateLimitCooldown === "number"
? record.rateLimitCooldown === 0
: fallback.useUpstreamRetryHints;
return {
baseCooldownMs: useUpstreamRetryHints
? transientCooldown
: Math.max(transientCooldown, legacyRateLimitCooldown),
useUpstreamRetryHints,
maxBackoffSteps: toInteger(record.maxBackoffLevel, fallback.maxBackoffSteps, {
min: 0,
max: 32,
}),
};
}
export function normalizeProviderBreakerProfile(
next: unknown,
fallback: ProviderBreakerProfileSettings
): ProviderBreakerProfileSettings {
const record = asRecord(next);
const failureThreshold = toInteger(record.failureThreshold, fallback.failureThreshold, {
min: 1,
max: 1000,
});
const degradationThreshold = Math.min(
toInteger(record.degradationThreshold, fallback.degradationThreshold, {
min: 1,
max: 1000,
}),
failureThreshold <= 1 ? 1 : failureThreshold - 1
);
return {
failureThreshold,
degradationThreshold,
resetTimeoutMs: toInteger(record.resetTimeoutMs, fallback.resetTimeoutMs, {
min: 1000,
max: 24 * 60 * 60 * 1000,
}),
};
}
export function normalizeProviderWindowDefaults(
next: unknown,
fallback: Record<string, Record<string, number>>
): Record<string, Record<string, number>> {
// Accept either an explicit object or fall back. Drop providers/windows
// whose values are not a valid 0-100 integer so a malformed setting can't
// accidentally disable cutoffs entirely.
const rawProviders = asRecord(next ?? fallback);
const out: Record<string, Record<string, number>> = {};
for (const [provider, windows] of Object.entries(rawProviders)) {
if (!provider || typeof windows !== "object" || windows === null) continue;
const windowMap: Record<string, number> = {};
for (const [windowName, percent] of Object.entries(windows as Record<string, unknown>)) {
if (!windowName) continue;
const parsed =
typeof percent === "number"
? percent
: typeof percent === "string" && percent.trim() !== ""
? Number(percent)
: NaN;
if (Number.isFinite(parsed)) {
const clamped = Math.min(100, Math.max(0, Math.trunc(parsed)));
windowMap[windowName] = clamped;
}
}
if (Object.keys(windowMap).length > 0) {
out[provider] = windowMap;
}
}
return out;
}
export function normalizeQuotaPreflightSettings(
next: unknown,
fallback: QuotaPreflightSettings
): QuotaPreflightSettings {
const record = asRecord(next);
// Remaining-% semantics: cutoff is the lowest acceptable remaining %, warn
// is the higher "you're getting close" remaining %. So warn MUST be greater
// than cutoff — otherwise the warn log would only fire after the request
// is already blocked.
const defaultThresholdPercent = toInteger(
record.defaultThresholdPercent,
fallback.defaultThresholdPercent,
{ min: 0, max: 99 }
);
const warnRaw = toInteger(record.warnThresholdPercent, fallback.warnThresholdPercent, {
min: 0,
max: 100,
});
const warnThresholdPercent =
warnRaw <= defaultThresholdPercent ? Math.min(100, defaultThresholdPercent + 1) : warnRaw;
const providerWindowDefaults = normalizeProviderWindowDefaults(
record.providerWindowDefaults,
fallback.providerWindowDefaults
);
const enabled = typeof record.enabled === "boolean" ? record.enabled : fallback.enabled;
return { enabled, defaultThresholdPercent, warnThresholdPercent, providerWindowDefaults };
}
export function normalizeWaitForCooldownSettings(
next: unknown,
fallback: WaitForCooldownSettings
): WaitForCooldownSettings {
const record = asRecord(next);
const maxRetryWaitSec = toInteger(record.maxRetryWaitSec, fallback.maxRetryWaitSec, {
min: 0,
max: 300,
});
const maxRetries = toInteger(record.maxRetries, fallback.maxRetries, { min: 0, max: 10 });
const enabled =
toBoolean(record.enabled, fallback.enabled) && maxRetries > 0 && maxRetryWaitSec > 0;
return {
enabled,
maxRetries,
maxRetryWaitSec,
maxRetryWaitMs: maxRetryWaitSec * 1000,
};
}
export function normalizeComboCooldownWaitSettings(
next: unknown,
fallback: ComboCooldownWaitSettings
): ComboCooldownWaitSettings {
const record = asRecord(next);
// Hard ceiling of 30s on a single wait — this layer only ever exists for
// SHORT transient cooldowns; anything longer should fall through to the
// existing 429 crystallization (and the cross-request cooldown layers).
const maxWaitMs = toInteger(record.maxWaitMs, fallback.maxWaitMs, { min: 0, max: 30000 });
const maxAttempts = toInteger(record.maxAttempts, fallback.maxAttempts, { min: 0, max: 10 });
// Budget can never be smaller than a single wait, otherwise no wait could
// ever fire; floor it at maxWaitMs.
const budgetMs = toInteger(record.budgetMs, fallback.budgetMs, {
min: maxWaitMs,
max: 5 * 60 * 1000,
});
const enabled = toBoolean(record.enabled, fallback.enabled) && maxWaitMs > 0 && maxAttempts > 0;
return { enabled, maxWaitMs, maxAttempts, budgetMs };
}
export function normalizeQuotaShareConcurrencyLimitSettings(
next: unknown,
fallback: QuotaShareConcurrencyLimitSettings
): QuotaShareConcurrencyLimitSettings {
const record = asRecord(next);
return { enabled: toBoolean(record.enabled, fallback.enabled) };
}
export function normalizeProviderCooldownSettings(
next: unknown,
fallback: ProviderCooldownSettings
): ProviderCooldownSettings {
const record = asRecord(next);
const enabled = toBoolean(record.enabled, fallback.enabled);
const minRetryCooldownMs = toInteger(record.minRetryCooldownMs, fallback.minRetryCooldownMs, {
min: 0,
max: 60 * 60 * 1000,
});
const maxRetryCooldownMs = toInteger(record.maxRetryCooldownMs, fallback.maxRetryCooldownMs, {
min: minRetryCooldownMs,
max: 24 * 60 * 60 * 1000,
});
return { enabled, minRetryCooldownMs, maxRetryCooldownMs };
}
export function normalizeStreamRecoverySettings(
next: unknown,
fallback: StreamRecoverySettings
): StreamRecoverySettings {
const record = asRecord(next);
return {
enabled: toBoolean(record.enabled, fallback.enabled),
continueMidStream: toBoolean(record.continueMidStream, fallback.continueMidStream),
};
}

View File

@@ -0,0 +1,182 @@
/**
* resilience/settings/types — resilience settings shape (pure types).
*
* Extracted verbatim from resilience/settings.ts. Zero imports, type-only.
* The host re-exports the public interfaces so its API is unchanged; the
* normalize layer imports these to type its coercion.
*
* @module lib/resilience/settings/types
*/
export type JsonRecord = Record<string, unknown>;
export type AuthCategory = "oauth" | "apikey";
export interface RequestQueueSettings {
autoEnableApiKeyProviders: boolean;
requestsPerMinute: number;
minTimeBetweenRequestsMs: number;
concurrentRequests: number;
maxWaitMs: number;
}
export interface ConnectionCooldownProfileSettings {
baseCooldownMs: number;
useUpstreamRetryHints: boolean;
/**
* Issue #2100 follow-up: opt-in toggle for upstream 429 hint trust at the
* circuit-breaker cooldown layer (independent of `useUpstreamRetryHints`
* which controls retry scheduling).
*
* Stored shape is intentionally optional / `boolean | undefined`: when
* unset, the per-provider default from `providerHints.ts` applies.
* Normalize/merge MUST preserve `undefined` — do not coerce via
* `toBoolean(value, fallback)`.
*/
useUpstream429BreakerHints?: boolean;
maxBackoffSteps: number;
}
export interface ProviderBreakerProfileSettings {
failureThreshold: number;
degradationThreshold: number;
resetTimeoutMs: number;
}
export interface WaitForCooldownSettings {
enabled: boolean;
maxRetries: number;
maxRetryWaitSec: number;
maxRetryWaitMs: number;
}
/**
* Quota-share combo cooldown-aware retry (Variante A). A quota-share (`qtSd/…`)
* combo that would crystallize a 429 `model_cooldown` for a SHORT transient
* cooldown waits it out and re-dispatches instead. Guards (gating + the
* `quota_exhausted`/auth/not-found exclusions) live in
* open-sse/services/combo/comboCooldownRetry.ts; `maxWaitMs`/`maxAttempts`/
* `budgetMs` bound a single wait, the retry cycles, and the total wait time.
*/
export interface ComboCooldownWaitSettings {
enabled: boolean;
maxWaitMs: number;
maxAttempts: number;
budgetMs: number;
}
/**
* Per-connection concurrency limit for quota-share (`qtSd/…`) combos (FASE 2.1).
* The quota-share gating in selectQuotaShareTarget is fail-open and cannot
* hard-limit a single-connection pool, so concurrent requests to one
* subscription account can still flood it (→ 429 + cooldown). When a connection
* declares a positive `max_concurrent` ceiling, this layer serializes concurrent
* requests to that account through a per-connection semaphore (excess requests
* wait in the queue instead of flooding). Kill-switch only: the cap itself comes
* from each connection's `max_concurrent`. Wiring lives in
* open-sse/services/combo/quotaShareConcurrency.ts.
*/
export interface QuotaShareConcurrencyLimitSettings {
enabled: boolean;
}
export interface ProviderCooldownSettings {
/**
* Minimum cooldown (ms) before a failed provider/connection can be retried.
* This prevents subsequent requests from immediately re-walking failing providers.
* Scaled exponentially with failure count: minRetryCooldownMs * 2^(failures-1).
* Default: 5000 (5 seconds).
*/
minRetryCooldownMs: number;
/**
* Maximum cooldown (ms) before a failed provider/connection is retried regardless.
* Hard cap to prevent providers from being skipped indefinitely.
* Default: 300000 (5 minutes).
*/
maxRetryCooldownMs: number;
/**
* Enable/disable global provider cooldown tracking.
* When disabled, only per-request cooldown state is used.
* Default: true.
*/
enabled: boolean;
}
export interface QuotaPreflightSettings {
/**
* Master switch for the auto-routing quota cutoff (buildAutoCandidates). When
* disabled (default), candidates are NOT dropped for low quota before scoring —
* the soft quota penalty + connection cooldown still apply, so behavior is
* unchanged. Opt-in because the hard cutoff interacts with the auto-routing
* scorer and must be validated per deployment. Default: false.
*/
enabled: boolean;
/**
* Global minimum-remaining cutoff (percent, 0-100). A connection is skipped
* when its remaining quota drops to this value or below. Matches the
* dashboard's quota bars (which show REMAINING %, not used %), so the
* number means the same thing in both places. Default: 2 (stop at 2%
* remaining = 98% used).
*/
defaultThresholdPercent: number;
/**
* Global warn threshold (percent, 0-100 remaining %). Fires when remaining
* quota drops to this value or below. Must be HIGHER than the cutoff so
* warnings appear before the block point. Default: 20 (warn at 20%
* remaining = 80% used).
*/
warnThresholdPercent: number;
/**
* Per-(provider, window) defaults for providers that expose multiple quota
* windows (e.g. Codex's session + weekly). Values are minimum-remaining %
* cutoffs. Resolution order, low-to-high precedence:
* defaultThresholdPercent
* → providerWindowDefaults[provider][window]
* → connection.quotaWindowThresholds[window]
*/
providerWindowDefaults: Record<string, Record<string, number>>;
}
export interface StreamRecoverySettings {
/**
* Opt-in transparent recovery of truncated upstream streams (free-claude-code port).
* When enabled, the opening SSE window is briefly held (see STREAM_RECOVERY in
* open-sse/config/constants.ts) so an early cutoff can be retried before any byte
* reaches the client. OFF by default because holding the window adds up to
* STREAM_RECOVERY.HOLDBACK_MS of time-to-first-token latency on every stream.
* Default seeds from the STREAM_RECOVERY_ENABLED feature flag / env var.
*/
enabled: boolean;
/**
* Opt-in mid-stream continuation (Fase 4.4): when an upstream stream truncates AFTER
* bytes already reached the client, re-request with the partial text as an assistant
* prefill and stitch the missing suffix (plain-text OpenAI-compatible streams only;
* never with a tool call in flight). OFF by default because the recovered tail arrives
* as one burst rather than token-by-token. Default seeds from the
* STREAM_RECOVERY_MIDSTREAM_ENABLED feature flag / env var.
*/
continueMidStream: boolean;
}
export interface ResilienceSettings {
requestQueue: RequestQueueSettings;
connectionCooldown: Record<AuthCategory, ConnectionCooldownProfileSettings>;
providerBreaker: Record<AuthCategory, ProviderBreakerProfileSettings>;
waitForCooldown: WaitForCooldownSettings;
comboCooldownWait: ComboCooldownWaitSettings;
quotaShareConcurrencyLimit: QuotaShareConcurrencyLimitSettings;
providerCooldown: ProviderCooldownSettings;
quotaPreflight: QuotaPreflightSettings;
streamRecovery: StreamRecoverySettings;
}
export interface ResilienceSettingsPatch {
requestQueue?: Partial<RequestQueueSettings>;
connectionCooldown?: Partial<Record<AuthCategory, Partial<ConnectionCooldownProfileSettings>>>;
providerBreaker?: Partial<Record<AuthCategory, Partial<ProviderBreakerProfileSettings>>>;
waitForCooldown?: Partial<WaitForCooldownSettings>;
comboCooldownWait?: Partial<ComboCooldownWaitSettings>;
quotaShareConcurrencyLimit?: Partial<QuotaShareConcurrencyLimitSettings>;
providerCooldown?: Partial<ProviderCooldownSettings>;
quotaPreflight?: Partial<QuotaPreflightSettings>;
streamRecovery?: Partial<StreamRecoverySettings>;
}

View File

@@ -0,0 +1,124 @@
/**
* Split-guard — resilience/settings ↔ settings/types + settings/normalize
*
* Guards the decomposition of the (fully pure) resilience settings module into
* two leaves: settings/types.ts (the shape) and settings/normalize.ts (coercion
* + per-section normalizers). Characterizes the coercers/normalizers and proves
* the host still exposes DEFAULT_RESILIENCE_SETTINGS + resolve/merge/compat with
* the re-exported types. DB-free — the whole layer is pure.
*/
import { describe, it } from "node:test";
import assert from "node:assert/strict";
import {
asRecord,
toInteger,
toBoolean,
normalizeProviderBreakerProfile,
normalizeWaitForCooldownSettings,
} from "../../src/lib/resilience/settings/normalize.ts";
import {
DEFAULT_RESILIENCE_SETTINGS,
resolveResilienceSettings,
mergeResilienceSettings,
buildLegacyResilienceCompat,
} from "../../src/lib/resilience/settings.ts";
import type { ResilienceSettings } from "../../src/lib/resilience/settings.ts";
describe("resilience/settings normalize split-guard", () => {
it("asRecord returns {} for non-objects and passes objects through", () => {
assert.deepEqual(asRecord(null), {});
assert.deepEqual(asRecord(42), {});
assert.deepEqual(asRecord([1, 2]), {});
assert.deepEqual(asRecord({ a: 1 }), { a: 1 });
});
it("toInteger clamps to [min,max], truncates, and falls back on NaN", () => {
assert.equal(toInteger(5.9, 0), 5);
assert.equal(toInteger("7", 0), 7);
assert.equal(toInteger("nope", 3), 3);
assert.equal(toInteger(-10, 0, { min: 1, max: 100 }), 1);
assert.equal(toInteger(9999, 0, { min: 1, max: 100 }), 100);
});
it("toBoolean only accepts real booleans", () => {
assert.equal(toBoolean(true, false), true);
assert.equal(toBoolean("true", false), false);
assert.equal(toBoolean(undefined, true), true);
});
it("normalizeProviderBreakerProfile keeps degradation below failure threshold", () => {
const out = normalizeProviderBreakerProfile(
{ failureThreshold: 3, degradationThreshold: 10, resetTimeoutMs: 60000 },
{ failureThreshold: 5, degradationThreshold: 4, resetTimeoutMs: 30000 }
);
assert.equal(out.failureThreshold, 3);
assert.equal(out.degradationThreshold, 2); // clamped to failureThreshold - 1
assert.equal(out.resetTimeoutMs, 60000);
});
it("normalizeWaitForCooldownSettings derives ms and disables on zero retries/wait", () => {
const on = normalizeWaitForCooldownSettings(
{ enabled: true, maxRetries: 2, maxRetryWaitSec: 30 },
{ enabled: true, maxRetries: 3, maxRetryWaitSec: 30, maxRetryWaitMs: 30000 }
);
assert.equal(on.enabled, true);
assert.equal(on.maxRetryWaitMs, 30000);
const off = normalizeWaitForCooldownSettings(
{ enabled: true, maxRetries: 0, maxRetryWaitSec: 30 },
{ enabled: true, maxRetries: 3, maxRetryWaitSec: 30, maxRetryWaitMs: 30000 }
);
assert.equal(off.enabled, false); // maxRetries 0 forces disabled
});
it("host exposes DEFAULT_RESILIENCE_SETTINGS with the full section set", () => {
const keys = Object.keys(DEFAULT_RESILIENCE_SETTINGS).sort();
assert.deepEqual(keys, [
"comboCooldownWait",
"connectionCooldown",
"providerBreaker",
"providerCooldown",
"quotaPreflight",
"quotaShareConcurrencyLimit",
"requestQueue",
"streamRecovery",
"waitForCooldown",
]);
});
it("host resolveResilienceSettings(null) yields normalized defaults", () => {
const resolved: ResilienceSettings = resolveResilienceSettings(null);
assert.equal(typeof resolved.requestQueue.requestsPerMinute, "number");
assert.equal(typeof resolved.providerBreaker.oauth.failureThreshold, "number");
// degradation is always kept below failure by the normalizer.
assert.ok(
resolved.providerBreaker.oauth.degradationThreshold <
resolved.providerBreaker.oauth.failureThreshold ||
resolved.providerBreaker.oauth.failureThreshold <= 1
);
});
it("host mergeResilienceSettings applies a partial patch", () => {
const base = resolveResilienceSettings(null);
const merged = mergeResilienceSettings(base, {
requestQueue: { concurrentRequests: 7 },
});
assert.equal(merged.requestQueue.concurrentRequests, 7);
assert.equal(merged.requestQueue.requestsPerMinute, base.requestQueue.requestsPerMinute);
});
it("host buildLegacyResilienceCompat round-trips connection cooldown into profiles", () => {
const compat = buildLegacyResilienceCompat(DEFAULT_RESILIENCE_SETTINGS);
assert.equal(
compat.profiles.oauth.transientCooldown,
DEFAULT_RESILIENCE_SETTINGS.connectionCooldown.oauth.baseCooldownMs
);
assert.equal(
compat.defaults.requestsPerMinute,
DEFAULT_RESILIENCE_SETTINGS.requestQueue.requestsPerMinute
);
});
});