mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-06 07:12:12 +03:00
refactor(resilience): split settings.ts into types + normalize leaves (#5745)
Decompose the (fully pure) resilience settings module into two sibling leaves: - src/lib/resilience/settings/types.ts: the settings shape (11 public interfaces + JsonRecord/AuthCategory), zero imports. - src/lib/resilience/settings/normalize.ts: the coercers (asRecord/toInteger/ toBoolean/feature-flag resolvers) + the 11 per-section normalize* functions. settings.ts keeps DEFAULT_RESILIENCE_SETTINGS, DEFAULT_REQUEST_QUEUE_MAX_WAIT_MS, buildLegacyFallback, and the public orchestrators (resolveResilienceSettings, mergeResilienceSettings, buildLegacyResilienceCompat); it imports the coercers/normalizers for internal use and re-exports the 11 settings interfaces, so the public API is unchanged. Host 840->363 LOC; leaves 182 + 359 LOC (< 800 cap); host was frozen-satisfied (841), so this is debt reduction. 472 moved lines are byte-identical; no cycles (leaves never import the host). New split-guard test characterizes the coercers/normalizers and the host resolve/merge/compat orchestration.
This commit is contained in:
committed by
GitHub
parent
027e0e57d8
commit
d5b0ee9394
@@ -1,241 +1,37 @@
|
||||
import { DEFAULT_API_LIMITS, PROVIDER_PROFILES } from "@omniroute/open-sse/config/constants";
|
||||
import { resolveFeatureFlag } from "@/shared/utils/featureFlags";
|
||||
|
||||
type JsonRecord = Record<string, unknown>;
|
||||
type AuthCategory = "oauth" | "apikey";
|
||||
import type { JsonRecord, ResilienceSettings, ResilienceSettingsPatch } from "./settings/types";
|
||||
import {
|
||||
asRecord,
|
||||
toInteger,
|
||||
resolveStreamRecoveryDefaults,
|
||||
normalizeLegacyConnectionCooldownProfile,
|
||||
normalizeRequestQueueSettings,
|
||||
normalizeConnectionCooldownProfile,
|
||||
normalizeProviderBreakerProfile,
|
||||
normalizeWaitForCooldownSettings,
|
||||
normalizeComboCooldownWaitSettings,
|
||||
normalizeQuotaShareConcurrencyLimitSettings,
|
||||
normalizeProviderCooldownSettings,
|
||||
normalizeQuotaPreflightSettings,
|
||||
normalizeStreamRecoverySettings,
|
||||
} from "./settings/normalize";
|
||||
|
||||
export interface RequestQueueSettings {
|
||||
autoEnableApiKeyProviders: boolean;
|
||||
requestsPerMinute: number;
|
||||
minTimeBetweenRequestsMs: number;
|
||||
concurrentRequests: number;
|
||||
maxWaitMs: number;
|
||||
}
|
||||
|
||||
export interface ConnectionCooldownProfileSettings {
|
||||
baseCooldownMs: number;
|
||||
useUpstreamRetryHints: boolean;
|
||||
/**
|
||||
* Issue #2100 follow-up: opt-in toggle for upstream 429 hint trust at the
|
||||
* circuit-breaker cooldown layer (independent of `useUpstreamRetryHints`
|
||||
* which controls retry scheduling).
|
||||
*
|
||||
* Stored shape is intentionally optional / `boolean | undefined`: when
|
||||
* unset, the per-provider default from `providerHints.ts` applies.
|
||||
* Normalize/merge MUST preserve `undefined` — do not coerce via
|
||||
* `toBoolean(value, fallback)`.
|
||||
*/
|
||||
useUpstream429BreakerHints?: boolean;
|
||||
maxBackoffSteps: number;
|
||||
}
|
||||
|
||||
export interface ProviderBreakerProfileSettings {
|
||||
failureThreshold: number;
|
||||
degradationThreshold: number;
|
||||
resetTimeoutMs: number;
|
||||
}
|
||||
|
||||
export interface WaitForCooldownSettings {
|
||||
enabled: boolean;
|
||||
maxRetries: number;
|
||||
maxRetryWaitSec: number;
|
||||
maxRetryWaitMs: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Quota-share combo cooldown-aware retry (Variante A). A quota-share (`qtSd/…`)
|
||||
* combo that would crystallize a 429 `model_cooldown` for a SHORT transient
|
||||
* cooldown waits it out and re-dispatches instead. Guards (gating + the
|
||||
* `quota_exhausted`/auth/not-found exclusions) live in
|
||||
* open-sse/services/combo/comboCooldownRetry.ts; `maxWaitMs`/`maxAttempts`/
|
||||
* `budgetMs` bound a single wait, the retry cycles, and the total wait time.
|
||||
*/
|
||||
export interface ComboCooldownWaitSettings {
|
||||
enabled: boolean;
|
||||
maxWaitMs: number;
|
||||
maxAttempts: number;
|
||||
budgetMs: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-connection concurrency limit for quota-share (`qtSd/…`) combos (FASE 2.1).
|
||||
* The quota-share gating in selectQuotaShareTarget is fail-open and cannot
|
||||
* hard-limit a single-connection pool, so concurrent requests to one
|
||||
* subscription account can still flood it (→ 429 + cooldown). When a connection
|
||||
* declares a positive `max_concurrent` ceiling, this layer serializes concurrent
|
||||
* requests to that account through a per-connection semaphore (excess requests
|
||||
* wait in the queue instead of flooding). Kill-switch only: the cap itself comes
|
||||
* from each connection's `max_concurrent`. Wiring lives in
|
||||
* open-sse/services/combo/quotaShareConcurrency.ts.
|
||||
*/
|
||||
export interface QuotaShareConcurrencyLimitSettings {
|
||||
enabled: boolean;
|
||||
}
|
||||
|
||||
export interface ProviderCooldownSettings {
|
||||
/**
|
||||
* Minimum cooldown (ms) before a failed provider/connection can be retried.
|
||||
* This prevents subsequent requests from immediately re-walking failing providers.
|
||||
* Scaled exponentially with failure count: minRetryCooldownMs * 2^(failures-1).
|
||||
* Default: 5000 (5 seconds).
|
||||
*/
|
||||
minRetryCooldownMs: number;
|
||||
/**
|
||||
* Maximum cooldown (ms) before a failed provider/connection is retried regardless.
|
||||
* Hard cap to prevent providers from being skipped indefinitely.
|
||||
* Default: 300000 (5 minutes).
|
||||
*/
|
||||
maxRetryCooldownMs: number;
|
||||
/**
|
||||
* Enable/disable global provider cooldown tracking.
|
||||
* When disabled, only per-request cooldown state is used.
|
||||
* Default: true.
|
||||
*/
|
||||
enabled: boolean;
|
||||
}
|
||||
|
||||
export interface QuotaPreflightSettings {
|
||||
/**
|
||||
* Master switch for the auto-routing quota cutoff (buildAutoCandidates). When
|
||||
* disabled (default), candidates are NOT dropped for low quota before scoring —
|
||||
* the soft quota penalty + connection cooldown still apply, so behavior is
|
||||
* unchanged. Opt-in because the hard cutoff interacts with the auto-routing
|
||||
* scorer and must be validated per deployment. Default: false.
|
||||
*/
|
||||
enabled: boolean;
|
||||
/**
|
||||
* Global minimum-remaining cutoff (percent, 0-100). A connection is skipped
|
||||
* when its remaining quota drops to this value or below. Matches the
|
||||
* dashboard's quota bars (which show REMAINING %, not used %), so the
|
||||
* number means the same thing in both places. Default: 2 (stop at 2%
|
||||
* remaining = 98% used).
|
||||
*/
|
||||
defaultThresholdPercent: number;
|
||||
/**
|
||||
* Global warn threshold (percent, 0-100 remaining %). Fires when remaining
|
||||
* quota drops to this value or below. Must be HIGHER than the cutoff so
|
||||
* warnings appear before the block point. Default: 20 (warn at 20%
|
||||
* remaining = 80% used).
|
||||
*/
|
||||
warnThresholdPercent: number;
|
||||
/**
|
||||
* Per-(provider, window) defaults for providers that expose multiple quota
|
||||
* windows (e.g. Codex's session + weekly). Values are minimum-remaining %
|
||||
* cutoffs. Resolution order, low-to-high precedence:
|
||||
* defaultThresholdPercent
|
||||
* → providerWindowDefaults[provider][window]
|
||||
* → connection.quotaWindowThresholds[window]
|
||||
*/
|
||||
providerWindowDefaults: Record<string, Record<string, number>>;
|
||||
}
|
||||
|
||||
export interface StreamRecoverySettings {
|
||||
/**
|
||||
* Opt-in transparent recovery of truncated upstream streams (free-claude-code port).
|
||||
* When enabled, the opening SSE window is briefly held (see STREAM_RECOVERY in
|
||||
* open-sse/config/constants.ts) so an early cutoff can be retried before any byte
|
||||
* reaches the client. OFF by default because holding the window adds up to
|
||||
* STREAM_RECOVERY.HOLDBACK_MS of time-to-first-token latency on every stream.
|
||||
* Default seeds from the STREAM_RECOVERY_ENABLED feature flag / env var.
|
||||
*/
|
||||
enabled: boolean;
|
||||
/**
|
||||
* Opt-in mid-stream continuation (Fase 4.4): when an upstream stream truncates AFTER
|
||||
* bytes already reached the client, re-request with the partial text as an assistant
|
||||
* prefill and stitch the missing suffix (plain-text OpenAI-compatible streams only;
|
||||
* never with a tool call in flight). OFF by default because the recovered tail arrives
|
||||
* as one burst rather than token-by-token. Default seeds from the
|
||||
* STREAM_RECOVERY_MIDSTREAM_ENABLED feature flag / env var.
|
||||
*/
|
||||
continueMidStream: boolean;
|
||||
}
|
||||
|
||||
export interface ResilienceSettings {
|
||||
requestQueue: RequestQueueSettings;
|
||||
connectionCooldown: Record<AuthCategory, ConnectionCooldownProfileSettings>;
|
||||
providerBreaker: Record<AuthCategory, ProviderBreakerProfileSettings>;
|
||||
waitForCooldown: WaitForCooldownSettings;
|
||||
comboCooldownWait: ComboCooldownWaitSettings;
|
||||
quotaShareConcurrencyLimit: QuotaShareConcurrencyLimitSettings;
|
||||
providerCooldown: ProviderCooldownSettings;
|
||||
quotaPreflight: QuotaPreflightSettings;
|
||||
streamRecovery: StreamRecoverySettings;
|
||||
}
|
||||
|
||||
export interface ResilienceSettingsPatch {
|
||||
requestQueue?: Partial<RequestQueueSettings>;
|
||||
connectionCooldown?: Partial<Record<AuthCategory, Partial<ConnectionCooldownProfileSettings>>>;
|
||||
providerBreaker?: Partial<Record<AuthCategory, Partial<ProviderBreakerProfileSettings>>>;
|
||||
waitForCooldown?: Partial<WaitForCooldownSettings>;
|
||||
comboCooldownWait?: Partial<ComboCooldownWaitSettings>;
|
||||
quotaShareConcurrencyLimit?: Partial<QuotaShareConcurrencyLimitSettings>;
|
||||
providerCooldown?: Partial<ProviderCooldownSettings>;
|
||||
quotaPreflight?: Partial<QuotaPreflightSettings>;
|
||||
streamRecovery?: Partial<StreamRecoverySettings>;
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): JsonRecord {
|
||||
return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {};
|
||||
}
|
||||
|
||||
function toInteger(
|
||||
value: unknown,
|
||||
fallback: number,
|
||||
options: { min?: number; max?: number } = {}
|
||||
): number {
|
||||
const min = options.min ?? 0;
|
||||
const max = options.max ?? Number.MAX_SAFE_INTEGER;
|
||||
const parsed =
|
||||
typeof value === "number"
|
||||
? value
|
||||
: typeof value === "string" && value.trim().length > 0
|
||||
? Number(value)
|
||||
: Number.NaN;
|
||||
|
||||
if (!Number.isFinite(parsed)) {
|
||||
return fallback;
|
||||
}
|
||||
|
||||
return Math.min(max, Math.max(min, Math.trunc(parsed)));
|
||||
}
|
||||
|
||||
function toBoolean(value: unknown, fallback: boolean): boolean {
|
||||
return typeof value === "boolean" ? value : fallback;
|
||||
}
|
||||
|
||||
function parseFeatureFlagBoolean(value: string, fallback: boolean): boolean {
|
||||
const normalized = value.trim().toLowerCase();
|
||||
if (normalized === "true" || normalized === "1" || normalized === "yes" || normalized === "on") {
|
||||
return true;
|
||||
}
|
||||
if (normalized === "false" || normalized === "0" || normalized === "no" || normalized === "off") {
|
||||
return false;
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
|
||||
function resolveBooleanFeatureFlag(key: string, fallback: boolean): boolean {
|
||||
try {
|
||||
return parseFeatureFlagBoolean(resolveFeatureFlag(key), fallback);
|
||||
} catch (error) {
|
||||
const envValue = process.env[key];
|
||||
if (typeof envValue === "string" && envValue.trim() !== "") {
|
||||
return parseFeatureFlagBoolean(envValue, fallback);
|
||||
}
|
||||
console.error(
|
||||
`[resilience] Failed to resolve ${key}, falling back to ${String(fallback)}:`,
|
||||
error instanceof Error ? error.message : error
|
||||
);
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
function resolveStreamRecoveryDefaults(): StreamRecoverySettings {
|
||||
return {
|
||||
enabled: resolveBooleanFeatureFlag("STREAM_RECOVERY_ENABLED", false),
|
||||
continueMidStream: resolveBooleanFeatureFlag("STREAM_RECOVERY_MIDSTREAM_ENABLED", false),
|
||||
};
|
||||
}
|
||||
// Re-export the settings shape (moved to ./settings/types) so this module's
|
||||
// public API is unchanged.
|
||||
export type {
|
||||
RequestQueueSettings,
|
||||
ConnectionCooldownProfileSettings,
|
||||
ProviderBreakerProfileSettings,
|
||||
WaitForCooldownSettings,
|
||||
ComboCooldownWaitSettings,
|
||||
QuotaShareConcurrencyLimitSettings,
|
||||
ProviderCooldownSettings,
|
||||
QuotaPreflightSettings,
|
||||
StreamRecoverySettings,
|
||||
ResilienceSettings,
|
||||
ResilienceSettingsPatch,
|
||||
} from "./settings/types";
|
||||
|
||||
export const DEFAULT_REQUEST_QUEUE_MAX_WAIT_MS = (() => {
|
||||
const parsed = Number(process.env.RATE_LIMIT_MAX_WAIT_MS || "120000");
|
||||
@@ -338,279 +134,6 @@ export const DEFAULT_RESILIENCE_SETTINGS: ResilienceSettings = {
|
||||
},
|
||||
};
|
||||
|
||||
function normalizeRequestQueueSettings(
|
||||
next: unknown,
|
||||
fallback: RequestQueueSettings
|
||||
): RequestQueueSettings {
|
||||
const record = asRecord(next);
|
||||
const requestsPerMinute = toInteger(record.requestsPerMinute, fallback.requestsPerMinute, {
|
||||
min: 1,
|
||||
max: 1_000_000,
|
||||
});
|
||||
const minTimeBetweenRequestsMs = toInteger(
|
||||
record.minTimeBetweenRequestsMs,
|
||||
fallback.minTimeBetweenRequestsMs,
|
||||
{ min: 0, max: 60 * 60 * 1000 }
|
||||
);
|
||||
const concurrentRequests = toInteger(record.concurrentRequests, fallback.concurrentRequests, {
|
||||
min: 1,
|
||||
max: 10_000,
|
||||
});
|
||||
const maxWaitMs = toInteger(record.maxWaitMs, fallback.maxWaitMs, {
|
||||
min: 1,
|
||||
max: 24 * 60 * 60 * 1000,
|
||||
});
|
||||
|
||||
return {
|
||||
autoEnableApiKeyProviders: toBoolean(
|
||||
record.autoEnableApiKeyProviders,
|
||||
fallback.autoEnableApiKeyProviders
|
||||
),
|
||||
requestsPerMinute,
|
||||
minTimeBetweenRequestsMs,
|
||||
concurrentRequests,
|
||||
maxWaitMs,
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeConnectionCooldownProfile(
|
||||
next: unknown,
|
||||
fallback: ConnectionCooldownProfileSettings
|
||||
): ConnectionCooldownProfileSettings {
|
||||
const record = asRecord(next);
|
||||
// useUpstream429BreakerHints uses a 3-state input contract:
|
||||
// - boolean → user override, store as-is
|
||||
// - null → explicit unset sentinel, drop key so the per-provider
|
||||
// default in `providerHints.ts` resolves at runtime
|
||||
// - omitted → leave existing fallback value unchanged (partial-merge)
|
||||
// Never coerce via `toBoolean(value, fallback)` because that would
|
||||
// collapse the unset state.
|
||||
const hasHintsKey = Object.prototype.hasOwnProperty.call(record, "useUpstream429BreakerHints");
|
||||
const rawHints = record.useUpstream429BreakerHints;
|
||||
let useUpstream429BreakerHints: boolean | undefined;
|
||||
if (!hasHintsKey) {
|
||||
useUpstream429BreakerHints = fallback.useUpstream429BreakerHints;
|
||||
} else if (rawHints === null) {
|
||||
useUpstream429BreakerHints = undefined;
|
||||
} else if (typeof rawHints === "boolean") {
|
||||
useUpstream429BreakerHints = rawHints;
|
||||
} else {
|
||||
useUpstream429BreakerHints = fallback.useUpstream429BreakerHints;
|
||||
}
|
||||
const out: ConnectionCooldownProfileSettings = {
|
||||
baseCooldownMs: toInteger(record.baseCooldownMs, fallback.baseCooldownMs, {
|
||||
min: 0,
|
||||
max: 24 * 60 * 60 * 1000,
|
||||
}),
|
||||
useUpstreamRetryHints: toBoolean(record.useUpstreamRetryHints, fallback.useUpstreamRetryHints),
|
||||
maxBackoffSteps: toInteger(record.maxBackoffSteps, fallback.maxBackoffSteps, {
|
||||
min: 0,
|
||||
max: 32,
|
||||
}),
|
||||
};
|
||||
// Only attach the key when defined — preserves omission across round-trips.
|
||||
if (useUpstream429BreakerHints !== undefined) {
|
||||
out.useUpstream429BreakerHints = useUpstream429BreakerHints;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function normalizeLegacyConnectionCooldownProfile(
|
||||
next: unknown,
|
||||
fallback: ConnectionCooldownProfileSettings
|
||||
): ConnectionCooldownProfileSettings {
|
||||
const record = asRecord(next);
|
||||
const transientCooldown = toInteger(record.transientCooldown, fallback.baseCooldownMs, {
|
||||
min: 0,
|
||||
max: 24 * 60 * 60 * 1000,
|
||||
});
|
||||
const legacyRateLimitCooldown = toInteger(record.rateLimitCooldown, transientCooldown, {
|
||||
min: 0,
|
||||
max: 24 * 60 * 60 * 1000,
|
||||
});
|
||||
const useUpstreamRetryHints =
|
||||
typeof record.rateLimitCooldown === "number"
|
||||
? record.rateLimitCooldown === 0
|
||||
: fallback.useUpstreamRetryHints;
|
||||
|
||||
return {
|
||||
baseCooldownMs: useUpstreamRetryHints
|
||||
? transientCooldown
|
||||
: Math.max(transientCooldown, legacyRateLimitCooldown),
|
||||
useUpstreamRetryHints,
|
||||
maxBackoffSteps: toInteger(record.maxBackoffLevel, fallback.maxBackoffSteps, {
|
||||
min: 0,
|
||||
max: 32,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeProviderBreakerProfile(
|
||||
next: unknown,
|
||||
fallback: ProviderBreakerProfileSettings
|
||||
): ProviderBreakerProfileSettings {
|
||||
const record = asRecord(next);
|
||||
const failureThreshold = toInteger(record.failureThreshold, fallback.failureThreshold, {
|
||||
min: 1,
|
||||
max: 1000,
|
||||
});
|
||||
const degradationThreshold = Math.min(
|
||||
toInteger(record.degradationThreshold, fallback.degradationThreshold, {
|
||||
min: 1,
|
||||
max: 1000,
|
||||
}),
|
||||
failureThreshold <= 1 ? 1 : failureThreshold - 1
|
||||
);
|
||||
|
||||
return {
|
||||
failureThreshold,
|
||||
degradationThreshold,
|
||||
resetTimeoutMs: toInteger(record.resetTimeoutMs, fallback.resetTimeoutMs, {
|
||||
min: 1000,
|
||||
max: 24 * 60 * 60 * 1000,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeProviderWindowDefaults(
|
||||
next: unknown,
|
||||
fallback: Record<string, Record<string, number>>
|
||||
): Record<string, Record<string, number>> {
|
||||
// Accept either an explicit object or fall back. Drop providers/windows
|
||||
// whose values are not a valid 0-100 integer so a malformed setting can't
|
||||
// accidentally disable cutoffs entirely.
|
||||
const rawProviders = asRecord(next ?? fallback);
|
||||
const out: Record<string, Record<string, number>> = {};
|
||||
for (const [provider, windows] of Object.entries(rawProviders)) {
|
||||
if (!provider || typeof windows !== "object" || windows === null) continue;
|
||||
const windowMap: Record<string, number> = {};
|
||||
for (const [windowName, percent] of Object.entries(windows as Record<string, unknown>)) {
|
||||
if (!windowName) continue;
|
||||
const parsed =
|
||||
typeof percent === "number"
|
||||
? percent
|
||||
: typeof percent === "string" && percent.trim() !== ""
|
||||
? Number(percent)
|
||||
: NaN;
|
||||
if (Number.isFinite(parsed)) {
|
||||
const clamped = Math.min(100, Math.max(0, Math.trunc(parsed)));
|
||||
windowMap[windowName] = clamped;
|
||||
}
|
||||
}
|
||||
if (Object.keys(windowMap).length > 0) {
|
||||
out[provider] = windowMap;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function normalizeQuotaPreflightSettings(
|
||||
next: unknown,
|
||||
fallback: QuotaPreflightSettings
|
||||
): QuotaPreflightSettings {
|
||||
const record = asRecord(next);
|
||||
// Remaining-% semantics: cutoff is the lowest acceptable remaining %, warn
|
||||
// is the higher "you're getting close" remaining %. So warn MUST be greater
|
||||
// than cutoff — otherwise the warn log would only fire after the request
|
||||
// is already blocked.
|
||||
const defaultThresholdPercent = toInteger(
|
||||
record.defaultThresholdPercent,
|
||||
fallback.defaultThresholdPercent,
|
||||
{ min: 0, max: 99 }
|
||||
);
|
||||
const warnRaw = toInteger(record.warnThresholdPercent, fallback.warnThresholdPercent, {
|
||||
min: 0,
|
||||
max: 100,
|
||||
});
|
||||
const warnThresholdPercent =
|
||||
warnRaw <= defaultThresholdPercent ? Math.min(100, defaultThresholdPercent + 1) : warnRaw;
|
||||
const providerWindowDefaults = normalizeProviderWindowDefaults(
|
||||
record.providerWindowDefaults,
|
||||
fallback.providerWindowDefaults
|
||||
);
|
||||
const enabled = typeof record.enabled === "boolean" ? record.enabled : fallback.enabled;
|
||||
return { enabled, defaultThresholdPercent, warnThresholdPercent, providerWindowDefaults };
|
||||
}
|
||||
|
||||
function normalizeWaitForCooldownSettings(
|
||||
next: unknown,
|
||||
fallback: WaitForCooldownSettings
|
||||
): WaitForCooldownSettings {
|
||||
const record = asRecord(next);
|
||||
const maxRetryWaitSec = toInteger(record.maxRetryWaitSec, fallback.maxRetryWaitSec, {
|
||||
min: 0,
|
||||
max: 300,
|
||||
});
|
||||
const maxRetries = toInteger(record.maxRetries, fallback.maxRetries, { min: 0, max: 10 });
|
||||
const enabled =
|
||||
toBoolean(record.enabled, fallback.enabled) && maxRetries > 0 && maxRetryWaitSec > 0;
|
||||
|
||||
return {
|
||||
enabled,
|
||||
maxRetries,
|
||||
maxRetryWaitSec,
|
||||
maxRetryWaitMs: maxRetryWaitSec * 1000,
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeComboCooldownWaitSettings(
|
||||
next: unknown,
|
||||
fallback: ComboCooldownWaitSettings
|
||||
): ComboCooldownWaitSettings {
|
||||
const record = asRecord(next);
|
||||
// Hard ceiling of 30s on a single wait — this layer only ever exists for
|
||||
// SHORT transient cooldowns; anything longer should fall through to the
|
||||
// existing 429 crystallization (and the cross-request cooldown layers).
|
||||
const maxWaitMs = toInteger(record.maxWaitMs, fallback.maxWaitMs, { min: 0, max: 30000 });
|
||||
const maxAttempts = toInteger(record.maxAttempts, fallback.maxAttempts, { min: 0, max: 10 });
|
||||
// Budget can never be smaller than a single wait, otherwise no wait could
|
||||
// ever fire; floor it at maxWaitMs.
|
||||
const budgetMs = toInteger(record.budgetMs, fallback.budgetMs, {
|
||||
min: maxWaitMs,
|
||||
max: 5 * 60 * 1000,
|
||||
});
|
||||
const enabled = toBoolean(record.enabled, fallback.enabled) && maxWaitMs > 0 && maxAttempts > 0;
|
||||
|
||||
return { enabled, maxWaitMs, maxAttempts, budgetMs };
|
||||
}
|
||||
|
||||
function normalizeQuotaShareConcurrencyLimitSettings(
|
||||
next: unknown,
|
||||
fallback: QuotaShareConcurrencyLimitSettings
|
||||
): QuotaShareConcurrencyLimitSettings {
|
||||
const record = asRecord(next);
|
||||
return { enabled: toBoolean(record.enabled, fallback.enabled) };
|
||||
}
|
||||
|
||||
function normalizeProviderCooldownSettings(
|
||||
next: unknown,
|
||||
fallback: ProviderCooldownSettings
|
||||
): ProviderCooldownSettings {
|
||||
const record = asRecord(next);
|
||||
const enabled = toBoolean(record.enabled, fallback.enabled);
|
||||
const minRetryCooldownMs = toInteger(record.minRetryCooldownMs, fallback.minRetryCooldownMs, {
|
||||
min: 0,
|
||||
max: 60 * 60 * 1000,
|
||||
});
|
||||
const maxRetryCooldownMs = toInteger(record.maxRetryCooldownMs, fallback.maxRetryCooldownMs, {
|
||||
min: minRetryCooldownMs,
|
||||
max: 24 * 60 * 60 * 1000,
|
||||
});
|
||||
|
||||
return { enabled, minRetryCooldownMs, maxRetryCooldownMs };
|
||||
}
|
||||
|
||||
function normalizeStreamRecoverySettings(
|
||||
next: unknown,
|
||||
fallback: StreamRecoverySettings
|
||||
): StreamRecoverySettings {
|
||||
const record = asRecord(next);
|
||||
return {
|
||||
enabled: toBoolean(record.enabled, fallback.enabled),
|
||||
continueMidStream: toBoolean(record.continueMidStream, fallback.continueMidStream),
|
||||
};
|
||||
}
|
||||
|
||||
function buildLegacyFallback(settings: JsonRecord): ResilienceSettings {
|
||||
const profiles = asRecord(settings.providerProfiles);
|
||||
const defaults = asRecord(settings.rateLimitDefaults);
|
||||
|
||||
359
src/lib/resilience/settings/normalize.ts
Normal file
359
src/lib/resilience/settings/normalize.ts
Normal file
@@ -0,0 +1,359 @@
|
||||
/**
|
||||
* resilience/settings/normalize — coercion + per-section normalizers (pure).
|
||||
*
|
||||
* Extracted verbatim from resilience/settings.ts. Pure functions: no DB, no
|
||||
* module state. Depends only on the feature-flag resolver and the settings
|
||||
* types. The host imports these for its resolve/merge/legacy orchestration.
|
||||
*
|
||||
* @module lib/resilience/settings/normalize
|
||||
*/
|
||||
|
||||
import { resolveFeatureFlag } from "@/shared/utils/featureFlags";
|
||||
import type {
|
||||
JsonRecord,
|
||||
RequestQueueSettings,
|
||||
ConnectionCooldownProfileSettings,
|
||||
ProviderBreakerProfileSettings,
|
||||
WaitForCooldownSettings,
|
||||
ComboCooldownWaitSettings,
|
||||
QuotaShareConcurrencyLimitSettings,
|
||||
ProviderCooldownSettings,
|
||||
QuotaPreflightSettings,
|
||||
StreamRecoverySettings,
|
||||
} from "./types";
|
||||
|
||||
export function asRecord(value: unknown): JsonRecord {
|
||||
return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {};
|
||||
}
|
||||
|
||||
export function toInteger(
|
||||
value: unknown,
|
||||
fallback: number,
|
||||
options: { min?: number; max?: number } = {}
|
||||
): number {
|
||||
const min = options.min ?? 0;
|
||||
const max = options.max ?? Number.MAX_SAFE_INTEGER;
|
||||
const parsed =
|
||||
typeof value === "number"
|
||||
? value
|
||||
: typeof value === "string" && value.trim().length > 0
|
||||
? Number(value)
|
||||
: Number.NaN;
|
||||
|
||||
if (!Number.isFinite(parsed)) {
|
||||
return fallback;
|
||||
}
|
||||
|
||||
return Math.min(max, Math.max(min, Math.trunc(parsed)));
|
||||
}
|
||||
|
||||
export function toBoolean(value: unknown, fallback: boolean): boolean {
|
||||
return typeof value === "boolean" ? value : fallback;
|
||||
}
|
||||
|
||||
export function parseFeatureFlagBoolean(value: string, fallback: boolean): boolean {
|
||||
const normalized = value.trim().toLowerCase();
|
||||
if (normalized === "true" || normalized === "1" || normalized === "yes" || normalized === "on") {
|
||||
return true;
|
||||
}
|
||||
if (normalized === "false" || normalized === "0" || normalized === "no" || normalized === "off") {
|
||||
return false;
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
|
||||
export function resolveBooleanFeatureFlag(key: string, fallback: boolean): boolean {
|
||||
try {
|
||||
return parseFeatureFlagBoolean(resolveFeatureFlag(key), fallback);
|
||||
} catch (error) {
|
||||
const envValue = process.env[key];
|
||||
if (typeof envValue === "string" && envValue.trim() !== "") {
|
||||
return parseFeatureFlagBoolean(envValue, fallback);
|
||||
}
|
||||
console.error(
|
||||
`[resilience] Failed to resolve ${key}, falling back to ${String(fallback)}:`,
|
||||
error instanceof Error ? error.message : error
|
||||
);
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
export function resolveStreamRecoveryDefaults(): StreamRecoverySettings {
|
||||
return {
|
||||
enabled: resolveBooleanFeatureFlag("STREAM_RECOVERY_ENABLED", false),
|
||||
continueMidStream: resolveBooleanFeatureFlag("STREAM_RECOVERY_MIDSTREAM_ENABLED", false),
|
||||
};
|
||||
}
|
||||
|
||||
export function normalizeRequestQueueSettings(
|
||||
next: unknown,
|
||||
fallback: RequestQueueSettings
|
||||
): RequestQueueSettings {
|
||||
const record = asRecord(next);
|
||||
const requestsPerMinute = toInteger(record.requestsPerMinute, fallback.requestsPerMinute, {
|
||||
min: 1,
|
||||
max: 1_000_000,
|
||||
});
|
||||
const minTimeBetweenRequestsMs = toInteger(
|
||||
record.minTimeBetweenRequestsMs,
|
||||
fallback.minTimeBetweenRequestsMs,
|
||||
{ min: 0, max: 60 * 60 * 1000 }
|
||||
);
|
||||
const concurrentRequests = toInteger(record.concurrentRequests, fallback.concurrentRequests, {
|
||||
min: 1,
|
||||
max: 10_000,
|
||||
});
|
||||
const maxWaitMs = toInteger(record.maxWaitMs, fallback.maxWaitMs, {
|
||||
min: 1,
|
||||
max: 24 * 60 * 60 * 1000,
|
||||
});
|
||||
|
||||
return {
|
||||
autoEnableApiKeyProviders: toBoolean(
|
||||
record.autoEnableApiKeyProviders,
|
||||
fallback.autoEnableApiKeyProviders
|
||||
),
|
||||
requestsPerMinute,
|
||||
minTimeBetweenRequestsMs,
|
||||
concurrentRequests,
|
||||
maxWaitMs,
|
||||
};
|
||||
}
|
||||
|
||||
export function normalizeConnectionCooldownProfile(
|
||||
next: unknown,
|
||||
fallback: ConnectionCooldownProfileSettings
|
||||
): ConnectionCooldownProfileSettings {
|
||||
const record = asRecord(next);
|
||||
// useUpstream429BreakerHints uses a 3-state input contract:
|
||||
// - boolean → user override, store as-is
|
||||
// - null → explicit unset sentinel, drop key so the per-provider
|
||||
// default in `providerHints.ts` resolves at runtime
|
||||
// - omitted → leave existing fallback value unchanged (partial-merge)
|
||||
// Never coerce via `toBoolean(value, fallback)` because that would
|
||||
// collapse the unset state.
|
||||
const hasHintsKey = Object.prototype.hasOwnProperty.call(record, "useUpstream429BreakerHints");
|
||||
const rawHints = record.useUpstream429BreakerHints;
|
||||
let useUpstream429BreakerHints: boolean | undefined;
|
||||
if (!hasHintsKey) {
|
||||
useUpstream429BreakerHints = fallback.useUpstream429BreakerHints;
|
||||
} else if (rawHints === null) {
|
||||
useUpstream429BreakerHints = undefined;
|
||||
} else if (typeof rawHints === "boolean") {
|
||||
useUpstream429BreakerHints = rawHints;
|
||||
} else {
|
||||
useUpstream429BreakerHints = fallback.useUpstream429BreakerHints;
|
||||
}
|
||||
const out: ConnectionCooldownProfileSettings = {
|
||||
baseCooldownMs: toInteger(record.baseCooldownMs, fallback.baseCooldownMs, {
|
||||
min: 0,
|
||||
max: 24 * 60 * 60 * 1000,
|
||||
}),
|
||||
useUpstreamRetryHints: toBoolean(record.useUpstreamRetryHints, fallback.useUpstreamRetryHints),
|
||||
maxBackoffSteps: toInteger(record.maxBackoffSteps, fallback.maxBackoffSteps, {
|
||||
min: 0,
|
||||
max: 32,
|
||||
}),
|
||||
};
|
||||
// Only attach the key when defined — preserves omission across round-trips.
|
||||
if (useUpstream429BreakerHints !== undefined) {
|
||||
out.useUpstream429BreakerHints = useUpstream429BreakerHints;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function normalizeLegacyConnectionCooldownProfile(
|
||||
next: unknown,
|
||||
fallback: ConnectionCooldownProfileSettings
|
||||
): ConnectionCooldownProfileSettings {
|
||||
const record = asRecord(next);
|
||||
const transientCooldown = toInteger(record.transientCooldown, fallback.baseCooldownMs, {
|
||||
min: 0,
|
||||
max: 24 * 60 * 60 * 1000,
|
||||
});
|
||||
const legacyRateLimitCooldown = toInteger(record.rateLimitCooldown, transientCooldown, {
|
||||
min: 0,
|
||||
max: 24 * 60 * 60 * 1000,
|
||||
});
|
||||
const useUpstreamRetryHints =
|
||||
typeof record.rateLimitCooldown === "number"
|
||||
? record.rateLimitCooldown === 0
|
||||
: fallback.useUpstreamRetryHints;
|
||||
|
||||
return {
|
||||
baseCooldownMs: useUpstreamRetryHints
|
||||
? transientCooldown
|
||||
: Math.max(transientCooldown, legacyRateLimitCooldown),
|
||||
useUpstreamRetryHints,
|
||||
maxBackoffSteps: toInteger(record.maxBackoffLevel, fallback.maxBackoffSteps, {
|
||||
min: 0,
|
||||
max: 32,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
export function normalizeProviderBreakerProfile(
|
||||
next: unknown,
|
||||
fallback: ProviderBreakerProfileSettings
|
||||
): ProviderBreakerProfileSettings {
|
||||
const record = asRecord(next);
|
||||
const failureThreshold = toInteger(record.failureThreshold, fallback.failureThreshold, {
|
||||
min: 1,
|
||||
max: 1000,
|
||||
});
|
||||
const degradationThreshold = Math.min(
|
||||
toInteger(record.degradationThreshold, fallback.degradationThreshold, {
|
||||
min: 1,
|
||||
max: 1000,
|
||||
}),
|
||||
failureThreshold <= 1 ? 1 : failureThreshold - 1
|
||||
);
|
||||
|
||||
return {
|
||||
failureThreshold,
|
||||
degradationThreshold,
|
||||
resetTimeoutMs: toInteger(record.resetTimeoutMs, fallback.resetTimeoutMs, {
|
||||
min: 1000,
|
||||
max: 24 * 60 * 60 * 1000,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
export function normalizeProviderWindowDefaults(
|
||||
next: unknown,
|
||||
fallback: Record<string, Record<string, number>>
|
||||
): Record<string, Record<string, number>> {
|
||||
// Accept either an explicit object or fall back. Drop providers/windows
|
||||
// whose values are not a valid 0-100 integer so a malformed setting can't
|
||||
// accidentally disable cutoffs entirely.
|
||||
const rawProviders = asRecord(next ?? fallback);
|
||||
const out: Record<string, Record<string, number>> = {};
|
||||
for (const [provider, windows] of Object.entries(rawProviders)) {
|
||||
if (!provider || typeof windows !== "object" || windows === null) continue;
|
||||
const windowMap: Record<string, number> = {};
|
||||
for (const [windowName, percent] of Object.entries(windows as Record<string, unknown>)) {
|
||||
if (!windowName) continue;
|
||||
const parsed =
|
||||
typeof percent === "number"
|
||||
? percent
|
||||
: typeof percent === "string" && percent.trim() !== ""
|
||||
? Number(percent)
|
||||
: NaN;
|
||||
if (Number.isFinite(parsed)) {
|
||||
const clamped = Math.min(100, Math.max(0, Math.trunc(parsed)));
|
||||
windowMap[windowName] = clamped;
|
||||
}
|
||||
}
|
||||
if (Object.keys(windowMap).length > 0) {
|
||||
out[provider] = windowMap;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function normalizeQuotaPreflightSettings(
|
||||
next: unknown,
|
||||
fallback: QuotaPreflightSettings
|
||||
): QuotaPreflightSettings {
|
||||
const record = asRecord(next);
|
||||
// Remaining-% semantics: cutoff is the lowest acceptable remaining %, warn
|
||||
// is the higher "you're getting close" remaining %. So warn MUST be greater
|
||||
// than cutoff — otherwise the warn log would only fire after the request
|
||||
// is already blocked.
|
||||
const defaultThresholdPercent = toInteger(
|
||||
record.defaultThresholdPercent,
|
||||
fallback.defaultThresholdPercent,
|
||||
{ min: 0, max: 99 }
|
||||
);
|
||||
const warnRaw = toInteger(record.warnThresholdPercent, fallback.warnThresholdPercent, {
|
||||
min: 0,
|
||||
max: 100,
|
||||
});
|
||||
const warnThresholdPercent =
|
||||
warnRaw <= defaultThresholdPercent ? Math.min(100, defaultThresholdPercent + 1) : warnRaw;
|
||||
const providerWindowDefaults = normalizeProviderWindowDefaults(
|
||||
record.providerWindowDefaults,
|
||||
fallback.providerWindowDefaults
|
||||
);
|
||||
const enabled = typeof record.enabled === "boolean" ? record.enabled : fallback.enabled;
|
||||
return { enabled, defaultThresholdPercent, warnThresholdPercent, providerWindowDefaults };
|
||||
}
|
||||
|
||||
export function normalizeWaitForCooldownSettings(
|
||||
next: unknown,
|
||||
fallback: WaitForCooldownSettings
|
||||
): WaitForCooldownSettings {
|
||||
const record = asRecord(next);
|
||||
const maxRetryWaitSec = toInteger(record.maxRetryWaitSec, fallback.maxRetryWaitSec, {
|
||||
min: 0,
|
||||
max: 300,
|
||||
});
|
||||
const maxRetries = toInteger(record.maxRetries, fallback.maxRetries, { min: 0, max: 10 });
|
||||
const enabled =
|
||||
toBoolean(record.enabled, fallback.enabled) && maxRetries > 0 && maxRetryWaitSec > 0;
|
||||
|
||||
return {
|
||||
enabled,
|
||||
maxRetries,
|
||||
maxRetryWaitSec,
|
||||
maxRetryWaitMs: maxRetryWaitSec * 1000,
|
||||
};
|
||||
}
|
||||
|
||||
export function normalizeComboCooldownWaitSettings(
|
||||
next: unknown,
|
||||
fallback: ComboCooldownWaitSettings
|
||||
): ComboCooldownWaitSettings {
|
||||
const record = asRecord(next);
|
||||
// Hard ceiling of 30s on a single wait — this layer only ever exists for
|
||||
// SHORT transient cooldowns; anything longer should fall through to the
|
||||
// existing 429 crystallization (and the cross-request cooldown layers).
|
||||
const maxWaitMs = toInteger(record.maxWaitMs, fallback.maxWaitMs, { min: 0, max: 30000 });
|
||||
const maxAttempts = toInteger(record.maxAttempts, fallback.maxAttempts, { min: 0, max: 10 });
|
||||
// Budget can never be smaller than a single wait, otherwise no wait could
|
||||
// ever fire; floor it at maxWaitMs.
|
||||
const budgetMs = toInteger(record.budgetMs, fallback.budgetMs, {
|
||||
min: maxWaitMs,
|
||||
max: 5 * 60 * 1000,
|
||||
});
|
||||
const enabled = toBoolean(record.enabled, fallback.enabled) && maxWaitMs > 0 && maxAttempts > 0;
|
||||
|
||||
return { enabled, maxWaitMs, maxAttempts, budgetMs };
|
||||
}
|
||||
|
||||
export function normalizeQuotaShareConcurrencyLimitSettings(
|
||||
next: unknown,
|
||||
fallback: QuotaShareConcurrencyLimitSettings
|
||||
): QuotaShareConcurrencyLimitSettings {
|
||||
const record = asRecord(next);
|
||||
return { enabled: toBoolean(record.enabled, fallback.enabled) };
|
||||
}
|
||||
|
||||
export function normalizeProviderCooldownSettings(
|
||||
next: unknown,
|
||||
fallback: ProviderCooldownSettings
|
||||
): ProviderCooldownSettings {
|
||||
const record = asRecord(next);
|
||||
const enabled = toBoolean(record.enabled, fallback.enabled);
|
||||
const minRetryCooldownMs = toInteger(record.minRetryCooldownMs, fallback.minRetryCooldownMs, {
|
||||
min: 0,
|
||||
max: 60 * 60 * 1000,
|
||||
});
|
||||
const maxRetryCooldownMs = toInteger(record.maxRetryCooldownMs, fallback.maxRetryCooldownMs, {
|
||||
min: minRetryCooldownMs,
|
||||
max: 24 * 60 * 60 * 1000,
|
||||
});
|
||||
|
||||
return { enabled, minRetryCooldownMs, maxRetryCooldownMs };
|
||||
}
|
||||
|
||||
export function normalizeStreamRecoverySettings(
|
||||
next: unknown,
|
||||
fallback: StreamRecoverySettings
|
||||
): StreamRecoverySettings {
|
||||
const record = asRecord(next);
|
||||
return {
|
||||
enabled: toBoolean(record.enabled, fallback.enabled),
|
||||
continueMidStream: toBoolean(record.continueMidStream, fallback.continueMidStream),
|
||||
};
|
||||
}
|
||||
182
src/lib/resilience/settings/types.ts
Normal file
182
src/lib/resilience/settings/types.ts
Normal file
@@ -0,0 +1,182 @@
|
||||
/**
|
||||
* resilience/settings/types — resilience settings shape (pure types).
|
||||
*
|
||||
* Extracted verbatim from resilience/settings.ts. Zero imports, type-only.
|
||||
* The host re-exports the public interfaces so its API is unchanged; the
|
||||
* normalize layer imports these to type its coercion.
|
||||
*
|
||||
* @module lib/resilience/settings/types
|
||||
*/
|
||||
|
||||
export type JsonRecord = Record<string, unknown>;
|
||||
export type AuthCategory = "oauth" | "apikey";
|
||||
|
||||
export interface RequestQueueSettings {
|
||||
autoEnableApiKeyProviders: boolean;
|
||||
requestsPerMinute: number;
|
||||
minTimeBetweenRequestsMs: number;
|
||||
concurrentRequests: number;
|
||||
maxWaitMs: number;
|
||||
}
|
||||
|
||||
export interface ConnectionCooldownProfileSettings {
|
||||
baseCooldownMs: number;
|
||||
useUpstreamRetryHints: boolean;
|
||||
/**
|
||||
* Issue #2100 follow-up: opt-in toggle for upstream 429 hint trust at the
|
||||
* circuit-breaker cooldown layer (independent of `useUpstreamRetryHints`
|
||||
* which controls retry scheduling).
|
||||
*
|
||||
* Stored shape is intentionally optional / `boolean | undefined`: when
|
||||
* unset, the per-provider default from `providerHints.ts` applies.
|
||||
* Normalize/merge MUST preserve `undefined` — do not coerce via
|
||||
* `toBoolean(value, fallback)`.
|
||||
*/
|
||||
useUpstream429BreakerHints?: boolean;
|
||||
maxBackoffSteps: number;
|
||||
}
|
||||
|
||||
export interface ProviderBreakerProfileSettings {
|
||||
failureThreshold: number;
|
||||
degradationThreshold: number;
|
||||
resetTimeoutMs: number;
|
||||
}
|
||||
|
||||
export interface WaitForCooldownSettings {
|
||||
enabled: boolean;
|
||||
maxRetries: number;
|
||||
maxRetryWaitSec: number;
|
||||
maxRetryWaitMs: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Quota-share combo cooldown-aware retry (Variante A). A quota-share (`qtSd/…`)
|
||||
* combo that would crystallize a 429 `model_cooldown` for a SHORT transient
|
||||
* cooldown waits it out and re-dispatches instead. Guards (gating + the
|
||||
* `quota_exhausted`/auth/not-found exclusions) live in
|
||||
* open-sse/services/combo/comboCooldownRetry.ts; `maxWaitMs`/`maxAttempts`/
|
||||
* `budgetMs` bound a single wait, the retry cycles, and the total wait time.
|
||||
*/
|
||||
export interface ComboCooldownWaitSettings {
|
||||
enabled: boolean;
|
||||
maxWaitMs: number;
|
||||
maxAttempts: number;
|
||||
budgetMs: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-connection concurrency limit for quota-share (`qtSd/…`) combos (FASE 2.1).
|
||||
* The quota-share gating in selectQuotaShareTarget is fail-open and cannot
|
||||
* hard-limit a single-connection pool, so concurrent requests to one
|
||||
* subscription account can still flood it (→ 429 + cooldown). When a connection
|
||||
* declares a positive `max_concurrent` ceiling, this layer serializes concurrent
|
||||
* requests to that account through a per-connection semaphore (excess requests
|
||||
* wait in the queue instead of flooding). Kill-switch only: the cap itself comes
|
||||
* from each connection's `max_concurrent`. Wiring lives in
|
||||
* open-sse/services/combo/quotaShareConcurrency.ts.
|
||||
*/
|
||||
export interface QuotaShareConcurrencyLimitSettings {
|
||||
enabled: boolean;
|
||||
}
|
||||
|
||||
export interface ProviderCooldownSettings {
|
||||
/**
|
||||
* Minimum cooldown (ms) before a failed provider/connection can be retried.
|
||||
* This prevents subsequent requests from immediately re-walking failing providers.
|
||||
* Scaled exponentially with failure count: minRetryCooldownMs * 2^(failures-1).
|
||||
* Default: 5000 (5 seconds).
|
||||
*/
|
||||
minRetryCooldownMs: number;
|
||||
/**
|
||||
* Maximum cooldown (ms) before a failed provider/connection is retried regardless.
|
||||
* Hard cap to prevent providers from being skipped indefinitely.
|
||||
* Default: 300000 (5 minutes).
|
||||
*/
|
||||
maxRetryCooldownMs: number;
|
||||
/**
|
||||
* Enable/disable global provider cooldown tracking.
|
||||
* When disabled, only per-request cooldown state is used.
|
||||
* Default: true.
|
||||
*/
|
||||
enabled: boolean;
|
||||
}
|
||||
|
||||
export interface QuotaPreflightSettings {
|
||||
/**
|
||||
* Master switch for the auto-routing quota cutoff (buildAutoCandidates). When
|
||||
* disabled (default), candidates are NOT dropped for low quota before scoring —
|
||||
* the soft quota penalty + connection cooldown still apply, so behavior is
|
||||
* unchanged. Opt-in because the hard cutoff interacts with the auto-routing
|
||||
* scorer and must be validated per deployment. Default: false.
|
||||
*/
|
||||
enabled: boolean;
|
||||
/**
|
||||
* Global minimum-remaining cutoff (percent, 0-100). A connection is skipped
|
||||
* when its remaining quota drops to this value or below. Matches the
|
||||
* dashboard's quota bars (which show REMAINING %, not used %), so the
|
||||
* number means the same thing in both places. Default: 2 (stop at 2%
|
||||
* remaining = 98% used).
|
||||
*/
|
||||
defaultThresholdPercent: number;
|
||||
/**
|
||||
* Global warn threshold (percent, 0-100 remaining %). Fires when remaining
|
||||
* quota drops to this value or below. Must be HIGHER than the cutoff so
|
||||
* warnings appear before the block point. Default: 20 (warn at 20%
|
||||
* remaining = 80% used).
|
||||
*/
|
||||
warnThresholdPercent: number;
|
||||
/**
|
||||
* Per-(provider, window) defaults for providers that expose multiple quota
|
||||
* windows (e.g. Codex's session + weekly). Values are minimum-remaining %
|
||||
* cutoffs. Resolution order, low-to-high precedence:
|
||||
* defaultThresholdPercent
|
||||
* → providerWindowDefaults[provider][window]
|
||||
* → connection.quotaWindowThresholds[window]
|
||||
*/
|
||||
providerWindowDefaults: Record<string, Record<string, number>>;
|
||||
}
|
||||
|
||||
export interface StreamRecoverySettings {
|
||||
/**
|
||||
* Opt-in transparent recovery of truncated upstream streams (free-claude-code port).
|
||||
* When enabled, the opening SSE window is briefly held (see STREAM_RECOVERY in
|
||||
* open-sse/config/constants.ts) so an early cutoff can be retried before any byte
|
||||
* reaches the client. OFF by default because holding the window adds up to
|
||||
* STREAM_RECOVERY.HOLDBACK_MS of time-to-first-token latency on every stream.
|
||||
* Default seeds from the STREAM_RECOVERY_ENABLED feature flag / env var.
|
||||
*/
|
||||
enabled: boolean;
|
||||
/**
|
||||
* Opt-in mid-stream continuation (Fase 4.4): when an upstream stream truncates AFTER
|
||||
* bytes already reached the client, re-request with the partial text as an assistant
|
||||
* prefill and stitch the missing suffix (plain-text OpenAI-compatible streams only;
|
||||
* never with a tool call in flight). OFF by default because the recovered tail arrives
|
||||
* as one burst rather than token-by-token. Default seeds from the
|
||||
* STREAM_RECOVERY_MIDSTREAM_ENABLED feature flag / env var.
|
||||
*/
|
||||
continueMidStream: boolean;
|
||||
}
|
||||
|
||||
export interface ResilienceSettings {
|
||||
requestQueue: RequestQueueSettings;
|
||||
connectionCooldown: Record<AuthCategory, ConnectionCooldownProfileSettings>;
|
||||
providerBreaker: Record<AuthCategory, ProviderBreakerProfileSettings>;
|
||||
waitForCooldown: WaitForCooldownSettings;
|
||||
comboCooldownWait: ComboCooldownWaitSettings;
|
||||
quotaShareConcurrencyLimit: QuotaShareConcurrencyLimitSettings;
|
||||
providerCooldown: ProviderCooldownSettings;
|
||||
quotaPreflight: QuotaPreflightSettings;
|
||||
streamRecovery: StreamRecoverySettings;
|
||||
}
|
||||
|
||||
export interface ResilienceSettingsPatch {
|
||||
requestQueue?: Partial<RequestQueueSettings>;
|
||||
connectionCooldown?: Partial<Record<AuthCategory, Partial<ConnectionCooldownProfileSettings>>>;
|
||||
providerBreaker?: Partial<Record<AuthCategory, Partial<ProviderBreakerProfileSettings>>>;
|
||||
waitForCooldown?: Partial<WaitForCooldownSettings>;
|
||||
comboCooldownWait?: Partial<ComboCooldownWaitSettings>;
|
||||
quotaShareConcurrencyLimit?: Partial<QuotaShareConcurrencyLimitSettings>;
|
||||
providerCooldown?: Partial<ProviderCooldownSettings>;
|
||||
quotaPreflight?: Partial<QuotaPreflightSettings>;
|
||||
streamRecovery?: Partial<StreamRecoverySettings>;
|
||||
}
|
||||
124
tests/unit/resilience-settings-normalize-split.test.ts
Normal file
124
tests/unit/resilience-settings-normalize-split.test.ts
Normal file
@@ -0,0 +1,124 @@
|
||||
/**
|
||||
* Split-guard — resilience/settings ↔ settings/types + settings/normalize
|
||||
*
|
||||
* Guards the decomposition of the (fully pure) resilience settings module into
|
||||
* two leaves: settings/types.ts (the shape) and settings/normalize.ts (coercion
|
||||
* + per-section normalizers). Characterizes the coercers/normalizers and proves
|
||||
* the host still exposes DEFAULT_RESILIENCE_SETTINGS + resolve/merge/compat with
|
||||
* the re-exported types. DB-free — the whole layer is pure.
|
||||
*/
|
||||
|
||||
import { describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import {
|
||||
asRecord,
|
||||
toInteger,
|
||||
toBoolean,
|
||||
normalizeProviderBreakerProfile,
|
||||
normalizeWaitForCooldownSettings,
|
||||
} from "../../src/lib/resilience/settings/normalize.ts";
|
||||
|
||||
import {
|
||||
DEFAULT_RESILIENCE_SETTINGS,
|
||||
resolveResilienceSettings,
|
||||
mergeResilienceSettings,
|
||||
buildLegacyResilienceCompat,
|
||||
} from "../../src/lib/resilience/settings.ts";
|
||||
import type { ResilienceSettings } from "../../src/lib/resilience/settings.ts";
|
||||
|
||||
describe("resilience/settings normalize split-guard", () => {
|
||||
it("asRecord returns {} for non-objects and passes objects through", () => {
|
||||
assert.deepEqual(asRecord(null), {});
|
||||
assert.deepEqual(asRecord(42), {});
|
||||
assert.deepEqual(asRecord([1, 2]), {});
|
||||
assert.deepEqual(asRecord({ a: 1 }), { a: 1 });
|
||||
});
|
||||
|
||||
it("toInteger clamps to [min,max], truncates, and falls back on NaN", () => {
|
||||
assert.equal(toInteger(5.9, 0), 5);
|
||||
assert.equal(toInteger("7", 0), 7);
|
||||
assert.equal(toInteger("nope", 3), 3);
|
||||
assert.equal(toInteger(-10, 0, { min: 1, max: 100 }), 1);
|
||||
assert.equal(toInteger(9999, 0, { min: 1, max: 100 }), 100);
|
||||
});
|
||||
|
||||
it("toBoolean only accepts real booleans", () => {
|
||||
assert.equal(toBoolean(true, false), true);
|
||||
assert.equal(toBoolean("true", false), false);
|
||||
assert.equal(toBoolean(undefined, true), true);
|
||||
});
|
||||
|
||||
it("normalizeProviderBreakerProfile keeps degradation below failure threshold", () => {
|
||||
const out = normalizeProviderBreakerProfile(
|
||||
{ failureThreshold: 3, degradationThreshold: 10, resetTimeoutMs: 60000 },
|
||||
{ failureThreshold: 5, degradationThreshold: 4, resetTimeoutMs: 30000 }
|
||||
);
|
||||
assert.equal(out.failureThreshold, 3);
|
||||
assert.equal(out.degradationThreshold, 2); // clamped to failureThreshold - 1
|
||||
assert.equal(out.resetTimeoutMs, 60000);
|
||||
});
|
||||
|
||||
it("normalizeWaitForCooldownSettings derives ms and disables on zero retries/wait", () => {
|
||||
const on = normalizeWaitForCooldownSettings(
|
||||
{ enabled: true, maxRetries: 2, maxRetryWaitSec: 30 },
|
||||
{ enabled: true, maxRetries: 3, maxRetryWaitSec: 30, maxRetryWaitMs: 30000 }
|
||||
);
|
||||
assert.equal(on.enabled, true);
|
||||
assert.equal(on.maxRetryWaitMs, 30000);
|
||||
|
||||
const off = normalizeWaitForCooldownSettings(
|
||||
{ enabled: true, maxRetries: 0, maxRetryWaitSec: 30 },
|
||||
{ enabled: true, maxRetries: 3, maxRetryWaitSec: 30, maxRetryWaitMs: 30000 }
|
||||
);
|
||||
assert.equal(off.enabled, false); // maxRetries 0 forces disabled
|
||||
});
|
||||
|
||||
it("host exposes DEFAULT_RESILIENCE_SETTINGS with the full section set", () => {
|
||||
const keys = Object.keys(DEFAULT_RESILIENCE_SETTINGS).sort();
|
||||
assert.deepEqual(keys, [
|
||||
"comboCooldownWait",
|
||||
"connectionCooldown",
|
||||
"providerBreaker",
|
||||
"providerCooldown",
|
||||
"quotaPreflight",
|
||||
"quotaShareConcurrencyLimit",
|
||||
"requestQueue",
|
||||
"streamRecovery",
|
||||
"waitForCooldown",
|
||||
]);
|
||||
});
|
||||
|
||||
it("host resolveResilienceSettings(null) yields normalized defaults", () => {
|
||||
const resolved: ResilienceSettings = resolveResilienceSettings(null);
|
||||
assert.equal(typeof resolved.requestQueue.requestsPerMinute, "number");
|
||||
assert.equal(typeof resolved.providerBreaker.oauth.failureThreshold, "number");
|
||||
// degradation is always kept below failure by the normalizer.
|
||||
assert.ok(
|
||||
resolved.providerBreaker.oauth.degradationThreshold <
|
||||
resolved.providerBreaker.oauth.failureThreshold ||
|
||||
resolved.providerBreaker.oauth.failureThreshold <= 1
|
||||
);
|
||||
});
|
||||
|
||||
it("host mergeResilienceSettings applies a partial patch", () => {
|
||||
const base = resolveResilienceSettings(null);
|
||||
const merged = mergeResilienceSettings(base, {
|
||||
requestQueue: { concurrentRequests: 7 },
|
||||
});
|
||||
assert.equal(merged.requestQueue.concurrentRequests, 7);
|
||||
assert.equal(merged.requestQueue.requestsPerMinute, base.requestQueue.requestsPerMinute);
|
||||
});
|
||||
|
||||
it("host buildLegacyResilienceCompat round-trips connection cooldown into profiles", () => {
|
||||
const compat = buildLegacyResilienceCompat(DEFAULT_RESILIENCE_SETTINGS);
|
||||
assert.equal(
|
||||
compat.profiles.oauth.transientCooldown,
|
||||
DEFAULT_RESILIENCE_SETTINGS.connectionCooldown.oauth.baseCooldownMs
|
||||
);
|
||||
assert.equal(
|
||||
compat.defaults.requestsPerMinute,
|
||||
DEFAULT_RESILIENCE_SETTINGS.requestQueue.requestsPerMinute
|
||||
);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user