mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-07 07:42:13 +03:00
* fix(cli-tools): guard modelId type before calling indexOf E2E shakedown v3.8.0: cli-tools quebrava com TypeError quando dynamicModels continha entradas sem .id (objeto retornado diretamente em vez de string). * fix(offline): avoid SSR/CSR hydration mismatch on navigator.onLine Replace useState+lazy-initializer with useSyncExternalStore so the server snapshot (() => false) and client snapshot (() => navigator.onLine) are declared separately. React hydrates with the server value and switches to the real online status client-side without a mismatch. * chore(i18n): add missing en.json keys for translator, cli-tools, memory, onboarding Adds 58 missing keys identified by the new dashboard audit script: - cliTools: 18 custom CLI builder keys (CustomCliCard) - translator: 24 keys covering stream transformer, live monitor, test bench - memory: 12 health/pagination/dialog keys - onboarding.tier: 8 keys for the tier tour walkthrough Also adds scripts/i18n/audit-dashboard-pages.mjs which scans all dashboard pages, reports t() calls referencing missing en.json keys, and flags candidate hardcoded JSX/attribute strings. * chore(i18n): replace hardcoded UI text with t() calls across dashboard (round 1) Subagents refactored 8 high-impact dashboard pages, replacing 81 of the 407 hardcoded English/PT strings flagged by the audit with proper useTranslations() lookups. Added 73 corresponding keys to en.json across the home, apiManager, providers, settings, and usage namespaces. Pages affected: - BudgetTab (27 → 0) - HomePageClient (2 → 0) - RoutingTab (25 → 7) - ResilienceTab (38 → 18) - SystemStorageTab (42 → 21) - providers/[id] (17 → 15) - ApiManagerPageClient (14 → 13) - OneproxyTab (13 → 10) Also adds two helper scripts: - scripts/i18n/extract-keys-from-diff.mjs — extracts new keys from git diff - scripts/i18n/merge-keys.mjs — merges a pending-keys JSON into en.json Remaining hardcoded strings will be addressed in follow-up rounds. * chore(i18n): replace hardcoded UI text with t() calls across dashboard (round 2) Continues round 1 (commit8d34f4c65). Round-2 subagents refactored additional dashboard pages, replacing 77 more hardcoded strings with useTranslations() lookups. Added 79 corresponding keys to en.json across the a2aDashboard, agents, analytics, apiManager, cliTools, common, and settings namespaces. Pages affected: - a2a/page (new useTranslations + 6 keys) - agent-skills/page (new useTranslations + 9 keys) - AutoRoutingAnalyticsTab (new useTranslations + 6 keys) - AppearanceTab (8 → 6 remaining) - OneproxyTab (10 → 0) - ResilienceTab (18 → 0 missing key) - RoutingTab (7 → 0 missing key) - VisionBridgeSettingsTab (new useTranslations + 6 keys) - CopilotToolCard (7 → 0 missing key) - ApiManagerPageClient (13 → 0 missing key) - gamification/admin (new useTranslations + 7 keys) Hardcoded total: 326 → 249. Real missing keys: 0 (the 6 still flagged are false positives in exampleTemplates.tsx where t is passed as a parameter — keys exist at translator.templatePayloads.*). * chore(i18n): replace hardcoded UI text with t() calls across dashboard (round 3) Round-3 subagents and manual edits refactored 9 more dashboard pages (plus 2 small extras), replacing ~80 hardcoded strings with useTranslations() lookups. Added 79 corresponding keys to en.json across analytics, cloudAgents, combos, common, health, settings, and usage namespaces. Pages affected: - analytics/ComboHealthTab (new useTranslations + 15 keys) - analytics/CompressionAnalyticsTab (new useTranslations + 11 keys) - settings/SystemStorageTab (21 → 0 missing key) - tokens/page (new useTranslations + 13 keys) - usage/BudgetTab (9 missing fixed) - health/page (manual: 6 keys) - cloud-agents/page (manual: 3 keys) - combos/page (manual: 1 key) Hardcoded total: 249 → 164. Real missing keys: 0 (6 remaining are exampleTemplates.tsx false positives). Also adds scripts/i18n/build-pending-from-missing.mjs which reads _audit.json and locates English values from HEAD to rebuild _pending-keys.json after race-condition resets between subagent edits. * chore(i18n): localize remaining dashboard settings labels Replace hardcoded labels in compression and resilience settings with translation lookups to continue the dashboard i18n cleanup. Add the v3.8.0 dashboard shakedown runbook to document the manual smoke-test process and known dev environment pitfalls. * chore(i18n): replace hardcoded UI text with t() calls across dashboard (round 4) Round-4 subagent + manual key-resolution refactored remaining strings in 3 high-traffic settings/API tabs, plus extracted English values for keys that were already added as t() calls but lost during the previous en.json race-condition resets. Pages affected: - api-manager/ApiManagerPageClient (7 → 0 missing key) - settings/CompressionSettingsTab (8 → 0 missing key) - settings/MemorySkillsTab (8 → 0 missing key) - settings/ResilienceTab (4 more keys recovered) Hardcoded total: 164 → 140. Real missing keys: 0 (6 remaining are the exampleTemplates.tsx false positives — t passed as parameter). * chore(i18n): replace hardcoded UI text with t() calls across dashboard (round 5) Round-5 agent began processing the remaining smaller dashboard files. Added 5 more keys to en.json for providers/[id]/page.tsx OAuth flow labels and the cross-OS auto-detection hint. Pages affected: - providers/[id]/page.tsx (5 keys) Hardcoded total: 140 → 136. Real missing keys: 0. * chore(i18n): resolve last 2 missing providers/[id] keys Adds providerDetailMyClaudeAccountPlaceholder and providerDetailPathAutoDetected — the final user-visible labels in the providers/[id] page that the round-5 subagent rewrote to t() calls without yet adding to en.json. Real missing keys: 0 (6 remaining are exampleTemplates.tsx false positives — t is passed as a parameter so the audit cannot resolve the namespace; keys do exist at translator.templatePayloads.*). * chore(i18n): replace hardcoded UI text with t() calls across dashboard (round 6 — 10 parallel agents) Round-6 dispatched 10 parallel subagents covering all 57 remaining dashboard files. Each agent worked on a disjoint file set to avoid en.json race conditions. Added ~60 new i18n keys across 9 namespaces covering small UI labels, table headers, search placeholders, and empty-state messages. Major changes: - analytics: SearchAnalyticsTab, ProviderUtilizationTab, DiversityScoreCard, CompressionAnalyticsTab (new useTranslations + keys) - batch: BatchDetailModal, BatchListTab, FileDetailModal, FilesListTab (new useTranslations + keys) - settings: CliproxyapiSettingsTab, PayloadRulesTab, ModelCooldownsCard, AppearanceTab, PricingTab (mostly new useTranslations) - endpoint: TokenSaverCard, ApiEndpointsTab, EndpointPageClient - cache: CachePerformance, IdempotencyLayer, ReasoningCacheTab, MediaPageClient, page - combos: IntelligentComboPanel, page - playground: ChatPlayground, SearchPlayground - providers: ProviderCard - onboarding: TierFlowDiagram - changelog: ChangelogViewer - home: ProviderTopology, TierCoverageWidget, BootstrapBanner, BadgeToast - usage: BudgetTab, BudgetTelemetryCards, QuotaTable - quotaShare: QuotaSharePageClient - profile: page - leaderboard: page - skills: page Hardcoded total: 131 → 60. Real missing keys: 0 plus 1 false-positive for combos.modePack (lookup via prop-passed t). * chore(i18n): finalize round-6 keys for batch/cache/endpoint/usage Adds the remaining keys produced by parallel agents A4, A6, A8, A9: - common: batch-related labels (BatchDetailModal, BatchListTab, FileDetailModal, FilesListTab, page) + profile/leaderboard - cache: hit rate, latency, retry, avg chars - endpoint: token saver, API endpoints, copy URL, cloud/local labels - usage: noSpend, activeSessions, quotaAlerts, budget timing - skills: install/marketplace/filter - proxyRegistry/quotaShare/mcpDashboard: misc labels Hardcoded total: 60 → 48. Real missing keys: 0 (modePack remaining is a false positive — combos.modePack exists but the audit can't resolve it since IntelligentComboPanel receives t as a prop). * fix(playground): dedupe filteredModels to avoid duplicate React key warning The /v1/models endpoint can return the same model id twice (e.g., when a model is listed by both an alias and its canonical provider), which made the <Select> emit two <option> elements with the same key — triggering "Encountered two children with the same key, codex/gpt-5.5". Replace the chained filter + map with a single pass that skips ids already added. * fix(playground): guard against non-string model ids before .split/.startsWith The /v1/models endpoint can include synthetic entries (combos, locals, in-progress imports) with a null/undefined id. The playground used to call m.id.split("/") in the provider-discovery loop, which threw on the first non-string entry; the surrounding .catch(() => {}) silently swallowed the error, so the provider/model/account dropdowns ended up empty even though /v1/models returned thousands of valid entries. - Skip entries without a string id before split/startsWith. - Log the rejection in the .catch handler so future regressions are visible in DevTools instead of silently emptying the UI. * fix(playground): guard ChatPlayground filteredModels for non-string ids Same root cause as commit49fe356b9: ChatPlayground filtered models with m.id.startsWith(...) which crashed on null/undefined ids returned by /v1/models (synthetic combo entries). Apply the same defensive guard and dedupe used in the parent page. * fix(claude): drop orphan tool_result after fixToolAdjacency strip (discussion #2410) Discussion #2410 reports Claude returning 400 for sequences like: assistant: tool_use(id=X) user: <plain text> ← breaks adjacency user: tool_result(id=X) The previous round added `fixToolAdjacency` (commit44d9abac9) which correctly strips the orphan tool_use from the assistant message. But that left the now-unmatched tool_result intact, so the upstream rejected the request with: messages.N.content.M: unexpected `tool_use_id` found in `tool_result` blocks: X. Each tool_result block must have a corresponding tool_use block in the previous message. Fix: after running `fixToolAdjacency`, re-run `fixToolPairs` to drop the orphaned tool_result blocks. All three call sites updated: - contextManager.purifyHistory (both inside the binary-search loop and the final pass) - BaseExecutor message-prep (Claude path) - claudeCodeCompatible request signer Also tightens an unrelated dynamic-key access in readNestedString (claudeCodeCompatible) to satisfy the prototype- pollution scanner triggered by the post-tool semgrep hook. * fix(mitm): point runtime manager re-export to js entrypoint Use the emitted `.js` path for the runtime manager re-export so dynamic runtime loading resolves correctly outside the Turbopack alias handling. * docs: add AgentRouter setup guide (#2422) Integrated into release/v3.8.0 — AgentRouter setup guide docs. * feat: add new feature on combos - falloverBeforeRetry (#2417) Integrated into release/v3.8.0 — falloverBeforeRetry for per-model quota skipping in combos. * feat(batch): implement 10 feature requests harvested (#2414) Integrated into release/v3.8.0 — batch of 10 feature requests: llama.cpp local provider, upstream error exposure, Termux detection, providers rotate CLI, t3.chat web skeleton, Zed Docker integration, Kiro multi-account OAuth isolation, auto-combo cost blending, auto-combo context filter, combo provider-level exhaustion tracking (#1731). Conflicts with #2417 (falloverBeforeRetry) resolved. * fix(gamification): resolve SQL bug, auth gap, pagination, and anomaly scoring (#2421) Integrated into release/v3.8.0 — 6 critical gamification bug fixes: SQL SELECT in checkActionCountBadges, federation auth enforcement, leaderboard pagination offset, real z-score computation, addXp level calculation, and barrel index.ts * docs(changelog): add post-release entries for #2414 #2417 #2421 #2422 - feat(batch): T3-Chat-Web executor, exhaustedProviders set (#1731), Zed Docker - feat(combos): falloverBeforeRetry + setTry loop (#2417 — @hartmark) - fix(gamification): SQL SELECT bug, federation auth, pagination, z-score (#2421 — @oyi77) - docs: AgentRouter setup guide (#2422 — @leninejunior) * fix(security): resolve CodeQL random/password-hash alerts and sync docs & tests --------- Co-authored-by: diegosouzapw <diego.souza.pw@gmail.com> Co-authored-by: Lenine Júnior <lenine@engrene.com.br> Co-authored-by: Markus Hartung <mail@hartmark.se> Co-authored-by: Paijo <14921983+oyi77@users.noreply.github.com>
1381 lines
48 KiB
TypeScript
1381 lines
48 KiB
TypeScript
import {
|
|
BACKOFF_STEPS_MS,
|
|
PROVIDER_PROFILES,
|
|
RateLimitReason,
|
|
HTTP_STATUS,
|
|
} from "../config/constants.ts";
|
|
import {
|
|
BACKOFF_CONFIG,
|
|
COOLDOWN_MS,
|
|
calculateBackoffCooldown,
|
|
findMatchingErrorRule,
|
|
matchErrorRuleByText,
|
|
matchErrorRuleByStatus,
|
|
} from "../config/errorConfig.ts";
|
|
import { getPassthroughProviders, getProviderCategory } from "../config/providerRegistry.ts";
|
|
import {
|
|
DEFAULT_RESILIENCE_SETTINGS,
|
|
resolveResilienceSettings,
|
|
} from "../../src/lib/resilience/settings";
|
|
import {
|
|
getAllCircuitBreakerStatuses,
|
|
getCircuitBreaker,
|
|
STATE,
|
|
} from "../../src/shared/utils/circuitBreaker";
|
|
import { classify429FromError, type FailureKind } from "../../src/shared/utils/classify429";
|
|
import { resolveUseUpstream429BreakerHints } from "../../src/shared/utils/providerHints";
|
|
|
|
type ProviderProfile = {
|
|
baseCooldownMs: number;
|
|
useUpstreamRetryHints: boolean;
|
|
/** Issue #2100 follow-up. Stored override; undefined → per-provider default. */
|
|
useUpstream429BreakerHints?: boolean;
|
|
maxBackoffSteps: number;
|
|
failureThreshold: number;
|
|
resetTimeoutMs: number;
|
|
transientCooldown: number;
|
|
rateLimitCooldown: number;
|
|
maxBackoffLevel: number;
|
|
circuitBreakerThreshold: number;
|
|
circuitBreakerReset: number;
|
|
// Provider-level circuit breaker fields
|
|
providerFailureThreshold: number;
|
|
providerFailureWindowMs: number;
|
|
providerCooldownMs: number;
|
|
};
|
|
type JsonRecord = Record<string, unknown>;
|
|
type RateLimitReasonValue = (typeof RateLimitReason)[keyof typeof RateLimitReason];
|
|
type ModelLockoutEntry = {
|
|
reason: string;
|
|
until: number;
|
|
lockedAt: number;
|
|
failureCount: number;
|
|
lastFailureAt: number;
|
|
resetAfterMs: number;
|
|
};
|
|
type ModelFailureState = {
|
|
failureCount: number;
|
|
lastFailureAt: number;
|
|
resetAfterMs: number;
|
|
};
|
|
type AccountState = JsonRecord & {
|
|
id?: string | null;
|
|
rateLimitedUntil?: string | null;
|
|
backoffLevel?: number | null;
|
|
lastError?: unknown;
|
|
status?: string;
|
|
};
|
|
|
|
function toJsonRecord(value: unknown): JsonRecord {
|
|
return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {};
|
|
}
|
|
|
|
// Provider-level failure tracking for circuit breaker behavior
|
|
// Error codes that count toward provider-level failure threshold
|
|
// 429 (rate limit) is intentionally excluded: rate limits are connection-scoped
|
|
// and handled via Connection Cooldown, not provider-wide circuit breaker.
|
|
// Counting 429 toward provider failure causes cascading provider trips at scale
|
|
// when many connections hit rate limits simultaneously (Issue #1846).
|
|
const PROVIDER_FAILURE_ERROR_CODES = new Set([408, 500, 502, 503, 504]);
|
|
|
|
// Per-connection failure deduplication: prevents rapid-fire failures from the
|
|
// same connection from counting multiple times toward the provider breaker.
|
|
const CONNECTION_FAILURE_DEDUP_MS = 5000;
|
|
const lastConnectionFailure = new Map<string, number>();
|
|
|
|
// T06 (sub2api PR #1037): Signals that indicate permanent account deactivation.
|
|
// When a 401 body contains these strings, the account is permanently dead
|
|
// and should NOT be retried after token refresh.
|
|
export const ACCOUNT_DEACTIVATED_SIGNALS = [
|
|
"account_deactivated",
|
|
"account has been deactivated",
|
|
"account has been disabled",
|
|
"your account has been suspended",
|
|
"this account is deactivated",
|
|
// AG (Antigravity/Google Cloud Code) permanent ban signals
|
|
"verify your account to continue",
|
|
"this service has been disabled in this account for violation",
|
|
"this service has been disabled in this account",
|
|
];
|
|
|
|
// T10 (sub2api PR #1169): Signals that indicate billing credits are exhausted.
|
|
// Distinct from rate-limit 429 — the account won't recover until credits are added.
|
|
export const CREDITS_EXHAUSTED_SIGNALS = [
|
|
"insufficient_quota",
|
|
"billing_hard_limit_reached",
|
|
"exceeded your current quota",
|
|
"credit_balance_too_low",
|
|
"your credit balance is too low",
|
|
"credits exhausted",
|
|
"out of credits",
|
|
"payment required",
|
|
];
|
|
|
|
// T11: Signals that indicate OAuth token is invalid/expired (not permanent deactivation)
|
|
export const OAUTH_INVALID_TOKEN_SIGNALS = [
|
|
"invalid authentication credentials",
|
|
"oauth 2",
|
|
"login cookie",
|
|
"valid authentication credential",
|
|
"invalid credentials",
|
|
];
|
|
|
|
// Context overflow patterns — the prompt exceeds the model's maximum context length.
|
|
// Different providers phrase this differently. Used to decide whether a 400 error
|
|
// should trigger combo fallback (a different model may have a larger context window).
|
|
const CONTEXT_OVERFLOW_PATTERNS = [
|
|
/\binput is too long\b/i,
|
|
/\binput too long\b/i,
|
|
/\bcontext.*(too long|exceeded|overflow|limit)/i,
|
|
/\btoo many tokens\b/i,
|
|
/\bprompt is too long\b/i,
|
|
/\bcontext window/i,
|
|
/\bmaximum context/i,
|
|
/\bmax.*token/i,
|
|
/\btoken limit/i,
|
|
/\brequest too large\b/i,
|
|
];
|
|
|
|
// Malformed request patterns — the model rejected the message format but a different
|
|
// provider/model in the combo may accept it.
|
|
const MALFORMED_REQUEST_PATTERNS = [
|
|
/\bimproperly formed request\b/i,
|
|
/\binvalid.*message.*format/i,
|
|
/\bmessages must alternate\b/i,
|
|
/\bempty (message|content)\b/i,
|
|
// Tool call function name errors
|
|
/\bfunction'?s? name (?:can't|can not|is|has) (?:blank|empty|missing)/i,
|
|
/function.*name.*(?:blank|empty|missing)/i,
|
|
/tool_call.*name.*(?:blank|empty|missing)/i,
|
|
];
|
|
|
|
/**
|
|
* T06: Returns true if response body indicates the account is permanently deactivated.
|
|
*/
|
|
export function isAccountDeactivated(errorText: string): boolean {
|
|
const lower = String(errorText || "").toLowerCase();
|
|
return ACCOUNT_DEACTIVATED_SIGNALS.some((sig) => lower.includes(sig));
|
|
}
|
|
|
|
/**
|
|
* T10: Returns true if response body indicates credits/quota are permanently exhausted.
|
|
*/
|
|
export function isCreditsExhausted(errorText: string): boolean {
|
|
const lower = String(errorText || "").toLowerCase();
|
|
return CREDITS_EXHAUSTED_SIGNALS.some((sig) => lower.includes(sig));
|
|
}
|
|
|
|
/**
|
|
* T11: Returns true if response body indicates OAuth token is invalid/expired.
|
|
* This is different from permanent account deactivation - token refresh can recover.
|
|
*/
|
|
export function isOAuthInvalidToken(errorText: string): boolean {
|
|
const lower = String(errorText || "").toLowerCase();
|
|
return OAUTH_INVALID_TOKEN_SIGNALS.some((sig) => lower.includes(sig));
|
|
}
|
|
|
|
// ─── Resilience Profile Helper ──────────────────────────────────────────────
|
|
|
|
function asRecord(value: unknown): JsonRecord {
|
|
return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {};
|
|
}
|
|
|
|
function isCompatibleProvider(provider: string | null | undefined): boolean {
|
|
return (
|
|
typeof provider === "string" &&
|
|
(provider.startsWith("openai-compatible-") || provider.startsWith("anthropic-compatible-"))
|
|
);
|
|
}
|
|
|
|
function buildProviderProfile(
|
|
category: "oauth" | "apikey",
|
|
settings?: Record<string, unknown> | null
|
|
) {
|
|
const resilience = settings ? resolveResilienceSettings(settings) : DEFAULT_RESILIENCE_SETTINGS;
|
|
const connectionCooldown = resilience.connectionCooldown[category];
|
|
const providerBreaker = resilience.providerBreaker[category];
|
|
|
|
return {
|
|
baseCooldownMs: connectionCooldown.baseCooldownMs,
|
|
useUpstreamRetryHints: connectionCooldown.useUpstreamRetryHints,
|
|
// Issue #2100 follow-up: propagate stored override (boolean | undefined)
|
|
// so the runtime resolver picks user setting first, then per-provider default.
|
|
useUpstream429BreakerHints: connectionCooldown.useUpstream429BreakerHints,
|
|
maxBackoffSteps: connectionCooldown.maxBackoffSteps,
|
|
failureThreshold: providerBreaker.failureThreshold,
|
|
resetTimeoutMs: providerBreaker.resetTimeoutMs,
|
|
transientCooldown: connectionCooldown.baseCooldownMs,
|
|
rateLimitCooldown: connectionCooldown.useUpstreamRetryHints
|
|
? 0
|
|
: connectionCooldown.baseCooldownMs,
|
|
maxBackoffLevel: connectionCooldown.maxBackoffSteps,
|
|
circuitBreakerThreshold: providerBreaker.failureThreshold,
|
|
circuitBreakerReset: providerBreaker.resetTimeoutMs,
|
|
// Provider-level circuit breaker fields (not configurable via settings, use PROVIDER_PROFILES defaults)
|
|
providerFailureThreshold: PROVIDER_PROFILES[category].providerFailureThreshold,
|
|
providerFailureWindowMs: PROVIDER_PROFILES[category].providerFailureWindowMs,
|
|
providerCooldownMs: PROVIDER_PROFILES[category].providerCooldownMs,
|
|
} satisfies ProviderProfile;
|
|
}
|
|
|
|
/**
|
|
* Get the resilience profile for a provider (oauth or apikey).
|
|
* @param {string} provider - Provider ID or alias
|
|
*/
|
|
export function getProviderProfile(provider: string): ProviderProfile {
|
|
const category = getProviderCategory(provider);
|
|
return buildProviderProfile(category);
|
|
}
|
|
|
|
function shouldPreserveQuotaSignalsFor429(provider: string | null | undefined): boolean {
|
|
if (!provider) return true;
|
|
return getProviderCategory(provider) === "oauth";
|
|
}
|
|
|
|
export async function getRuntimeProviderProfile(provider: string | null | undefined) {
|
|
try {
|
|
const { getCachedSettings } = await import("@/lib/db/readCache");
|
|
const settings = await getCachedSettings();
|
|
const category = getProviderCategory(provider || "");
|
|
return buildProviderProfile(category, settings);
|
|
} catch {
|
|
return getProviderProfile(provider || "");
|
|
}
|
|
}
|
|
|
|
// ─── Per-Model Lockout Tracking ─────────────────────────────────────────────
|
|
// In-memory map: "provider:connectionId:model" → { reason, until, lockedAt }
|
|
const modelLockouts = new Map<string, ModelLockoutEntry>();
|
|
const modelFailureState = new Map<string, ModelFailureState>();
|
|
|
|
function getModelLockKey(provider: string, connectionId: string, model: string) {
|
|
return `${provider}:${connectionId}:${model}`;
|
|
}
|
|
|
|
function getFailureWindowMs(profile: ProviderProfile | null = null, fallbackMs = 30 * 60 * 1000) {
|
|
const configured = profile?.resetTimeoutMs;
|
|
return typeof configured === "number" && configured > 0 ? configured : fallbackMs;
|
|
}
|
|
|
|
function cleanupModelLockKey(key: string, now = Date.now()) {
|
|
const entry = modelLockouts.get(key);
|
|
if (entry && now > entry.until) {
|
|
modelLockouts.delete(key);
|
|
}
|
|
|
|
const failure = modelFailureState.get(key);
|
|
if (!failure) return;
|
|
if (now - failure.lastFailureAt <= failure.resetAfterMs) return;
|
|
if (modelLockouts.has(key)) return;
|
|
modelFailureState.delete(key);
|
|
}
|
|
|
|
function getModelLockBaseCooldown(
|
|
status: number,
|
|
fallbackCooldownMs: number,
|
|
profile: ProviderProfile | null = null
|
|
) {
|
|
if (Number.isFinite(fallbackCooldownMs) && fallbackCooldownMs > 0) {
|
|
return fallbackCooldownMs;
|
|
}
|
|
if (typeof profile?.baseCooldownMs === "number" && profile.baseCooldownMs >= 0) {
|
|
return profile.baseCooldownMs;
|
|
}
|
|
return status === HTTP_STATUS.RATE_LIMITED ? getQuotaCooldown(0) : COOLDOWN_MS.transientInitial;
|
|
}
|
|
|
|
function getScaledCooldown(
|
|
baseCooldownMs: number,
|
|
failureCount: number,
|
|
maxBackoffLevel = BACKOFF_CONFIG.maxLevel
|
|
) {
|
|
const safeBase = Number.isFinite(baseCooldownMs) && baseCooldownMs > 0 ? baseCooldownMs : 1000;
|
|
const exponent = Math.min(Math.max(0, failureCount - 1), Math.max(0, maxBackoffLevel));
|
|
return safeBase * Math.pow(2, exponent);
|
|
}
|
|
|
|
// Auto-cleanup expired lockouts every 15 seconds (lazy init for Cloudflare Workers compatibility)
|
|
let _cleanupTimer: ReturnType<typeof setInterval> | null = null;
|
|
|
|
function ensureCleanupTimer() {
|
|
if (_cleanupTimer) return;
|
|
try {
|
|
_cleanupTimer = setInterval(() => {
|
|
const now = Date.now();
|
|
for (const key of modelLockouts.keys()) cleanupModelLockKey(key, now);
|
|
for (const key of modelFailureState.keys()) cleanupModelLockKey(key, now);
|
|
}, 15_000);
|
|
if (typeof _cleanupTimer === "object" && "unref" in _cleanupTimer) {
|
|
(_cleanupTimer as { unref?: () => void }).unref?.(); // Don't prevent process exit (Node.js only)
|
|
}
|
|
} catch {
|
|
// Cloudflare Workers may not support setInterval outside handlers — skip cleanup timer
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Lock a specific model on a specific account
|
|
* @param {string} provider
|
|
* @param {string} connectionId
|
|
* @param {string} model
|
|
* @param {string} reason - from RateLimitReason
|
|
* @param {number} cooldownMs
|
|
*/
|
|
export function lockModel(
|
|
provider: string,
|
|
connectionId: string,
|
|
model: string | null | undefined,
|
|
reason: string,
|
|
cooldownMs: number,
|
|
metadata: Partial<ModelLockoutEntry> = {}
|
|
): void {
|
|
if (!model) return; // No model → skip model-level locking
|
|
ensureCleanupTimer();
|
|
const key = getModelLockKey(provider, connectionId, model);
|
|
cleanupModelLockKey(key);
|
|
const newUntil = Date.now() + cooldownMs;
|
|
// Preserve the longer cooldown if an existing lock has more time remaining.
|
|
// Safe without a mutex: no await between get/set, so this runs atomically
|
|
// within Node.js's single-threaded event loop.
|
|
const existing = modelLockouts.get(key);
|
|
if (existing && existing.until > newUntil) {
|
|
if (metadata.failureCount && metadata.failureCount > existing.failureCount) {
|
|
existing.failureCount = metadata.failureCount;
|
|
existing.lastFailureAt = metadata.lastFailureAt ?? existing.lastFailureAt;
|
|
existing.resetAfterMs = metadata.resetAfterMs ?? existing.resetAfterMs;
|
|
modelLockouts.set(key, existing);
|
|
}
|
|
return;
|
|
}
|
|
const now = Date.now();
|
|
modelLockouts.set(key, {
|
|
reason,
|
|
until: newUntil,
|
|
lockedAt: now,
|
|
failureCount: metadata.failureCount ?? existing?.failureCount ?? 1,
|
|
lastFailureAt: metadata.lastFailureAt ?? now,
|
|
resetAfterMs: metadata.resetAfterMs ?? existing?.resetAfterMs ?? 0,
|
|
});
|
|
}
|
|
|
|
export function recordModelLockoutFailure(
|
|
provider: string,
|
|
connectionId: string,
|
|
model: string,
|
|
reason: string,
|
|
status: number,
|
|
fallbackCooldownMs: number,
|
|
profile: ProviderProfile | null = null,
|
|
options: { exactCooldownMs?: number | null } = {}
|
|
) {
|
|
ensureCleanupTimer();
|
|
const key = getModelLockKey(provider, connectionId, model);
|
|
const now = Date.now();
|
|
cleanupModelLockKey(key, now);
|
|
|
|
// For daily quota exhaustion (quota_exhausted), set cooldown until tomorrow 00:00
|
|
// Use exactCooldownMs to bypass exponential backoff, ensuring precise lock until midnight
|
|
if (reason === "quota_exhausted" && typeof options.exactCooldownMs !== "number") {
|
|
options = { ...options, exactCooldownMs: getMsUntilTomorrow() };
|
|
}
|
|
|
|
const resetAfterMs = getFailureWindowMs(profile);
|
|
const previous = modelFailureState.get(key);
|
|
const withinWindow = previous && now - previous.lastFailureAt <= previous.resetAfterMs;
|
|
const failureCount = withinWindow ? previous.failureCount + 1 : 1;
|
|
modelFailureState.set(key, {
|
|
failureCount,
|
|
lastFailureAt: now,
|
|
resetAfterMs,
|
|
});
|
|
|
|
const baseCooldownMs = getModelLockBaseCooldown(status, fallbackCooldownMs, profile);
|
|
const cooldownMs =
|
|
typeof options.exactCooldownMs === "number" && options.exactCooldownMs > 0
|
|
? options.exactCooldownMs
|
|
: getScaledCooldown(
|
|
baseCooldownMs,
|
|
failureCount,
|
|
profile?.maxBackoffSteps ?? BACKOFF_CONFIG.maxLevel
|
|
);
|
|
|
|
lockModel(provider, connectionId, model, reason, cooldownMs, {
|
|
failureCount,
|
|
lastFailureAt: now,
|
|
resetAfterMs,
|
|
});
|
|
|
|
return {
|
|
cooldownMs,
|
|
failureCount,
|
|
resetAfterMs,
|
|
};
|
|
}
|
|
|
|
export function clearModelLock(
|
|
provider: string,
|
|
connectionId: string,
|
|
model: string | null | undefined
|
|
): boolean {
|
|
if (!model) return false;
|
|
const key = getModelLockKey(provider, connectionId, model);
|
|
const hadLock = modelLockouts.delete(key);
|
|
const hadFailureState = modelFailureState.delete(key);
|
|
return hadLock || hadFailureState;
|
|
}
|
|
|
|
/**
|
|
* Whether a provider should use per-model lockouts instead of connection-wide cooldowns.
|
|
* Compatible and passthrough providers multiplex multiple upstream models behind one
|
|
* connection, so transient 404/429 responses should stay model-scoped instead of
|
|
* poisoning the whole connection.
|
|
*
|
|
* @param provider - Provider ID
|
|
* @param _model - Model ID (reserved for future use)
|
|
* @param connectionPassthroughModels - Optional per-connection override from providerSpecificData.
|
|
* When provided, takes precedence over registry/provider-level logic.
|
|
*/
|
|
export function hasPerModelQuota(
|
|
provider: string | null | undefined,
|
|
_model: string | null | undefined = null,
|
|
connectionPassthroughModels?: boolean
|
|
): boolean {
|
|
// Connection-level override takes precedence (e.g., user-configured ModelScope)
|
|
if (typeof connectionPassthroughModels === "boolean") {
|
|
return connectionPassthroughModels;
|
|
}
|
|
if (!provider) return false;
|
|
if (provider === "gemini" || provider === "github") return true;
|
|
if (getPassthroughProviders().has(provider)) return true;
|
|
if (isCompatibleProvider(provider)) return true;
|
|
return false;
|
|
}
|
|
|
|
/**
|
|
* Lock a model (not connection) for a provider with per-model quotas.
|
|
* No-ops for providers that don't use per-model lockouts.
|
|
*/
|
|
export function lockModelIfPerModelQuota(
|
|
provider: string,
|
|
connectionId: string,
|
|
model: string | null,
|
|
reason: string,
|
|
cooldownMs: number,
|
|
connectionPassthroughModels?: boolean
|
|
): boolean {
|
|
if (!hasPerModelQuota(provider, model, connectionPassthroughModels) || !model) return false;
|
|
// Skip model-level lock if the entire provider is in circuit-breaker cooldown.
|
|
// The provider cooldown already prevents all requests, so a model lock is redundant.
|
|
if (isProviderInCooldown(provider)) return false;
|
|
lockModel(provider, connectionId, model, reason, cooldownMs);
|
|
return true;
|
|
}
|
|
|
|
export function shouldMarkAccountExhaustedFrom429(
|
|
provider: string | null | undefined,
|
|
model: string | null | undefined = null,
|
|
connectionPassthroughModels?: boolean,
|
|
failureKind?: FailureKind
|
|
): boolean {
|
|
// A plain 429 means transient rate limiting / high traffic for many OAuth providers.
|
|
// Only connection-poison the quota cache when the upstream body explicitly says
|
|
// the long-window quota is exhausted; otherwise fallback should try another account
|
|
// without making this one look quota-depleted for 5 minutes.
|
|
if (failureKind === "rate_limit" || failureKind === "transient") return false;
|
|
return (
|
|
shouldPreserveQuotaSignalsFor429(provider) &&
|
|
!hasPerModelQuota(provider, model, connectionPassthroughModels)
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Clear all in-memory model lockouts and failure state (for tests / full reset).
|
|
*/
|
|
export function clearAllModelLockouts(): void {
|
|
modelLockouts.clear();
|
|
modelFailureState.clear();
|
|
}
|
|
|
|
/**
|
|
* Check if a specific model on a specific account is locked
|
|
* @returns {boolean}
|
|
*/
|
|
export function isModelLocked(
|
|
provider: string,
|
|
connectionId: string,
|
|
model: string | null | undefined
|
|
): boolean {
|
|
if (!model) return false;
|
|
const key = getModelLockKey(provider, connectionId, model);
|
|
cleanupModelLockKey(key);
|
|
const entry = modelLockouts.get(key);
|
|
return Boolean(entry);
|
|
}
|
|
|
|
/**
|
|
* Get model lockout info (for debugging/dashboard)
|
|
*/
|
|
export function getModelLockoutInfo(
|
|
provider: string,
|
|
connectionId: string,
|
|
model: string | null | undefined
|
|
) {
|
|
if (!model) return null;
|
|
const key = getModelLockKey(provider, connectionId, model);
|
|
cleanupModelLockKey(key);
|
|
const entry = modelLockouts.get(key);
|
|
if (!entry) return null;
|
|
return {
|
|
reason: entry.reason,
|
|
remainingMs: entry.until - Date.now(),
|
|
lockedAt: new Date(entry.lockedAt).toISOString(),
|
|
failureCount: entry.failureCount,
|
|
};
|
|
}
|
|
|
|
type ModelLockoutInfo = {
|
|
provider: string;
|
|
connectionId: string;
|
|
model: string;
|
|
reason: string;
|
|
remainingMs: number;
|
|
failureCount: number;
|
|
};
|
|
|
|
/**
|
|
* Get all active model lockouts (for dashboard)
|
|
*/
|
|
export function getAllModelLockouts(): ModelLockoutInfo[] {
|
|
const now = Date.now();
|
|
const active: ModelLockoutInfo[] = [];
|
|
for (const key of modelLockouts.keys()) {
|
|
cleanupModelLockKey(key, now);
|
|
}
|
|
for (const [key, entry] of modelLockouts) {
|
|
const [provider, connectionId, model] = key.split(":");
|
|
active.push({
|
|
provider,
|
|
connectionId,
|
|
model,
|
|
reason: entry.reason,
|
|
remainingMs: entry.until - now,
|
|
failureCount: entry.failureCount,
|
|
});
|
|
}
|
|
return active;
|
|
}
|
|
|
|
// ─── Provider Breaker Compatibility Wrappers ────────────────────────────────
|
|
// Legacy helpers now delegate to the shared provider circuit breaker.
|
|
|
|
type ProviderBreakerProfile = Partial<
|
|
Pick<
|
|
ProviderProfile,
|
|
"failureThreshold" | "resetTimeoutMs" | "circuitBreakerThreshold" | "circuitBreakerReset"
|
|
>
|
|
>;
|
|
|
|
function getProviderBreaker(provider: string | null | undefined) {
|
|
return provider ? getCircuitBreaker(provider) : null;
|
|
}
|
|
|
|
function configureProviderBreaker(
|
|
provider: string | null | undefined,
|
|
profile?: ProviderBreakerProfile | null
|
|
) {
|
|
if (!provider) return null;
|
|
|
|
const resolvedProfile = { ...getProviderProfile(provider), ...(profile ?? {}) };
|
|
// Issue #2100 follow-up: resolve useUpstream429BreakerHints from the
|
|
// provider profile (stored override) or fall back to per-provider default.
|
|
// Stored value type is `boolean | undefined` — never `null` after PATCH.
|
|
const userValue = resolvedProfile.useUpstream429BreakerHints;
|
|
const useHints = resolveUseUpstream429BreakerHints(provider, userValue);
|
|
return getCircuitBreaker(provider, {
|
|
failureThreshold: resolvedProfile.failureThreshold ?? resolvedProfile.circuitBreakerThreshold,
|
|
resetTimeout: resolvedProfile.resetTimeoutMs ?? resolvedProfile.circuitBreakerReset,
|
|
...(useHints
|
|
? {
|
|
cooldownByKind: {
|
|
rate_limit: 60_000,
|
|
quota_exhausted: 3_600_000,
|
|
} satisfies Partial<Record<FailureKind, number>>,
|
|
classifyError: classify429FromError,
|
|
}
|
|
: {}),
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Check if a provider is currently blocked by the shared circuit breaker.
|
|
*/
|
|
export function isProviderInCooldown(provider: string | null | undefined): boolean {
|
|
const breaker = getProviderBreaker(provider);
|
|
return breaker ? !breaker.canExecute() : false;
|
|
}
|
|
|
|
/**
|
|
* Get remaining retry-after time for a provider breaker.
|
|
*/
|
|
export function getProviderCooldownRemainingMs(provider: string | null | undefined): number | null {
|
|
const breaker = getProviderBreaker(provider);
|
|
if (!breaker || breaker.canExecute()) return null;
|
|
const remaining = breaker.getRetryAfterMs();
|
|
return remaining > 0 ? remaining : null;
|
|
}
|
|
|
|
/**
|
|
* Record a provider failure against the shared circuit breaker.
|
|
* Delegates to the existing CircuitBreaker utility which handles
|
|
* failure counting, threshold detection, and state transitions.
|
|
*
|
|
* IMPORTANT: If the breaker is already OPEN (in cooldown), we skip
|
|
* recording the failure to prevent resetting the cooldown timer.
|
|
* This matches the original behavior where failures during cooldown
|
|
* were ignored to avoid indefinite lockout.
|
|
*/
|
|
export function recordProviderFailure(
|
|
provider: string | null | undefined,
|
|
log?: { warn?: (...args: unknown[]) => void },
|
|
connectionId?: string | null,
|
|
profile?: ProviderBreakerProfile | null
|
|
): void {
|
|
if (!provider) return;
|
|
|
|
// Deduplicate rapid-fire failures from the same connection
|
|
if (connectionId) {
|
|
const dedupKey = `${provider}:${connectionId}`;
|
|
const now = Date.now();
|
|
const lastFailure = lastConnectionFailure.get(dedupKey);
|
|
if (lastFailure && now - lastFailure < CONNECTION_FAILURE_DEDUP_MS) {
|
|
return;
|
|
}
|
|
// Prevent memory leak by clearing map if it grows too large
|
|
if (lastConnectionFailure.size > 10000) {
|
|
lastConnectionFailure.clear();
|
|
}
|
|
lastConnectionFailure.set(dedupKey, now);
|
|
}
|
|
|
|
const breaker = configureProviderBreaker(provider, profile);
|
|
if (!breaker) return;
|
|
|
|
if (!breaker.canExecute()) return;
|
|
|
|
breaker._onFailure();
|
|
|
|
if (!breaker.canExecute()) {
|
|
log?.warn?.(`[ProviderFailure] ${provider}: circuit breaker opened after repeated failures`);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Reset the shared provider breaker.
|
|
*/
|
|
export function clearProviderFailure(provider: string | null | undefined): void {
|
|
const breaker = getProviderBreaker(provider);
|
|
breaker?.reset();
|
|
}
|
|
|
|
/**
|
|
* Get all providers currently blocked by the shared breaker.
|
|
*/
|
|
export function getProvidersInCooldown(): Array<{
|
|
provider: string;
|
|
failureCount: number;
|
|
cooldownRemainingMs: number | null;
|
|
lastFailureAt: number | null;
|
|
}> {
|
|
return getAllCircuitBreakerStatuses()
|
|
.filter((status) => {
|
|
const breaker = getProviderBreaker(status.name);
|
|
return Boolean(breaker && !breaker.canExecute());
|
|
})
|
|
.map((status) => ({
|
|
provider: status.name,
|
|
failureCount: status.failureCount,
|
|
cooldownRemainingMs: status.retryAfterMs || null,
|
|
lastFailureAt: status.lastFailureTime,
|
|
}));
|
|
}
|
|
|
|
/**
|
|
* Check if a status code should be counted toward provider failure threshold
|
|
*/
|
|
export function isProviderFailureCode(status: number): boolean {
|
|
return PROVIDER_FAILURE_ERROR_CODES.has(status);
|
|
}
|
|
|
|
/**
|
|
* Returns true when a checkFallbackError result signals that the entire provider
|
|
* quota is exhausted for this request, so the combo router can skip remaining
|
|
* targets from the same provider (#1731).
|
|
*
|
|
* Covers:
|
|
* - reason === "quota_exhausted" (subscription, daily, credits)
|
|
* - creditsExhausted flag
|
|
* - dailyQuotaExhausted flag
|
|
*/
|
|
export function isProviderExhaustedReason(result: {
|
|
reason?: string;
|
|
creditsExhausted?: boolean;
|
|
dailyQuotaExhausted?: boolean;
|
|
}): boolean {
|
|
if (result.creditsExhausted || result.dailyQuotaExhausted) return true;
|
|
return result.reason === RateLimitReason.QUOTA_EXHAUSTED;
|
|
}
|
|
|
|
// ─── Retry-After Parsing ────────────────────────────────────────────────────
|
|
|
|
/**
|
|
* Parse retry-after information from JSON error response bodies.
|
|
* Providers embed retry info in different formats.
|
|
*
|
|
* @param {string|object} responseBody - Raw response body or parsed JSON
|
|
* @returns {{ retryAfterMs: number|null, reason: string }}
|
|
*/
|
|
export function parseRetryAfterFromBody(responseBody: unknown): {
|
|
retryAfterMs: number | null;
|
|
reason: RateLimitReasonValue;
|
|
} {
|
|
let body: JsonRecord;
|
|
try {
|
|
body = toJsonRecord(typeof responseBody === "string" ? JSON.parse(responseBody) : responseBody);
|
|
} catch {
|
|
return { retryAfterMs: null, reason: RateLimitReason.UNKNOWN };
|
|
}
|
|
|
|
if (Object.keys(body).length === 0) {
|
|
return { retryAfterMs: null, reason: RateLimitReason.UNKNOWN };
|
|
}
|
|
|
|
// Gemini: { error: { details: [{ retryDelay: "33s" }] } }
|
|
const error = toJsonRecord(body.error);
|
|
const details = error.details || body.details || [];
|
|
for (const detail of Array.isArray(details) ? details : []) {
|
|
const detailRecord = toJsonRecord(detail);
|
|
if (detailRecord.retryDelay) {
|
|
return {
|
|
retryAfterMs: parseDelayString(detailRecord.retryDelay),
|
|
reason: RateLimitReason.RATE_LIMIT_EXCEEDED,
|
|
};
|
|
}
|
|
}
|
|
|
|
// OpenAI: "Please retry after 20s" in message
|
|
const msg = String(error.message || body.message || "");
|
|
const retryMatch = msg.match(/retry\s+after\s+(\d+)\s*s/i);
|
|
if (retryMatch) {
|
|
return {
|
|
retryAfterMs: parseInt(retryMatch[1], 10) * 1000,
|
|
reason: RateLimitReason.RATE_LIMIT_EXCEEDED,
|
|
};
|
|
}
|
|
|
|
// Anthropic: error type classification
|
|
const errorType = String(error.type || body.type || "");
|
|
if (errorType === "rate_limit_error") {
|
|
return { retryAfterMs: null, reason: RateLimitReason.RATE_LIMIT_EXCEEDED };
|
|
}
|
|
|
|
// Classify by error message keywords
|
|
const reason = classifyErrorText(msg || errorType);
|
|
return { retryAfterMs: null, reason };
|
|
}
|
|
|
|
/**
|
|
* Parse delay strings like "33s", "2m", "1h", "1500ms"
|
|
*/
|
|
function parseDelayString(value: unknown): number | null {
|
|
if (!value) return null;
|
|
const str = String(value).trim();
|
|
const msMatch = str.match(/^(\d+)\s*ms$/i);
|
|
if (msMatch) return parseInt(msMatch[1], 10);
|
|
const secMatch = str.match(/^(\d+)\s*s$/i);
|
|
if (secMatch) return parseInt(secMatch[1], 10) * 1000;
|
|
const minMatch = str.match(/^(\d+)\s*m$/i);
|
|
if (minMatch) return parseInt(minMatch[1], 10) * 60 * 1000;
|
|
const hrMatch = str.match(/^(\d+)\s*h$/i);
|
|
if (hrMatch) return parseInt(hrMatch[1], 10) * 3600 * 1000;
|
|
// Bare number → seconds
|
|
const num = parseInt(str, 10);
|
|
return isNaN(num) ? null : num * 1000;
|
|
}
|
|
|
|
/**
|
|
* T07: Parse retry time from error text body with combined "XhYmZs" format.
|
|
* Examples: "Your quota will reset after 2h30m14s", "reset after 45m", "reset after 30s"
|
|
* Returns milliseconds or null if not parseable.
|
|
*
|
|
* @param {string} errorText - Error message text from response body
|
|
* @returns {number|null} Retry duration in milliseconds
|
|
*/
|
|
export function parseRetryFromErrorText(errorText: unknown): number | null {
|
|
if (!errorText || typeof errorText !== "string") return null;
|
|
|
|
// Issue #2321: Anthropic OAuth occasionally embeds an absolute ISO 8601
|
|
// timestamp instead of a relative duration (e.g. "Try again at
|
|
// 2026-05-17T10:00:00Z" or "Please wait until 2026-05-17T10:00:00.000Z").
|
|
// Convert to a future-duration in milliseconds if it parses.
|
|
const isoMatch = errorText.match(
|
|
/\b(?:try again at|wait until|reset(?:s)? at|available at|retry after)\s+(\d{4}-\d{2}-\d{2}[Tt ]\d{2}:\d{2}(?::\d{2})?(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?)/i
|
|
);
|
|
if (isoMatch) {
|
|
const parsedTs = Date.parse(isoMatch[1]);
|
|
if (Number.isFinite(parsedTs)) {
|
|
const waitMs = parsedTs - Date.now();
|
|
if (waitMs > 0) return waitMs;
|
|
}
|
|
}
|
|
|
|
const match = errorText.match(/reset after (\d+h)?(\d+m)?(\d+s)?/i);
|
|
if (!match) {
|
|
// Also try the variant without "reset after": "will reset after XhYmZs"
|
|
const altMatch = errorText.match(/will reset after (\d+h)?(\d+m)?(\d+s)?/i);
|
|
if (!altMatch) return null;
|
|
return computeDurationMs(altMatch);
|
|
}
|
|
|
|
return computeDurationMs(match);
|
|
}
|
|
|
|
/**
|
|
* Compute total milliseconds from regex match groups (Xh)(Ym)(Zs)
|
|
*/
|
|
function computeDurationMs(match: RegExpMatchArray): number | null {
|
|
let totalMs = 0;
|
|
if (match[1]) totalMs += parseInt(match[1], 10) * 3600 * 1000; // hours
|
|
if (match[2]) totalMs += parseInt(match[2], 10) * 60 * 1000; // minutes
|
|
if (match[3]) totalMs += parseInt(match[3], 10) * 1000; // seconds
|
|
return totalMs > 0 ? totalMs : null;
|
|
}
|
|
|
|
// ─── Error Classification ───────────────────────────────────────────────────
|
|
|
|
/**
|
|
* Classify error text into RateLimitReason
|
|
*/
|
|
export function classifyErrorText(errorText: unknown): RateLimitReasonValue {
|
|
if (!errorText) return RateLimitReason.UNKNOWN;
|
|
const lower = String(errorText).toLowerCase();
|
|
|
|
if (
|
|
lower.includes("quota exceeded") ||
|
|
lower.includes("quota depleted") ||
|
|
lower.includes("quota will reset") ||
|
|
lower.includes("your quota will reset") ||
|
|
lower.includes("quota has been exceeded") ||
|
|
lower.includes("hour quota") ||
|
|
lower.includes("billing") ||
|
|
// Issue #2321: Anthropic OAuth (Claude Code Pro/Team) 429 bodies surface
|
|
// the subscription quota with phrases that contain neither "quota" nor
|
|
// "billing". Without these patterns the error was classified as a
|
|
// transient RATE_LIMIT_EXCEEDED (~5s base cooldown), which cascades all
|
|
// Pro accounts into a tight retry loop until the 5h window resets.
|
|
lower.includes("usage limit reached") ||
|
|
lower.includes("usage limit has been") ||
|
|
lower.includes("claude pro usage limit") ||
|
|
lower.includes("you've reached your usage limit") ||
|
|
lower.includes("you have reached your usage limit")
|
|
) {
|
|
return RateLimitReason.QUOTA_EXHAUSTED;
|
|
}
|
|
// T10: credits_exhausted signals
|
|
if (isCreditsExhausted(lower)) {
|
|
return RateLimitReason.QUOTA_EXHAUSTED;
|
|
}
|
|
// T06: account_deactivated signals
|
|
if (isAccountDeactivated(lower)) {
|
|
return RateLimitReason.AUTH_ERROR;
|
|
}
|
|
const configuredRule = matchErrorRuleByText(errorText);
|
|
if (configuredRule?.reason) return configuredRule.reason;
|
|
if (lower.includes("rate_limit")) return RateLimitReason.RATE_LIMIT_EXCEEDED;
|
|
if (lower.includes("resource exhausted")) return RateLimitReason.MODEL_CAPACITY;
|
|
if (
|
|
lower.includes("unauthorized") ||
|
|
lower.includes("invalid api key") ||
|
|
lower.includes("authentication")
|
|
) {
|
|
return RateLimitReason.AUTH_ERROR;
|
|
}
|
|
if (lower.includes("server error") || lower.includes("internal error")) {
|
|
return RateLimitReason.SERVER_ERROR;
|
|
}
|
|
return RateLimitReason.UNKNOWN;
|
|
}
|
|
|
|
/**
|
|
* Classify HTTP status + error text into RateLimitReason
|
|
*/
|
|
export function classifyError(status: number, errorText: unknown): RateLimitReasonValue {
|
|
// Text classification takes priority (more specific)
|
|
const textReason = classifyErrorText(errorText);
|
|
if (textReason !== RateLimitReason.UNKNOWN) return textReason;
|
|
|
|
// Fall back to status code
|
|
if (status === HTTP_STATUS.UNAUTHORIZED || status === HTTP_STATUS.FORBIDDEN) {
|
|
return RateLimitReason.AUTH_ERROR;
|
|
}
|
|
if (status === HTTP_STATUS.PAYMENT_REQUIRED) {
|
|
return RateLimitReason.QUOTA_EXHAUSTED;
|
|
}
|
|
if (status === HTTP_STATUS.RATE_LIMITED) {
|
|
return RateLimitReason.RATE_LIMIT_EXCEEDED;
|
|
}
|
|
if (status === HTTP_STATUS.SERVICE_UNAVAILABLE || status === 529) {
|
|
return RateLimitReason.MODEL_CAPACITY;
|
|
}
|
|
if (status >= 500) {
|
|
return RateLimitReason.SERVER_ERROR;
|
|
}
|
|
return RateLimitReason.UNKNOWN;
|
|
}
|
|
|
|
// ─── Daily Quota Helpers ────────────────────────────────────────────────────
|
|
|
|
/**
|
|
* Calculate milliseconds from now until tomorrow at midnight (00:00:00).
|
|
* Used to lock a model until the next day when daily quota is exhausted.
|
|
* @returns {number} Milliseconds until tomorrow
|
|
*/
|
|
export function getMsUntilTomorrow(): number {
|
|
const nowMs = Date.now();
|
|
const now = new Date(nowMs);
|
|
const tomorrow = new Date(nowMs);
|
|
tomorrow.setDate(tomorrow.getDate() + 1);
|
|
tomorrow.setHours(0, 0, 0, 0);
|
|
const ms = tomorrow.getTime() - nowMs;
|
|
// Guard against DST edge cases: if ms is negative (shouldn't happen) or
|
|
// unreasonably large (>25h due to spring-forward), cap at 24 hours.
|
|
return ms > 0 && ms <= 25 * 60 * 60 * 1000 ? ms : 24 * 60 * 60 * 1000;
|
|
}
|
|
|
|
/**
|
|
* Check if error text indicates daily quota exhaustion (as opposed to rate limiting).
|
|
* Daily quota errors typically mention "today's quota" or "try again tomorrow".
|
|
* @param {string} errorText - Error message text
|
|
* @returns {boolean} True if daily quota is exhausted
|
|
*/
|
|
export function isDailyQuotaExhausted(errorText: string): boolean {
|
|
if (!errorText) return false;
|
|
const lower = errorText.toLowerCase();
|
|
return (
|
|
lower.includes("today's quota") ||
|
|
lower.includes("daily quota") ||
|
|
lower.includes("try again tomorrow")
|
|
);
|
|
}
|
|
|
|
// ─── Configurable Backoff ───────────────────────────────────────────────────
|
|
|
|
/**
|
|
* Get backoff duration from configurable steps.
|
|
* @param {number} failureCount - Number of consecutive failures
|
|
* @returns {number} Duration in ms
|
|
*/
|
|
export function getBackoffDuration(failureCount: number): number {
|
|
const idx = Math.min(failureCount, BACKOFF_STEPS_MS.length - 1);
|
|
return BACKOFF_STEPS_MS[idx];
|
|
}
|
|
|
|
// ─── Original API (Backward Compatible) ────────────────────────────────────
|
|
|
|
/**
|
|
* Calculate exponential backoff cooldown for rate limits (429)
|
|
* Level 0: 1s, Level 1: 2s, Level 2: 4s... → max 2 min
|
|
* @param {number} backoffLevel - Current backoff level
|
|
* @returns {number} Cooldown in milliseconds
|
|
*/
|
|
export function getQuotaCooldown(backoffLevel = 0) {
|
|
return calculateBackoffCooldown(backoffLevel);
|
|
}
|
|
|
|
/**
|
|
* Check if error should trigger account fallback (switch to next account)
|
|
* @param {number} status - HTTP status code
|
|
* @param {string} errorText - Error message text
|
|
* @param {number} backoffLevel - Current backoff level for exponential backoff
|
|
* @param {string} [model] - Optional model name for model-level lockout
|
|
* @param {string} [provider] - Provider ID for profile-aware cooldowns
|
|
* @returns {{ shouldFallback: boolean, cooldownMs: number, newBackoffLevel?: number, reason?: string }}
|
|
*/
|
|
export function checkFallbackError(
|
|
status: number,
|
|
errorText: string | null,
|
|
backoffLevel: number = 0,
|
|
_model: string | null = null,
|
|
provider: string | null = null,
|
|
headers: Headers | Record<string, string> | null = null,
|
|
profileOverride: ProviderProfile | null = null
|
|
): {
|
|
shouldFallback: boolean;
|
|
cooldownMs: number;
|
|
baseCooldownMs?: number;
|
|
newBackoffLevel?: number;
|
|
usedUpstreamRetryHint?: boolean;
|
|
reason?: string;
|
|
permanent?: boolean;
|
|
creditsExhausted?: boolean;
|
|
dailyQuotaExhausted?: boolean;
|
|
} {
|
|
const errorStr = (errorText || "").toString();
|
|
const profile = profileOverride ?? (provider ? getProviderProfile(provider) : null);
|
|
const maxBackoffSteps = profile?.maxBackoffSteps ?? BACKOFF_CONFIG.maxLevel;
|
|
const retryableStatuses = new Set([
|
|
HTTP_STATUS.REQUEST_TIMEOUT,
|
|
HTTP_STATUS.RATE_LIMITED,
|
|
HTTP_STATUS.SERVER_ERROR,
|
|
HTTP_STATUS.BAD_GATEWAY,
|
|
HTTP_STATUS.SERVICE_UNAVAILABLE,
|
|
HTTP_STATUS.GATEWAY_TIMEOUT,
|
|
]);
|
|
|
|
function parseResetFromHeaders(headers: Headers | Record<string, string> | null): number | null {
|
|
if (!headers) return null;
|
|
const recordHeaders = headers as Record<string, string>;
|
|
|
|
// Retry-After header
|
|
const retryAfter =
|
|
typeof (headers as Headers).get === "function"
|
|
? (headers as Headers).get("retry-after")
|
|
: recordHeaders["retry-after"] || recordHeaders["Retry-After"];
|
|
|
|
if (retryAfter) {
|
|
const seconds = parseInt(retryAfter, 10);
|
|
if (!isNaN(seconds) && String(seconds) === String(retryAfter).trim()) {
|
|
return Date.now() + seconds * 1000;
|
|
}
|
|
const date = new Date(retryAfter);
|
|
if (!isNaN(date.getTime())) return date.getTime();
|
|
}
|
|
|
|
// X-RateLimit-Reset
|
|
const rlReset =
|
|
typeof (headers as Headers).get === "function"
|
|
? (headers as Headers).get("x-ratelimit-reset")
|
|
: recordHeaders["x-ratelimit-reset"] || recordHeaders["X-RateLimit-Reset"];
|
|
|
|
if (rlReset) {
|
|
const ts = parseInt(rlReset, 10);
|
|
if (!isNaN(ts)) {
|
|
return ts > 10000000000 ? ts : ts * 1000;
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function getUpstreamRetryHintMs() {
|
|
if (!profile?.useUpstreamRetryHints) return null;
|
|
const resetTime = parseResetFromHeaders(headers);
|
|
if (resetTime) {
|
|
const waitMs = Math.max(resetTime - Date.now(), 0);
|
|
if (waitMs > 0) return waitMs;
|
|
}
|
|
|
|
const retryFromErrorText = parseRetryFromErrorText(errorStr);
|
|
if (retryFromErrorText && retryFromErrorText > 0) {
|
|
return retryFromErrorText;
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
function getScaledBaseCooldown(reason: RateLimitReasonValue, level = backoffLevel) {
|
|
void reason;
|
|
const baseCooldownMs =
|
|
typeof profile?.baseCooldownMs === "number" && profile.baseCooldownMs >= 0
|
|
? profile.baseCooldownMs
|
|
: COOLDOWN_MS.transientInitial;
|
|
return {
|
|
baseCooldownMs,
|
|
cooldownMs: getScaledCooldown(baseCooldownMs, level + 1, maxBackoffSteps),
|
|
newBackoffLevel: Math.min(level + 1, maxBackoffSteps),
|
|
};
|
|
}
|
|
|
|
function buildRetryableFallback(reason: RateLimitReasonValue) {
|
|
const upstreamRetryHintMs = getUpstreamRetryHintMs();
|
|
if (typeof upstreamRetryHintMs === "number" && upstreamRetryHintMs > 0) {
|
|
return {
|
|
shouldFallback: true,
|
|
cooldownMs: upstreamRetryHintMs,
|
|
baseCooldownMs: upstreamRetryHintMs,
|
|
newBackoffLevel: 0,
|
|
usedUpstreamRetryHint: true,
|
|
reason,
|
|
};
|
|
}
|
|
|
|
const scaled = getScaledBaseCooldown(reason, backoffLevel);
|
|
return {
|
|
shouldFallback: true,
|
|
cooldownMs: scaled.cooldownMs,
|
|
baseCooldownMs: scaled.baseCooldownMs,
|
|
newBackoffLevel: scaled.newBackoffLevel,
|
|
usedUpstreamRetryHint: false,
|
|
reason,
|
|
};
|
|
}
|
|
|
|
const isRateLimitStatus = status === HTTP_STATUS.RATE_LIMITED;
|
|
const preserveQuota429 = shouldPreserveQuotaSignalsFor429(provider);
|
|
const shouldUseQuotaSignal = !isRateLimitStatus || preserveQuota429;
|
|
|
|
// Check error message FIRST - specific patterns take priority over status codes
|
|
if (errorText) {
|
|
// T06 (sub2api #1037): Permanent account deactivation — do NOT retry, mark as permanent failure
|
|
if (isAccountDeactivated(errorStr)) {
|
|
return {
|
|
shouldFallback: true,
|
|
cooldownMs: 365 * 24 * 60 * 60 * 1000, // 1 year = effectively permanent
|
|
reason: RateLimitReason.AUTH_ERROR,
|
|
permanent: true,
|
|
};
|
|
}
|
|
|
|
// T10 (sub2api #1169): Credits/quota exhausted — long cooldown, distinct from rate limit
|
|
if (shouldUseQuotaSignal && isCreditsExhausted(errorStr)) {
|
|
return {
|
|
shouldFallback: true,
|
|
cooldownMs: COOLDOWN_MS.paymentRequired ?? 3600 * 1000, // 1h cooldown
|
|
reason: RateLimitReason.QUOTA_EXHAUSTED,
|
|
creditsExhausted: true,
|
|
};
|
|
}
|
|
|
|
// Daily quota exhausted — lock model until tomorrow
|
|
if (shouldUseQuotaSignal && isDailyQuotaExhausted(errorStr)) {
|
|
const msUntilTomorrow = getMsUntilTomorrow();
|
|
// Cap at 24 hours to handle timezone edge cases
|
|
const cooldownMs = Math.min(msUntilTomorrow, 24 * 60 * 60 * 1000);
|
|
return {
|
|
shouldFallback: true,
|
|
cooldownMs,
|
|
reason: RateLimitReason.QUOTA_EXHAUSTED,
|
|
dailyQuotaExhausted: true,
|
|
};
|
|
}
|
|
|
|
// Issue #2321: Anthropic OAuth (Claude Pro/Team) returns 429 with
|
|
// "Usage Limit Reached" for the 5-hour subscription quota. The
|
|
// pattern-based classifier now flags these as QUOTA_EXHAUSTED, but
|
|
// without a dedicated branch the request would still fall through to
|
|
// the generic 429 retry path (~5s base cooldown). Honor any
|
|
// upstream retry hint (Retry-After header or ISO timestamp in the
|
|
// body) when present, otherwise apply a 1h cooldown so all Pro
|
|
// accounts on the same subscription tier stop cycling through tight
|
|
// retries until the window genuinely resets. (We deliberately do not
|
|
// use COOLDOWN_MS.paymentRequired here — that constant is 2 minutes,
|
|
// which is shorter than the recovery time of a subscription quota.)
|
|
if (
|
|
shouldUseQuotaSignal &&
|
|
!isCreditsExhausted(errorStr) &&
|
|
!isDailyQuotaExhausted(errorStr) &&
|
|
classifyErrorText(errorStr) === RateLimitReason.QUOTA_EXHAUSTED
|
|
) {
|
|
// For a quota error the upstream reset hint (Retry-After header or
|
|
// ISO timestamp embedded in the body) is the most accurate wait.
|
|
// We honor it even when the resilience profile does not opt-in to
|
|
// generic upstream retry hints — a subscription quota has a
|
|
// definite recovery time, not a best-effort transient backoff.
|
|
const hintMs = getUpstreamRetryHintMs() ?? parseRetryFromErrorText(errorStr) ?? null;
|
|
const SUBSCRIPTION_QUOTA_COOLDOWN_MS = 60 * 60 * 1000; // 1 hour
|
|
return {
|
|
shouldFallback: true,
|
|
cooldownMs: hintMs ?? SUBSCRIPTION_QUOTA_COOLDOWN_MS,
|
|
reason: RateLimitReason.QUOTA_EXHAUSTED,
|
|
usedUpstreamRetryHint: Boolean(hintMs),
|
|
};
|
|
}
|
|
|
|
if (
|
|
status === HTTP_STATUS.FORBIDDEN &&
|
|
provider &&
|
|
getProviderCategory(provider) === "apikey" &&
|
|
!errorStr.toLowerCase().includes("has not been used in project") &&
|
|
!errorStr.toLowerCase().includes("hour quota") &&
|
|
!errorStr.toLowerCase().includes("quota has been exceeded")
|
|
) {
|
|
return buildRetryableFallback(RateLimitReason.AUTH_ERROR);
|
|
}
|
|
}
|
|
|
|
const configuredRule =
|
|
isRateLimitStatus && !preserveQuota429
|
|
? matchErrorRuleByStatus(status)
|
|
: findMatchingErrorRule(status, errorStr);
|
|
if (configuredRule) {
|
|
if (configuredRule.backoff) {
|
|
return buildRetryableFallback(configuredRule.reason ?? classifyError(status, errorStr));
|
|
}
|
|
const cooldownMs = configuredRule.cooldownMs ?? 0;
|
|
return {
|
|
shouldFallback: true,
|
|
cooldownMs,
|
|
baseCooldownMs: cooldownMs,
|
|
reason: configuredRule.reason ?? RateLimitReason.UNKNOWN,
|
|
};
|
|
}
|
|
|
|
if (status === HTTP_STATUS.NOT_ACCEPTABLE || retryableStatuses.has(status)) {
|
|
return buildRetryableFallback(RateLimitReason.SERVER_ERROR);
|
|
}
|
|
|
|
// 400 — context overflow / malformed request may succeed on another model in the combo
|
|
if (status === HTTP_STATUS.BAD_REQUEST) {
|
|
const isOverflow = CONTEXT_OVERFLOW_PATTERNS.some((p) => p.test(errorStr));
|
|
const isMalformed = MALFORMED_REQUEST_PATTERNS.some((p) => p.test(errorStr));
|
|
|
|
if (isOverflow || isMalformed) {
|
|
return {
|
|
shouldFallback: true,
|
|
cooldownMs: 0,
|
|
reason: RateLimitReason.MODEL_CAPACITY,
|
|
};
|
|
}
|
|
|
|
// Generic 400 is not account-fallback-worthy. Combo routing may still try a
|
|
// different provider/model because combo fallback is target-level orchestration.
|
|
return { shouldFallback: false, cooldownMs: 0, reason: RateLimitReason.UNKNOWN };
|
|
}
|
|
|
|
// All other errors - fallback with transient cooldown
|
|
return {
|
|
shouldFallback: true,
|
|
cooldownMs: profile?.baseCooldownMs ?? COOLDOWN_MS.transient,
|
|
baseCooldownMs: profile?.baseCooldownMs ?? COOLDOWN_MS.transient,
|
|
reason: RateLimitReason.UNKNOWN,
|
|
};
|
|
}
|
|
|
|
// ─── Account State Management ───────────────────────────────────────────────
|
|
|
|
/**
|
|
* Check if account is currently unavailable (cooldown not expired)
|
|
*/
|
|
export function isAccountUnavailable(unavailableUntil: string | Date | null | undefined): boolean {
|
|
if (!unavailableUntil) return false;
|
|
return new Date(unavailableUntil).getTime() > Date.now();
|
|
}
|
|
|
|
/**
|
|
* Calculate unavailable until timestamp
|
|
*/
|
|
export function getUnavailableUntil(cooldownMs: number): string {
|
|
return new Date(Date.now() + cooldownMs).toISOString();
|
|
}
|
|
|
|
/**
|
|
* Get the earliest rateLimitedUntil from a list of accounts
|
|
*/
|
|
export function getEarliestRateLimitedUntil(
|
|
accounts: Array<{ rateLimitedUntil?: string | null }>
|
|
): string | null {
|
|
let earliest: number | null = null;
|
|
const now = Date.now();
|
|
for (const acc of accounts) {
|
|
if (!acc.rateLimitedUntil) continue;
|
|
const until = new Date(acc.rateLimitedUntil).getTime();
|
|
if (until <= now) continue;
|
|
if (!earliest || until < earliest) earliest = until;
|
|
}
|
|
if (!earliest) return null;
|
|
return new Date(earliest).toISOString();
|
|
}
|
|
|
|
/**
|
|
* Format rateLimitedUntil to human-readable "reset after Xm Ys"
|
|
*/
|
|
export function formatRetryAfter(rateLimitedUntil: string | Date | null | undefined): string {
|
|
if (!rateLimitedUntil) return "";
|
|
const diffMs = new Date(rateLimitedUntil).getTime() - Date.now();
|
|
if (diffMs <= 0) return "reset after 0s";
|
|
const totalSec = Math.ceil(diffMs / 1000);
|
|
const h = Math.floor(totalSec / 3600);
|
|
const m = Math.floor((totalSec % 3600) / 60);
|
|
const s = totalSec % 60;
|
|
const parts: string[] = [];
|
|
if (h > 0) parts.push(`${h}h`);
|
|
if (m > 0) parts.push(`${m}m`);
|
|
if (s > 0 || parts.length === 0) parts.push(`${s}s`);
|
|
return `reset after ${parts.join(" ")}`;
|
|
}
|
|
|
|
/**
|
|
* Filter available accounts (not in cooldown)
|
|
*/
|
|
export function filterAvailableAccounts<T extends AccountState>(
|
|
accounts: T[],
|
|
excludeId: string | null = null
|
|
): T[] {
|
|
const now = Date.now();
|
|
return accounts.filter((acc) => {
|
|
if (excludeId && acc.id === excludeId) return false;
|
|
if (acc.rateLimitedUntil) {
|
|
const until = new Date(acc.rateLimitedUntil).getTime();
|
|
if (until > now) return false;
|
|
}
|
|
return true;
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Reset account state when request succeeds
|
|
*/
|
|
export function resetAccountState<T extends AccountState | null | undefined>(
|
|
account: T
|
|
): T | AccountState {
|
|
if (!account) return account;
|
|
return {
|
|
...account,
|
|
rateLimitedUntil: null,
|
|
backoffLevel: 0,
|
|
lastError: null,
|
|
status: "active",
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Apply error state to account
|
|
*/
|
|
export function applyErrorState<T extends AccountState | null | undefined>(
|
|
account: T,
|
|
status: number,
|
|
errorText: string | null,
|
|
provider: string | null = null
|
|
): T | AccountState {
|
|
if (!account) return account;
|
|
|
|
const backoffLevel = account.backoffLevel || 0;
|
|
const fallbackDecision = checkFallbackError(status, errorText, backoffLevel, null, provider);
|
|
const { cooldownMs, reason } = fallbackDecision;
|
|
const newBackoffLevel =
|
|
"newBackoffLevel" in fallbackDecision ? fallbackDecision.newBackoffLevel : undefined;
|
|
|
|
return {
|
|
...account,
|
|
rateLimitedUntil: cooldownMs > 0 ? getUnavailableUntil(cooldownMs) : null,
|
|
backoffLevel: newBackoffLevel ?? backoffLevel,
|
|
lastError: { status, message: errorText, timestamp: new Date().toISOString(), reason },
|
|
status: "error",
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Get account health score (0-100) for P2C selection (Phase 9)
|
|
* @param {object} account
|
|
* @returns {number} score 0 = unhealthy, 100 = perfectly healthy
|
|
*/
|
|
export function getAccountHealth(
|
|
account: AccountState | null | undefined,
|
|
model?: unknown
|
|
): number {
|
|
if (!account) return 0;
|
|
let score = 100;
|
|
score -= (account.backoffLevel || 0) * 10;
|
|
if (account.lastError) score -= 20;
|
|
if (account.rateLimitedUntil && isAccountUnavailable(account.rateLimitedUntil)) score -= 30;
|
|
return Math.max(0, score);
|
|
}
|