import test from "node:test"; import assert from "node:assert/strict"; const accountFallback = await import("../../open-sse/services/accountFallback.ts"); const accountSelector = await import("../../open-sse/services/accountSelector.ts"); const { RateLimitReason, COOLDOWN_MS, PROVIDER_PROFILES } = await import("../../open-sse/config/constants.ts"); const { getCircuitBreaker } = await import("../../src/shared/utils/circuitBreaker.ts"); const { isOAuthInvalidToken, parseRetryFromErrorText, checkFallbackError, filterAvailableAccounts, getEarliestRateLimitedUntil, formatRetryAfter, applyErrorState, lockModelIfPerModelQuota, isModelLocked, getModelLockoutInfo, hasPerModelQuota, getProviderProfile, recordModelLockoutFailure, clearModelLock, shouldMarkAccountExhaustedFrom429, recordProviderFailure, isProviderInCooldown, getProviderCooldownRemainingMs, clearProviderFailure, isProviderFailureCode, getProvidersInCooldown, } = accountFallback; const { selectAccount } = accountSelector; /** Build a full ProviderProfile from partial overrides (test helper). */ function makeProfile(overrides: Record = {}): any { return { baseCooldownMs: 125, useUpstreamRetryHints: false, maxBackoffSteps: 3, failureThreshold: 60, resetTimeoutMs: 5000, transientCooldown: 125, rateLimitCooldown: 125, maxBackoffLevel: 3, circuitBreakerThreshold: 60, circuitBreakerReset: 5000, providerFailureThreshold: 5, providerFailureWindowMs: 300000, providerCooldownMs: 60000, ...overrides, }; } function withMockedNow(now, fn) { const originalNow = Date.now; Date.now = () => now; try { return fn(); } finally { Date.now = originalNow; } } test("isOAuthInvalidToken detects refreshable oauth failures", () => { assert.equal( isOAuthInvalidToken("Invalid authentication credentials for this OAuth 2 session"), true ); assert.equal(isOAuthInvalidToken("plain rate limit"), false); }); test("parseRetryFromErrorText parses both compact reset formats", () => { assert.equal(parseRetryFromErrorText("Your quota will reset after 2h30m14s"), 9_014_000); assert.equal(parseRetryFromErrorText("The pool will reset after 45m"), 2_700_000); assert.equal(parseRetryFromErrorText("This will reset after 30s"), 30_000); assert.equal(parseRetryFromErrorText("No reset metadata"), null); }); test("checkFallbackError marks deactivated accounts as permanent auth failures", () => { const result = checkFallbackError(401, "This account has been deactivated"); assert.equal(result.shouldFallback, true); assert.equal(result.reason, RateLimitReason.AUTH_ERROR); assert.equal(result.permanent, true); assert.ok(result.cooldownMs >= 300 * 24 * 60 * 60 * 1000); }); test("checkFallbackError treats non-429 exhausted credits as long quota cooldowns", () => { const result = checkFallbackError(402, "credit_balance_too_low"); assert.equal(result.shouldFallback, true); assert.equal(result.reason, RateLimitReason.QUOTA_EXHAUSTED); assert.equal(result.creditsExhausted, true); assert.equal(result.cooldownMs, COOLDOWN_MS.paymentRequired ?? 3600 * 1000); }); test("checkFallbackError keeps API-key 429 exhausted-credit text on the resilience cooldown path", () => { const result = checkFallbackError( 429, "credit_balance_too_low", 0, null, "openai", null, makeProfile() ); assert.equal(result.shouldFallback, true); assert.equal(result.reason, RateLimitReason.RATE_LIMIT_EXCEEDED); assert.equal(result.creditsExhausted, undefined); assert.equal(result.cooldownMs, 125); }); test("checkFallbackError preserves OAuth 429 exhausted-credit semantics", () => { const result = checkFallbackError( 429, "credit_balance_too_low", 0, null, "codex", null, makeProfile() ); assert.equal(result.shouldFallback, true); assert.equal(result.reason, RateLimitReason.QUOTA_EXHAUSTED); assert.equal(result.creditsExhausted, true); assert.equal(result.cooldownMs, COOLDOWN_MS.paymentRequired ?? 3600 * 1000); }); test("checkFallbackError keeps API-key 429 quota text on the status-based resilience path", () => { const result = checkFallbackError(429, "quota exceeded", 0, null, "openai", null, makeProfile()); assert.equal(result.shouldFallback, true); assert.equal(result.reason, RateLimitReason.RATE_LIMIT_EXCEEDED); assert.equal(result.cooldownMs, 125); }); test("checkFallbackError honors Retry-After header for rate limits", () => { withMockedNow(1_700_000_000_000, () => { const headers = new Headers({ "retry-after": "120" }); const result = checkFallbackError(429, "Rate limit hit", 3, null, "openai", headers); assert.equal(result.shouldFallback, true); assert.equal(result.reason, RateLimitReason.RATE_LIMIT_EXCEEDED); assert.equal(result.newBackoffLevel, 0); assert.equal(result.cooldownMs, 120_000); }); }); test("checkFallbackError honors x-ratelimit-reset for transient 5xx errors", () => { withMockedNow(1_700_000_000_000, () => { const resetSeconds = Math.floor((Date.now() + 90_000) / 1000); const headers = new Headers({ "x-ratelimit-reset": String(resetSeconds) }); const result = checkFallbackError(503, "upstream unavailable", 1, null, "openai", headers); assert.equal(result.shouldFallback, true); assert.equal(result.reason, RateLimitReason.SERVER_ERROR); assert.equal(result.newBackoffLevel, 0); assert.ok(result.cooldownMs >= 89_000); assert.ok(result.cooldownMs <= 90_000); }); }); test("checkFallbackError keeps generic 400 client errors terminal", () => { const result = checkFallbackError(400, "bad request payload"); assert.deepEqual(result, { shouldFallback: false, cooldownMs: 0, reason: RateLimitReason.UNKNOWN, }); }); test("checkFallbackError treats a genuine 400 model-access error as combo fallback", () => { const result = checkFallbackError(400, "The model `foo` does not exist or is not available"); assert.equal(result.shouldFallback, true); assert.equal(result.reason, RateLimitReason.MODEL_CAPACITY); }); test("checkFallbackError does NOT treat a bad-credential 400 as model-access fallback", () => { // Phrased so it would otherwise match MODEL_ACCESS_DENIED_PATTERNS ("...api key // ... model"), but the bad-credential signal must keep it terminal so the real // auth error surfaces instead of silently exhausting every combo target. const result = checkFallbackError(400, "Invalid API key provided for model gpt-4o"); assert.deepEqual(result, { shouldFallback: false, cooldownMs: 0, reason: RateLimitReason.UNKNOWN, }); }); test("checkFallbackError still honors structured model_not_found even with credential-like text", () => { // Structured codes are authoritative and unaffected by the credential guard. const result = checkFallbackError(400, "unauthorized-ish blob", 0, null, "openai", null, null, { code: "model_not_found", }); assert.equal(result.shouldFallback, true); assert.equal(result.reason, RateLimitReason.MODEL_CAPACITY); }); test("filterAvailableAccounts skips exclusion and active cooldowns but keeps recovered ones", () => { withMockedNow(1_700_000_000_000, () => { const accounts = [ { id: "exclude-me" }, { id: "cooling", rateLimitedUntil: new Date(Date.now() + 60_000).toISOString() }, { id: "recovered", rateLimitedUntil: new Date(Date.now() - 1_000).toISOString() }, { id: "healthy" }, ]; const available = filterAvailableAccounts(accounts, "exclude-me"); assert.deepEqual( available.map((account) => account.id), ["recovered", "healthy"] ); }); }); test("getEarliestRateLimitedUntil returns the shortest future cooldown and formatRetryAfter humanizes it", () => { withMockedNow(1_700_000_000_000, () => { const earliest = getEarliestRateLimitedUntil([ { rateLimitedUntil: new Date(Date.now() - 5_000).toISOString() }, { rateLimitedUntil: new Date(Date.now() + 90_000).toISOString() }, { rateLimitedUntil: new Date(Date.now() + 30_000).toISOString() }, ]); assert.equal(earliest, new Date(Date.now() + 30_000).toISOString()); assert.equal(formatRetryAfter(earliest), "reset after 30s"); }); }); test("applyErrorState and selectAccount advance to the next account after an auth failure", () => { withMockedNow(1_700_000_000_000, () => { const accounts = [ { id: "conn-a", backoffLevel: 0 }, { id: "conn-b", backoffLevel: 0 }, ]; const firstSelection = selectAccount(accounts, "fill-first"); assert.equal(firstSelection.account.id, "conn-a"); const failedFirst = applyErrorState(firstSelection.account, 401, "Unauthorized", "claude"); assert.equal(failedFirst.status, "error"); assert.equal(failedFirst.lastError.reason, RateLimitReason.AUTH_ERROR); const candidates = filterAvailableAccounts([failedFirst, accounts[1]], failedFirst.id); const nextSelection = selectAccount(candidates, "fill-first"); assert.equal(nextSelection.account.id, "conn-b"); }); }); test("lockModelIfPerModelQuota only locks supported providers and real models", () => { const geminiConnectionId = `gemini-${Date.now()}`; const openAiConnectionId = `openai-${Date.now()}`; const compatibleConnectionId = `compatible-${Date.now()}`; const compatibleProvider = "openai-compatible-custom-node"; const compatibleModel = "custom-model-a"; assert.equal(hasPerModelQuota("gemini"), true); assert.equal(hasPerModelQuota("openai"), false); assert.equal(hasPerModelQuota(compatibleProvider, compatibleModel), true); assert.equal( lockModelIfPerModelQuota( "gemini", geminiConnectionId, "gemini-2.5-pro", RateLimitReason.RATE_LIMIT_EXCEEDED, 30_000 ), true ); assert.equal(isModelLocked("gemini", geminiConnectionId, "gemini-2.5-pro"), true); assert.equal( lockModelIfPerModelQuota( "openai", openAiConnectionId, "gpt-5-mini", RateLimitReason.RATE_LIMIT_EXCEEDED, 30_000 ), false ); assert.equal(isModelLocked("openai", openAiConnectionId, "gpt-5-mini"), false); assert.equal( lockModelIfPerModelQuota( compatibleProvider, compatibleConnectionId, compatibleModel, RateLimitReason.RATE_LIMIT_EXCEEDED, 30_000 ), true ); assert.equal(isModelLocked(compatibleProvider, compatibleConnectionId, compatibleModel), true); }); test("getProviderProfile differentiates oauth and api-key providers", () => { const oauthProfile = getProviderProfile("claude"); assert.equal(oauthProfile.transientCooldown, PROVIDER_PROFILES.oauth.transientCooldown); assert.equal( oauthProfile.rateLimitCooldown, oauthProfile.useUpstreamRetryHints ? 0 : oauthProfile.baseCooldownMs ); assert.equal(oauthProfile.maxBackoffLevel, PROVIDER_PROFILES.oauth.maxBackoffLevel); assert.equal( oauthProfile.circuitBreakerThreshold, PROVIDER_PROFILES.oauth.circuitBreakerThreshold ); assert.equal(oauthProfile.circuitBreakerReset, PROVIDER_PROFILES.oauth.circuitBreakerReset); assert.equal(oauthProfile.baseCooldownMs, PROVIDER_PROFILES.oauth.transientCooldown); assert.equal(oauthProfile.failureThreshold, PROVIDER_PROFILES.oauth.circuitBreakerThreshold); assert.equal(oauthProfile.resetTimeoutMs, PROVIDER_PROFILES.oauth.circuitBreakerReset); const apiKeyProfile = getProviderProfile("openai"); assert.equal(apiKeyProfile.transientCooldown, PROVIDER_PROFILES.apikey.transientCooldown); assert.equal( apiKeyProfile.rateLimitCooldown, apiKeyProfile.useUpstreamRetryHints ? 0 : apiKeyProfile.baseCooldownMs ); assert.equal(apiKeyProfile.maxBackoffLevel, PROVIDER_PROFILES.apikey.maxBackoffLevel); assert.equal( apiKeyProfile.circuitBreakerThreshold, PROVIDER_PROFILES.apikey.circuitBreakerThreshold ); assert.equal(apiKeyProfile.circuitBreakerReset, PROVIDER_PROFILES.apikey.circuitBreakerReset); assert.equal(apiKeyProfile.baseCooldownMs, PROVIDER_PROFILES.apikey.transientCooldown); assert.equal(apiKeyProfile.failureThreshold, PROVIDER_PROFILES.apikey.circuitBreakerThreshold); assert.equal(apiKeyProfile.resetTimeoutMs, PROVIDER_PROFILES.apikey.circuitBreakerReset); }); test("shouldMarkAccountExhaustedFrom429 skips connection poisoning for compatible providers", () => { assert.equal(shouldMarkAccountExhaustedFrom429("gemini", "gemini-2.5-pro"), false); assert.equal( shouldMarkAccountExhaustedFrom429("openai-compatible-custom-node", "any-model"), false ); assert.equal(shouldMarkAccountExhaustedFrom429("openai", "gpt-4o-mini"), false); assert.equal(shouldMarkAccountExhaustedFrom429("claude", "claude-sonnet-4-6"), true); }); test("shouldMarkAccountExhaustedFrom429 does not poison quota cache for transient 429s", () => { assert.equal( shouldMarkAccountExhaustedFrom429("kiro", "claude-opus-4.7", undefined, "rate_limit"), false ); assert.equal( shouldMarkAccountExhaustedFrom429("kiro", "claude-opus-4.7", undefined, "transient"), false ); assert.equal( shouldMarkAccountExhaustedFrom429("kiro", "claude-opus-4.7", undefined, "quota_exhausted"), true ); }); test("hasPerModelQuota returns true for GitHub Copilot provider (#1624)", () => { assert.equal(hasPerModelQuota("github"), true); assert.equal(hasPerModelQuota("github", "gpt-5.1-codex-max"), true); assert.equal(hasPerModelQuota("github", "gpt-5-mini"), true); }); test("shouldMarkAccountExhaustedFrom429 skips connection-wide lockout for GitHub (#1624)", () => { assert.equal(shouldMarkAccountExhaustedFrom429("github", "gpt-5.1-codex-max"), false); assert.equal(shouldMarkAccountExhaustedFrom429("github", "gpt-5-mini"), false); assert.equal(shouldMarkAccountExhaustedFrom429("github", "claude-haiku-4.5"), false); }); test("lockModelIfPerModelQuota locks individual GitHub models without poisoning the connection (#1624)", () => { const connectionId = `github-${Date.now()}`; // A 429 on a high-PRU model should lock ONLY that model assert.equal( lockModelIfPerModelQuota( "github", connectionId, "gpt-5.1-codex-max", RateLimitReason.RATE_LIMIT_EXCEEDED, 30_000 ), true ); assert.equal(isModelLocked("github", connectionId, "gpt-5.1-codex-max"), true); // Other models on the same connection should remain unlocked assert.equal(isModelLocked("github", connectionId, "gpt-5-mini"), false); assert.equal(isModelLocked("github", connectionId, "claude-haiku-4.5"), false); }); test("recordModelLockoutFailure uses provider profile cooldowns, backoff, and reset window", () => { const originalNow = Date.now; let now = 1_700_000_000_000; Date.now = () => now; try { const compatibleProvider = "openai-compatible-custom-node"; const compatibleModel = "custom-model-a"; const profile = makeProfile({ maxBackoffSteps: 2, maxBackoffLevel: 2, resetTimeoutMs: 500, }); const first = recordModelLockoutFailure( compatibleProvider, "conn-compatible", compatibleModel, "rate_limited", 429, 0, profile ); now += 50; const second = recordModelLockoutFailure( compatibleProvider, "conn-compatible", compatibleModel, "rate_limited", 429, 0, profile ); now += 50; const third = recordModelLockoutFailure( compatibleProvider, "conn-compatible", compatibleModel, "rate_limited", 429, 0, profile ); const info = getModelLockoutInfo(compatibleProvider, "conn-compatible", compatibleModel); assert.equal(first.failureCount, 1); assert.equal(first.cooldownMs, 125); assert.equal(second.failureCount, 2); assert.equal(second.cooldownMs, 250); assert.equal(third.failureCount, 3); assert.equal(third.cooldownMs, 500); assert.equal(info.failureCount, 3); clearModelLock(compatibleProvider, "conn-compatible", compatibleModel); now += 600; const afterReset = recordModelLockoutFailure( compatibleProvider, "conn-compatible", compatibleModel, "rate_limited", 429, 0, profile ); assert.equal(afterReset.failureCount, 1); assert.equal(afterReset.cooldownMs, 125); } finally { Date.now = originalNow; clearModelLock("openai-compatible-custom-node", "conn-compatible", "custom-model-a"); } }); // Provider-level failure circuit breaker tests test("isProviderFailureCode correctly identifies provider-wide transient error codes", () => { assert.equal(isProviderFailureCode(429), false); assert.equal(isProviderFailureCode(408), true); assert.equal(isProviderFailureCode(500), true); assert.equal(isProviderFailureCode(502), true); assert.equal(isProviderFailureCode(503), true); assert.equal(isProviderFailureCode(504), true); assert.equal(isProviderFailureCode(401), false); assert.equal(isProviderFailureCode(403), false); assert.equal(isProviderFailureCode(400), false); assert.equal(isProviderFailureCode(404), false); assert.equal(isProviderFailureCode(200), false); }); test("recordProviderFailure tracks failures and triggers cooldown after threshold", () => { const originalNow = Date.now; let now = 1_700_000_000_000; Date.now = () => now; try { const provider = "test-provider"; // Clear any existing state clearProviderFailure(provider); assert.equal(isProviderInCooldown(provider), false); assert.equal(getProviderCooldownRemainingMs(provider), null); const threshold = PROVIDER_PROFILES.apikey.circuitBreakerThreshold; // Record failures up to threshold - 1 for (let i = 0; i < threshold - 1; i++) { recordProviderFailure(provider); now += 1000; // 1 second between failures } assert.equal(isProviderInCooldown(provider), false); // Final failure to trigger threshold recordProviderFailure(provider); assert.equal(isProviderInCooldown(provider), true); const remaining = getProviderCooldownRemainingMs(provider); assert.ok(remaining !== null); assert.ok(remaining > 0); assert.ok(remaining <= 10 * 60 * 1000); // 10 minutes max // Check getProvidersInCooldown returns the provider const inCooldown = getProvidersInCooldown(); assert.ok(inCooldown.some((p) => p.provider === provider)); assert.equal(inCooldown.find((p) => p.provider === provider)?.failureCount, threshold); // Simulate cooldown expiration now += 11 * 60 * 1000; // 11 minutes later assert.equal(isProviderInCooldown(provider), false); assert.equal(getProviderCooldownRemainingMs(provider), null); assert.equal( getProvidersInCooldown().some((p) => p.provider === provider), false ); } finally { Date.now = originalNow; clearProviderFailure("test-provider"); } }); test("recordProviderFailure honors runtime provider breaker profile", () => { const provider = "test-provider-runtime-profile"; clearProviderFailure(provider); try { const runtimeProfile = { failureThreshold: PROVIDER_PROFILES.apikey.circuitBreakerThreshold + 7, resetTimeoutMs: PROVIDER_PROFILES.apikey.circuitBreakerReset + 45_000, }; recordProviderFailure(provider, undefined, "conn-runtime-profile", runtimeProfile); const breaker = getCircuitBreaker(provider); assert.equal(breaker.failureThreshold, runtimeProfile.failureThreshold); assert.equal(breaker.resetTimeout, runtimeProfile.resetTimeoutMs); assert.equal(isProviderInCooldown(provider), false); const breakerAfterStatusCheck = getCircuitBreaker(provider); assert.equal(breakerAfterStatusCheck.failureThreshold, runtimeProfile.failureThreshold); assert.equal(breakerAfterStatusCheck.resetTimeout, runtimeProfile.resetTimeoutMs); } finally { clearProviderFailure(provider); } }); test("checkFallbackError no longer mutates provider breaker state on per-connection failures", () => { const provider = "test-provider-check"; clearProviderFailure(provider); for (let i = 0; i < 5; i++) { checkFallbackError(429, "rate limited", 0, null, provider); } assert.equal(isProviderInCooldown(provider), false); clearProviderFailure(provider); }); test("checkFallbackError does not record provider failure for non-transient errors", () => { const originalNow = Date.now; let now = 1_700_000_000_000; Date.now = () => now; try { const provider = "test-provider-no-record"; clearProviderFailure(provider); // Simulate 5 auth errors (401) - should NOT trigger provider cooldown for (let i = 0; i < 5; i++) { checkFallbackError(401, "unauthorized", 0, null, provider); now += 1000; } // Provider should NOT be in cooldown assert.equal(isProviderInCooldown(provider), false); } finally { Date.now = originalNow; clearProviderFailure("test-provider-no-record"); } }); test("clearProviderFailure removes provider from cooldown", () => { const originalNow = Date.now; let now = 1_700_000_000_000; Date.now = () => now; try { const provider = "test-provider-clear"; clearProviderFailure(provider); // Trigger cooldown const threshold = PROVIDER_PROFILES.apikey.circuitBreakerThreshold; for (let i = 0; i < threshold; i++) { recordProviderFailure(provider); now += 1000; } assert.equal(isProviderInCooldown(provider), true); // Clear the failure state clearProviderFailure(provider); assert.equal(isProviderInCooldown(provider), false); assert.equal(getProviderCooldownRemainingMs(provider), null); } finally { Date.now = originalNow; clearProviderFailure("test-provider-clear"); } }); // Daily quota exhausted detection tests test("isDailyQuotaExhausted detects today's quota errors", () => { const { isDailyQuotaExhausted } = accountFallback; assert.equal(isDailyQuotaExhausted("You have exceeded today's quota for model X"), true); assert.equal(isDailyQuotaExhausted("exceeded your daily quota"), true); assert.equal(isDailyQuotaExhausted("Please try again tomorrow"), true); assert.equal(isDailyQuotaExhausted("rate limit exceeded"), false); assert.equal(isDailyQuotaExhausted(""), false); assert.equal(isDailyQuotaExhausted(null), false); }); test("getMsUntilTomorrow returns positive value less than 24 hours", () => { const { getMsUntilTomorrow } = accountFallback; const ms = getMsUntilTomorrow(); assert.ok(ms > 0, "should be positive"); assert.ok(ms <= 24 * 60 * 60 * 1000, "should be <= 24 hours"); }); test("checkFallbackError locks model until tomorrow for non-429 daily quota exhaustion", () => { const result = checkFallbackError( 402, "You have exceeded today's quota for model moonshotai/Kimi-K2.5, please try again tomorrow" ); assert.equal(result.shouldFallback, true); assert.equal(result.reason, RateLimitReason.QUOTA_EXHAUSTED); assert.equal(result.dailyQuotaExhausted, true); assert.ok(result.cooldownMs > 0, "cooldown should be positive"); assert.ok(result.cooldownMs <= 24 * 60 * 60 * 1000, "cooldown should be <= 24 hours"); }); test("checkFallbackError routes API-key 429 'try again tomorrow' through resilience cooldown", () => { const result = checkFallbackError( 429, "Please try again tomorrow", 0, null, "openai", null, makeProfile() ); assert.equal(result.shouldFallback, true); assert.equal(result.dailyQuotaExhausted, undefined); assert.equal(result.cooldownMs, 125); }); test("checkFallbackError routes API-key 429 'daily quota' text through resilience cooldown", () => { const result = checkFallbackError( 429, "You have exceeded your daily quota", 0, null, "openai", null, makeProfile() ); assert.equal(result.shouldFallback, true); assert.equal(result.dailyQuotaExhausted, undefined); assert.equal(result.cooldownMs, 125); }); test("checkFallbackError preserves OAuth 429 daily quota semantics", () => { const result = checkFallbackError( 429, "You have exceeded your daily quota", 0, null, "codex", null, makeProfile() ); assert.equal(result.shouldFallback, true); assert.equal(result.reason, RateLimitReason.QUOTA_EXHAUSTED); assert.equal(result.dailyQuotaExhausted, true); assert.ok(result.cooldownMs > 0); }); // ModelScope daily quota lockout tests (commit 0456a1f5) test("recordModelLockoutFailure sets cooldown until tomorrow 0:00 for quota_exhausted reason", () => { const originalNow = Date.now; // Use a fixed local time (noon) to ensure predictable results const testDate = new Date(); testDate.setHours(12, 0, 0, 0); // Set to noon today const now = testDate.getTime(); Date.now = () => now; try { const provider = "modelscope"; const connectionId = "test-conn-modelscope-1"; const model = "qwen/Qwen2.5-Coder-32B-Instruct"; // Clear any existing state clearModelLock(provider, connectionId, model); const profile = makeProfile(); // Calculate milliseconds until tomorrow 00:00 local time const tomorrow = new Date(now); tomorrow.setDate(tomorrow.getDate() + 1); tomorrow.setHours(0, 0, 0, 0); const expectedMsUntilTomorrow = tomorrow.getTime() - now; // Account for timezone offset: function uses local time, test env may use UTC const timezoneOffset = new Date().getTimezoneOffset() * 60 * 1000; // Record failure with quota_exhausted reason const result = recordModelLockoutFailure( provider, connectionId, model, "quota_exhausted", 429, 0, // fallbackCooldownMs should be overridden to ms until tomorrow profile ); // Verify the cooldown is set to ms until tomorrow 0:00 (with tolerance) // The cooldown should be close to expectedMsUntilTomorrow const tolerance = 60 * 1000; // 1 minute tolerance // Calculate difference between actual and expected values const diff = Math.abs(result.cooldownMs - expectedMsUntilTomorrow); // Allow ±5 minutes tolerance (300,000 ms) assert.ok( diff <= 300_000, `cooldown should be ms until tomorrow 0:00 (expected ${expectedMsUntilTomorrow}ms, got ${result.cooldownMs}ms, diff ${diff}ms)` ); // Verify model is locked assert.equal(isModelLocked(provider, connectionId, model), true); const lockInfo = getModelLockoutInfo(provider, connectionId, model); assert.ok(lockInfo !== null, "lockInfo should not be null"); assert.ok(lockInfo.remainingMs > 0, "remaining time should be positive"); clearModelLock(provider, connectionId, model); } finally { Date.now = originalNow; clearModelLock("modelscope", "test-conn-modelscope-1", "qwen/Qwen2.5-Coder-32B-Instruct"); } }); test("recordModelLockoutFailure uses regular backoff for non-quota reasons", () => { const originalNow = Date.now; const now = 1_700_000_000_000; Date.now = () => now; try { const provider = "modelscope"; const connectionId = "test-conn-modelscope-2"; const model = "qwen/Qwen2.5-Coder-32B-Instruct"; clearModelLock(provider, connectionId, model); const profile = makeProfile({ baseCooldownMs: 5000, transientCooldown: 5000, rateLimitCooldown: 5000, }); // Record failure with rate_limited reason (not quota_exhausted) const result = recordModelLockoutFailure( provider, connectionId, model, "rate_limited", 429, 0, profile ); // Verify the cooldown uses regular profile baseCooldownMs (5000ms) assert.ok( result.cooldownMs < 24 * 60 * 60 * 1000, "cooldown should be less than 24h for non-quota reasons" ); assert.equal(result.cooldownMs, 5000, "cooldown should use profile baseCooldownMs"); clearModelLock(provider, connectionId, model); } finally { Date.now = originalNow; clearModelLock("modelscope", "test-conn-modelscope-2", "qwen/Qwen2.5-Coder-32B-Instruct"); } }); // Test for hour quota related error messages test("checkFallbackError classifies hour quota errors correctly", () => { // For OAuth providers (e.g., codex), hour quota errors should be QUOTA_EXHAUSTED const result1 = checkFallbackError( 429, "Coding Plan hour quota has been exceeded", 0, null, "codex" ); assert.equal(result1.shouldFallback, true); assert.equal(result1.reason, RateLimitReason.QUOTA_EXHAUSTED); const result2 = checkFallbackError(429, "hour quota exceeded", 0, null, "codex"); assert.equal(result2.shouldFallback, true); assert.equal(result2.reason, RateLimitReason.QUOTA_EXHAUSTED); const result3 = checkFallbackError(429, "Your hour quota is exceeded", 0, null, "codex"); assert.equal(result3.shouldFallback, true); assert.equal(result3.reason, RateLimitReason.QUOTA_EXHAUSTED); const result4 = checkFallbackError(429, "hour quota depleted", 0, null, "codex"); assert.equal(result4.shouldFallback, true); assert.equal(result4.reason, RateLimitReason.QUOTA_EXHAUSTED); // For API-key providers with 402 status, hour quota errors should be QUOTA_EXHAUSTED const result5 = checkFallbackError(402, "hour quota has been exceeded", 0, null, "openai"); assert.equal(result5.shouldFallback, true); assert.equal(result5.reason, RateLimitReason.QUOTA_EXHAUSTED); const result6 = checkFallbackError( 403, "Coding Plan hour quota has been exceeded", 0, null, "openai" ); assert.equal(result6.shouldFallback, true); assert.equal(result6.reason, RateLimitReason.QUOTA_EXHAUSTED); }); // Test for classifyErrorText function with hour quota test("classifyErrorText handles hour quota messages", () => { const { classifyErrorText } = accountFallback; assert.equal( classifyErrorText("Coding Plan hour quota has been exceeded"), RateLimitReason.QUOTA_EXHAUSTED ); assert.equal(classifyErrorText("hour quota exceeded"), RateLimitReason.QUOTA_EXHAUSTED); assert.equal(classifyErrorText("Your hour quota is exceeded"), RateLimitReason.QUOTA_EXHAUSTED); assert.equal(classifyErrorText("hour quota has been exceeded"), RateLimitReason.QUOTA_EXHAUSTED); assert.equal(classifyErrorText("quota has been exceeded"), RateLimitReason.QUOTA_EXHAUSTED); }); // ─── Model Access Denied (structured error codes + regex fallback) ───── test("checkFallbackError detects model access denied via structured error code (OpenAI)", () => { const result = checkFallbackError( 400, "The model `gpt-5` does not exist", 0, null, "openai", null, null, { code: "model_not_found", type: null } ); assert.equal(result.shouldFallback, true); assert.equal(result.cooldownMs, 0); assert.equal(result.reason, RateLimitReason.MODEL_CAPACITY); }); test("checkFallbackError detects model access denied via structured error type (Anthropic not_found_error)", () => { const result = checkFallbackError( 400, "model: claude-sonnet-4-7-20260515", 0, null, "anthropic", null, null, { code: null, type: "not_found_error" } ); assert.equal(result.shouldFallback, true); assert.equal(result.cooldownMs, 0); assert.equal(result.reason, RateLimitReason.MODEL_CAPACITY); }); test("checkFallbackError detects model access denied via structured error type (Anthropic permission_error) when the message confirms the model", () => { const result = checkFallbackError( 400, "you do not have access to the requested model", 0, null, "anthropic", null, null, { code: null, type: "permission_error" } ); assert.equal(result.shouldFallback, true); assert.equal(result.cooldownMs, 0); assert.equal(result.reason, RateLimitReason.MODEL_CAPACITY); }); test("checkFallbackError does NOT fallback on a permission_error that is a key/feature scope issue (not model access)", () => { // permission_error is ambiguous on Anthropic — also raised for API-key scope, // org restrictions and feature gating. Without a model-related message it must // surface the real error instead of silently exhausting every combo target. const result = checkFallbackError( 400, "Your API key does not have permission to use the Message Batches API", 0, null, "anthropic", null, null, { code: null, type: "permission_error" } ); assert.equal(result.shouldFallback, false); }); test("checkFallbackError detects model access denied via regex fallback (invalid model)", () => { const result = checkFallbackError( 400, "Invalid model: gpt-5-turbo", 0, null, "some-provider", null, null ); assert.equal(result.shouldFallback, true); assert.equal(result.cooldownMs, 0); assert.equal(result.reason, RateLimitReason.MODEL_CAPACITY); }); test("checkFallbackError does NOT fallback on generic 400 without model access denied", () => { const result = checkFallbackError(400, "bad request payload", 0, null, "openai", null, null); assert.equal(result.shouldFallback, false); }); test("checkFallbackError ignores structured error with unrelated code on 400", () => { const result = checkFallbackError(400, "something went wrong", 0, null, "openai", null, null, { code: "invalid_api_key", type: null, }); // "invalid_api_key" is not in MODEL_ACCESS_DENIED_CODES, // no MODEL_ACCESS_DENIED_PATTERNS match either → shouldFallback: false assert.equal(result.shouldFallback, false); }); // ─── G-02: X-Omni-Fallback-Hint: connection_cooldown ───────────────────────── // When 9router executor signals a supervisor-not-running 503, checkFallbackError // must return 5s cooldown with skipProviderBreaker:true — not trip the circuit breaker. test("G-02: X-Omni-Fallback-Hint connection_cooldown on 503 returns 5s cooldown + skipProviderBreaker", () => { const headers = new Headers({ "X-Omni-Fallback-Hint": "connection_cooldown" }); const result = checkFallbackError( 503, "9router is not running (state: stopped)", 0, null, "9router", headers ); assert.equal(result.shouldFallback, true); assert.equal(result.cooldownMs, 5_000); assert.equal(result.skipProviderBreaker, true); assert.equal(result.newBackoffLevel, 0); assert.equal(result.reason, "service_not_running"); }); test("G-02: X-Omni-Fallback-Hint connection_cooldown header lookup is case-insensitive (lowercase header key)", () => { // Headers object normalises keys to lowercase — test the plain-object path const headers: Record = { "x-omni-fallback-hint": "connection_cooldown" }; const result = checkFallbackError( 503, "9router is not running (state: stopped)", 0, null, "9router", headers ); assert.equal(result.skipProviderBreaker, true); assert.equal(result.cooldownMs, 5_000); }); test("G-02: hint header is ignored for non-503 status codes", () => { const headers = new Headers({ "X-Omni-Fallback-Hint": "connection_cooldown" }); // 502 should NOT trigger the hint path even if the header is present const result = checkFallbackError(502, "bad gateway", 0, null, "9router", headers); assert.equal(result.skipProviderBreaker, undefined); // normal path, no skip flag }); test("G-02: 503 without hint header follows normal circuit-breaker path", () => { // A plain 503 from a real upstream must still feed the circuit breaker const result = checkFallbackError(503, "service unavailable", 0, null, "openai", null); assert.equal(result.skipProviderBreaker, undefined); assert.ok(result.cooldownMs > 0); }); test("G-02: five consecutive 503 service_not_running do NOT trip provider circuit breaker (flag)", () => { // Verify that every call returns skipProviderBreaker:true so callers can skip recordProviderFailure const headers = new Headers({ "X-Omni-Fallback-Hint": "connection_cooldown" }); for (let i = 0; i < 5; i++) { const result = checkFallbackError( 503, "9router is not running (state: stopped)", 0, null, "9router", headers ); assert.equal( result.skipProviderBreaker, true, `call ${i + 1} should have skipProviderBreaker:true` ); } // Verify the circuit breaker for 9router is NOT open after those 5 calls const { isProviderInCooldown, clearProviderFailure } = accountFallback; assert.equal( isProviderInCooldown("9router"), false, "9router circuit breaker must remain closed" ); clearProviderFailure("9router"); // cleanup });