mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-02 05:12:11 +03:00
* chore(release): open v3.8.23 development cycle
* fix(anthropic): strip top_p when temperature is set to avoid 400 (#3691)
Integrated into release/v3.8.23
* fix(vertex): support Vertex AI Express-mode API keys (#3690)
Integrated into release/v3.8.23
* fix(stream): error on empty Claude SSE instead of synthetic success (#3689)
Integrated into release/v3.8.23
* fix(oauth): stop token-refresh invalidation loop + harden proxy resolution (#3692)
Integrated into release/v3.8.23
* docs: add FUNDING.yml and Support section to README (#3698)
Integrated into release/v3.8.23
* feat: gemini - handle known ratelimits (#3686)
Integrated into release/v3.8.23
* fix: stream combo fails over on empty content-filtered response (#3685) (#3702)
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
* fix(antigravity): preserve gemini-3.1-pro high/low budget tiers (#3696) (#3703)
Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
* feat(auto-combo): add auto-updating model intelligence scoring (#3660)
Integrated into release/v3.8.23
* fix(gemini): context-mode fallback for signatureless tool calls (#3688) (#3704)
* chore(quality-gate): reconcile file-size baseline (27 files + providerLimits.ts) (#3705)
* feat(vertex): dynamic model discovery via Generative Language models API (#3712)
Integrated into release/v3.8.23. Vertex dynamic model discovery — surfaces image models (imagen-*, gemini-*-image), embeddings and audio from the live Generative Language catalog, with cached→static fallback and the shared parseGeminiModelsList helper. Validated: parser test 5/5, typecheck:core clean.
* fix(combo): gate reasoning token buffer (#3700)
Integrated into release/v3.8.23. Makes the #3588 reasoning token buffer safe and configurable: only inflates max_tokens when the model has a known, non-default output cap and the buffered value fits inside it; otherwise preserves/clamps the client limit. Adds the reasoningTokenBufferEnabled kill switch (default ON). Validated: combo-routing-engine 81/81, combo-config 25/25, combo-quality-validator-reasoning 12/12, phase1f 10/10, typecheck:core clean.
* refactor(#3501): god-component Phase 1g-1j — client 4062→3408 LOC (-654) (#3717)
Phase 1g-1j of #3501: client 4062→3408 LOC. Pure extraction (ProviderPlaygroundPanel, useCommandCodeAuth, useExternalLinkFlow+ExternalLinkModal, useAuthFileHandlers) + loadConnProxies ReferenceError fix + phase1f test path fix.
Co-authored-by: oyi77 <14921983+oyi77@users.noreply.github.com>
* refactor(#3501): god-component Phase 1k-1m — client 3408→2553 LOC (-855) (#3721)
Phase 1k-1m of #3501: client 3408→2553 LOC. Pure extraction (useModelImportHandlers+ImportProgressModal, useModelVisibilityHandlers, ProviderModelsSection).
Co-authored-by: oyi77 <14921983+oyi77@users.noreply.github.com>
* docs(changelog): restore #3590 bullet lost on the v3.8.20 release branch
The fix itself reached main pre-tag via cherry-pick #3591, but its changelog
bullet (commit e33fdd4ab) only ever existed on release/v3.8.20 after the
squash-merge. Restored under [3.8.20] per the 2026-06-12 release-branch
leftover audit (_tasks/release-audit/release-leftovers-audit-2026-06-12.md).
* fix(kiro): resolve quota for IAM Identity Center accounts missing a profileArn (#3722)
Integrated into release/v3.8.23
* refactor(#3501): god-component Phase 1n-1s — client 2553→1376 LOC (-1177) (#3725)
Phase 1n-1s of #3501: client 2553→1376 LOC. Pure extraction (ConnectionsListPanel, ConnectionsHeaderToolbar, ZedImportCard, BatchTestResultsModal, AdaptaTutorialModal, useApiKeySave + helpers).
Co-authored-by: oyi77 <14921983+oyi77@users.noreply.github.com>
* feat(model-lockout): settings UI, backend integration, error classification, and success-decay recovery (#3629)
Integrated into release/v3.8.23
* refactor(#3501): god-component Phase 1t — client 1376→781 LOC (≤800 TARGET REACHED ✅) (#3727)
Phase 1t of #3501: client 1376→781 LOC (≤800 reached). Original god-component 12,882→781 (−94%).
Co-authored-by: oyi77 <14921983+oyi77@users.noreply.github.com>
* fix: bundle @omniroute/opencode-plugin inside omniroute + add 'setup opencode' CLI command (#3726)
Integrated into release/v3.8.23
* feat(vertex): self-tracked USD spend since account added (#3724)
Integrated into release/v3.8.23
* fix(qwen-web): migrate to v2 chat API with full cookie-jar replay (#3288) (#3723)
Integrated into release/v3.8.23
* fix(sse): make safeLogEvents async — 'await' in a sync function broke every chatHelpers import
#3692 added a lazy 'await import(proxyEgress)' for egress-IP visibility inside
safeLogEvents, which is a sync function — an ES syntax error. It went unnoticed
because typecheck:core does not cover src/sse and no test in the merge gates
loaded chatHelpers via tsx; any consumer that did (chat-context-relay and
chat-route-coverage suites, integration harnesses) failed at module load with
'await can only be used inside an async function'.
safeLogEvents is fire-and-forget logging with an outer try/catch, so making it
async (and 'void'-ing the single chat.ts call site) preserves behavior exactly.
Validation: tests/unit/chat-context-relay.test.ts + chat-route-coverage.test.ts
went from failing-at-load to green (+14 tests destravados).
* fix(sse): remove cross-provider credential leak in emergency fallback + combo/proxy audit fixes (#3699)
Integrated into release/v3.8.23
* fix(executors): inject MiMoCode anti-abuse marker so free endpoint stops 403ing (#3728)
Integrated into release/v3.8.23
* fix(dashboard): repair "Test all models" — toast crash, status icons, auto-hide (#3729)
Integrated into release/v3.8.23
* chore(deps): bump actions/upload-artifact from 4 to 7 (#3735)
Integrated into release/v3.8.23 — aligns upload-artifact to v7 (already used across ci.yml).
* chore(deps): bump actions/cache from 4 to 5 (#3734)
Integrated into release/v3.8.23 — actions/cache v4→v5.
* chore(deps): bump actions/download-artifact from 4 to 8 (#3733)
Integrated into release/v3.8.23 — download-artifact v4→v8.
* feat(fallback): add OMNIROUTE_EMERGENCY_FALLBACK env switch (#3741)
Adds an OMNIROUTE_EMERGENCY_FALLBACK env switch to disable the emergency budget-exhaustion fallback (reroute to free nvidia/gpt-oss-120b). Default unchanged (enabled). Closes #3739, related #2879.
Integrated into release/v3.8.23.
* i18n: comprehensive zh-CN translation improvements (#3736)
Aligns zh-CN to en (hundreds of entries), translates batch-action labels + settings sidebar menu, adds categoryConfig/endpointTokenSaver keys, resolves __MISSING__ stubs. Sidebar/SidebarTab hardcoded strings replaced with t(). en.json purely additive (8 new sidebar.* keys, 0 removed); cli-i18n gate green.
Integrated into release/v3.8.23.
* chore(release): v3.8.23 — 2026-06-12
- CHANGELOG: complete v3.8.23 section (28 bullets, 27 commits)
- fix(webdav): resolve promise on writeStream finish, not req end — eliminates
intermittent 500 on PUT update (writeStream may not have flushed at rename time)
- test(autoCombo): stub DB calls from PR #3660 in tieredRotation.test.ts to prevent
5s timeout in vitest (getModelIntelligenceBySource DB init path)
- chore(env-sync): add XDG_DATA_HOME + OMNIROUTE_OPENCODE_PLUGIN_DIR to IGNORE_FROM_CODE
allowlist (introduced by PR #3726 setup-open-code.mjs, not OmniRoute config vars)
- chore(cli): regenerated bin/cli/api-commands/*.mjs (7 new, 27 updated)
* fix(model-family): fallback lookup also tries bare model name with dots
getNextFamilyFallback normalized dots-to-hyphens ("gemini-3.1-pro-high" →
"gemini-3-1-pro-high") but MODEL_FAMILIES keys use the literal dot form. The
lookup always missed, returning null for any model whose dots are part of the
name rather than a version separator.
Fallback: try MODEL_FAMILIES[lookupKey] ?? MODEL_FAMILIES[bareModel] so both
naming conventions are covered. Fixes T30 test (pre-existing since v3.8.22).
* feat: expose API key cost drilldown + quota % used (#3742)
Adds all-time USD cost per API key in the API Key Manager, a per-key deep-link into the Cost Explorer (filtered + grouped by model), URL-param hydration of range/groupBy/apiKeyIds, and a '% used' quota display. Review adjustments: extracted URL-param parsers to a tested module (Rule #18), i18n'd the new strings (en + zh-CN), dropped the redundant webdav-handler entry already on release.
Integrated into release/v3.8.23.
* feat: add provider display modes — All / Configured / Compact (#3743)
Replaces the Providers page configured-only toggle with All/Configured/Compact display modes (Compact = flat deduped grid, no-auth last). Persists the preference and migrates the legacy localStorage key. Rebased onto release/v3.8.23.
Integrated into release/v3.8.23.
* fix(cache): scope semantic-cache signature to API key (#3740)
Adds the api_key_id dimension to generateSignature's SHA-256 hash so two callers with different API keys never receive each other's cached responses. Threads apiKeyId through checkSemanticCache + both write sites; migration 098 clears pre-existing key-less entries; unauthenticated requests stay isolated from keyed ones. 3 TDD tests.
Integrated into release/v3.8.23.
* fix(responses): apply OpenAI Responses API stream=false spec default (#3708)
resolveStreamFlag now applies the stream=false-when-omitted default for sourceFormat=openai-responses (same as the existing claude path), so spec-compliant /v1/responses upstreams that return JSON no longer fall through to the wildcard-Accept heuristic and trigger STREAM_EARLY_EOF / 502. Codex CLI (stream:true) and explicit text/event-stream clients unaffected.
Integrated into release/v3.8.23.
* chore(release): reconcile CI gates for v3.8.23
- file-size baseline: re-freeze 8 files grown by PRs #3742/#3743/#3740
(cost drilldown, provider display modes, cache key isolation)
- ARCHITECTURE.md: update executor count 55→60 (check:docs-counts drift)
- .env.example: add OMNIROUTE_EMERGENCY_FALLBACK (#3741, env-doc-sync)
- CHANGELOG: add formatted bullets for #3742, #3743, #3708, #3740,
model-family-fallback fix; remove duplicate raw ### Fixed section
* test: restore assert count to satisfy check:test-masking gate
Three test files had net assertion removals after behavior-changing PRs:
- chatcore-translation-paths: emergency fallback moved to routing layer
(#3699) — add body error assertion + model-name guard
- executor-vertex-extended: non-JSON is now Express API key (#3690) —
add projects/-path guard to the express-key URL test
- stream-utils: empty streams now emit error (#3685) — add code/message/
status/completePayload guards to both passthrough and translate variants
All new assertions are meaningful (code enum value, 5xx range, non-empty
message, onComplete must-not-fire contract).
* fix(ci): move rtl-logical-classes test to ui/ so vitest:ui runner collects it
---------
Co-authored-by: Felipe Almeman <4226997+zhiru@users.noreply.github.com>
Co-authored-by: NOXX - Commiter <artur1992123@mail.ru>
Co-authored-by: Nick Sullivan <142708+TechNickAI@users.noreply.github.com>
Co-authored-by: Markus Hartung <mail@hartmark.se>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: PizzaV <103120356+pizzav-xyz@users.noreply.github.com>
Co-authored-by: Randi <55005611+rdself@users.noreply.github.com>
Co-authored-by: oyi77 <14921983+oyi77@users.noreply.github.com>
Co-authored-by: Chewji <126886556+Chewji9875@users.noreply.github.com>
Co-authored-by: Hernan Javier Ardila Sanchez <hjasgr@gmail.com>
Co-authored-by: Felipe Sartori <felipesartori.ti@gmail.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Zois Pagoulatos <zpagoulatos@hotmail.com>
Co-authored-by: sdfsdfw2 <167810361+sdfsdfw2@users.noreply.github.com>
Co-authored-by: Witroch4 <witalo_rocha@hotmail.com>
1527 lines
51 KiB
TypeScript
1527 lines
51 KiB
TypeScript
import test from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import fs from "node:fs";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
|
|
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-sse-auth-"));
|
|
process.env.DATA_DIR = TEST_DATA_DIR;
|
|
process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "sse-auth-test-secret";
|
|
|
|
const core = await import("../../src/lib/db/core.ts");
|
|
const providersDb = await import("../../src/lib/db/providers.ts");
|
|
const settingsDb = await import("../../src/lib/db/settings.ts");
|
|
const apiKeysDb = await import("../../src/lib/db/apiKeys.ts");
|
|
const auth = await import("../../src/sse/services/auth.ts");
|
|
const quotaCache = await import("../../src/domain/quotaCache.ts");
|
|
const fallback = await import("../../open-sse/services/accountFallback.ts");
|
|
|
|
async function resetStorage() {
|
|
core.resetDbInstance();
|
|
apiKeysDb.resetApiKeyState();
|
|
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
|
|
fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
|
|
}
|
|
|
|
function futureIso(ms = 60_000) {
|
|
return new Date(Date.now() + ms).toISOString();
|
|
}
|
|
|
|
async function seedConnection(provider: string, overrides: any = {}) {
|
|
return providersDb.createProviderConnection({
|
|
provider,
|
|
authType: overrides.authType || "apikey",
|
|
name: overrides.name || `${provider}-${Math.random().toString(16).slice(2, 8)}`,
|
|
email: overrides.email,
|
|
// Unique per connection by default — real accounts have distinct keys, and
|
|
// createProviderConnection dedups by decrypted key value (#3023), so a shared
|
|
// default would collapse multiple seeded connections into one and break
|
|
// round-robin / least-used / fallback selection tests.
|
|
apiKey: overrides.apiKey || `sk-test-${Math.random().toString(16).slice(2, 10)}`,
|
|
accessToken: overrides.accessToken,
|
|
refreshToken: overrides.refreshToken,
|
|
isActive: overrides.isActive ?? true,
|
|
testStatus: overrides.testStatus || "active",
|
|
priority: overrides.priority,
|
|
rateLimitedUntil: overrides.rateLimitedUntil,
|
|
lastError: overrides.lastError,
|
|
lastErrorType: overrides.lastErrorType,
|
|
lastErrorSource: overrides.lastErrorSource,
|
|
errorCode: overrides.errorCode,
|
|
backoffLevel: overrides.backoffLevel,
|
|
providerSpecificData: overrides.providerSpecificData || {},
|
|
lastUsedAt: overrides.lastUsedAt,
|
|
consecutiveUseCount: overrides.consecutiveUseCount,
|
|
});
|
|
}
|
|
|
|
function msUntil(timestamp) {
|
|
return new Date(timestamp).getTime() - Date.now();
|
|
}
|
|
|
|
async function flushWrites() {
|
|
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
}
|
|
|
|
test.beforeEach(async () => {
|
|
await resetStorage();
|
|
});
|
|
|
|
test.after(async () => {
|
|
core.resetDbInstance();
|
|
apiKeysDb.resetApiKeyState();
|
|
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
|
|
});
|
|
|
|
test("extractApiKey parses bearer headers and isValidApiKey validates persisted keys", async () => {
|
|
const created = await apiKeysDb.createApiKey("auth-check", "machine-auth-check");
|
|
|
|
const request = new Request("http://localhost/v1/chat/completions", {
|
|
headers: { Authorization: `Bearer ${created.key}` },
|
|
});
|
|
|
|
assert.equal(auth.extractApiKey(request), created.key);
|
|
assert.equal(
|
|
auth.extractApiKey(
|
|
new Request("http://localhost/v1/chat/completions", {
|
|
headers: { Authorization: "Basic abc123" },
|
|
})
|
|
),
|
|
null
|
|
);
|
|
// Security follow-up (#3300): query-string token fallbacks were removed — a
|
|
// credential in `?token=` must NOT be extracted (it leaks into logs/Referer).
|
|
assert.equal(
|
|
auth.extractApiKey(new Request(`http://localhost/v1/chat/completions?token=${created.key}`)),
|
|
null
|
|
);
|
|
// The path-scoped `/vscode/<token>/…` form (VS Code integration) still works.
|
|
assert.equal(
|
|
auth.extractApiKey(
|
|
new Request(`http://localhost/api/v1/vscode/${created.key}/chat/completions`)
|
|
),
|
|
created.key
|
|
);
|
|
// …but never when the caller opts out of URL extraction (management auth path).
|
|
assert.equal(
|
|
auth.extractApiKey(
|
|
new Request(`http://localhost/api/v1/vscode/${created.key}/chat/completions`),
|
|
{ allowUrl: false }
|
|
),
|
|
null
|
|
);
|
|
assert.equal(await auth.isValidApiKey(created.key), true);
|
|
assert.equal(await auth.isValidApiKey("sk-missing"), false);
|
|
assert.equal(await auth.isValidApiKey(""), false);
|
|
});
|
|
|
|
test("getProviderCredentials reports rate limiting when only inactive suppressed records remain", async () => {
|
|
const retryAfter = futureIso();
|
|
await seedConnection("openai", {
|
|
name: "inactive-rate-limited",
|
|
isActive: false,
|
|
rateLimitedUntil: retryAfter,
|
|
});
|
|
|
|
const result = await auth.getProviderCredentials("openai");
|
|
|
|
assert.equal(result.allRateLimited, true);
|
|
assert.equal(result.retryAfter, retryAfter);
|
|
assert.match(String(result.retryAfterHuman), /reset after/i);
|
|
});
|
|
|
|
test("codex session account affinity is opt-in and honors TTL", async () => {
|
|
const affinityDb = await import("../../src/lib/db/sessionAccountAffinity.ts");
|
|
await settingsDb.updateSettings({ fallbackStrategy: "least-used" });
|
|
await seedConnection("codex", { name: "codex-affinity-a", priority: 1 });
|
|
await seedConnection("codex", { name: "codex-affinity-b", priority: 2 });
|
|
|
|
const withoutAffinityA = await auth.getProviderCredentials("codex", null, null, "gpt-5", {
|
|
sessionKey: "session-without-affinity",
|
|
});
|
|
const withoutAffinityB = await auth.getProviderCredentials("codex", null, null, "gpt-5", {
|
|
sessionKey: "session-without-affinity",
|
|
});
|
|
|
|
assert.equal(typeof withoutAffinityA.connectionId, "string");
|
|
assert.equal(typeof withoutAffinityB.connectionId, "string");
|
|
assert.equal(
|
|
affinityDb.getSessionAccountAffinity("session-without-affinity", "codex", 60_000),
|
|
null
|
|
);
|
|
|
|
await settingsDb.updateSettings({ codexSessionAffinityTtlMs: 60_000 });
|
|
|
|
const withAffinityA = await auth.getProviderCredentials("codex", null, null, "gpt-5", {
|
|
sessionKey: "session-with-affinity",
|
|
});
|
|
const withAffinityB = await auth.getProviderCredentials("codex", null, null, "gpt-5", {
|
|
sessionKey: "session-with-affinity",
|
|
});
|
|
|
|
assert.equal(withAffinityB.connectionId, withAffinityA.connectionId);
|
|
assert.equal(
|
|
affinityDb.getSessionAccountAffinity("session-with-affinity", "codex", 60_000)?.connectionId,
|
|
withAffinityA.connectionId
|
|
);
|
|
});
|
|
|
|
test("session account affinity expires when TTL has passed", async () => {
|
|
const affinityDb = await import("../../src/lib/db/sessionAccountAffinity.ts");
|
|
const now = Date.now();
|
|
|
|
affinityDb.upsertSessionAccountAffinity("expiring-session", "codex", "conn-a", now, 1000);
|
|
|
|
assert.equal(
|
|
affinityDb.getSessionAccountAffinity("expiring-session", "codex", 1000, now + 500)
|
|
?.connectionId,
|
|
"conn-a"
|
|
);
|
|
assert.equal(
|
|
affinityDb.getSessionAccountAffinity("expiring-session", "codex", 1000, now + 1001),
|
|
null
|
|
);
|
|
});
|
|
|
|
test("getProviderCredentials returns last error metadata when active accounts are all rate limited", async () => {
|
|
const retryAfter = futureIso();
|
|
await seedConnection("openai", {
|
|
name: "active-rate-limited",
|
|
rateLimitedUntil: retryAfter,
|
|
lastError: "provider rate limit",
|
|
errorCode: 429,
|
|
});
|
|
|
|
const result = await auth.getProviderCredentials("openai");
|
|
|
|
assert.equal(result.allRateLimited, true);
|
|
assert.equal(result.retryAfter, retryAfter);
|
|
assert.equal(Number(result.lastErrorCode), 429);
|
|
assert.equal(result.lastError, "provider rate limit");
|
|
});
|
|
|
|
test("getProviderCredentials enforces generic quota policy unless explicitly bypassed", async () => {
|
|
const connection = await seedConnection("openai", {
|
|
name: "quota-policy",
|
|
providerSpecificData: {
|
|
limitPolicy: {
|
|
enabled: true,
|
|
thresholdPercent: 75,
|
|
windows: ["daily"],
|
|
},
|
|
},
|
|
});
|
|
const resetAt = futureIso();
|
|
quotaCache.setQuotaCache(connection.id, "openai", {
|
|
daily: { remainingPercentage: 10, resetAt },
|
|
});
|
|
|
|
const blocked = await auth.getProviderCredentials("openai");
|
|
const bypassed = await auth.getProviderCredentials("openai", null, null, null, {
|
|
bypassQuotaPolicy: true,
|
|
});
|
|
|
|
assert.equal(blocked.allRateLimited, true);
|
|
assert.equal(blocked.lastErrorCode, 429);
|
|
assert.match(blocked.lastError, /configured quota threshold/i);
|
|
assert.equal(blocked.retryAfter, resetAt);
|
|
assert.equal(bypassed.connectionId, connection.id);
|
|
});
|
|
|
|
test("getProviderCredentialsWithQuotaPreflight skips exhausted preflight accounts and selects the next healthy one", async () => {
|
|
const blocked = await seedConnection("openai", {
|
|
name: "quota-preflight-blocked",
|
|
apiKey: "sk-preflight-blocked",
|
|
providerSpecificData: {
|
|
quotaPreflightEnabled: true,
|
|
},
|
|
});
|
|
const healthy = await seedConnection("openai", {
|
|
name: "quota-preflight-healthy",
|
|
apiKey: "sk-preflight-healthy",
|
|
providerSpecificData: {
|
|
quotaPreflightEnabled: true,
|
|
},
|
|
});
|
|
|
|
const quotaPreflight = await import("../../open-sse/services/quotaPreflight.ts");
|
|
quotaPreflight.registerQuotaFetcher("openai", async (connectionId) => ({
|
|
used: connectionId === blocked.id ? 100 : 40,
|
|
total: 100,
|
|
percentUsed: connectionId === blocked.id ? 1.0 : 0.4,
|
|
}));
|
|
|
|
const selected = await auth.getProviderCredentialsWithQuotaPreflight("openai");
|
|
|
|
assert.equal((selected as any).connectionId, healthy.id);
|
|
});
|
|
|
|
test("getProviderCredentialsWithQuotaPreflight returns allRateLimited when a forced connection is blocked by preflight", async () => {
|
|
const blocked = await seedConnection("openai", {
|
|
name: "quota-preflight-forced",
|
|
apiKey: "sk-preflight-forced",
|
|
providerSpecificData: {
|
|
quotaPreflightEnabled: true,
|
|
},
|
|
});
|
|
|
|
const quotaPreflight = await import("../../open-sse/services/quotaPreflight.ts");
|
|
quotaPreflight.registerQuotaFetcher("openai", async (connectionId) => ({
|
|
used: connectionId === blocked.id ? 100 : 20,
|
|
total: 100,
|
|
percentUsed: connectionId === blocked.id ? 1.0 : 0.2,
|
|
resetAt: futureIso(120_000),
|
|
}));
|
|
|
|
const selected = await auth.getProviderCredentialsWithQuotaPreflight("openai", null, null, null, {
|
|
forcedConnectionId: (blocked as any).id,
|
|
});
|
|
|
|
assert.equal(selected.allRateLimited, true);
|
|
assert.equal(selected.lastErrorCode, 429);
|
|
assert.match(selected.lastError, /quota preflight/i);
|
|
});
|
|
|
|
test("getProviderCredentialsWithQuotaPreflight skips the upstream fetcher when no limits are configured", async () => {
|
|
// Latency gate regression test. When a connection has no per-window
|
|
// overrides AND its provider has no per-(provider, window) defaults seeded
|
|
// AND the legacy quotaPreflightEnabled flag isn't set, the dispatch loop
|
|
// must NOT call the registered quota fetcher. We assert by registering a
|
|
// fetcher that throws if invoked — any invocation surfaces as a test
|
|
// failure either via the thrown error or via the connection getting
|
|
// skipped (we expect it to pass through cleanly).
|
|
const conn = await seedConnection("openai", {
|
|
name: "quota-preflight-no-limits",
|
|
apiKey: "sk-no-limits",
|
|
// Crucially: no quotaPreflightEnabled flag, no overrides.
|
|
});
|
|
|
|
const quotaPreflight = await import("../../open-sse/services/quotaPreflight.ts");
|
|
let fetcherCalls = 0;
|
|
quotaPreflight.registerQuotaFetcher("openai", async () => {
|
|
fetcherCalls++;
|
|
throw new Error(
|
|
"quota fetcher must not run when no per-window overrides or provider defaults are set"
|
|
);
|
|
});
|
|
|
|
const selected = await auth.getProviderCredentialsWithQuotaPreflight("openai");
|
|
|
|
assert.equal((selected as any).connectionId, conn.id);
|
|
assert.equal(fetcherCalls, 0, "fetcher should not have been invoked");
|
|
});
|
|
|
|
test("getProviderCredentialsWithQuotaPreflight invokes the fetcher when the global default is restrictive", async () => {
|
|
// No per-connection override and no provider-window defaults — but the
|
|
// operator has raised the global default cutoff above the factory no-op
|
|
// level (2% remaining). Preflight must run so the tighter floor applies.
|
|
const conn = await seedConnection("openai", {
|
|
name: "quota-preflight-restrictive-global",
|
|
apiKey: "sk-restrictive-global",
|
|
});
|
|
await settingsDb.updateSettings({
|
|
resilienceSettings: {
|
|
quotaPreflight: {
|
|
defaultThresholdPercent: 20, // stop at 20% remaining = 80% used
|
|
warnThresholdPercent: 30,
|
|
providerWindowDefaults: {},
|
|
},
|
|
},
|
|
});
|
|
|
|
const quotaPreflight = await import("../../open-sse/services/quotaPreflight.ts");
|
|
let fetcherCalls = 0;
|
|
quotaPreflight.registerQuotaFetcher("openai", async () => {
|
|
fetcherCalls++;
|
|
return null;
|
|
});
|
|
|
|
await auth.getProviderCredentialsWithQuotaPreflight("openai");
|
|
assert.equal(
|
|
fetcherCalls,
|
|
1,
|
|
"fetcher should run when global default is stricter than the factory no-op level"
|
|
);
|
|
|
|
// Reset settings so subsequent tests see factory defaults.
|
|
await settingsDb.updateSettings({ resilienceSettings: {} });
|
|
// Verify the gate immediately returns to skip mode.
|
|
fetcherCalls = 0;
|
|
quotaPreflight.registerQuotaFetcher("openai", async () => {
|
|
fetcherCalls++;
|
|
throw new Error("must not run with factory global default");
|
|
});
|
|
await auth.getProviderCredentialsWithQuotaPreflight("openai");
|
|
assert.equal(fetcherCalls, 0, "fetcher should not run after settings reset to factory default");
|
|
});
|
|
|
|
test("getProviderCredentialsWithQuotaPreflight invokes the fetcher when an override IS set", async () => {
|
|
// Counterpart to the no-limits test: if the connection has a
|
|
// quotaWindowThresholds override, preflight must run.
|
|
const conn = await seedConnection("openai", {
|
|
name: "quota-preflight-with-override",
|
|
apiKey: "sk-with-override",
|
|
});
|
|
const updated = await providersDb.updateProviderConnection(conn.id, {
|
|
quotaWindowThresholds: { primary: 50 },
|
|
});
|
|
// Sanity: the override must be readable on the connection row (this is
|
|
// what the dispatch loop reads through getProviderCredentials).
|
|
assert.deepEqual(
|
|
(updated as any)?.quotaWindowThresholds,
|
|
{ primary: 50 },
|
|
"override must be persisted on the connection row"
|
|
);
|
|
const refetched = await providersDb.getProviderConnectionById(conn.id);
|
|
assert.deepEqual(
|
|
(refetched as any)?.quotaWindowThresholds,
|
|
{ primary: 50 },
|
|
"override must round-trip through getProviderConnectionById"
|
|
);
|
|
|
|
const quotaPreflight = await import("../../open-sse/services/quotaPreflight.ts");
|
|
let fetcherCalls = 0;
|
|
quotaPreflight.registerQuotaFetcher("openai", async () => {
|
|
fetcherCalls++;
|
|
return null; // null → preflight proceeds, no skip
|
|
});
|
|
|
|
await auth.getProviderCredentialsWithQuotaPreflight("openai");
|
|
assert.equal(fetcherCalls, 1, "fetcher should have been invoked exactly once");
|
|
});
|
|
|
|
test("getProviderCredentialsWithQuotaPreflight: explicit quotaPreflightEnabled:false bypasses preflight even when provider has global window defaults (#2831)", async () => {
|
|
// Regression: when a provider has providerWindowDefaults set AND a connection
|
|
// carries quotaWindowThresholds, the AND-of-negations gate in auth.ts would
|
|
// proceed to preflight even if the connection explicitly opted out with
|
|
// providerSpecificData.quotaPreflightEnabled === false.
|
|
const conn = await seedConnection("github", {
|
|
name: "github-explicit-opt-out",
|
|
apiKey: "ghp-opt-out-test",
|
|
providerSpecificData: { quotaPreflightEnabled: false },
|
|
});
|
|
// Give the connection per-window overrides (simulates a user-configured
|
|
// threshold) — this is the field that previously caused the gate to keep going.
|
|
await (
|
|
await import("../../src/lib/db/providers.ts")
|
|
).updateProviderConnection(conn.id, {
|
|
quotaWindowThresholds: { primary: 50 },
|
|
});
|
|
|
|
// Seed provider-level window defaults so providerHasDefaults === true.
|
|
await settingsDb.updateSettings({
|
|
resilienceSettings: {
|
|
quotaPreflight: {
|
|
defaultThresholdPercent: 2,
|
|
warnThresholdPercent: 20,
|
|
providerWindowDefaults: { github: { primary: 10 } },
|
|
},
|
|
},
|
|
});
|
|
|
|
const quotaPreflight = await import("../../open-sse/services/quotaPreflight.ts");
|
|
let fetcherCalls = 0;
|
|
quotaPreflight.registerQuotaFetcher("github", async () => {
|
|
fetcherCalls++;
|
|
// Return a quota that would block if preflight actually ran.
|
|
return { used: 98, total: 100, percentUsed: 0.98 };
|
|
});
|
|
|
|
const selected = await auth.getProviderCredentialsWithQuotaPreflight("github");
|
|
|
|
assert.equal(
|
|
fetcherCalls,
|
|
0,
|
|
"fetcher must NOT run when connection explicitly opts out with quotaPreflightEnabled: false"
|
|
);
|
|
assert.equal(
|
|
(selected as any).connectionId,
|
|
conn.id,
|
|
"opted-out connection must be returned directly without being blocked by preflight"
|
|
);
|
|
|
|
// Cleanup: reset settings so subsequent tests see factory defaults.
|
|
await settingsDb.updateSettings({ resilienceSettings: {} });
|
|
});
|
|
|
|
test("getProviderCredentials keeps separate codex affinity per session", async () => {
|
|
await settingsDb.updateSettings({
|
|
fallbackStrategy: "round-robin",
|
|
stickyRoundRobinLimit: 10,
|
|
codexSessionAffinityTtlMs: 60_000,
|
|
});
|
|
const first = await seedConnection("codex", {
|
|
name: "codex-affinity-a",
|
|
lastUsedAt: new Date(Date.now() - 20_000).toISOString(),
|
|
});
|
|
const second = await seedConnection("codex", {
|
|
name: "codex-affinity-b",
|
|
lastUsedAt: new Date(Date.now() - 10_000).toISOString(),
|
|
});
|
|
|
|
const sessionA1 = await auth.getProviderCredentials("codex", null, null, "gpt-5.5", {
|
|
sessionKey: "session-a",
|
|
});
|
|
const sessionB1 = await auth.getProviderCredentials("codex", null, null, "gpt-5.5", {
|
|
sessionKey: "session-b",
|
|
});
|
|
const sessionA2 = await auth.getProviderCredentials("codex", null, null, "gpt-5.5", {
|
|
sessionKey: "session-a",
|
|
});
|
|
const sessionB2 = await auth.getProviderCredentials("codex", null, null, "gpt-5.5", {
|
|
sessionKey: "session-b",
|
|
});
|
|
|
|
assert.equal(sessionA1.connectionId, first.id);
|
|
assert.equal(sessionB1.connectionId, second.id);
|
|
assert.equal(sessionA2.connectionId, first.id);
|
|
assert.equal(sessionB2.connectionId, second.id);
|
|
});
|
|
|
|
test("getProviderCredentials rebinds codex session when affinity connection is excluded", async () => {
|
|
await settingsDb.updateSettings({
|
|
fallbackStrategy: "round-robin",
|
|
stickyRoundRobinLimit: 10,
|
|
codexSessionAffinityTtlMs: 60_000,
|
|
});
|
|
const first = await seedConnection("codex", {
|
|
name: "codex-affinity-excluded-a",
|
|
lastUsedAt: new Date(Date.now() - 20_000).toISOString(),
|
|
});
|
|
const second = await seedConnection("codex", {
|
|
name: "codex-affinity-excluded-b",
|
|
lastUsedAt: new Date(Date.now() - 10_000).toISOString(),
|
|
});
|
|
|
|
const initial = await auth.getProviderCredentials("codex", null, null, "gpt-5.5", {
|
|
sessionKey: "session-excluded",
|
|
});
|
|
const rebound = await auth.getProviderCredentials("codex", first.id, null, "gpt-5.5", {
|
|
sessionKey: "session-excluded",
|
|
});
|
|
const sticky = await auth.getProviderCredentials("codex", null, null, "gpt-5.5", {
|
|
sessionKey: "session-excluded",
|
|
});
|
|
|
|
assert.equal(initial.connectionId, first.id);
|
|
assert.equal(rebound.connectionId, second.id);
|
|
assert.equal(sticky.connectionId, second.id);
|
|
});
|
|
|
|
test("resolveQuotaLimitPolicy normalizes Codex windows, thresholds, and defaults", () => {
|
|
const normalized = auth.resolveQuotaLimitPolicy("codex", {
|
|
limitPolicy: {
|
|
windows: [" Session (5h) ", "weekly (7d)", "custom-window", "", 42],
|
|
thresholdPercent: "250",
|
|
},
|
|
codexLimitPolicy: {
|
|
use5h: false,
|
|
useWeekly: true,
|
|
},
|
|
});
|
|
const defaults = auth.resolveQuotaLimitPolicy("codex", {});
|
|
const generic = auth.resolveQuotaLimitPolicy("openai", {
|
|
limitPolicy: {
|
|
enabled: "maybe",
|
|
thresholdPercent: "0",
|
|
windows: [" Daily ", "", null],
|
|
},
|
|
});
|
|
|
|
assert.deepEqual(normalized, {
|
|
enabled: true,
|
|
thresholdPercent: 100,
|
|
windows: ["weekly", "custom-window"],
|
|
});
|
|
assert.deepEqual(defaults, {
|
|
enabled: true,
|
|
thresholdPercent: 99,
|
|
windows: ["session", "weekly"],
|
|
});
|
|
assert.deepEqual(generic, {
|
|
enabled: false,
|
|
thresholdPercent: 1,
|
|
windows: ["daily"],
|
|
});
|
|
});
|
|
|
|
test("evaluateQuotaLimitPolicy aggregates reasons and keeps the earliest valid future reset", () => {
|
|
const connection = {
|
|
id: "quota-eval-connection",
|
|
providerSpecificData: {
|
|
limitPolicy: {
|
|
enabled: true,
|
|
thresholdPercent: 75,
|
|
windows: ["weekly", "session", "daily"],
|
|
},
|
|
},
|
|
};
|
|
const earliestReset = futureIso(90_000);
|
|
|
|
quotaCache.setQuotaCache(connection.id, "openai", {
|
|
weekly: { remainingPercentage: 20, resetAt: "not-a-date" },
|
|
session: { remainingPercentage: 5, resetAt: earliestReset },
|
|
daily: { remainingPercentage: 90, resetAt: futureIso(180_000) },
|
|
});
|
|
|
|
const evaluation = auth.evaluateQuotaLimitPolicy("openai", connection as any);
|
|
|
|
assert.equal(evaluation.blocked, true);
|
|
assert.deepEqual(evaluation.reasons, ["weekly usage 80%", "session usage 95%"]);
|
|
assert.equal(evaluation.resetAt, earliestReset);
|
|
});
|
|
|
|
test("getProviderCredentials round-robin stays on the current account while below the sticky limit", async () => {
|
|
await settingsDb.updateSettings({
|
|
fallbackStrategy: "round-robin",
|
|
stickyRoundRobinLimit: 3,
|
|
});
|
|
const current = await seedConnection("openai", {
|
|
name: "round-robin-current",
|
|
priority: 1,
|
|
});
|
|
const other = await seedConnection("openai", {
|
|
name: "round-robin-other",
|
|
priority: 2,
|
|
});
|
|
|
|
await providersDb.updateProviderConnection(current.id, {
|
|
lastUsedAt: new Date().toISOString(),
|
|
consecutiveUseCount: 1,
|
|
});
|
|
await providersDb.updateProviderConnection(other.id, {
|
|
lastUsedAt: new Date(Date.now() - 60_000).toISOString(),
|
|
consecutiveUseCount: 0,
|
|
});
|
|
|
|
const selected = await auth.getProviderCredentials("openai");
|
|
const updated = await providersDb.getProviderConnectionById(current.id);
|
|
|
|
assert.equal(selected.connectionId, current.id);
|
|
assert.equal(updated.consecutiveUseCount, 2);
|
|
});
|
|
|
|
test("getProviderCredentials returns null when only inactive non-rate-limited records remain", async () => {
|
|
await seedConnection("openai", {
|
|
name: "inactive-no-limit",
|
|
isActive: false,
|
|
testStatus: "active",
|
|
});
|
|
|
|
const result = await auth.getProviderCredentials("openai");
|
|
|
|
assert.equal(result, null);
|
|
});
|
|
|
|
test("getProviderCredentials honors allowedConnections filters", async () => {
|
|
const skipped = await seedConnection("openai", {
|
|
name: "allowed-skip",
|
|
apiKey: "sk-skip",
|
|
});
|
|
const selectedConn = await seedConnection("openai", {
|
|
name: "allowed-select",
|
|
apiKey: "sk-selected",
|
|
});
|
|
|
|
const selected = await auth.getProviderCredentials("openai", null, [(selectedConn as any).id]);
|
|
|
|
assert.equal(selected.connectionId, selectedConn.id);
|
|
assert.equal(selected.apiKey, "sk-selected");
|
|
assert.notEqual(selected.connectionId, skipped.id);
|
|
});
|
|
|
|
test("getProviderCredentials honors forcedConnectionId even when another account is preferred", async () => {
|
|
await seedConnection("openai", {
|
|
name: "forced-default",
|
|
priority: 1,
|
|
apiKey: "sk-default",
|
|
});
|
|
const forcedConn = await seedConnection("openai", {
|
|
name: "forced-target",
|
|
priority: 99,
|
|
apiKey: "sk-forced",
|
|
});
|
|
|
|
const selected = await auth.getProviderCredentials("openai", null, null, null, {
|
|
forcedConnectionId: (forcedConn as any).id,
|
|
});
|
|
|
|
assert.equal(selected.connectionId, forcedConn.id);
|
|
assert.equal(selected.apiKey, "sk-forced");
|
|
});
|
|
|
|
test("getProviderCredentials intersects forcedConnectionId with allowedConnections", async () => {
|
|
const allowedConn = await seedConnection("openai", {
|
|
name: "forced-allowed",
|
|
apiKey: "sk-allowed",
|
|
});
|
|
const blockedConn = await seedConnection("openai", {
|
|
name: "forced-blocked",
|
|
apiKey: "sk-blocked",
|
|
});
|
|
|
|
const selected = await auth.getProviderCredentials(
|
|
"openai",
|
|
null,
|
|
[(allowedConn as any).id],
|
|
null,
|
|
{
|
|
forcedConnectionId: (blockedConn as any).id,
|
|
}
|
|
);
|
|
|
|
assert.equal(selected, null);
|
|
});
|
|
|
|
test("getProviderCredentials retains rate-limited accounts when allowSuppressedConnections is enabled", async () => {
|
|
const connection = await seedConnection("openai", {
|
|
name: "suppressed-rate-limit",
|
|
rateLimitedUntil: futureIso(),
|
|
});
|
|
|
|
const blocked = await auth.getProviderCredentials("openai");
|
|
const bypassed = await auth.getProviderCredentials("openai", null, null, null, {
|
|
allowSuppressedConnections: true,
|
|
});
|
|
|
|
assert.equal(blocked.allRateLimited, true);
|
|
assert.equal(bypassed.connectionId, connection.id);
|
|
});
|
|
|
|
test("getProviderCredentials retains rate-limited accounts when allowRateLimitedConnections is enabled", async () => {
|
|
const connection = await seedConnection("openai", {
|
|
name: "allow-rate-limit-option",
|
|
rateLimitedUntil: futureIso(),
|
|
});
|
|
|
|
const blocked = await auth.getProviderCredentials("openai");
|
|
const bypassed = await auth.getProviderCredentials("openai", null, null, null, {
|
|
allowRateLimitedConnections: true,
|
|
});
|
|
|
|
assert.equal(blocked.allRateLimited, true);
|
|
assert.equal(bypassed.connectionId, connection.id);
|
|
});
|
|
|
|
test("getProviderCredentials retains terminal accounts for combo live tests", async () => {
|
|
const connection = await seedConnection("openai", {
|
|
name: "suppressed-terminal",
|
|
testStatus: "banned",
|
|
backoffLevel: 4,
|
|
});
|
|
|
|
const blocked = await auth.getProviderCredentials("openai");
|
|
const bypassed = await auth.getProviderCredentials("openai", null, null, null, {
|
|
allowSuppressedConnections: true,
|
|
});
|
|
const updated = await providersDb.getProviderConnectionById(connection.id);
|
|
|
|
assert.equal(blocked, null);
|
|
assert.equal(bypassed.connectionId, connection.id);
|
|
assert.equal(updated.testStatus, "banned");
|
|
});
|
|
|
|
test("getProviderCredentials skips codex scope-limited accounts unless suppression is allowed", async () => {
|
|
const retryAfter = futureIso();
|
|
const connection = await seedConnection("codex", {
|
|
authType: "oauth",
|
|
name: "codex-scope-limited",
|
|
email: "scope-limited@example.com",
|
|
apiKey: null,
|
|
accessToken: "scope-access",
|
|
refreshToken: "scope-refresh",
|
|
providerSpecificData: {
|
|
codexScopeRateLimitedUntil: {
|
|
spark: retryAfter,
|
|
},
|
|
},
|
|
});
|
|
|
|
const blocked = await auth.getProviderCredentials("codex", null, null, "codex-spark-mini");
|
|
const bypassed = await auth.getProviderCredentials("codex", null, null, "codex-spark-mini", {
|
|
allowSuppressedConnections: true,
|
|
});
|
|
|
|
assert.equal(blocked.allRateLimited, true);
|
|
assert.equal(blocked.retryAfter, retryAfter);
|
|
assert.equal(bypassed.connectionId, connection.id);
|
|
});
|
|
|
|
test("getProviderCredentials reports allRateLimited when every account is model-locked", async () => {
|
|
const first = await seedConnection("gemini", {
|
|
name: "gemini-model-lock-first",
|
|
});
|
|
const second = await seedConnection("gemini", {
|
|
name: "gemini-model-lock-second",
|
|
});
|
|
|
|
await auth.markAccountUnavailable(first.id, 429, "too many requests", "gemini", "gemini-2.5-pro");
|
|
await auth.markAccountUnavailable(
|
|
second.id,
|
|
429,
|
|
"too many requests",
|
|
"gemini",
|
|
"gemini-2.5-pro"
|
|
);
|
|
|
|
const blocked = await auth.getProviderCredentials("gemini", null, null, "gemini-2.5-pro");
|
|
|
|
assert.equal(blocked.allRateLimited, true);
|
|
assert.equal(Number(blocked.lastErrorCode), 429);
|
|
assert.ok(typeof blocked.retryAfter === "string" && blocked.retryAfter.length > 0);
|
|
assert.match(String(blocked.retryAfterHuman), /reset after/i);
|
|
});
|
|
|
|
test("getProviderCredentials auto-decays stale backoff metadata for recovered accounts", async () => {
|
|
const connection = await seedConnection("openai", {
|
|
name: "stale-backoff",
|
|
testStatus: "unavailable",
|
|
rateLimitedUntil: new Date(Date.now() - 60_000).toISOString(),
|
|
lastError: "old error",
|
|
errorCode: 429,
|
|
backoffLevel: 3,
|
|
});
|
|
|
|
const selected = await auth.getProviderCredentials("openai");
|
|
await flushWrites();
|
|
const updated = await providersDb.getProviderConnectionById(connection.id);
|
|
|
|
assert.equal(selected.connectionId, connection.id);
|
|
assert.equal(updated.backoffLevel, 0);
|
|
});
|
|
|
|
test("getProviderCredentials falls back to a five-minute retry window when quota policy has no reset", async () => {
|
|
const connection = await seedConnection("openai", {
|
|
name: "quota-no-reset",
|
|
providerSpecificData: {
|
|
limitPolicy: {
|
|
enabled: true,
|
|
thresholdPercent: 50,
|
|
windows: ["daily"],
|
|
},
|
|
},
|
|
});
|
|
|
|
quotaCache.setQuotaCache(connection.id, "openai", {
|
|
daily: { remainingPercentage: 0, resetAt: null },
|
|
});
|
|
|
|
const result = await auth.getProviderCredentials("openai");
|
|
|
|
assert.equal(result.allRateLimited, true);
|
|
assert.equal(result.lastErrorCode, 429);
|
|
assert.match(result.lastError, /configured quota threshold/i);
|
|
assert.ok(msUntil(result.retryAfter) > 240_000);
|
|
assert.ok(msUntil(result.retryAfter) <= 305_000);
|
|
});
|
|
|
|
test("getProviderCredentials prioritizes accounts that still have quota available", async () => {
|
|
const exhausted = await seedConnection("openai", {
|
|
name: "quota-exhausted",
|
|
priority: 1,
|
|
apiKey: "sk-exhausted",
|
|
});
|
|
const available = await seedConnection("openai", {
|
|
name: "quota-available",
|
|
priority: 9,
|
|
apiKey: "sk-available",
|
|
});
|
|
|
|
quotaCache.setQuotaCache(exhausted.id, "openai", {
|
|
daily: { remainingPercentage: 0, resetAt: futureIso() },
|
|
});
|
|
quotaCache.setQuotaCache(available.id, "openai", {
|
|
daily: { remainingPercentage: 65, resetAt: futureIso() },
|
|
});
|
|
|
|
const selected = await auth.getProviderCredentials("openai");
|
|
|
|
assert.equal(selected.connectionId, available.id);
|
|
assert.equal(selected.apiKey, "sk-available");
|
|
});
|
|
|
|
test("getProviderCredentials round-robin switches to the least recently used account after the sticky limit", async () => {
|
|
await settingsDb.updateSettings({
|
|
fallbackStrategy: "round-robin",
|
|
stickyRoundRobinLimit: 2,
|
|
});
|
|
const current = await seedConnection("openai", {
|
|
name: "round-robin-limit-current",
|
|
priority: 1,
|
|
});
|
|
const fallback = await seedConnection("openai", {
|
|
name: "round-robin-limit-fallback",
|
|
priority: 2,
|
|
});
|
|
|
|
await providersDb.updateProviderConnection(current.id, {
|
|
lastUsedAt: new Date().toISOString(),
|
|
consecutiveUseCount: 2,
|
|
});
|
|
await providersDb.updateProviderConnection(fallback.id, {
|
|
lastUsedAt: new Date(Date.now() - 120_000).toISOString(),
|
|
consecutiveUseCount: 0,
|
|
});
|
|
|
|
const selected = await auth.getProviderCredentials("openai");
|
|
const updated = await providersDb.getProviderConnectionById(fallback.id);
|
|
|
|
assert.equal(selected.connectionId, fallback.id);
|
|
assert.equal(updated.consecutiveUseCount, 1);
|
|
});
|
|
|
|
test("getProviderCredentials round-robin fallback mode excludes the failed account and picks the LRU peer", async () => {
|
|
await settingsDb.updateSettings({
|
|
fallbackStrategy: "round-robin",
|
|
stickyRoundRobinLimit: 2,
|
|
});
|
|
const failed = await seedConnection("openai", {
|
|
name: "round-robin-failed",
|
|
priority: 1,
|
|
});
|
|
const fallback = await seedConnection("openai", {
|
|
name: "round-robin-fallback",
|
|
priority: 2,
|
|
});
|
|
|
|
await providersDb.updateProviderConnection(failed.id, {
|
|
lastUsedAt: new Date().toISOString(),
|
|
consecutiveUseCount: 3,
|
|
});
|
|
await providersDb.updateProviderConnection(fallback.id, {
|
|
lastUsedAt: new Date(Date.now() - 120_000).toISOString(),
|
|
consecutiveUseCount: 0,
|
|
});
|
|
|
|
const selected = await auth.getProviderCredentials("openai", failed.id);
|
|
const updated = await providersDb.getProviderConnectionById(fallback.id);
|
|
|
|
assert.equal(selected.connectionId, fallback.id);
|
|
assert.equal(updated.consecutiveUseCount, 1);
|
|
});
|
|
|
|
for (const strategy of ["random", "p2c", "least-used", "cost-optimized", "strict-random"]) {
|
|
test(`getProviderCredentials supports the ${strategy} selection strategy`, async () => {
|
|
await settingsDb.updateSettings({ fallbackStrategy: strategy });
|
|
const connection = await seedConnection("openai", {
|
|
name: `strategy-${strategy}`,
|
|
priority: 7,
|
|
});
|
|
|
|
const selected = await auth.getProviderCredentials("openai");
|
|
|
|
assert.equal(selected.connectionId, connection.id);
|
|
});
|
|
}
|
|
|
|
test("getProviderCredentials least-used prefers accounts that were never used", async () => {
|
|
await settingsDb.updateSettings({ fallbackStrategy: "least-used" });
|
|
const recentlyUsed = await seedConnection("openai", {
|
|
name: "least-used-recent",
|
|
priority: 1,
|
|
});
|
|
const neverUsed = await seedConnection("openai", {
|
|
name: "least-used-never",
|
|
priority: 9,
|
|
});
|
|
await providersDb.updateProviderConnection(recentlyUsed.id, {
|
|
lastUsedAt: new Date().toISOString(),
|
|
});
|
|
await providersDb.updateProviderConnection(neverUsed.id, {
|
|
lastUsedAt: null,
|
|
});
|
|
|
|
const selected = await auth.getProviderCredentials("openai");
|
|
|
|
assert.equal(selected.connectionId, neverUsed.id);
|
|
assert.notEqual(selected.connectionId, recentlyUsed.id);
|
|
});
|
|
|
|
test("getProviderCredentials least-used prefers the oldest timestamp when all accounts were used", async () => {
|
|
await settingsDb.updateSettings({ fallbackStrategy: "least-used" });
|
|
const oldest = await seedConnection("openai", {
|
|
name: "least-used-oldest",
|
|
priority: 9,
|
|
});
|
|
const newest = await seedConnection("openai", {
|
|
name: "least-used-newest",
|
|
priority: 1,
|
|
});
|
|
|
|
await providersDb.updateProviderConnection(oldest.id, {
|
|
lastUsedAt: new Date(Date.now() - 120_000).toISOString(),
|
|
});
|
|
await providersDb.updateProviderConnection(newest.id, {
|
|
lastUsedAt: new Date().toISOString(),
|
|
});
|
|
|
|
const selected = await auth.getProviderCredentials("openai");
|
|
|
|
assert.equal(selected.connectionId, oldest.id);
|
|
});
|
|
|
|
test("getProviderCredentials cost-optimized selects the lowest priority account", async () => {
|
|
await settingsDb.updateSettings({ fallbackStrategy: "cost-optimized" });
|
|
const cheapest = await seedConnection("openai", {
|
|
name: "cost-low",
|
|
priority: 1,
|
|
});
|
|
await seedConnection("openai", {
|
|
name: "cost-high",
|
|
priority: 8,
|
|
});
|
|
|
|
const selected = await auth.getProviderCredentials("openai");
|
|
|
|
assert.equal(selected.connectionId, cheapest.id);
|
|
});
|
|
|
|
test("getProviderCredentials p2c prefers the account with more quota headroom over raw priority", async () => {
|
|
await settingsDb.updateSettings({ fallbackStrategy: "p2c" });
|
|
const nearLimit = await seedConnection("openai", {
|
|
name: "p2c-near-limit",
|
|
priority: 1,
|
|
apiKey: "sk-p2c-near-limit",
|
|
providerSpecificData: {
|
|
limitPolicy: {
|
|
enabled: true,
|
|
thresholdPercent: 80,
|
|
windows: ["daily"],
|
|
},
|
|
},
|
|
});
|
|
const healthy = await seedConnection("openai", {
|
|
name: "p2c-healthy-headroom",
|
|
priority: 9,
|
|
apiKey: "sk-p2c-healthy",
|
|
providerSpecificData: {
|
|
limitPolicy: {
|
|
enabled: true,
|
|
thresholdPercent: 80,
|
|
windows: ["daily"],
|
|
},
|
|
},
|
|
});
|
|
|
|
quotaCache.setQuotaCache(nearLimit.id, "openai", {
|
|
daily: { remainingPercentage: 12, resetAt: futureIso(180_000) },
|
|
});
|
|
quotaCache.setQuotaCache(healthy.id, "openai", {
|
|
daily: { remainingPercentage: 78, resetAt: futureIso(180_000) },
|
|
});
|
|
|
|
const selected = await auth.getProviderCredentials("openai");
|
|
|
|
assert.equal(selected.connectionId, healthy.id);
|
|
});
|
|
|
|
test("getProviderCredentials p2c deprioritizes accounts with recent rate-limit/backoff signals", async () => {
|
|
await settingsDb.updateSettings({ fallbackStrategy: "p2c" });
|
|
await seedConnection("openai", {
|
|
name: "p2c-rate-limited-history",
|
|
priority: 1,
|
|
apiKey: "sk-p2c-rate-limited",
|
|
lastError: "rate limit",
|
|
lastErrorType: "rate_limited",
|
|
errorCode: 429,
|
|
backoffLevel: 3,
|
|
});
|
|
const healthy = await seedConnection("openai", {
|
|
name: "p2c-clean-account",
|
|
priority: 8,
|
|
apiKey: "sk-p2c-clean",
|
|
});
|
|
|
|
const selected = await auth.getProviderCredentials("openai");
|
|
|
|
assert.equal(selected.connectionId, healthy.id);
|
|
});
|
|
|
|
test("getProviderCredentials resolves the nvidia special alias pool", async () => {
|
|
const connection = await seedConnection("nvidia_nim", {
|
|
name: "nvidia-special-alias",
|
|
});
|
|
|
|
const selected = await auth.getProviderCredentials("nvidia");
|
|
|
|
assert.equal(selected.connectionId, connection.id);
|
|
});
|
|
|
|
test("getProviderCredentials exposes copilotToken when present in providerSpecificData", async () => {
|
|
const connection = await seedConnection("codex", {
|
|
authType: "oauth",
|
|
name: "codex-copilot-token",
|
|
email: "copilot@example.com",
|
|
apiKey: null,
|
|
accessToken: "codex-access",
|
|
refreshToken: "codex-refresh",
|
|
providerSpecificData: {
|
|
copilotToken: "copilot-token-value",
|
|
},
|
|
});
|
|
|
|
const selected = await auth.getProviderCredentials("codex");
|
|
|
|
assert.equal(selected.connectionId, connection.id);
|
|
assert.equal(selected.copilotToken, "copilot-token-value");
|
|
});
|
|
|
|
test("markAccountUnavailable uses configured cooldowns for local 404 model lockouts", async () => {
|
|
await settingsDb.updateSettings({
|
|
providerProfiles: {
|
|
apikey: {
|
|
transientCooldown: 250,
|
|
rateLimitCooldown: 125,
|
|
maxBackoffLevel: 3,
|
|
circuitBreakerThreshold: 60,
|
|
circuitBreakerReset: 5000,
|
|
},
|
|
},
|
|
modelLockout: {
|
|
enabled: true,
|
|
baseCooldownMs: 250,
|
|
maxCooldownMs: 1000,
|
|
maxBackoffSteps: 3,
|
|
useExponentialBackoff: true,
|
|
errorCodes: [404],
|
|
},
|
|
});
|
|
|
|
const connection = await seedConnection("openai", {
|
|
name: "local-openai",
|
|
providerSpecificData: {
|
|
baseUrl: "http://127.0.0.1:8080/v1",
|
|
},
|
|
});
|
|
|
|
const result = await auth.markAccountUnavailable(
|
|
connection.id,
|
|
404,
|
|
"model not found",
|
|
"openai",
|
|
"local-model"
|
|
);
|
|
const updated = await providersDb.getProviderConnectionById(connection.id);
|
|
|
|
assert.equal(result.shouldFallback, true);
|
|
assert.equal(result.cooldownMs, 250);
|
|
assert.equal(updated.testStatus, "active");
|
|
assert.equal(updated.rateLimitedUntil, undefined);
|
|
assert.equal(updated.lastErrorType, "not_found");
|
|
assert.equal(Number(updated.errorCode), 404);
|
|
|
|
await settingsDb.updateSettings({ modelLockout: null });
|
|
});
|
|
|
|
test("markAccountUnavailable applies a model-only lockout for Gemini 429 responses", async () => {
|
|
const connection = await seedConnection("gemini", {
|
|
name: "gemini-model-limit",
|
|
});
|
|
|
|
const result = await auth.markAccountUnavailable(
|
|
connection.id,
|
|
429,
|
|
"too many requests",
|
|
"gemini",
|
|
"gemini-2.5-pro"
|
|
);
|
|
await flushWrites();
|
|
const updated = await providersDb.getProviderConnectionById(connection.id);
|
|
|
|
assert.equal(result.shouldFallback, true);
|
|
assert.ok(result.cooldownMs > 0);
|
|
assert.equal(updated.testStatus, "active");
|
|
assert.equal(updated.rateLimitedUntil, undefined);
|
|
assert.equal(updated.lastErrorType, "rate_limited");
|
|
assert.equal(Number(updated.errorCode), 429);
|
|
});
|
|
|
|
test("markAccountUnavailable applies a model-only lockout for compatible provider 429 responses", async () => {
|
|
const connection = await seedConnection("openai-compatible-custom-node", {
|
|
name: "compatible-model-limit",
|
|
});
|
|
|
|
const result = await auth.markAccountUnavailable(
|
|
connection.id,
|
|
429,
|
|
"The upstream compatible service exhausted its capacity",
|
|
"openai-compatible-custom-node",
|
|
"custom-model-a"
|
|
);
|
|
await flushWrites();
|
|
const updated = await providersDb.getProviderConnectionById(connection.id);
|
|
|
|
assert.equal(result.shouldFallback, true);
|
|
assert.ok(result.cooldownMs > 0);
|
|
assert.equal(updated.testStatus, "active");
|
|
assert.equal(updated.rateLimitedUntil, undefined);
|
|
assert.equal(updated.lastErrorType, "rate_limited");
|
|
assert.equal(Number(updated.errorCode), 429);
|
|
});
|
|
|
|
// #3027 — a per-model subscription/permission 403 from a passthrough provider
|
|
// (ollama-cloud) must lock only the paid model, not the whole connection.
|
|
test("markAccountUnavailable: ollama-cloud per-model subscription 403 locks the model, not the connection (#3027)", async () => {
|
|
fallback.clearAllModelLockouts();
|
|
const connection = await seedConnection("ollama-cloud", {
|
|
name: "ollama-paid-model",
|
|
providerSpecificData: { passthroughModels: true },
|
|
});
|
|
|
|
const result = await auth.markAccountUnavailable(
|
|
connection.id,
|
|
403,
|
|
"this model requires a subscription, upgrade for access: https://ollama.com/upgrade",
|
|
"ollama-cloud",
|
|
"deepseek-v4-pro"
|
|
);
|
|
await flushWrites();
|
|
const updated = await providersDb.getProviderConnectionById(connection.id);
|
|
|
|
// Connection stays eligible — only the paid model is cooled down.
|
|
assert.equal(result.shouldFallback, true);
|
|
assert.ok(result.cooldownMs > 0);
|
|
assert.equal(updated.testStatus, "active");
|
|
assert.equal(updated.rateLimitedUntil, undefined);
|
|
assert.equal(updated.lastErrorType, "forbidden");
|
|
assert.equal(Number(updated.errorCode), 403);
|
|
|
|
assert.equal(fallback.isModelLocked("ollama-cloud", connection.id, "deepseek-v4-pro"), true);
|
|
// A free model on the same key is still usable.
|
|
assert.equal(fallback.isModelLocked("ollama-cloud", connection.id, "gemma4:31b"), false);
|
|
});
|
|
|
|
// #3027 regression — a genuine whole-key 403 (deactivated/banned key) must NOT
|
|
// be downgraded to a model lockout; the connection still becomes terminal.
|
|
test("markAccountUnavailable: a whole-key 403 still deactivates the ollama-cloud connection (#3027 regression)", async () => {
|
|
fallback.clearAllModelLockouts();
|
|
const connection = await seedConnection("ollama-cloud", {
|
|
name: "ollama-banned-key",
|
|
providerSpecificData: { passthroughModels: true },
|
|
});
|
|
|
|
await auth.markAccountUnavailable(
|
|
connection.id,
|
|
403,
|
|
"account has been deactivated",
|
|
"ollama-cloud",
|
|
"deepseek-v4-pro"
|
|
);
|
|
await flushWrites();
|
|
const updated = await providersDb.getProviderConnectionById(connection.id);
|
|
|
|
assert.equal(fallback.isModelLocked("ollama-cloud", connection.id, "deepseek-v4-pro"), false);
|
|
assert.ok(
|
|
["banned", "expired", "credits_exhausted"].includes(updated.testStatus),
|
|
`expected a terminal connection status, got ${updated.testStatus}`
|
|
);
|
|
});
|
|
|
|
// #3027 — repeated subscription 403s on the paid model must never escalate a
|
|
// connection-wide cooldown/backoff (only the model lockout escalates).
|
|
test("markAccountUnavailable: repeated ollama-cloud subscription 403s never escalate connection backoff (#3027)", async () => {
|
|
fallback.clearAllModelLockouts();
|
|
const connection = await seedConnection("ollama-cloud", {
|
|
name: "ollama-repeat-403",
|
|
providerSpecificData: { passthroughModels: true },
|
|
});
|
|
|
|
for (let i = 0; i < 3; i++) {
|
|
await auth.markAccountUnavailable(
|
|
connection.id,
|
|
403,
|
|
"this model requires a subscription, upgrade for access",
|
|
"ollama-cloud",
|
|
"deepseek-v4-pro"
|
|
);
|
|
await flushWrites();
|
|
}
|
|
const updated = await providersDb.getProviderConnectionById(connection.id);
|
|
|
|
assert.equal(updated.rateLimitedUntil, undefined);
|
|
assert.notEqual(updated.testStatus, "unavailable");
|
|
assert.ok(!updated.backoffLevel, "connection backoffLevel must not escalate");
|
|
assert.equal(fallback.isModelLocked("ollama-cloud", connection.id, "deepseek-v4-pro"), true);
|
|
});
|
|
|
|
test("markAccountUnavailable honors configured api-key rate-limit cooldowns", async () => {
|
|
await settingsDb.updateSettings({
|
|
providerProfiles: {
|
|
apikey: {
|
|
transientCooldown: 125,
|
|
rateLimitCooldown: 125,
|
|
maxBackoffLevel: 3,
|
|
circuitBreakerThreshold: 60,
|
|
circuitBreakerReset: 5000,
|
|
},
|
|
},
|
|
});
|
|
|
|
const connection = await seedConnection("openai", {
|
|
name: "configured-rate-limit-cooldown",
|
|
});
|
|
|
|
const result = await auth.markAccountUnavailable(
|
|
connection.id,
|
|
429,
|
|
"too many requests",
|
|
"openai",
|
|
"gpt-4o-mini"
|
|
);
|
|
|
|
assert.equal(result.shouldFallback, true);
|
|
assert.equal(result.cooldownMs, 125);
|
|
});
|
|
|
|
test("markAccountUnavailable stores Codex scope-specific cooldowns without a global rate limit", async () => {
|
|
const connection = await seedConnection("codex", {
|
|
authType: "oauth",
|
|
name: "codex-scope",
|
|
email: "codex@example.com",
|
|
apiKey: null,
|
|
accessToken: "codex-access",
|
|
refreshToken: "codex-refresh",
|
|
});
|
|
|
|
const result = await auth.markAccountUnavailable(
|
|
connection.id,
|
|
429,
|
|
"quota reached",
|
|
"codex",
|
|
"codex-spark-mini"
|
|
);
|
|
const updated = await providersDb.getProviderConnectionById(connection.id);
|
|
const selected = await auth.getProviderCredentials("codex", null, null, "codex-spark-mini");
|
|
|
|
assert.equal(result.shouldFallback, true);
|
|
assert.ok(result.cooldownMs > 0);
|
|
assert.equal(updated.testStatus, "unavailable");
|
|
assert.equal(updated.rateLimitedUntil, undefined);
|
|
assert.ok(updated.providerSpecificData.codexScopeRateLimitedUntil.spark);
|
|
assert.equal(selected.allRateLimited, true);
|
|
});
|
|
|
|
test("markAccountUnavailable returns without fallback on bad requests", async () => {
|
|
const connection = await seedConnection("openai", {
|
|
name: "bad-request-no-fallback",
|
|
});
|
|
|
|
const result = await auth.markAccountUnavailable(
|
|
connection.id,
|
|
400,
|
|
"schema mismatch",
|
|
"openai",
|
|
"gpt-4o"
|
|
);
|
|
const updated = await providersDb.getProviderConnectionById(connection.id);
|
|
|
|
assert.deepEqual(result, { shouldFallback: false, cooldownMs: 0 });
|
|
assert.equal(updated.testStatus, "active");
|
|
assert.equal(updated.rateLimitedUntil, undefined);
|
|
});
|
|
|
|
test("markAccountUnavailable preserves terminal statuses without overwriting them", async () => {
|
|
const connection = await seedConnection("openai", {
|
|
name: "terminal-status",
|
|
testStatus: "expired",
|
|
rateLimitedUntil: null,
|
|
});
|
|
|
|
const result = await auth.markAccountUnavailable(connection.id, 503, "upstream error", "openai");
|
|
const updated = await providersDb.getProviderConnectionById(connection.id);
|
|
|
|
assert.equal(result.shouldFallback, true);
|
|
assert.equal(result.cooldownMs, 0);
|
|
assert.equal(updated.testStatus, "expired");
|
|
assert.equal(updated.rateLimitedUntil, undefined);
|
|
});
|
|
|
|
test("markAccountUnavailable reuses an existing connection-wide cooldown", async () => {
|
|
const retryAfter = futureIso(90_000);
|
|
const connection = await seedConnection("openai", {
|
|
name: "existing-cooldown",
|
|
rateLimitedUntil: retryAfter,
|
|
});
|
|
|
|
const result = await auth.markAccountUnavailable(connection.id, 503, "upstream error", "openai");
|
|
const updated = await providersDb.getProviderConnectionById(connection.id);
|
|
|
|
assert.equal(result.shouldFallback, true);
|
|
assert.ok(result.cooldownMs > 0);
|
|
assert.equal(updated.rateLimitedUntil, retryAfter);
|
|
});
|
|
|
|
test("markAccountUnavailable reuses an existing Codex scope cooldown", async () => {
|
|
const retryAfter = futureIso(90_000);
|
|
const connection = await seedConnection("codex", {
|
|
authType: "oauth",
|
|
name: "codex-existing-scope",
|
|
email: "codex-existing-scope@example.com",
|
|
apiKey: null,
|
|
accessToken: "scope-access",
|
|
refreshToken: "scope-refresh",
|
|
providerSpecificData: {
|
|
codexScopeRateLimitedUntil: {
|
|
spark: retryAfter,
|
|
},
|
|
},
|
|
});
|
|
|
|
const result = await auth.markAccountUnavailable(
|
|
connection.id,
|
|
429,
|
|
"quota reached",
|
|
"codex",
|
|
"codex-spark-mini"
|
|
);
|
|
const updated = await providersDb.getProviderConnectionById(connection.id);
|
|
|
|
assert.equal(result.shouldFallback, true);
|
|
assert.ok(result.cooldownMs > 0);
|
|
assert.equal(updated.rateLimitedUntil, undefined);
|
|
assert.equal(updated.providerSpecificData.codexScopeRateLimitedUntil.spark, retryAfter);
|
|
});
|
|
|
|
test("markAccountUnavailable uses a connection-wide cooldown for non-local 404 errors", async () => {
|
|
const connection = await seedConnection("openai", {
|
|
name: "remote-404",
|
|
providerSpecificData: {
|
|
baseUrl: "https://api.openai.com/v1",
|
|
},
|
|
});
|
|
|
|
const result = await auth.markAccountUnavailable(
|
|
connection.id,
|
|
404,
|
|
"model not found",
|
|
"openai",
|
|
"gpt-missing"
|
|
);
|
|
const updated = await providersDb.getProviderConnectionById(connection.id);
|
|
|
|
assert.equal(result.shouldFallback, true);
|
|
assert.ok(result.cooldownMs > 0);
|
|
assert.equal(updated.testStatus, "unavailable");
|
|
assert.ok(updated.rateLimitedUntil);
|
|
});
|
|
|
|
test("markAccountUnavailable auto-disables permanently banned accounts when the setting is enabled", async () => {
|
|
await settingsDb.updateSettings({ autoDisableBannedAccounts: true });
|
|
const connection = await seedConnection("openai", {
|
|
name: "permanent-ban",
|
|
});
|
|
|
|
const result = await auth.markAccountUnavailable(
|
|
connection.id,
|
|
401,
|
|
"Verify your account to continue",
|
|
"openai",
|
|
"gpt-4o"
|
|
);
|
|
const updated = await providersDb.getProviderConnectionById(connection.id);
|
|
|
|
assert.equal(result.shouldFallback, true);
|
|
assert.equal(updated.isActive, false);
|
|
assert.equal(updated.testStatus, "banned");
|
|
});
|
|
|
|
test("markAccountUnavailable leaves permanently banned accounts active when auto-disable is disabled", async () => {
|
|
await settingsDb.updateSettings({ autoDisableBannedAccounts: false });
|
|
const connection = await seedConnection("openai", {
|
|
name: "permanent-ban-disabled",
|
|
});
|
|
|
|
const result = await auth.markAccountUnavailable(
|
|
connection.id,
|
|
401,
|
|
"Verify your account to continue",
|
|
"openai",
|
|
"gpt-4o"
|
|
);
|
|
const updated = await providersDb.getProviderConnectionById(connection.id);
|
|
|
|
assert.equal(result.shouldFallback, true);
|
|
assert.equal(updated.isActive, true);
|
|
assert.equal(updated.testStatus, "banned");
|
|
});
|
|
|
|
test("markAccountUnavailable swallows auto-disable persistence errors", async () => {
|
|
await settingsDb.updateSettings({ autoDisableBannedAccounts: true });
|
|
const connection = await seedConnection("openai", {
|
|
name: "permanent-ban-update-fails",
|
|
});
|
|
|
|
const db = core.getDbInstance();
|
|
const originalPrepare = db.prepare.bind(db);
|
|
db.prepare = (sql) => {
|
|
const statement = originalPrepare(sql);
|
|
if (!String(sql).includes("UPDATE provider_connections SET")) {
|
|
return statement;
|
|
}
|
|
|
|
return new Proxy(statement, {
|
|
get(target, prop, receiver) {
|
|
if (prop === "run") {
|
|
return (params) => {
|
|
if (params && typeof params === "object" && params.isActive === 0) {
|
|
throw new Error("persist disable failed");
|
|
}
|
|
return target.run(params);
|
|
};
|
|
}
|
|
return Reflect.get(target, prop, receiver);
|
|
},
|
|
});
|
|
};
|
|
|
|
try {
|
|
const result = await auth.markAccountUnavailable(
|
|
connection.id,
|
|
401,
|
|
"Verify your account to continue",
|
|
"openai",
|
|
"gpt-4o"
|
|
);
|
|
const updated = await providersDb.getProviderConnectionById(connection.id);
|
|
|
|
assert.equal(result.shouldFallback, true);
|
|
assert.equal(updated.isActive, true);
|
|
assert.equal(updated.testStatus, "banned");
|
|
} finally {
|
|
db.prepare = originalPrepare;
|
|
}
|
|
});
|
|
|
|
test("markAccountUnavailable persists in-memory model lockout for combo transient 429 when persistUnavailableState=false", async () => {
|
|
const connection = await seedConnection("openai", {
|
|
name: "combo-transient-test",
|
|
});
|
|
const model = "gpt-4o";
|
|
const connId = connection.id as string;
|
|
|
|
assert.equal(fallback.isModelLocked("openai", connId, model), false);
|
|
|
|
await auth.markAccountUnavailable(
|
|
connId,
|
|
429,
|
|
"Rate limit exceeded",
|
|
"openai",
|
|
model,
|
|
null,
|
|
{ persistUnavailableState: false }
|
|
);
|
|
|
|
assert.equal(fallback.isModelLocked("openai", connId, model), true);
|
|
|
|
assert.equal(fallback.isModelLocked("openai", connId, "gpt-4o-mini"), false);
|
|
|
|
const otherConn = await seedConnection("openai", {
|
|
name: "other-conn",
|
|
});
|
|
assert.equal(fallback.isModelLocked("openai", (otherConn.id as string), model), false);
|
|
|
|
const updated = await providersDb.getProviderConnectionById(connId);
|
|
assert.equal(updated.rateLimitedUntil == null, true);
|
|
assert.notEqual(updated.testStatus, "unavailable");
|
|
});
|