fix(resilience): route remaining credential-selection call sites through quota preflight (#6686) (#6742)

* fix(resilience): route remaining credential-selection call sites through quota preflight (#6686)

* chore(6742): re-sync onto release tip; CHANGELOG entry → changelog.d fragment (fragments-first)
This commit is contained in:
Diego Rodrigues de Sa e Souza
2026-07-10 19:45:06 -03:00
committed by GitHub
parent 2ef88763e4
commit 249462d1ff
16 changed files with 218 additions and 34 deletions

View File

@@ -0,0 +1 @@
- **fix(resilience):** account selection could pick an account already out of quota upstream on every credentialed route except `chat`/`codex` ([#6686](https://github.com/diegosouzapw/OmniRoute/issues/6686)) — `getProviderCredentials()` (`src/sse/services/auth.ts`) only skips a connection when a *local cache* already flags it exhausted (`isQuotaExhaustedForRequest`/`src/domain/quotaCache.ts`); it never itself calls the registered upstream `QuotaFetcher`. Only `getProviderCredentialsWithQuotaPreflight()` performs that live upstream check, and it was wired into exactly 2 call sites (`src/sse/handlers/chat.ts`, `src/app/api/internal/codex-responses-ws/route.ts`) — every other credentialed route (`rerank`, `images/generations`, `images/edits`, `audio/transcriptions|speech|translations`, `videos/generations`, `music/generations`, `ocr`, `providers/[provider]/embeddings`, `providers/[provider]/images/generations`, `web/fetch`, `moderations`, `search`) called the plain, cache-only selector, so an account whose cache entry was never populated (e.g. its first request landed on one of these routes) could be selected even at 0% quota remaining. Those 14 call sites now go through `getProviderCredentialsWithQuotaPreflight()` instead, matching chat/codex coverage. Regression guard: `tests/unit/issue-6686-quota-preflight-coverage.test.ts` (static check that none of the routes call the plain selector anymore + a behavioral check that the preflight-aware selector blocks a 100%-used account).

View File

@@ -1,6 +1,9 @@
import { handleAudioSpeech } from "@omniroute/open-sse/handlers/audioSpeech.ts";
import { withInjectionGuard } from "@/middleware/promptInjectionGuard";
import { getProviderCredentials, clearRecoveredProviderState } from "@/sse/services/auth";
import {
getProviderCredentialsWithQuotaPreflight,
clearRecoveredProviderState,
} from "@/sse/services/auth";
import {
parseSpeechModel,
getSpeechProvider,
@@ -95,7 +98,7 @@ async function postHandler(request, context) {
// Get credentials — skip for local providers (authType: "none")
let credentials = null;
if (providerConfig && providerConfig.authType !== "none") {
credentials = await getProviderCredentials(provider);
credentials = await getProviderCredentialsWithQuotaPreflight(provider);
if (!credentials) {
return errorResponse(HTTP_STATUS.BAD_REQUEST, `No credentials for provider: ${provider}`);
}

View File

@@ -1,7 +1,10 @@
// Allow large audio/video file uploads — 5min for processing large files (up to 2GB)
export const maxDuration = 300;
import { handleAudioTranscription } from "@omniroute/open-sse/handlers/audioTranscription.ts";
import { getProviderCredentials, clearRecoveredProviderState } from "@/sse/services/auth";
import {
getProviderCredentialsWithQuotaPreflight,
clearRecoveredProviderState,
} from "@/sse/services/auth";
import {
parseTranscriptionModel,
getTranscriptionProvider,
@@ -96,7 +99,7 @@ export async function POST(request) {
// Get credentials — skip for local providers (authType: "none")
let credentials = null;
if (providerConfig && providerConfig.authType !== "none") {
credentials = await getProviderCredentials(provider);
credentials = await getProviderCredentialsWithQuotaPreflight(provider);
if (!credentials) {
return errorResponse(HTTP_STATUS.BAD_REQUEST, `No credentials for provider: ${provider}`);
}

View File

@@ -1,7 +1,10 @@
// Allow large audio/video file uploads — 5min for processing large files (up to 2GB)
export const maxDuration = 300;
import { handleAudioTranslation } from "@omniroute/open-sse/handlers/audioTranslation.ts";
import { getProviderCredentials, clearRecoveredProviderState } from "@/sse/services/auth";
import {
getProviderCredentialsWithQuotaPreflight,
clearRecoveredProviderState,
} from "@/sse/services/auth";
import {
parseTranslationModel,
getTranslationProvider,
@@ -98,7 +101,7 @@ export async function POST(request) {
// Get credentials — skip for local providers (authType: "none")
let credentials = null;
if (providerConfig && providerConfig.authType !== "none") {
credentials = await getProviderCredentials(provider);
credentials = await getProviderCredentialsWithQuotaPreflight(provider);
if (!credentials) {
return errorResponse(HTTP_STATUS.BAD_REQUEST, `No credentials for provider: ${provider}`);
}

View File

@@ -3,7 +3,10 @@ import {
handleOpenAIImageEdit,
} from "@omniroute/open-sse/handlers/imageGeneration.ts";
import { withInjectionGuard } from "@/middleware/promptInjectionGuard";
import { getProviderCredentials, clearRecoveredProviderState } from "@/sse/services/auth";
import {
getProviderCredentialsWithQuotaPreflight,
clearRecoveredProviderState,
} from "@/sse/services/auth";
import { parseImageModel, getImageProvider } from "@omniroute/open-sse/config/imageRegistry.ts";
import { errorResponse, unavailableResponse } from "@omniroute/open-sse/utils/error.ts";
import { HTTP_STATUS } from "@omniroute/open-sse/config/constants.ts";
@@ -172,7 +175,7 @@ async function postHandler(request: Request, context) {
// chatgpt-web keeps its conversation-continuation edit flow unchanged.
if (providerConfig?.format === "chatgpt-web") {
const credentials = await getProviderCredentials(
const credentials = await getProviderCredentialsWithQuotaPreflight(
parsed.provider,
null,
allowedConnections,
@@ -241,7 +244,7 @@ async function postHandler(request: Request, context) {
);
}
const credentials = await getProviderCredentials(
const credentials = await getProviderCredentialsWithQuotaPreflight(
customProviderId,
null,
allowedConnections,

View File

@@ -1,7 +1,7 @@
import { handleImageGeneration } from "@omniroute/open-sse/handlers/imageGeneration.ts";
import { withInjectionGuard } from "@/middleware/promptInjectionGuard";
import {
getProviderCredentials,
getProviderCredentialsWithQuotaPreflight,
clearRecoveredProviderState,
extractApiKey,
isValidApiKey,
@@ -171,7 +171,7 @@ async function postHandler(request, context) {
// Get credentials — skip for local providers (authType: "none")
let credentials = null;
if (providerConfig && providerConfig.authType !== "none") {
credentials = await getProviderCredentials(provider);
credentials = await getProviderCredentialsWithQuotaPreflight(provider);
if (!credentials) {
return errorResponse(
HTTP_STATUS.BAD_REQUEST,
@@ -187,7 +187,7 @@ async function postHandler(request, context) {
);
}
} else if (isCustomModel) {
credentials = await getProviderCredentials(provider);
credentials = await getProviderCredentialsWithQuotaPreflight(provider);
if (!credentials) {
return errorResponse(
HTTP_STATUS.BAD_REQUEST,

View File

@@ -1,5 +1,8 @@
import { handleModeration } from "@omniroute/open-sse/handlers/moderations.ts";
import { getProviderCredentials, clearRecoveredProviderState } from "@/sse/services/auth";
import {
getProviderCredentialsWithQuotaPreflight,
clearRecoveredProviderState,
} from "@/sse/services/auth";
import { withInjectionGuard } from "@/middleware/promptInjectionGuard";
import { parseModerationModel } from "@omniroute/open-sse/config/moderationRegistry.ts";
import { errorResponse } from "@omniroute/open-sse/utils/error.ts";
@@ -52,7 +55,7 @@ async function postHandler(request, context) {
// Default to openai if no provider prefix
const resolvedProvider = provider || "openai";
const credentials = await getProviderCredentials(resolvedProvider);
const credentials = await getProviderCredentialsWithQuotaPreflight(resolvedProvider);
if (!credentials) {
return errorResponse(
HTTP_STATUS.BAD_REQUEST,

View File

@@ -1,6 +1,9 @@
import { handleMusicGeneration } from "@omniroute/open-sse/handlers/musicGeneration.ts";
import { withInjectionGuard } from "@/middleware/promptInjectionGuard";
import { getProviderCredentials, clearRecoveredProviderState } from "@/sse/services/auth";
import {
getProviderCredentialsWithQuotaPreflight,
clearRecoveredProviderState,
} from "@/sse/services/auth";
import { parseMusicModel, getMusicProvider } from "@omniroute/open-sse/config/musicRegistry.ts";
import { errorResponse } from "@omniroute/open-sse/utils/error.ts";
import { HTTP_STATUS } from "@omniroute/open-sse/config/constants.ts";
@@ -72,7 +75,7 @@ async function postHandler(request, context) {
// Get credentials — skip for local providers (authType: "none")
let credentials = null;
if (providerConfig && providerConfig.authType !== "none") {
credentials = await getProviderCredentials(provider);
credentials = await getProviderCredentialsWithQuotaPreflight(provider);
if (!credentials) {
return errorResponse(
HTTP_STATUS.BAD_REQUEST,

View File

@@ -1,5 +1,8 @@
import { handleOcr } from "@omniroute/open-sse/handlers/ocr.ts";
import { getProviderCredentials, clearRecoveredProviderState } from "@/sse/services/auth";
import {
getProviderCredentialsWithQuotaPreflight,
clearRecoveredProviderState,
} from "@/sse/services/auth";
import { withInjectionGuard } from "@/middleware/promptInjectionGuard";
import { parseOcrModel } from "@omniroute/open-sse/config/ocrRegistry.ts";
import { errorResponse } from "@omniroute/open-sse/utils/error.ts";
@@ -52,7 +55,7 @@ async function postHandler(request, context) {
// Default to mistral if no provider prefix
const resolvedProvider = provider || "mistral";
const credentials = await getProviderCredentials(resolvedProvider);
const credentials = await getProviderCredentialsWithQuotaPreflight(resolvedProvider);
if (!credentials) {
return errorResponse(
HTTP_STATUS.BAD_REQUEST,

View File

@@ -2,7 +2,7 @@ import { errorResponse, unavailableResponse } from "@omniroute/open-sse/utils/er
import { HTTP_STATUS } from "@omniroute/open-sse/config/constants.ts";
import { getRegistryEntry } from "@omniroute/open-sse/config/providerRegistry.ts";
import {
getProviderCredentials,
getProviderCredentialsWithQuotaPreflight,
clearRecoveredProviderState,
extractApiKey,
isValidApiKey,
@@ -71,7 +71,7 @@ export async function POST(request, { params }) {
}
}
const credentials = await getProviderCredentials(providerEntry.id);
const credentials = await getProviderCredentialsWithQuotaPreflight(providerEntry.id);
if (!credentials) {
return errorResponse(HTTP_STATUS.BAD_REQUEST, `No credentials for provider: ${rawProvider}`);
}

View File

@@ -2,7 +2,7 @@ import { handleImageGeneration } from "@omniroute/open-sse/handlers/imageGenerat
import { errorResponse, unavailableResponse } from "@omniroute/open-sse/utils/error.ts";
import { HTTP_STATUS } from "@omniroute/open-sse/config/constants.ts";
import {
getProviderCredentials,
getProviderCredentialsWithQuotaPreflight,
clearRecoveredProviderState,
extractApiKey,
isValidApiKey,
@@ -68,7 +68,7 @@ export async function POST(request, { params }) {
);
}
const credentials = await getProviderCredentials(rawProvider);
const credentials = await getProviderCredentialsWithQuotaPreflight(rawProvider);
if (!credentials) {
return errorResponse(
HTTP_STATUS.BAD_REQUEST,

View File

@@ -1,5 +1,8 @@
import { handleRerank } from "@omniroute/open-sse/handlers/rerank.ts";
import { getProviderCredentials, clearRecoveredProviderState } from "@/sse/services/auth";
import {
getProviderCredentialsWithQuotaPreflight,
clearRecoveredProviderState,
} from "@/sse/services/auth";
import { withInjectionGuard } from "@/middleware/promptInjectionGuard";
import { parseRerankModel, getRerankProvider } from "@omniroute/open-sse/config/rerankRegistry.ts";
import { errorResponse } from "@omniroute/open-sse/utils/error.ts";
@@ -102,7 +105,7 @@ async function postHandler(request, context) {
if (provider) {
// Cloud provider matched
const credentials = await getProviderCredentials(provider);
const credentials = await getProviderCredentialsWithQuotaPreflight(provider);
if (!credentials) {
return errorResponse(HTTP_STATUS.BAD_REQUEST, `No credentials for provider: ${provider}`);
}
@@ -132,7 +135,7 @@ async function postHandler(request, context) {
const localProvider = localProviders.find((p) => p.id === prefix);
if (localProvider) {
const credentials = await getProviderCredentials(localProvider.providerId);
const credentials = await getProviderCredentialsWithQuotaPreflight(localProvider.providerId);
if (!credentials) {
return errorResponse(
HTTP_STATUS.BAD_REQUEST,

View File

@@ -1,5 +1,9 @@
import { handleSearch } from "@omniroute/open-sse/handlers/search.ts";
import { getProviderCredentials, extractApiKey, isValidApiKey } from "@/sse/services/auth";
import {
getProviderCredentialsWithQuotaPreflight,
extractApiKey,
isValidApiKey,
} from "@/sse/services/auth";
import {
getAllSearchProviders,
getSearchProvider,
@@ -64,13 +68,15 @@ type SearchCredentials = Record<string, any>;
type SearchCredentialLookup = SearchCredentials | RateLimitedCredentials | null;
async function resolveSearchCredentials(providerId: string): Promise<SearchCredentialLookup> {
const credentials = await getProviderCredentials(providerId).catch(() => null);
const credentials = await getProviderCredentialsWithQuotaPreflight(providerId).catch(() => null);
if (credentials && !isAllRateLimitedCredentials(credentials)) return credentials;
const fallbackId = SEARCH_CREDENTIAL_FALLBACKS[providerId];
if (!fallbackId) return credentials;
const fallbackCredentials = await getProviderCredentials(fallbackId).catch(() => null);
const fallbackCredentials = await getProviderCredentialsWithQuotaPreflight(fallbackId).catch(
() => null
);
if (fallbackCredentials && !isAllRateLimitedCredentials(fallbackCredentials)) {
return fallbackCredentials;
}
@@ -180,7 +186,9 @@ async function postHandler(request: Request, context: unknown) {
// Sort by cost to find cheapest with credentials (fallback-only providers
// are reached via the last-resort step below, never the primary pick).
const sortedIds = Object.values(SEARCH_PROVIDERS)
.filter((provider) => !provider.fallbackOnly && supportsSearchType(provider, body.search_type))
.filter(
(provider) => !provider.fallbackOnly && supportsSearchType(provider, body.search_type)
)
.sort((a, b) => a.costPerQuery - b.costPerQuery)
.map((p) => p.id);
@@ -216,7 +224,9 @@ async function postHandler(request: Request, context: unknown) {
// Find alternate for failover — must bind credentials to the matched provider.
// Exclude fallback-only providers; they are only used by the last-resort step.
const otherIds = Object.values(SEARCH_PROVIDERS)
.filter((provider) => !provider.fallbackOnly && supportsSearchType(provider, body.search_type))
.filter(
(provider) => !provider.fallbackOnly && supportsSearchType(provider, body.search_type)
)
.sort((a, b) => a.costPerQuery - b.costPerQuery)
.map((p) => p.id)
.filter((id) => id !== providerConfig.id);

View File

@@ -1,7 +1,10 @@
import { handleVideoGeneration } from "@omniroute/open-sse/handlers/videoGeneration.ts";
import { resolveVideoCredentialProvider } from "@omniroute/open-sse/handlers/videoGeneration/googleFlow.ts";
import { withInjectionGuard } from "@/middleware/promptInjectionGuard";
import { getProviderCredentials, clearRecoveredProviderState } from "@/sse/services/auth";
import {
getProviderCredentialsWithQuotaPreflight,
clearRecoveredProviderState,
} from "@/sse/services/auth";
import { parseVideoModel, getVideoProvider } from "@omniroute/open-sse/config/videoRegistry.ts";
import { errorResponse } from "@omniroute/open-sse/utils/error.ts";
import { HTTP_STATUS } from "@omniroute/open-sse/config/constants.ts";
@@ -75,7 +78,9 @@ async function postHandler(request, context) {
// OAuth credential (resolveVideoCredentialProvider maps googleflow → antigravity).
let credentials = null;
if (providerConfig && providerConfig.authType !== "none") {
credentials = await getProviderCredentials(resolveVideoCredentialProvider(provider));
credentials = await getProviderCredentialsWithQuotaPreflight(
resolveVideoCredentialProvider(provider)
);
if (!credentials) {
return errorResponse(
HTTP_STATUS.BAD_REQUEST,

View File

@@ -12,7 +12,11 @@ import { errorResponse } from "@omniroute/open-sse/utils/error.ts";
import { HTTP_STATUS } from "@omniroute/open-sse/config/constants.ts";
import { handleWebFetch } from "@omniroute/open-sse/handlers/webFetch.ts";
import * as log from "@/sse/utils/logger";
import { extractApiKey, isValidApiKey, getProviderCredentials } from "@/sse/services/auth";
import {
extractApiKey,
isValidApiKey,
getProviderCredentialsWithQuotaPreflight,
} from "@/sse/services/auth";
import { enforceApiKeyPolicy } from "@/shared/utils/apiKeyPolicy";
import { isRequireApiKeyEnabled } from "@/shared/utils/featureFlags";
import { v1WebFetchSchema } from "@/shared/validation/schemas";
@@ -38,7 +42,7 @@ async function resolveCredentials(
providerId: WebFetchProviderId
): Promise<{ apiKey?: string } | null> {
try {
const creds = await getProviderCredentials(providerId);
const creds = await getProviderCredentialsWithQuotaPreflight(providerId);
return creds ?? null;
} catch {
return null;

View File

@@ -0,0 +1,140 @@
import test from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import os from "node:os";
// Regression guard for issue #6686:
// "Account selection can pick accounts already out of quota (no live quota
// preflight outside chat/codex)."
//
// Root cause: getProviderCredentials() (src/sse/services/auth.ts) only skips
// a connection when a LOCAL CACHE already flags it exhausted
// (isQuotaExhaustedForRequest / src/domain/quotaCache.ts). It never itself
// calls the registered upstream QuotaFetcher. Only
// getProviderCredentialsWithQuotaPreflight() performs that live upstream
// check, and before this fix it was wired into exactly 2 call sites
// (src/sse/handlers/chat.ts, src/app/api/internal/codex-responses-ws).
// Every other credentialed route called the plain selector, so an account
// whose local cache entry was never populated (e.g. its first request landed
// on a non-chat/codex route) could be selected even at 0% quota remaining.
//
// The fix routes those remaining call sites through
// getProviderCredentialsWithQuotaPreflight() instead. This test has two
// parts:
// 1) A static, file-content check that every affected route no longer
// calls the plain, cache-only selector.
// 2) A behavioral check that the preflight-aware selector — now used by
// every credentialed route — genuinely blocks an account reported
// 100% used by a registered upstream quota fetcher.
const repoRoot = path.resolve(fileURLToPath(new URL("../../", import.meta.url)));
const ROUTES_REQUIRING_QUOTA_PREFLIGHT = [
"src/app/api/v1/rerank/route.ts",
"src/app/api/v1/images/generations/route.ts",
"src/app/api/v1/images/edits/route.ts",
"src/app/api/v1/audio/transcriptions/route.ts",
"src/app/api/v1/audio/speech/route.ts",
"src/app/api/v1/audio/translations/route.ts",
"src/app/api/v1/videos/generations/route.ts",
"src/app/api/v1/music/generations/route.ts",
"src/app/api/v1/ocr/route.ts",
"src/app/api/v1/providers/[provider]/embeddings/route.ts",
"src/app/api/v1/providers/[provider]/images/generations/route.ts",
"src/app/api/v1/web/fetch/route.ts",
"src/app/api/v1/moderations/route.ts",
"src/app/api/v1/search/route.ts",
];
test("#6686: previously-plain-selector routes must call the quota-preflight-aware selector, not the plain cache-only one", () => {
for (const relPath of ROUTES_REQUIRING_QUOTA_PREFLIGHT) {
const filePath = path.join(repoRoot, relPath);
const source = fs.readFileSync(filePath, "utf8");
// A bare `getProviderCredentials(` call (not immediately followed by
// `WithQuotaPreflight`) means live upstream quota is never checked before
// the account is used for this route — the exact #6686 gap.
const bareCalls = source.match(/getProviderCredentials(?!WithQuotaPreflight)\(/g) || [];
assert.equal(
bareCalls.length,
0,
`${relPath} must not call the plain getProviderCredentials() — use ` +
`getProviderCredentialsWithQuotaPreflight() instead (issue #6686)`
);
assert.match(
source,
/getProviderCredentialsWithQuotaPreflight/,
`${relPath} is expected to select credentials via ` +
`getProviderCredentialsWithQuotaPreflight (issue #6686)`
);
}
});
test("#6686: getProviderCredentialsWithQuotaPreflight (now used by every credentialed route) blocks an account already 100% out of quota upstream", async () => {
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-issue-6686-"));
process.env.DATA_DIR = TEST_DATA_DIR;
process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "issue-6686-secret";
const core = await import("../../src/lib/db/core.ts");
const providersDb = await import("../../src/lib/db/providers.ts");
const apiKeysDb = await import("../../src/lib/db/apiKeys.ts");
const auth = await import("../../src/sse/services/auth.ts");
const quotaPreflight = await import("../../open-sse/services/quotaPreflight.ts");
core.resetDbInstance();
apiKeysDb.resetApiKeyState();
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
try {
const provider = "issue6686";
const account = await providersDb.createProviderConnection({
provider,
authType: "apikey",
name: "issue-6686-exhausted",
apiKey: "sk-issue-6686-exhausted",
isActive: true,
testStatus: "active",
// Same shape used by every affected route's selection call — no
// cooldown, no rate-limit, nothing that would trip the reactive
// filters. Only a live upstream check can catch this account.
providerSpecificData: {
quotaPreflightEnabled: true,
},
});
// A registered upstream quota fetcher — what
// getProviderCredentialsWithQuotaPreflight() calls to discover the
// account is exhausted before the request is sent.
quotaPreflight.registerQuotaFetcher(provider, async () => ({
used: 100,
total: 100,
percentUsed: 1.0,
resetAt: new Date(Date.now() + 60_000).toISOString(),
}));
const preflightSelection = await auth.getProviderCredentialsWithQuotaPreflight(
provider,
null,
null,
null
);
const preflightResult = preflightSelection as {
allRateLimited?: boolean;
connectionId?: string;
} | null;
assert.ok(
preflightResult?.allRateLimited === true || preflightResult?.connectionId !== account.id,
"getProviderCredentialsWithQuotaPreflight should correctly block the exhausted account"
);
} finally {
core.resetDbInstance();
apiKeysDb.resetApiKeyState();
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
}
});