From dafeb42baf2e8ed312ee45f765dc8f183bf953cb Mon Sep 17 00:00:00 2001 From: WITALO ROCHA Date: Tue, 30 Jun 2026 23:45:29 -0300 Subject: [PATCH] feat(api): add opt-in API-key provider quota-policy bypass scope (#5731) Adds an opt-in per-API-key scope (policy:bypass-provider-quota) that lets a key skip provider/account-side quota cutoffs during routing. Operator USD budgets/usage limits still enforced unconditionally (fail-closed, before the bypass). Default-off; UI toggle + badge in API Manager. Integrated into release/v3.8.43. --- open-sse/services/combo.ts | 12 +++++- open-sse/services/combo/types.ts | 1 + .../api-manager/ApiManagerPageClient.tsx | 42 +++++++++++++++++-- .../dashboard/api-manager/apiManagerScopes.ts | 12 ++++-- src/shared/constants/apiKeyPolicyScopes.ts | 5 +++ src/shared/utils/apiKeyPolicy.ts | 1 + src/sse/handlers/chat.ts | 27 ++++++++---- ...pi-key-provider-quota-bypass-scope.test.ts | 34 +++++++++++++++ .../api-manager-scope-preservation.test.ts | 31 ++++++++++++++ 9 files changed, 150 insertions(+), 15 deletions(-) create mode 100644 src/shared/constants/apiKeyPolicyScopes.ts create mode 100644 tests/unit/api-key-provider-quota-bypass-scope.test.ts diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index c027f8213b..c99c9bbd1d 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -1278,12 +1278,22 @@ export async function handleComboChat({ log.warn("COMBO", "Failed to retrieve Last Known Good Provider. This is non-fatal.", { err }); } + const autoCandidateResilienceSettings = + relayOptions?.bypassProviderQuotaPolicy === true + ? { + ...resilienceSettings, + quotaPreflight: { + ...resilienceSettings.quotaPreflight, + enabled: false, + }, + } + : resilienceSettings; const candidates = await buildAutoCandidates( eligibleTargets, combo.name, relayOptions?.sessionId, resetWindowConfig, - resilienceSettings + autoCandidateResilienceSettings ); const routableCandidates = candidates.filter( (candidate) => candidate.quotaCutoffBlocked !== true diff --git a/open-sse/services/combo/types.ts b/open-sse/services/combo/types.ts index df42aef4a3..ba75daf328 100644 --- a/open-sse/services/combo/types.ts +++ b/open-sse/services/combo/types.ts @@ -63,6 +63,7 @@ export type IsModelAvailable = ( export type ComboRelayOptions = { sessionId?: string | null; config?: Record | null; + bypassProviderQuotaPolicy?: boolean; [key: string]: unknown; }; diff --git a/src/app/(dashboard)/dashboard/api-manager/ApiManagerPageClient.tsx b/src/app/(dashboard)/dashboard/api-manager/ApiManagerPageClient.tsx index 15224fb377..5097e4b94a 100644 --- a/src/app/(dashboard)/dashboard/api-manager/ApiManagerPageClient.tsx +++ b/src/app/(dashboard)/dashboard/api-manager/ApiManagerPageClient.tsx @@ -23,6 +23,7 @@ import { readActiveOnlyPreference, writeActiveOnlyPreference } from "./apiManage import { buildApiKeyCreateScopes, mergeApiKeyPermissionScopes } from "./apiManagerScopes"; import { SELF_ACCOUNT_QUOTA_SCOPE, SELF_USAGE_SCOPE } from "@/shared/constants/selfServiceScopes"; import { extractApiErrorMessage } from "@/shared/http/apiErrorMessage"; +import { hasProviderQuotaBypassScope } from "@/shared/constants/apiKeyPolicyScopes"; import { UsageLimitSettings } from "./components/UsageLimitSettings"; // Constants for validation @@ -958,6 +959,7 @@ export default function ApiManagerPageClient() { : 0; const hasThrottle = throttleDelayMs > 0; const hasManageScope = Array.isArray(key.scopes) && key.scopes.includes("manage"); + const hasProviderQuotaBypass = hasProviderQuotaBypassScope(key.scopes); const hasJsonStreamDefault = key.streamDefaultMode === "json"; const hasLocalUsageCommand = key.allowUsageCommand === true; const maxSessions = typeof key.maxSessions === "number" ? key.maxSessions : 0; @@ -985,9 +987,7 @@ export default function ApiManagerPageClient() {
- {visibleKeys.has(key.id) - ? (revealedKeys.get(key.id) ?? key.key) - : key.key} + {visibleKeys.has(key.id) ? (revealedKeys.get(key.id) ?? key.key) : key.key} {allowKeyReveal ? ( <> @@ -1114,6 +1114,12 @@ export default function ApiManagerPageClient() { USD quota )} + {hasProviderQuotaBypass && ( + + alt_route + Bypass quota policy + + )} {hasSessionLimit && ( group @@ -1588,6 +1594,9 @@ const PermissionsModal = memo(function PermissionsModal({ const [selfAccountQuotaEnabled, setSelfAccountQuotaEnabled] = useState( Array.isArray(apiKey?.scopes) && apiKey.scopes.includes(SELF_ACCOUNT_QUOTA_SCOPE) ); + const [bypassProviderQuotaPolicyEnabled, setBypassProviderQuotaPolicyEnabled] = useState( + hasProviderQuotaBypassScope(apiKey?.scopes) + ); const [maxSessions, setMaxSessions] = useState( typeof apiKey?.maxSessions === "number" && apiKey.maxSessions > 0 ? apiKey.maxSessions : 0 ); @@ -1822,6 +1831,7 @@ const PermissionsModal = memo(function PermissionsModal({ manageEnabled, selfUsageEnabled, selfAccountQuotaEnabled, + bypassProviderQuotaPolicyEnabled, }), allowAllEndpoints ? [] : selectedEndpoints, streamDefaultMode, @@ -1851,6 +1861,7 @@ const PermissionsModal = memo(function PermissionsModal({ manageEnabled, selfUsageEnabled, selfAccountQuotaEnabled, + bypassProviderQuotaPolicyEnabled, scheduleEnabled, scheduleFrom, scheduleUntil, @@ -2451,6 +2462,31 @@ const PermissionsModal = memo(function PermissionsModal({ />
+ {/* Advanced Provider Quota Policy Override */} +
+
+

Bypass provider quota cutoffs

+

+ Allows this key to ignore upstream provider/account cutoff policy during routing. API + key USD quotas still apply. +

+
+ +
+ {/* Disable Non-Public Models Toggle */}
diff --git a/src/app/(dashboard)/dashboard/api-manager/apiManagerScopes.ts b/src/app/(dashboard)/dashboard/api-manager/apiManagerScopes.ts index e8adb3546a..3e1692b179 100644 --- a/src/app/(dashboard)/dashboard/api-manager/apiManagerScopes.ts +++ b/src/app/(dashboard)/dashboard/api-manager/apiManagerScopes.ts @@ -1,7 +1,5 @@ -import { - SELF_ACCOUNT_QUOTA_SCOPE, - SELF_USAGE_SCOPE, -} from "@/shared/constants/selfServiceScopes"; +import { SELF_ACCOUNT_QUOTA_SCOPE, SELF_USAGE_SCOPE } from "@/shared/constants/selfServiceScopes"; +import { API_KEY_BYPASS_PROVIDER_QUOTA_SCOPE } from "@/shared/constants/apiKeyPolicyScopes"; const MANAGEMENT_SCOPE = "manage"; @@ -9,12 +7,14 @@ export interface CreateScopeOptions { manageEnabled: boolean; selfUsageEnabled?: boolean; selfAccountQuotaEnabled?: boolean; + bypassProviderQuotaPolicyEnabled?: boolean; } export interface PermissionScopeOptions { manageEnabled: boolean; selfUsageEnabled: boolean; selfAccountQuotaEnabled: boolean; + bypassProviderQuotaPolicyEnabled: boolean; } export function buildApiKeyCreateScopes(options: CreateScopeOptions): string[] { @@ -25,6 +25,9 @@ export function buildApiKeyCreateScopes(options: CreateScopeOptions): string[] { if (selfUsageEnabled && options.selfAccountQuotaEnabled === true) { scopes.push(SELF_ACCOUNT_QUOTA_SCOPE); } + if (options.bypassProviderQuotaPolicyEnabled === true) { + scopes.push(API_KEY_BYPASS_PROVIDER_QUOTA_SCOPE); + } return scopes; } @@ -41,6 +44,7 @@ export function mergeApiKeyPermissionScopes( SELF_ACCOUNT_QUOTA_SCOPE, options.selfUsageEnabled && options.selfAccountQuotaEnabled ); + setScope(scopes, API_KEY_BYPASS_PROVIDER_QUOTA_SCOPE, options.bypassProviderQuotaPolicyEnabled); return [...scopes]; } diff --git a/src/shared/constants/apiKeyPolicyScopes.ts b/src/shared/constants/apiKeyPolicyScopes.ts new file mode 100644 index 0000000000..b4ba9b931e --- /dev/null +++ b/src/shared/constants/apiKeyPolicyScopes.ts @@ -0,0 +1,5 @@ +export const API_KEY_BYPASS_PROVIDER_QUOTA_SCOPE = "policy:bypass-provider-quota"; + +export function hasProviderQuotaBypassScope(scopes: readonly string[] | null | undefined): boolean { + return Array.isArray(scopes) && scopes.includes(API_KEY_BYPASS_PROVIDER_QUOTA_SCOPE); +} diff --git a/src/shared/utils/apiKeyPolicy.ts b/src/shared/utils/apiKeyPolicy.ts index 7e93fb027d..a46c1cc8a7 100644 --- a/src/shared/utils/apiKeyPolicy.ts +++ b/src/shared/utils/apiKeyPolicy.ts @@ -90,6 +90,7 @@ export interface ApiKeyMetadata { throttleDelayMs?: number | null; maxSessions?: number | null; rateLimits?: RateLimitRule[] | null; + scopes?: string[]; allowedEndpoints?: string[]; disableNonPublicModels?: boolean; allowUsageCommand?: boolean; diff --git a/src/sse/handlers/chat.ts b/src/sse/handlers/chat.ts index a40a074377..9ab6445dc2 100644 --- a/src/sse/handlers/chat.ts +++ b/src/sse/handlers/chat.ts @@ -87,6 +87,7 @@ import { RequestTelemetry, recordTelemetry } from "../../shared/utils/requestTel import { generateRequestId } from "../../shared/utils/requestId"; import { logAuditEvent } from "../../lib/compliance/index"; import { enforceApiKeyPolicy } from "../../shared/utils/apiKeyPolicy"; +import { hasProviderQuotaBypassScope } from "../../shared/constants/apiKeyPolicyScopes"; import { cloneLogPayload } from "@/lib/logPayloads"; import { handleInternalUsageCommand } from "@/lib/usage/internalUsageCommand"; import { @@ -330,6 +331,7 @@ export async function handleChat( return policy.rejection; } const apiKeyInfo = policy.apiKeyInfo; + const bypassProviderQuotaPolicy = hasProviderQuotaBypassScope(apiKeyInfo?.scopes); telemetry.endPhase(); // Guardrail pre-call pipeline — prompt injection, PII masking, and future custom rules. @@ -627,6 +629,7 @@ export async function handleChat( sessionKey: sessionAffinityKey, ...(target?.allowRateLimitedConnection ? { allowRateLimitedConnections: true } : {}), ...(target?.connectionId ? { forcedConnectionId: target.connectionId } : {}), + ...(bypassProviderQuotaPolicy ? { bypassQuotaPolicy: true } : {}), } ); if (!creds || creds.allRateLimited) return false; @@ -642,6 +645,18 @@ export async function handleChat( ]); const relayConfig = combo.strategy === "context-relay" ? resolveComboConfig(combo, settings) : null; + const relayOptions = + combo.strategy === "context-relay" || bypassProviderQuotaPolicy + ? { + ...(combo.strategy === "context-relay" + ? { + sessionId, + config: relayConfig, + } + : {}), + ...(bypassProviderQuotaPolicy ? { bypassProviderQuotaPolicy: true } : {}), + } + : undefined; telemetry.endPhase(); // Context-relay keeps generation in combo.ts, but handoff injection lives here @@ -706,13 +721,7 @@ export async function handleChat( settings, allCombos, apiKeyAllowedConnections: apiKeyInfo?.allowedConnections ?? null, - relayOptions: - combo.strategy === "context-relay" - ? { - sessionId, - config: relayConfig, - } - : undefined, + relayOptions, signal: request?.signal ?? null, correlationId: reqId, }); @@ -956,6 +965,7 @@ async function handleSingleModelChat( return runtimeOptions.providerId; })(); const forceLiveComboTest = runtimeOptions.forceLiveComboTest === true; + const bypassProviderQuotaPolicy = hasProviderQuotaBypassScope(apiKeyInfo?.scopes); const hasForcedConnection = typeof runtimeOptions.forcedConnectionId === "string" && runtimeOptions.forcedConnectionId.trim().length > 0; @@ -1099,6 +1109,9 @@ async function handleSingleModelChat( bypassQuotaPolicy: true, } : {}), + ...(!forceLiveComboTest && bypassProviderQuotaPolicy + ? { bypassQuotaPolicy: true } + : {}), ...(runtimeOptions.forcedConnectionId ? { forcedConnectionId: runtimeOptions.forcedConnectionId } : {}), diff --git a/tests/unit/api-key-provider-quota-bypass-scope.test.ts b/tests/unit/api-key-provider-quota-bypass-scope.test.ts new file mode 100644 index 0000000000..c19f773394 --- /dev/null +++ b/tests/unit/api-key-provider-quota-bypass-scope.test.ts @@ -0,0 +1,34 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { + API_KEY_BYPASS_PROVIDER_QUOTA_SCOPE, + hasProviderQuotaBypassScope, +} from "../../src/shared/constants/apiKeyPolicyScopes.ts"; + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); + +test("provider quota bypass scope helper is explicit and strict", () => { + assert.equal(API_KEY_BYPASS_PROVIDER_QUOTA_SCOPE, "policy:bypass-provider-quota"); + assert.equal(hasProviderQuotaBypassScope([API_KEY_BYPASS_PROVIDER_QUOTA_SCOPE]), true); + assert.equal(hasProviderQuotaBypassScope(["policy:other"]), false); + assert.equal(hasProviderQuotaBypassScope(null), false); +}); + +test("chat handler maps API key provider quota bypass scope to auth bypass option", () => { + const source = fs.readFileSync(path.join(repoRoot, "src/sse/handlers/chat.ts"), "utf8"); + + assert.match(source, /hasProviderQuotaBypassScope\(apiKeyInfo\?\.scopes\)/); + assert.match(source, /bypassProviderQuotaPolicy[\s\S]*bypassQuotaPolicy: true/); + assert.match(source, /relayOptions[\s\S]*bypassProviderQuotaPolicy: true/); +}); + +test("auto combo disables hard provider quota cutoffs when relay requests bypass", () => { + const source = fs.readFileSync(path.join(repoRoot, "open-sse/services/combo.ts"), "utf8"); + + assert.match(source, /relayOptions\?\.bypassProviderQuotaPolicy === true/); + assert.match(source, /quotaPreflight:[\s\S]*enabled: false/); +}); diff --git a/tests/unit/api-manager-scope-preservation.test.ts b/tests/unit/api-manager-scope-preservation.test.ts index eb0363a1d2..e3da454809 100644 --- a/tests/unit/api-manager-scope-preservation.test.ts +++ b/tests/unit/api-manager-scope-preservation.test.ts @@ -9,10 +9,18 @@ import { SELF_ACCOUNT_QUOTA_SCOPE, SELF_USAGE_SCOPE, } from "../../src/shared/constants/selfServiceScopes.ts"; +import { API_KEY_BYPASS_PROVIDER_QUOTA_SCOPE } from "../../src/shared/constants/apiKeyPolicyScopes.ts"; test("create scopes enable own usage by default without shared account quota", () => { assert.deepEqual(buildApiKeyCreateScopes({ manageEnabled: false }), [SELF_USAGE_SCOPE]); assert.deepEqual(buildApiKeyCreateScopes({ manageEnabled: true }), ["manage", SELF_USAGE_SCOPE]); + assert.deepEqual( + buildApiKeyCreateScopes({ + manageEnabled: false, + bypassProviderQuotaPolicyEnabled: true, + }), + [SELF_USAGE_SCOPE, API_KEY_BYPASS_PROVIDER_QUOTA_SCOPE] + ); assert.deepEqual( buildApiKeyCreateScopes({ manageEnabled: false, @@ -28,6 +36,7 @@ test("permission scope merge preserves unrelated scopes while toggling managed s manageEnabled: true, selfUsageEnabled: true, selfAccountQuotaEnabled: true, + bypassProviderQuotaPolicyEnabled: true, }); assert.deepEqual(scopes, [ @@ -35,6 +44,7 @@ test("permission scope merge preserves unrelated scopes while toggling managed s SELF_USAGE_SCOPE, "manage", SELF_ACCOUNT_QUOTA_SCOPE, + API_KEY_BYPASS_PROVIDER_QUOTA_SCOPE, ]); }); @@ -45,8 +55,29 @@ test("permission scope merge removes shared quota visibility when own usage is d manageEnabled: false, selfUsageEnabled: false, selfAccountQuotaEnabled: true, + bypassProviderQuotaPolicyEnabled: false, } ); assert.deepEqual(scopes, ["custom:scope"]); }); + +test("permission scope merge toggles provider quota policy bypass without dropping custom scopes", () => { + const enabled = mergeApiKeyPermissionScopes(["custom:scope"], { + manageEnabled: false, + selfUsageEnabled: false, + selfAccountQuotaEnabled: false, + bypassProviderQuotaPolicyEnabled: true, + }); + + assert.deepEqual(enabled, ["custom:scope", API_KEY_BYPASS_PROVIDER_QUOTA_SCOPE]); + + const disabled = mergeApiKeyPermissionScopes(enabled, { + manageEnabled: false, + selfUsageEnabled: false, + selfAccountQuotaEnabled: false, + bypassProviderQuotaPolicyEnabled: false, + }); + + assert.deepEqual(disabled, ["custom:scope"]); +});