diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index 82e4e36d74..8b23ad3d33 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -220,6 +220,11 @@ import { } from "./combo/quotaExhaustionCutoff.ts"; import { expandTargetsByFingerprints } from "./combo/fingerprintExpansion.ts"; import { resolveComboTargetPipeline } from "./combo/targetResolution.ts"; +import { + isQuotaExhaustionResponse, + recordQuotaExhaustionClassification, + withQuotaExhaustionClassification, +} from "./combo/quotaExhaustion.ts"; export { RESET_WINDOW_NAMES, QUOTA_SOFT_DEPRIORITIZE_FACTOR, setCandidateQuotaSoftPenalty }; export { scoreAutoTargets, expandAutoComboCandidatePool }; @@ -817,6 +822,26 @@ export async function handleComboChat({ // Accumulator for per-model error details across targets in the current set try. // Reset at the start of each set retry (same lifecycle as lastError/recordedAttempts). let comboErrors: Array<{ model: string; status: number; error: string }> = []; + // Quota trust spans set retries and recursive cooldown re-dispatches. Once any + // failure is non-quota, a nested caller must never treat this dispatch as quota-only. + let observedFailure = false; + let allObservedFailuresQuota = true; + const targetFailureTrust = new Map< + string, + { observedFailure: boolean; allObservedFailuresQuota: boolean } + >(); + const observeFailure = (quotaExhausted: boolean, targetExecutionKey?: string) => { + observedFailure = true; + allObservedFailuresQuota &&= quotaExhausted; + if (!targetExecutionKey) return; + const trust = targetFailureTrust.get(targetExecutionKey) ?? { + observedFailure: false, + allObservedFailuresQuota: true, + }; + trust.observedFailure = true; + trust.allObservedFailuresQuota &&= quotaExhausted; + targetFailureTrust.set(targetExecutionKey, trust); + }; // FASE 2.1: per-connection concurrency limit for quota-share. The gating in // selectQuotaShareTarget is fail-open and cannot hard-limit a single-connection @@ -904,6 +929,9 @@ export async function handleComboChat({ let anySuccess = false; const abortControllers = new Map(); const zeroLatencyOptimizationsEnabled = config.zeroLatencyOptimizationsEnabled === true; + const hasProtectedPriorityTarget = + strategy === "priority" && + orderedTargets.some((target) => target.fallbackOnlyOnQuotaExhaustion === true); const executeTarget = async ( i: number @@ -912,12 +940,20 @@ export async function handleComboChat({ const modelStr = target.modelStr; const rawModel = parseModel(modelStr).model || modelStr; const provider = target.provider; + const protectedPriorityTarget = + strategy === "priority" && target.fallbackOnlyOnQuotaExhaustion === true; + const stopProtectedPriorityTarget = (message: string) => { + observeFailure(false, target.executionKey); + return protectedPriorityTarget + ? { ok: false, response: errorResponse(503, message) } + : null; + }; const cb = getCircuitBreaker(provider); if (cb.getStatus().state === "OPEN") { log.info("COMBO", `Skipping ${modelStr} — circuit breaker OPEN for ${provider}`); if (i > 0) fallbackCount++; - return null; + return stopProtectedPriorityTarget(`Provider ${provider} circuit breaker is open`); } if ( @@ -927,7 +963,7 @@ export async function handleComboChat({ ) { log.info("COMBO", `Skipping ${modelStr} — provider ${provider} in global cooldown`); if (i > 0) fallbackCount++; - return null; + return stopProtectedPriorityTarget(`Provider ${provider} is in cooldown`); } // Use pre-screened profile if available, otherwise fetch on demand @@ -954,14 +990,14 @@ export async function handleComboChat({ if (exhaustedSkip) { log.info("COMBO", exhaustedSkip); if (i > 0) fallbackCount++; - return null; + return stopProtectedPriorityTarget(`Target ${modelStr} is unavailable`); } // Pre-check: skip models locked by the resilience system (model-level lockout) if (provider && rawModel && isModelLocked(provider, target.connectionId || "", rawModel)) { log.info("COMBO", `Skipping ${modelStr} — model locked by resilience (cooldown active)`); if (i > 0) fallbackCount++; - return null; + return stopProtectedPriorityTarget(`Model ${modelStr} is locked`); } // #5923 (Finding #4) — honor the same opt-in quota-exhaustion cutoff the @@ -987,6 +1023,16 @@ export async function handleComboChat({ `Skipping ${modelStr} — quota exhaustion cutoff (${quotaCutoff.reason || "quota_exhausted"})` ); if (i > 0) fallbackCount++; + observeFailure(true, target.executionKey); + if (protectedPriorityTarget) { + const protectedTargetTrust = targetFailureTrust.get(target.executionKey); + if (!protectedTargetTrust?.allObservedFailuresQuota) { + return { + ok: false, + response: errorResponse(503, `Target ${modelStr} is unavailable`), + }; + } + } return null; } } @@ -1004,7 +1050,7 @@ export async function handleComboChat({ `Skipping ${modelStr} — no credentials available or model excluded` ); if (i > 0) fallbackCount++; - return null; + return stopProtectedPriorityTarget(`Model ${modelStr} is unavailable`); } } @@ -1015,7 +1061,7 @@ export async function handleComboChat({ if (gateResult.allowed === false) { logCredentialSkip(log, modelStr, gateResult.reason || "Credential gate blocked"); if (i > 0) fallbackCount++; - return null; + return stopProtectedPriorityTarget(`Credential gate blocked ${modelStr}`); } // Concurrency gate: fail-fast skip when connection is at max_concurrent capacity (e.g. Featherless 1/1) @@ -1029,7 +1075,7 @@ export async function handleComboChat({ `Skipping ${modelStr} — connection ${connectionId} is at max concurrency cap (${maxConcurrentCap})` ); if (i > 0) fallbackCount++; - return null; + return stopProtectedPriorityTarget(`Connection capacity reached for ${modelStr}`); } } @@ -1084,7 +1130,7 @@ export async function handleComboChat({ "COMBO", `Predictive TTFT Circuit Breaker: skipping ${modelStr} (avg ${m.avgLatencyMs}ms > max ${config.predictiveTtftMs}ms)` ); - return null; + return stopProtectedPriorityTarget(`Predictive latency check rejected ${modelStr}`); } } } @@ -1294,7 +1340,13 @@ export async function handleComboChat({ error: `Quality: ${quality.reason}`, latencyMs: Date.now() - startTime, }); - return null; + observeFailure(false, target.executionKey); + return protectedPriorityTarget + ? { + ok: false, + response: errorResponse(502, "Upstream response failed quality validation"), + } + : null; } // Success decay: a healthy response walks the model's lockout failure @@ -1645,7 +1697,7 @@ export async function handleComboChat({ result.status, errorText, 0, - null, + protectedPriorityTarget ? rawModel : null, provider, result.headers, profile, @@ -1776,6 +1828,15 @@ export async function handleComboChat({ recordProviderFailure(provider, log, targetWithConnection.connectionId, profile); } + const quotaExhausted = await isQuotaExhaustionResponse( + result, + provider, + rawModel, + profile + ); + recordQuotaExhaustionClassification(result, quotaExhausted); + observeFailure(quotaExhausted, target.executionKey); + // Check if this is a transient error worth retrying on same model. // A token-limit 429 is terminal for the client — never retry it. const isTransient = @@ -1785,6 +1846,7 @@ export async function handleComboChat({ [408, 429, 500, 502, 503, 504].includes(result.status); if (retry < maxRetries && isTransient && !providerExhausted) { if ( + !protectedPriorityTarget && provider && rawModel && isModelLocked(provider, targetWithConnection.connectionId || "", rawModel) @@ -1807,7 +1869,7 @@ export async function handleComboChat({ // once the model is cooling down, retrying it would waste an upstream // call and extend the cooldown via exponential backoff. let lockoutRecorded = false; - if (provider && rawModel && retry === 0 && !scopedFailure) { + if (!protectedPriorityTarget && provider && rawModel && retry === 0 && !scopedFailure) { const mlSettings = resolveModelLockoutSettings(settings); if (mlSettings.enabled && mlSettings.errorCodes.includes(result.status)) { recordModelLockoutFailure( @@ -1847,6 +1909,22 @@ export async function handleComboChat({ } // Done retrying this model + const protectedTargetTrust = targetFailureTrust.get(target.executionKey); + if ( + protectedPriorityTarget && + (!protectedTargetTrust?.observedFailure || + !protectedTargetTrust.allObservedFailuresQuota) + ) { + recordComboRequest(combo.name, modelStr, { + success: false, + latencyMs: Date.now() - startTime, + fallbackCount, + strategy, + target: toRecordedTarget(target), + }); + recordedAttempts++; + return { ok: false, response: result }; + } recordComboRequest(combo.name, modelStr, { success: false, latencyMs: Date.now() - startTime, @@ -1975,7 +2053,12 @@ export async function handleComboChat({ runningTasks.add(task); task.finally(() => runningTasks.delete(task)); - if (zeroLatencyOptimizationsEnabled && config.hedging && i + 1 < orderedTargets.length) { + if ( + zeroLatencyOptimizationsEnabled && + config.hedging && + !hasProtectedPriorityTarget && + i + 1 < orderedTargets.length + ) { const hedgeDelay = resolveDelayMs(config.hedgeDelayMs, 500); let timeoutResolve: () => void; const timeoutPromise = new Promise((r) => { @@ -2062,11 +2145,14 @@ export async function handleComboChat({ latencyMs, fallbackCount, }); - return errorResponseWithComboDiagnostics( - 503, - "Service temporarily unavailable: all targets were skipped by pre-dispatch filters", - buildComboDiag("all_targets_skipped"), - { code: "ALL_TARGETS_SKIPPED", type: "service_unavailable" } + return withQuotaExhaustionClassification( + errorResponseWithComboDiagnostics( + 503, + "Service temporarily unavailable: all targets were skipped by pre-dispatch filters", + buildComboDiag("all_targets_skipped"), + { code: "ALL_TARGETS_SKIPPED", type: "service_unavailable" } + ), + observedFailure ? allObservedFailuresQuota : null ); } notifyWebhookEvent("request.failed", { @@ -2157,7 +2243,10 @@ export async function handleComboChat({ if (earliestRetryAfter && isRetryAfterEligibleStatus(status)) { const retryHuman = formatRetryAfter(toRetryAfterDisplayValue(earliestRetryAfter)); log.warn("COMBO", `All models failed | ${msg} (${retryHuman})`); - return unavailableResponse(status, msg, earliestRetryAfter, retryHuman); + return withQuotaExhaustionClassification( + unavailableResponse(status, msg, earliestRetryAfter, retryHuman), + observedFailure ? allObservedFailuresQuota : null + ); } // Silent-stop fix: bump the failure counter (pin clears on 3rd consecutive) and emit @@ -2173,10 +2262,13 @@ export async function handleComboChat({ ); } const retryAfterSeconds = undefined; - return errorResponseWithComboDiagnostics( - status, - msg, - buildComboDiag(lastError ?? "all_models_failed", retryAfterSeconds) + return withQuotaExhaustionClassification( + errorResponseWithComboDiagnostics( + status, + msg, + buildComboDiag(lastError ?? "all_models_failed", retryAfterSeconds) + ), + observedFailure ? allObservedFailuresQuota : null ); } diff --git a/open-sse/services/combo/comboStructure.ts b/open-sse/services/combo/comboStructure.ts index 0430f03482..c125bc9052 100644 --- a/open-sse/services/combo/comboStructure.ts +++ b/open-sse/services/combo/comboStructure.ts @@ -115,6 +115,7 @@ function normalizeRuntimeStep( comboName: step.comboName, weight, label, + ...(step.fallbackOnlyOnQuotaExhaustion ? { fallbackOnlyOnQuotaExhaustion: true } : {}), }; } @@ -138,7 +139,12 @@ function normalizeRuntimeStep( : {}), weight, label, - prompt: step.kind === "model" ? step.prompt || null : null, + // `prompt` is a per-step pipeline input and only exists on a model step — + // #8894 widened the union with ComboProviderWildcardStep, which has no prompt. + prompt: (step.kind === "model" ? step.prompt : null) || null, + ...(step.kind === "model" && step.fallbackOnlyOnQuotaExhaustion + ? { fallbackOnlyOnQuotaExhaustion: true } + : {}), } satisfies ResolvedComboTarget; } diff --git a/open-sse/services/combo/quotaExhaustion.ts b/open-sse/services/combo/quotaExhaustion.ts new file mode 100644 index 0000000000..50e2d71863 --- /dev/null +++ b/open-sse/services/combo/quotaExhaustion.ts @@ -0,0 +1,107 @@ +import { checkFallbackError, type ProviderProfile } from "../accountFallback.ts"; +import { classifyGeminiQuotaMetricFromText } from "../geminiRateLimitTracker.ts"; + +const TERMINAL_QUOTA_CODES = new Set([ + "billing_hard_limit_reached", + "credits_exhausted", + "insufficient_quota", + "quota_exhausted", +]); + +const trustedClassifications = new WeakMap(); + +type ParsedError = { + text: string; + structuredError: { code?: string; type?: string } | null; +}; + +async function parseError(response: Response): Promise { + let text = response.statusText; + let structuredError: ParsedError["structuredError"] = null; + try { + const body = (await response.clone().json()) as { + error?: string | { message?: unknown; code?: unknown; type?: unknown }; + message?: unknown; + }; + if (typeof body.error === "string") text = body.error; + else if (body.error && typeof body.error === "object") { + if (typeof body.error.message === "string") text = body.error.message; + structuredError = { + ...(body.error.code == null ? {} : { code: String(body.error.code) }), + ...(body.error.type == null ? {} : { type: String(body.error.type) }), + }; + } else if (typeof body.message === "string") text = body.message; + } catch { + try { + text = await response.clone().text(); + } catch { + // The status and trusted in-process classification remain available. + } + } + return { text, structuredError }; +} + +export function recordQuotaExhaustionClassification(response: Response, exhausted: boolean): void { + trustedClassifications.set(response, exhausted); +} + +export function withQuotaExhaustionClassification( + response: Response, + exhausted: boolean | null +): Response { + if (exhausted !== null) recordQuotaExhaustionClassification(response, exhausted); + return response; +} + +export async function isQuotaExhaustionResponse( + response: Response, + provider: string | null, + model: string | null, + profile: ProviderProfile | null = null +): Promise { + const trusted = trustedClassifications.get(response); + if (trusted !== undefined) return trusted; + + if (response.status !== 402 && response.status !== 429) return false; + + const { text, structuredError } = await parseError(response); + if (provider === "gemini" && response.status === 429) { + const metric = classifyGeminiQuotaMetricFromText(text); + if (metric === "rpm" || metric === "tpm") return false; + if (metric === "rpd") return true; + } + const normalizedCode = structuredError?.code?.toLowerCase(); + const normalizedType = structuredError?.type?.toLowerCase(); + if ( + (normalizedCode && TERMINAL_QUOTA_CODES.has(normalizedCode)) || + (normalizedType && TERMINAL_QUOTA_CODES.has(normalizedType)) + ) { + return true; + } + + if ( + /\b(?:billing hard limit reached|credits? exhausted|subscription quota exhausted)\b/i.test(text) + ) { + return true; + } + + if ( + provider?.startsWith("openai-compatible-") || + provider?.startsWith("openai-compatible-chat-") + ) { + return false; + } + + return ( + checkFallbackError( + response.status, + text, + 0, + model, + provider, + response.headers, + profile, + structuredError + ).reason === "quota_exhausted" + ); +} diff --git a/open-sse/services/combo/runtimeUnits.ts b/open-sse/services/combo/runtimeUnits.ts index 90ad1c2cac..453ce38116 100644 --- a/open-sse/services/combo/runtimeUnits.ts +++ b/open-sse/services/combo/runtimeUnits.ts @@ -9,6 +9,7 @@ import { errorResponse } from "../../utils/error.ts"; import { recordComboRequest } from "../comboMetrics.ts"; import { resolveDelayMs } from "./comboPredicates.ts"; import { isRuntimeUnitAtConcurrencyCap } from "./runtimeUnitCapacity.ts"; +import { isQuotaExhaustionResponse, withQuotaExhaustionClassification } from "./quotaExhaustion.ts"; import { validateResponseQuality, releaseQualityClone } from "./validateQuality.ts"; import type { ResponseValidationConfig } from "./responseValidation.ts"; import type { @@ -197,8 +198,28 @@ export async function executeRuntimeUnitCombo(args: { const effectiveStrategy = args.effectiveComboStrategy ?? args.strategy; let lastResponse: Response | null = null; let fallbackCount = 0; + let observedFailure = false; + let allObservedFailuresQuota = true; + const targetFailureTrust = new Map< + string, + { observedFailure: boolean; allObservedFailuresQuota: boolean } + >(); + const observeFailure = async (response: Response, unit: ResolvedComboUnit): Promise => { + const quotaExhausted = await isQuotaExhaustionResponse( + response, + unit.kind === "model" ? unit.provider : null, + unit.kind === "model" ? unit.modelStr : null + ); + observedFailure = true; + allObservedFailuresQuota &&= quotaExhausted; + return quotaExhausted; + }; + const finalFailure = (response: Response): Response => + withQuotaExhaustionClassification(response, observedFailure ? allObservedFailuresQuota : null); for (const unit of orderedUnits) { + const protectedPriorityUnit = + effectiveStrategy === "priority" && unit.fallbackOnlyOnQuotaExhaustion === true; if ( await isRuntimeUnitAtConcurrencyCap( unit, @@ -211,16 +232,24 @@ export async function executeRuntimeUnitCombo(args: { "COMBO", `Skipping ${unit.kind} ${unitDisplayName(unit)} — concurrency cap reached` ); + lastResponse = errorResponse(503, `${unitDisplayName(unit)} is at concurrency capacity`); + await observeFailure(lastResponse, unit); + if (protectedPriorityUnit) return { response: finalFailure(lastResponse), unit }; fallbackCount += 1; continue; } for (let retry = 0; retry <= maxRetries; retry += 1) { - if (args.signal?.aborted) - return { response: errorResponse(499, "Client disconnected"), unit }; + if (args.signal?.aborted) { + lastResponse = errorResponse(499, "Client disconnected"); + await observeFailure(lastResponse, unit); + return { response: finalFailure(lastResponse), unit }; + } args.nesting.attemptBudget.count += 1; if (args.nesting.attemptBudget.count > args.nesting.attemptBudget.limit) { - return { response: errorResponse(503, "Maximum combo retry limit reached"), unit }; + lastResponse = errorResponse(503, "Maximum combo retry limit reached"); + await observeFailure(lastResponse, unit); + return { response: finalFailure(lastResponse), unit }; } if (retry > 0) { await new Promise((resolve) => setTimeout(resolve, retryDelayMs)); @@ -276,9 +305,31 @@ export async function executeRuntimeUnitCombo(args: { }); return { response, unit }; } + lastResponse = errorResponse(502, "Upstream response failed quality validation"); + } + if (lastResponse) { + const quotaExhausted = await observeFailure(lastResponse, unit); + if (protectedPriorityUnit) { + const trust = targetFailureTrust.get(unit.executionKey) ?? { + observedFailure: false, + allObservedFailuresQuota: true, + }; + trust.observedFailure = true; + trust.allObservedFailuresQuota &&= quotaExhausted; + targetFailureTrust.set(unit.executionKey, trust); + } } if (![408, 429, 500, 502, 503, 504].includes(response.status)) break; } + const protectedTargetTrust = targetFailureTrust.get(unit.executionKey); + if ( + protectedPriorityUnit && + protectedTargetTrust?.observedFailure && + !protectedTargetTrust.allObservedFailuresQuota && + lastResponse + ) { + return { response: finalFailure(lastResponse), unit }; + } fallbackCount += 1; } recordComboRequest(args.combo.name, null, { @@ -288,7 +339,9 @@ export async function executeRuntimeUnitCombo(args: { strategy: effectiveStrategy, }); return { - response: lastResponse || errorResponse(503, "All nested combo units unavailable"), + response: finalFailure( + lastResponse || errorResponse(503, "All nested combo units unavailable") + ), unit: null, }; } diff --git a/open-sse/services/combo/types.ts b/open-sse/services/combo/types.ts index 1252a87c90..71b4e849ec 100644 --- a/open-sse/services/combo/types.ts +++ b/open-sse/services/combo/types.ts @@ -174,6 +174,7 @@ export type ResolvedComboTarget = { label: string | null; prompt?: string | null; failoverBeforeRetry?: unknown; + fallbackOnlyOnQuotaExhaustion?: boolean; trafficType?: "production" | "shadow"; /** * Fingerprint-based account pin resolved from a combo builder composite @@ -200,6 +201,7 @@ export type ResolvedComboRefTarget = { comboName: string; weight: number; label: string | null; + fallbackOnlyOnQuotaExhaustion?: boolean; }; export type ResolvedComboUnit = ResolvedComboTarget | ResolvedComboRefTarget; diff --git a/src/app/(dashboard)/dashboard/combos/ComboQuotaOnlyFallbackToggle.tsx b/src/app/(dashboard)/dashboard/combos/ComboQuotaOnlyFallbackToggle.tsx new file mode 100644 index 0000000000..e1464f6ee6 --- /dev/null +++ b/src/app/(dashboard)/dashboard/combos/ComboQuotaOnlyFallbackToggle.tsx @@ -0,0 +1,60 @@ +import type { ComboStep } from "@/lib/combos/steps"; + +type Translate = (key: string, fallback: string) => string; + +type Props = { + strategy: string; + entry: ComboStep; + index: number; + hasPricing: boolean; + translate: Translate; + onCheckedChange: (index: number, enabled: boolean) => void; +}; + +export function ComboTargetOptions({ + strategy, + entry, + index, + hasPricing, + translate, + onCheckedChange, +}: Props) { + return ( + <> + {strategy === "cost-optimized" && ( + + {translate( + hasPricing ? "pricingAvailableShort" : "pricingMissingShort", + hasPricing ? "priced" : "no-price" + )} + + )} + {strategy === "priority" && (entry.kind === "model" || entry.kind === "combo-ref") && ( + + )} + + ); +} diff --git a/src/app/(dashboard)/dashboard/combos/comboQuotaOnlyFallback.ts b/src/app/(dashboard)/dashboard/combos/comboQuotaOnlyFallback.ts new file mode 100644 index 0000000000..8f879b3160 --- /dev/null +++ b/src/app/(dashboard)/dashboard/combos/comboQuotaOnlyFallback.ts @@ -0,0 +1,33 @@ +import type { ComboStep } from "@/lib/combos/steps"; + +export function setQuotaOnlyFallback( + entries: ComboStep[], + index: number, + enabled: boolean +): ComboStep[] { + const selected = entries[index]; + if (!selected || (selected.kind !== "model" && selected.kind !== "combo-ref")) return entries; + const next = [...entries]; + const { fallbackOnlyOnQuotaExhaustion: _removed, ...entry } = selected; + next[index] = enabled ? { ...entry, fallbackOnlyOnQuotaExhaustion: true } : entry; + return next; +} + +export function requiresExecuteMode(strategy: string, entries: ComboStep[]): boolean { + return ( + strategy === "priority" && + entries.some( + (entry) => entry.kind === "combo-ref" && entry.fallbackOnlyOnQuotaExhaustion === true + ) + ); +} + +export function applyQuotaOnlyFallbackConfig( + strategy: string, + entries: ComboStep[], + config: Record +): Record { + return requiresExecuteMode(strategy, entries) + ? { ...config, nestedComboMode: "execute" } + : config; +} diff --git a/src/app/(dashboard)/dashboard/combos/page.tsx b/src/app/(dashboard)/dashboard/combos/page.tsx index 93f77723f9..8d36151319 100644 --- a/src/app/(dashboard)/dashboard/combos/page.tsx +++ b/src/app/(dashboard)/dashboard/combos/page.tsx @@ -16,6 +16,8 @@ import Tooltip from "@/shared/components/Tooltip"; import { ComboCompressionModeSelect } from "@/shared/components/compression/ComboCompressionModeSelect"; import { useCopyToClipboard } from "@/shared/hooks/useCopyToClipboard"; import { FieldLabelWithHelp, WeightTotalBar } from "./parts"; +import { ComboTargetOptions } from "./ComboQuotaOnlyFallbackToggle"; +import { applyQuotaOnlyFallbackConfig, setQuotaOnlyFallback } from "./comboQuotaOnlyFallback"; import { useComboProxyAssignments } from "./useComboProxyAssignments"; import { ResponseValidationEditor, type ResponseValidationValue } from "./ResponseValidationEditor"; import ReasoningTokenBufferToggle from "./ReasoningTokenBufferToggle"; @@ -2793,7 +2795,11 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders, combo saveData.description = null; } - const configToSave = sanitizeComboRuntimeConfig(config); + const configToSave = applyQuotaOnlyFallbackConfig( + strategy, + models, + sanitizeComboRuntimeConfig(config) + ); if (strategy === "round-robin") { if (config.concurrencyPerModel !== undefined) configToSave.concurrencyPerModel = config.concurrencyPerModel; @@ -3477,24 +3483,16 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders, combo - {strategy === "cost-optimized" && ( - - {hasPricingForModel(entry.model) - ? getI18nOrFallback(t, "pricingAvailableShort", "priced") - : getI18nOrFallback(t, "pricingMissingShort", "no-price")} - - )} + + setModels(setQuotaOnlyFallback(models, stepIndex, enabled)) + } + hasPricing={hasPricingForModel(entry.model)} + translate={(key, fallback) => getI18nOrFallback(t, key, fallback)} + /> {/* Weight input (weighted mode only) */} {strategy === "weighted" && ( diff --git a/src/app/api/combos/[id]/route.ts b/src/app/api/combos/[id]/route.ts index a1ba03ee1b..869114486f 100644 --- a/src/app/api/combos/[id]/route.ts +++ b/src/app/api/combos/[id]/route.ts @@ -1,18 +1,13 @@ import { NextResponse } from "next/server"; -import { - getComboById, - updateCombo, - deleteCombo, - getComboByName, - getCombos, - isCloudEnabled, -} from "@/lib/localDb"; +import { getComboById, updateCombo, deleteCombo, getComboByName, getCombos } from "@/lib/db/combos"; +import { isCloudEnabled } from "@/lib/db/settings"; import { getConsistentMachineId } from "@/shared/utils/machineId"; import { syncToCloud } from "@/lib/cloudSync"; import { validateCompositeTiersConfig } from "@/lib/combos/compositeTiers"; import { normalizeComboModels } from "@/lib/combos/steps"; import { validateComboDAG, clampComboDepth } from "@omniroute/open-sse/services/combo.ts"; import { updateComboSchema } from "@/shared/validation/schemas"; +import { requiresQuotaOnlyComboRefExecute } from "@/shared/validation/schemas/combo"; import { isValidationFailure, validateBody } from "@/shared/validation/helpers"; import { requireManagementAuth } from "@/lib/api/requireManagementAuth"; import { QUOTA_MODEL_PREFIX } from "@/lib/quota/quotaModelNaming"; @@ -151,17 +146,17 @@ export async function PUT(request, { params }) { const normalizedUpdate = { ...validation.data }; if (normalizedUpdate.compressionOverride !== undefined) { const legacyCompressionOverride = normalizedUpdate.compressionOverride; - const nextConfig: Record = - currentCombo.config && - typeof currentCombo.config === "object" && - !Array.isArray(currentCombo.config) - ? { ...(currentCombo.config as Record) } - : {}; - if (legacyCompressionOverride) { - nextConfig.compressionMode = legacyCompressionOverride; - } else { - delete nextConfig.compressionMode; - } + const nextConfig: Record = + currentCombo.config && + typeof currentCombo.config === "object" && + !Array.isArray(currentCombo.config) + ? { ...(currentCombo.config as Record) } + : {}; + if (legacyCompressionOverride) { + nextConfig.compressionMode = legacyCompressionOverride; + } else { + delete nextConfig.compressionMode; + } normalizedUpdate.config = nextConfig; delete normalizedUpdate.compressionOverride; } @@ -187,6 +182,17 @@ export async function PUT(request, { params }) { ...body, name: comboName, }; + if (requiresQuotaOnlyComboRefExecute(nextComboState as never)) { + return comboErrorResponse( + "COMBO_002", + 400, + { + firstField: "config.nestedComboMode", + firstMessage: "Quota-only combo references require nestedComboMode execute", + }, + request + ); + } const compositeValidation = validateCompositeTiersConfig(nextComboState); if (compositeValidation.success === false) { const failure = compositeValidation as { @@ -235,12 +241,7 @@ export async function PUT(request, { params }) { : dagError instanceof Error && /depth/i.test(dagError.message) ? "max-depth-exceeded" : "invalid-graph"; - return comboErrorResponse( - "COMBO_005", - 400, - { comboName, reason }, - request - ); + return comboErrorResponse("COMBO_005", 400, { comboName, reason }, request); } } } @@ -253,9 +254,7 @@ export async function PUT(request, { params }) { // #8530: a combo renamed to a real model id is a supported pattern // (#6940 — bare-model-id provider fallback), so it is never rejected. // Surface it as a non-blocking warning instead of silently shadowing it. - const warning = comboName - ? buildComboNameCollisionWarning(String(comboName)) - : null; + const warning = comboName ? buildComboNameCollisionWarning(String(comboName)) : null; return NextResponse.json(warning ? { ...combo, warning } : combo); } catch (error) { if (error instanceof ComboInvariantError) { diff --git a/src/i18n/messages/en.json b/src/i18n/messages/en.json index e154bc29d7..8af252a53f 100644 --- a/src/i18n/messages/en.json +++ b/src/i18n/messages/en.json @@ -3248,6 +3248,8 @@ "pricingMissing": "No pricing", "pricingAvailableShort": "priced", "pricingMissingShort": "no-price", + "fallbackOnlyOnQuotaExhaustion": "Only advance on quota exhaustion", + "fallbackOnlyOnQuotaExhaustionShort": "quota-only fallback", "warningRoundRobinSingleModel": "Round-robin is most useful with at least 2 models.", "warningCostOptimizedPartialPricing": "Only {priced} of {total} models have pricing. Routing may be partially cost-aware.", "warningCostOptimizedNoPricing": "No pricing data found for this combo. Cost-optimized may route unexpectedly.", diff --git a/src/i18n/messages/zh-CN.json b/src/i18n/messages/zh-CN.json index d522262829..4666bb4183 100644 --- a/src/i18n/messages/zh-CN.json +++ b/src/i18n/messages/zh-CN.json @@ -3273,6 +3273,8 @@ "pricingMissing": "无定价", "pricingAvailableShort": "已定价", "pricingMissingShort": "无定价", + "fallbackOnlyOnQuotaExhaustion": "仅配额/额度耗尽时进入下一目标", + "fallbackOnlyOnQuotaExhaustionShort": "仅配额耗尽回退", "warningRoundRobinSingleModel": "轮询策略至少有 2 个模型时才最有价值。", "warningCostOptimizedPartialPricing": "在 {total} 个模型中,只有 {priced} 个有定价信息。路由可能只具备部分成本感知能力。", "warningCostOptimizedNoPricing": "该组合未找到任何定价数据。成本优化策略的路由结果可能不符合预期。", diff --git a/src/lib/combos/steps.ts b/src/lib/combos/steps.ts index d9f4ea7673..ed10e0ad76 100644 --- a/src/lib/combos/steps.ts +++ b/src/lib/combos/steps.ts @@ -25,6 +25,7 @@ export interface ComboModelStep { label?: string; prompt?: string | null; tags?: string[]; + fallbackOnlyOnQuotaExhaustion?: boolean; } export interface ComboRefStep { @@ -33,6 +34,7 @@ export interface ComboRefStep { comboName: string; weight: number; label?: string; + fallbackOnlyOnQuotaExhaustion?: boolean; } export interface ComboProviderWildcardStep { @@ -287,6 +289,7 @@ export function normalizeComboStep( const weight = toWeight(value.weight); const label = toTrimmedString(value.label); const prompt = toTrimmedString(value.prompt); + const fallbackOnlyOnQuotaExhaustion = value.fallbackOnlyOnQuotaExhaustion === true; if (value.kind === "combo-ref") { const comboRefName = toTrimmedString(value.comboName); @@ -297,6 +300,7 @@ export function normalizeComboStep( comboName: comboRefName, weight, ...(label ? { label } : {}), + ...(fallbackOnlyOnQuotaExhaustion ? { fallbackOnlyOnQuotaExhaustion: true } : {}), }; } @@ -409,6 +413,7 @@ export function normalizeComboStep( ...(prompt ? { prompt } : {}), ...(tags && tags.length > 0 ? { tags } : {}), ...(allowedConnectionIds && allowedConnectionIds.length > 0 ? { allowedConnectionIds } : {}), + ...(fallbackOnlyOnQuotaExhaustion ? { fallbackOnlyOnQuotaExhaustion: true } : {}), }; } diff --git a/src/shared/validation/schemas/combo.ts b/src/shared/validation/schemas/combo.ts index df9f9335fb..71d2a992de 100644 --- a/src/shared/validation/schemas/combo.ts +++ b/src/shared/validation/schemas/combo.ts @@ -20,6 +20,7 @@ export const comboStepMetaSchema = { id: z.string().trim().min(1).max(200).optional(), weight: z.number().min(0).max(100).optional().default(0), label: z.string().trim().min(1).max(200).optional(), + fallbackOnlyOnQuotaExhaustion: z.boolean().optional(), }; export const comboModelStepInputSchema = z.object({ @@ -284,29 +285,61 @@ export const comboNameSchema = z "Name can only contain letters, numbers, spaces, -, _, /, ., [ and ]." ); -export const createComboSchema = z.object({ - name: comboNameSchema, - description: z.string().max(2000).optional(), - models: z.array(comboModelEntry).optional().default([]), - strategy: comboStrategySchema.optional().default("priority"), - config: comboRuntimeConfigSchema.optional(), - allowedProviders: z.array(z.string().trim().min(1).max(200)).max(100).optional(), - allowedModelFamilies: z.array(z.string().trim().min(1).max(100)).max(100).optional(), - system_message: z.string().max(50000).optional(), - tool_filter_regex: z.string().max(1000).optional(), - context_cache_protection: z.boolean().optional(), - context_length: z.number().int().min(1000).max(2000000).optional(), - // Optional embedding dimensions override for embedding combos. - // When set, the value is injected into every upstream embedding request as - // the `dimensions` field (and translated to `outputDimensionality` for Gemini). - // Stored as a string to match the OpenAI API convention; coerced to number - // by the embedding handler. Leave unset to use each model's default. - dimensions: z - .string() - .regex(/^\d+$/, "dimensions must be a positive integer string") - .optional() - .nullable(), -}); +type QuotaOnlyComboRefState = { + models?: Array>; + strategy?: string; + config?: z.infer; +}; + +export function requiresQuotaOnlyComboRefExecute(value: QuotaOnlyComboRefState): boolean { + const hasProtectedComboRef = value.models?.some( + (step) => + typeof step === "object" && + step.kind === "combo-ref" && + step.fallbackOnlyOnQuotaExhaustion === true + ); + return ( + (value.strategy === undefined || value.strategy === "priority") && + hasProtectedComboRef === true && + value.config?.nestedComboMode !== "execute" + ); +} + +function validateQuotaOnlyComboRefs(value: QuotaOnlyComboRefState, ctx: z.RefinementCtx): void { + if (requiresQuotaOnlyComboRefExecute(value)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: "Quota-only combo references require nestedComboMode execute", + path: ["config", "nestedComboMode"], + }); + } +} + +export const createComboSchema = z + .object({ + name: comboNameSchema, + description: z.string().max(2000).optional(), + models: z.array(comboModelEntry).optional().default([]), + strategy: comboStrategySchema.optional().default("priority"), + config: comboRuntimeConfigSchema.optional(), + allowedProviders: z.array(z.string().trim().min(1).max(200)).max(100).optional(), + allowedModelFamilies: z.array(z.string().trim().min(1).max(100)).max(100).optional(), + system_message: z.string().max(50000).optional(), + tool_filter_regex: z.string().max(1000).optional(), + context_cache_protection: z.boolean().optional(), + context_length: z.number().int().min(1000).max(2000000).optional(), + // Optional embedding dimensions override for embedding combos. + // When set, the value is injected into every upstream embedding request as + // the `dimensions` field (and translated to `outputDimensionality` for Gemini). + // Stored as a string to match the OpenAI API convention; coerced to number + // by the embedding handler. Leave unset to use each model's default. + dimensions: z + .string() + .regex(/^\d+$/, "dimensions must be a positive integer string") + .optional() + .nullable(), + }) + .superRefine(validateQuotaOnlyComboRefs); export const updateComboDefaultsSchema = z .object({ diff --git a/tests/unit/chat-route-coverage.test.ts b/tests/unit/chat-route-coverage.test.ts index e249e5247a..b7174d6358 100644 --- a/tests/unit/chat-route-coverage.test.ts +++ b/tests/unit/chat-route-coverage.test.ts @@ -233,34 +233,48 @@ test("handleChat applies task-aware routing when a semantic override is enabled" assert.equal(json.choices[0].message.content, "Task-routed response"); }); -test("handleChat routes exact combo names and can recover via global fallback", async () => { +test("handleChat keeps protected combo fallback separate from Global Fallback Model", async () => { await seedConnection("openai", { apiKey: "sk-openai-combo-route" }); + const ordinaryTarget = await seedConnection("deepseek", { + apiKey: "sk-deepseek-combo-backup", + }); await seedConnection("claude", { apiKey: "sk-claude-global-fallback" }); await combosDb.createCombo({ name: "router-global-fallback", strategy: "priority", config: { maxRetries: 0, retryDelayMs: 0 }, - models: ["openai/gpt-4.1"], + models: [ + { + kind: "model", + model: "openai/gpt-4.1", + fallbackOnlyOnQuotaExhaustion: true, + }, + { + kind: "model", + model: "deepseek/deepseek-v4-flash", + connectionId: ordinaryTarget.id, + }, + ], }); await settingsDb.updateSettings({ globalFallbackModel: "claude/claude-3-5-sonnet-20241022", }); - let attempts = 0; - globalThis.fetch = async (_url, init = {}) => { - attempts += 1; - const headers = toPlainHeaders(init.headers); - if (attempts === 1) { - assert.equal(headers.Authorization ?? headers.authorization, "Bearer sk-openai-combo-route"); + const attemptedKeys: string[] = []; + globalThis.fetch = async (_url, init) => { + const headers = toPlainHeaders(init?.headers); + const key = headers["x-api-key"] ?? headers.Authorization ?? headers.authorization ?? ""; + attemptedKeys.push(key); + if (key === "sk-deepseek-combo-backup") { + assert.fail("protected primary must not invoke the ordinary combo target"); + } + if (key === "Bearer sk-openai-combo-route") { return new Response(JSON.stringify({ error: { message: "primary combo failed" } }), { status: 503, headers: { "Content-Type": "application/json" }, }); } - assert.equal( - headers["x-api-key"] ?? headers.Authorization ?? headers.authorization, - "sk-claude-global-fallback" - ); + assert.equal(key, "sk-claude-global-fallback"); return buildClaudeResponse("Global fallback answered"); }; @@ -276,7 +290,7 @@ test("handleChat routes exact combo names and can recover via global fallback", const json = (await response.json()) as any; assert.equal(response.status, 200); - assert.equal(attempts, 2); + assert.deepEqual(attemptedKeys, ["Bearer sk-openai-combo-route", "sk-claude-global-fallback"]); assert.equal(json.choices[0].message.content, "Global fallback answered"); }); diff --git a/tests/unit/combo-dispatch-prelude.test.ts b/tests/unit/combo-dispatch-prelude.test.ts index 688abc8b26..87d35ef25d 100644 --- a/tests/unit/combo-dispatch-prelude.test.ts +++ b/tests/unit/combo-dispatch-prelude.test.ts @@ -606,6 +606,82 @@ test("tryRuntimeUnitDispatch: round-robin rotates across successive dispatches", ); }); +test("tryRuntimeUnitDispatch: sticky weighted keeps quota-only fallback dormant", async () => { + const combo = { + ...twoRefCombo("weighted-quota-dormant", "weighted", { stickyWeightedLimit: 5 }), + models: [ + { kind: "combo-ref", comboName: "leafA", fallbackOnlyOnQuotaExhaustion: true }, + { kind: "combo-ref", comboName: "leafB" }, + ], + }; + const ctx = setup(combo); + const units = resolveComboRuntimeUnits(ctx.combo, [ctx.combo, LEAF_A, LEAF_B], "execute", 3); + const protectedUnit = units.find( + (unit) => unit.kind === "combo-ref" && unit.fallbackOnlyOnQuotaExhaustion === true + ); + assert.ok(protectedUnit); + recordStickyWeightedSuccess(combo.name, protectedUnit.executionKey, 5); + + const recursedInto: string[] = []; + const result = await tryRuntimeUnitDispatch({ + body: ctx.body, + combo: ctx.combo, + config: ctx.config, + strategy: "weighted", + allCombos: [ctx.combo, LEAF_A, LEAF_B], + handleSingleModel: async () => okResponse("raw"), + handleSingleModelWithTimeout: async () => okResponse("wrapped"), + log: ctx.log, + settings: {}, + runCombo: async (options) => { + recursedInto.push(options.combo.name); + return options.combo.name === "leafA" + ? new Response("unavailable", { status: 503 }) + : okResponse("backup"); + }, + }); + assert.equal(result?.status, 200); + assert.deepEqual(recursedInto, [protectedUnit.comboName, protectedUnit.comboName, "leafB"]); +}); + +test("tryRuntimeUnitDispatch: protected execute response validation returns local 502", async () => { + const combo = { + name: "protected-quality-execute", + strategy: "priority", + models: [ + { kind: "model", model: "p/invalid", fallbackOnlyOnQuotaExhaustion: true }, + COMBO_REF_STEP, + ], + config: { + nestedComboMode: "execute", + maxRetries: 0, + responseValidation: { minContentLength: 100 }, + }, + }; + const ctx = setup(combo); + const calls: string[] = []; + const result = await tryRuntimeUnitDispatch({ + body: ctx.body, + combo: ctx.combo, + config: ctx.config, + strategy: "priority", + allCombos: [ctx.combo, LEAF_COMBO], + handleSingleModel: async () => okResponse("raw"), + handleSingleModelWithTimeout: async (_body, modelStr) => { + calls.push(modelStr); + return okResponse("short"); + }, + log: ctx.log, + settings: {}, + runCombo: async () => { + calls.push("backup"); + return okResponse("backup"); + }, + }); + assert.equal(result?.status, 502); + assert.deepEqual(calls, ["p/invalid"]); +}); + test("tryRuntimeUnitDispatch: weighted honors a previously recorded sticky unit", async () => { const combo = twoRefCombo("weighted-sticky", "weighted", { stickyWeightedLimit: 5 }); const ctx = setup(combo); diff --git a/tests/unit/combo-quota-exhaustion-only-fallback.test.ts b/tests/unit/combo-quota-exhaustion-only-fallback.test.ts new file mode 100644 index 0000000000..eb4a7baaff --- /dev/null +++ b/tests/unit/combo-quota-exhaustion-only-fallback.test.ts @@ -0,0 +1,696 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-quota-only-")); +const ORIGINAL_DATA_DIR = process.env.DATA_DIR; +process.env.DATA_DIR = TEST_DATA_DIR; +const { handleComboChat } = await import("../../open-sse/services/combo.ts"); +const { isQuotaExhaustionResponse } = + await import("../../open-sse/services/combo/quotaExhaustion.ts"); +const { clearAllModelLockouts, lockModel } = + await import("../../open-sse/services/accountFallback.ts"); +const { recordComboRequest, resetComboMetrics } = + await import("../../open-sse/services/comboMetrics.ts"); +const { resetDbInstance } = await import("../../src/lib/db/core.ts"); +const { resetAllCircuitBreakers } = await import("../../src/shared/utils/circuitBreaker.ts"); + +test.afterEach(() => { + clearAllModelLockouts(); + resetAllCircuitBreakers(); +}); + +test.after(() => { + resetDbInstance(); + if (ORIGINAL_DATA_DIR === undefined) delete process.env.DATA_DIR; + else process.env.DATA_DIR = ORIGINAL_DATA_DIR; + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +function log() { + return { info() {}, warn() {}, debug() {}, error() {} }; +} + +function response( + status: number, + message: string, + options: { code?: string; type?: string; headers?: Record } = {} +) { + return Response.json( + { + error: { + message, + ...(options.code ? { code: options.code } : {}), + ...(options.type ? { type: options.type } : {}), + }, + }, + { status, headers: options.headers } + ); +} + +function ok(model: string) { + return Response.json({ model }); +} + +function model(model: string, fallbackOnlyOnQuotaExhaustion?: boolean) { + return { + kind: "model" as const, + model, + providerId: model.split("/")[0], + ...(fallbackOnlyOnQuotaExhaustion === undefined ? {} : { fallbackOnlyOnQuotaExhaustion }), + }; +} + +async function run( + first: Response | Response[], + options: { + enabled?: boolean; + maxRetries?: number; + strategy?: string; + primary?: string; + connectionId?: string; + config?: Record; + settings?: Record | null; + available?: (modelStr: string) => Promise | boolean; + } = {} +) { + const calls: string[] = []; + const primaryResponses = Array.isArray(first) ? first : [first]; + let primaryAttempt = 0; + const combo = { + name: `quota-only-${Math.random()}`, + strategy: options.strategy ?? "priority", + models: [ + { + ...model(options.primary ?? "openai/primary", options.enabled), + ...(options.connectionId ? { connectionId: options.connectionId } : {}), + }, + model("anthropic/backup"), + ], + config: { maxRetries: options.maxRetries ?? 0, retryDelayMs: 0, ...options.config }, + }; + const result = await handleComboChat({ + body: {}, + combo, + allCombos: [combo], + settings: options.settings ?? null, + log: log(), + isModelAvailable: options.available ?? (async () => true), + handleSingleModel: async (_body, modelStr) => { + calls.push(modelStr); + if (modelStr !== (options.primary ?? "openai/primary")) return ok(modelStr); + const result = primaryResponses[Math.min(primaryAttempt, primaryResponses.length - 1)]; + primaryAttempt += 1; + return result.clone(); + }, + }); + return { result, calls }; +} + +test("quota classifier rejects terminal-looking evidence on ineligible statuses", async () => { + for (const status of [400, 401, 403, 404, 408, 409, 422, 500, 502, 503, 504]) { + for (const terminal of ["insufficient_quota", "quota_exhausted", "credits_exhausted"]) { + assert.equal( + await isQuotaExhaustionResponse( + response(status, `${terminal}: billing hard limit reached`, { + code: terminal, + type: terminal, + }), + "openai", + "gpt-4o" + ), + false, + `${status} with ${terminal} must not classify as terminal depletion` + ); + } + } +}); + +test("quota classifier accepts explicit terminal depletion only on eligible statuses", async () => { + for (const status of [402, 429]) { + assert.equal( + await isQuotaExhaustionResponse( + response(status, "Payment required", { code: "insufficient_quota" }), + "openai", + "gpt-4o" + ), + true + ); + } +}); + +test("priority target advances on explicit quota exhaustion", async () => { + const { result, calls } = await run(response(429, "You have exceeded your current quota"), { + enabled: true, + }); + assert.equal(result.ok, true); + assert.deepEqual(calls, ["openai/primary", "anthropic/backup"]); +}); + +test("priority target retries plain 429 then returns the last response without advancing", async () => { + const { result, calls } = await run(response(429, "Rate limit exceeded; retry later"), { + enabled: true, + maxRetries: 1, + }); + assert.equal(result.status, 429); + assert.deepEqual(calls, ["openai/primary", "openai/primary"]); +}); + +test("priority target keeps a non-quota retry monotonic through terminal quota exhaustion", async () => { + const { result, calls } = await run( + [ + response(503, "Service unavailable"), + response(429, "Payment required", { code: "insufficient_quota" }), + ], + { enabled: true, maxRetries: 1 } + ); + assert.equal(result.status, 429); + assert.deepEqual(calls, ["openai/primary", "openai/primary"]); +}); + +test("priority target returns a later successful retry", async () => { + const { result, calls } = await run( + [response(503, "Service unavailable"), ok("openai/primary")], + { enabled: true, maxRetries: 1 } + ); + assert.equal(result.ok, true); + assert.deepEqual(calls, ["openai/primary", "openai/primary"]); +}); + +test("priority target advances when every retry is quota exhaustion", async () => { + const { result, calls } = await run( + [ + response(429, "Payment required", { code: "quota_exhausted" }), + response(429, "Billing hard limit reached", { code: "quota_exhausted" }), + ], + { enabled: true, maxRetries: 1 } + ); + assert.equal(result.ok, true); + assert.deepEqual(calls, ["openai/primary", "openai/primary", "anthropic/backup"]); +}); + +test("failures of another target do not contaminate protected target trust", async () => { + const calls: string[] = []; + const combo = { + name: `target-local-${Math.random()}`, + strategy: "priority", + models: [model("openai/unprotected"), model("anthropic/protected", true), model("paid/backup")], + config: { maxRetries: 0, retryDelayMs: 0 }, + }; + const result = await handleComboChat({ + body: {}, + combo, + allCombos: [combo], + settings: null, + log: log(), + isModelAvailable: async () => true, + handleSingleModel: async (_body, modelStr) => { + calls.push(modelStr); + return modelStr === "openai/unprotected" + ? response(503, "Service unavailable") + : modelStr === "anthropic/protected" + ? response(429, "Payment required", { code: "quota_exhausted" }) + : ok(modelStr); + }, + }); + assert.equal(result.ok, true); + assert.deepEqual(calls, ["openai/unprotected", "anthropic/protected", "paid/backup"]); +}); + +for (const [status, message] of [ + [408, "Request timeout"], + [502, "Bad gateway"], + [503, "Service unavailable"], + [503, "Local concurrency capacity reached"], + [504, "Gateway timeout"], + [401, "Invalid API key"], +] as const) { + test(`priority target does not advance on non-quota ${status} (${message})`, async () => { + const { result, calls } = await run(response(status, message), { enabled: true }); + assert.equal(result.status, status); + assert.deepEqual(calls, ["openai/primary"]); + }); +} + +test("protected non-quota response preserves status, body, and headers", async () => { + const first = response(502, "upstream detail", { headers: { "x-upstream": "kept" } }); + const { result } = await run(first, { enabled: true }); + assert.equal(result.status, 502); + assert.equal(result.headers.get("x-upstream"), "kept"); + assert.deepEqual(await result.json(), { error: { message: "upstream detail" } }); +}); + +for (const [metric, advances] of [ + ["generate_content_free_tier_requests", false], + ["generate_content_free_tier_input_token_count", false], + ["generate_content_free_tier_requests_per_day", true], +] as const) { + test(`Gemini ${metric} classification ${advances ? "advances" : "stops"} direct routing`, async () => { + const message = `You exceeded your current quota for metric generativelanguage.googleapis.com/${metric}`; + const { result, calls } = await run(response(429, message), { + enabled: true, + primary: "gemini/gemini-2.5-flash", + }); + assert.equal(result.ok, advances); + assert.deepEqual( + calls, + advances ? ["gemini/gemini-2.5-flash", "anthropic/backup"] : ["gemini/gemini-2.5-flash"] + ); + }); +} + +test("compatible provider advances only on explicit structured terminal depletion", async () => { + const plain = await run(response(402, "billing service unavailable"), { + enabled: true, + primary: "openai-compatible-chat-test/primary", + }); + assert.equal(plain.result.status, 402); + assert.deepEqual(plain.calls, ["openai-compatible-chat-test/primary"]); + + const terminal = await run(response(402, "Payment required", { code: "insufficient_quota" }), { + enabled: true, + primary: "openai-compatible-chat-test/primary", + }); + assert.equal(terminal.result.ok, true); + assert.deepEqual(terminal.calls, ["openai-compatible-chat-test/primary", "anthropic/backup"]); +}); + +test("unset Gemini RPM response retains baseline fallback", async () => { + const { result, calls } = await run( + response( + 429, + "You exceeded your current quota for metric generativelanguage.googleapis.com/generate_content_free_tier_requests" + ), + { primary: "gemini/gemini-2.5-flash" } + ); + assert.equal(result.ok, true); + assert.deepEqual(calls, ["gemini/gemini-2.5-flash", "anthropic/backup"]); +}); + +test("unset target retains generic fallback", async () => { + const { result, calls } = await run(response(503, "Service unavailable")); + assert.equal(result.ok, true); + assert.deepEqual(calls, ["openai/primary", "anthropic/backup"]); +}); + +test("option is dormant outside priority", async () => { + const { result, calls } = await run(response(503, "Service unavailable"), { + enabled: true, + strategy: "fill-first", + }); + assert.equal(result.ok, true); + assert.deepEqual(calls, ["openai/primary", "anthropic/backup"]); +}); + +test("authoritative local quota cutoff advances an opted-in priority target", async () => { + const calls: string[] = []; + const combo = { + name: `local-quota-${Math.random()}`, + strategy: "priority", + models: [ + { ...model("openai/primary", true), connectionId: "quota-empty" }, + model("anthropic/backup"), + ], + config: { maxRetries: 0, retryDelayMs: 0 }, + }; + const quotaPreflight = await import("../../open-sse/services/quotaPreflight.ts"); + quotaPreflight.registerQuotaFetcher("openai", async () => ({ + used: 100, + total: 100, + percentUsed: 1, + })); + const result = await handleComboChat({ + body: {}, + combo, + allCombos: [combo], + log: log(), + settings: { + resilienceSettings: { + quotaPreflight: { enabled: true, defaultThresholdPercent: 2, warnThresholdPercent: 20 }, + }, + }, + isModelAvailable: async () => true, + handleSingleModel: async (_body, modelStr) => { + calls.push(modelStr); + return ok(modelStr); + }, + }); + assert.equal(result.ok, true); + assert.deepEqual(calls, ["anthropic/backup"]); +}); + +test("protected pre-existing model lockout stops routing with a local 503", async () => { + lockModel("openai", "locked-connection", "primary", "rate_limited", 60_000); + const { result, calls } = await run(response(503, "unused"), { + enabled: true, + connectionId: "locked-connection", + }); + assert.equal(result.status, 503); + assert.deepEqual(calls, []); +}); + +test("protected target keeps configured retries after recording a transient model lockout", async () => { + const { result, calls } = await run(response(503, "Service unavailable"), { + enabled: true, + maxRetries: 1, + connectionId: "retry-connection", + settings: { + resilienceSettings: { + modelLockout: { enabled: true, errorCodes: [503], baseCooldownMs: 60_000 }, + }, + }, + }); + assert.equal(result.status, 503); + assert.deepEqual(calls, ["openai/primary", "openai/primary"]); +}); + +test("protected response-quality rejection returns a sanitized 502 without advancing", async () => { + const { result, calls } = await run(ok("openai/primary"), { + enabled: true, + config: { responseValidation: { minContentLength: 100 } }, + }); + assert.equal(result.status, 502); + assert.deepEqual(await result.json(), { + error: { + message: "Upstream response failed quality validation", + type: "server_error", + code: "bad_gateway", + }, + }); + assert.deepEqual(calls, ["openai/primary"]); +}); + +test("protected predictive TTFT skip returns local 503 without advancing", async () => { + const name = `predictive-${Math.random()}`; + resetComboMetrics(name); + for (let index = 0; index < 5; index += 1) { + recordComboRequest(name, "openai/primary", { success: true, latencyMs: 10_000 }); + } + const calls: string[] = []; + const combo = { + name, + strategy: "priority", + models: [model("openai/primary", true), model("anthropic/backup")], + config: { maxRetries: 0, zeroLatencyOptimizationsEnabled: true, predictiveTtftMs: 1 }, + }; + const result = await handleComboChat({ + body: {}, + combo, + allCombos: [combo], + log: log(), + settings: null, + isModelAvailable: async () => true, + handleSingleModel: async (_body, modelStr) => { + calls.push(modelStr); + return ok(modelStr); + }, + }); + assert.equal(result.status, 503); + assert.deepEqual(calls, []); +}); + +test("opted-in combo ref remains a black box and only quota exhaustion advances parent", async () => { + for (const childError of [ + response(429, "Billing hard limit reached"), + response(503, "Service unavailable"), + ]) { + const calls: string[] = []; + const child = { + name: `child-${childError.status}-${Math.random()}`, + strategy: "priority", + models: [model("openai/child")], + config: { maxRetries: 0, retryDelayMs: 0 }, + }; + const outer = { + name: `outer-${Math.random()}`, + strategy: "priority", + models: [ + { + kind: "combo-ref" as const, + comboName: child.name, + fallbackOnlyOnQuotaExhaustion: true, + }, + model("anthropic/backup"), + ], + config: { nestedComboMode: "execute", maxRetries: 0, retryDelayMs: 0 }, + }; + const result = await handleComboChat({ + body: {}, + combo: outer, + allCombos: [outer, child], + settings: null, + log: log(), + isModelAvailable: async () => true, + handleSingleModel: async (_body, modelStr) => { + calls.push(modelStr); + return modelStr === "openai/child" ? childError.clone() : ok(modelStr); + }, + }); + if (childError.status === 429) { + assert.equal(result.ok, true); + assert.deepEqual(calls, ["openai/child", "anthropic/backup"]); + } else { + assert.equal(result.status, 503); + assert.deepEqual(calls, ["openai/child"]); + } + } +}); + +test("protected parent combo-ref stops after normal child quality rejection then quota exhaustion", async () => { + const calls: string[] = []; + const child = { + name: `quality-child-${Math.random()}`, + strategy: "priority", + models: [model("openai/quality-invalid"), model("anthropic/quota")], + config: { + maxRetries: 0, + retryDelayMs: 0, + flattenSingleTargetNested: true, + responseValidation: { minContentLength: 100 }, + }, + }; + const outer = { + name: `quality-outer-${Math.random()}`, + strategy: "priority", + models: [ + { kind: "combo-ref" as const, comboName: child.name, fallbackOnlyOnQuotaExhaustion: true }, + model("paid/backup"), + ], + config: { nestedComboMode: "execute", maxRetries: 0, retryDelayMs: 0 }, + }; + const result = await handleComboChat({ + body: {}, + combo: outer, + allCombos: [outer, child], + settings: null, + log: log(), + isModelAvailable: async () => true, + handleSingleModel: async (_body, modelStr) => { + calls.push(modelStr); + return modelStr === "openai/quality-invalid" + ? Response.json({ choices: [{ message: {} }] }) + : modelStr === "anthropic/quota" + ? response(429, "Payment required", { code: "insufficient_quota" }) + : ok(modelStr); + }, + }); + assert.equal(result.status, 429); + assert.deepEqual(calls, ["openai/quality-invalid", "anthropic/quota"]); +}); + +test("protected parent combo-ref stops when a child has mixed quota and non-quota failures", async () => { + const calls: string[] = []; + const child = { + name: `mixed-child-${Math.random()}`, + strategy: "priority", + models: [model("openai/quota"), model("anthropic/unavailable")], + config: { maxRetries: 0, retryDelayMs: 0 }, + }; + const outer = { + name: `mixed-outer-${Math.random()}`, + strategy: "priority", + models: [ + { kind: "combo-ref" as const, comboName: child.name, fallbackOnlyOnQuotaExhaustion: true }, + model("anthropic/paid-backup"), + ], + config: { nestedComboMode: "execute", maxRetries: 0, retryDelayMs: 0 }, + }; + const result = await handleComboChat({ + body: {}, + combo: outer, + allCombos: [outer, child], + settings: null, + log: log(), + isModelAvailable: async (modelStr) => modelStr !== "anthropic/unavailable", + handleSingleModel: async (_body, modelStr) => { + calls.push(modelStr); + return modelStr === "openai/quota" + ? response(429, "Billing hard limit reached") + : ok(modelStr); + }, + }); + assert.equal(result.status, 429); + assert.deepEqual(calls, ["openai/quota"]); +}); + +test("protected parent combo-ref keeps a child's non-quota target retry monotonic", async () => { + const calls: string[] = []; + const child = { + name: `target-retry-child-${Math.random()}`, + strategy: "priority", + models: [model("retry-provider/child")], + config: { maxRetries: 1, retryDelayMs: 0 }, + }; + const outer = { + name: `target-retry-outer-${Math.random()}`, + strategy: "priority", + models: [ + { kind: "combo-ref" as const, comboName: child.name, fallbackOnlyOnQuotaExhaustion: true }, + model("anthropic/paid-backup"), + ], + config: { nestedComboMode: "execute", maxRetries: 0, retryDelayMs: 0 }, + }; + const result = await handleComboChat({ + body: {}, + combo: outer, + allCombos: [outer, child], + settings: null, + log: log(), + isModelAvailable: async () => true, + handleSingleModel: async (_body, modelStr) => { + calls.push(modelStr); + return calls.length === 1 + ? response(503, "Service unavailable") + : response(429, "Payment required", { code: "insufficient_quota" }); + }, + }); + assert.equal(result.status, 429); + assert.deepEqual(calls, ["retry-provider/child", "retry-provider/child"]); +}); + +test("protected parent combo-ref keeps a child's non-quota set failure monotonic", async () => { + const calls: string[] = []; + const child = { + name: `set-retry-child-${Math.random()}`, + strategy: "priority", + models: [model("openai/child")], + config: { maxRetries: 0, retryDelayMs: 0, maxSetRetries: 1, setRetryDelayMs: 0 }, + }; + const outer = { + name: `set-retry-outer-${Math.random()}`, + strategy: "priority", + models: [ + { kind: "combo-ref" as const, comboName: child.name, fallbackOnlyOnQuotaExhaustion: true }, + model("anthropic/paid-backup"), + ], + config: { nestedComboMode: "execute", maxRetries: 0, retryDelayMs: 0 }, + }; + const result = await handleComboChat({ + body: {}, + combo: outer, + allCombos: [outer, child], + settings: null, + log: log(), + isModelAvailable: async () => true, + handleSingleModel: async (_body, modelStr) => { + calls.push(modelStr); + return calls.length === 1 + ? response(503, "Service unavailable") + : response(429, "Payment required", { code: "insufficient_quota" }); + }, + }); + assert.equal(result.status, 429); + assert.deepEqual(calls, ["openai/child", "openai/child"]); +}); + +test("protected parent combo-ref advances on a single authoritative child quota cutoff", async () => { + const calls: string[] = []; + const child = { + name: `cutoff-child-${Math.random()}`, + strategy: "priority", + models: [{ ...model("openai/cutoff"), connectionId: "nested-quota-empty" }], + config: { maxRetries: 0, retryDelayMs: 0 }, + }; + const outer = { + name: `cutoff-outer-${Math.random()}`, + strategy: "priority", + models: [ + { kind: "combo-ref" as const, comboName: child.name, fallbackOnlyOnQuotaExhaustion: true }, + model("anthropic/paid-backup"), + ], + config: { nestedComboMode: "execute", maxRetries: 0, retryDelayMs: 0 }, + }; + const quotaPreflight = await import("../../open-sse/services/quotaPreflight.ts"); + quotaPreflight.registerQuotaFetcher("openai", async () => ({ + used: 100, + total: 100, + percentUsed: 1, + })); + const result = await handleComboChat({ + body: {}, + combo: outer, + allCombos: [outer, child], + settings: { + resilienceSettings: { + quotaPreflight: { enabled: true, defaultThresholdPercent: 2, warnThresholdPercent: 20 }, + }, + }, + log: log(), + isModelAvailable: async () => true, + handleSingleModel: async (_body, modelStr) => { + calls.push(modelStr); + return ok(modelStr); + }, + }); + assert.equal(result.ok, true); + assert.deepEqual(calls, ["anthropic/paid-backup"]); +}); + +for (const [metric, advances] of [ + ["generate_content_free_tier_requests", false], + ["generate_content_free_tier_input_token_count", false], + ["generate_content_free_tier_requests_per_day", true], +] as const) { + test(`nested Gemini ${metric} ${advances ? "advances" : "stops"} parent routing`, async () => { + const calls: string[] = []; + const child = { + name: `gemini-child-${Math.random()}`, + strategy: "priority", + models: [model("gemini/gemini-2.5-flash")], + config: { maxRetries: 0, retryDelayMs: 0 }, + }; + const outer = { + name: `gemini-outer-${Math.random()}`, + strategy: "priority", + models: [ + { kind: "combo-ref" as const, comboName: child.name, fallbackOnlyOnQuotaExhaustion: true }, + model("anthropic/backup"), + ], + config: { nestedComboMode: "execute", maxRetries: 0, retryDelayMs: 0 }, + }; + const result = await handleComboChat({ + body: {}, + combo: outer, + allCombos: [outer, child], + settings: null, + log: log(), + isModelAvailable: async () => true, + handleSingleModel: async (_body, modelStr) => { + calls.push(modelStr); + return modelStr.startsWith("gemini/") + ? response( + 429, + `You exceeded your current quota for metric generativelanguage.googleapis.com/${metric}` + ) + : ok(modelStr); + }, + }); + assert.equal(result.ok, advances); + assert.deepEqual( + calls, + advances ? ["gemini/gemini-2.5-flash", "anthropic/backup"] : ["gemini/gemini-2.5-flash"] + ); + }); +} diff --git a/tests/unit/combo-quota-exhaustion-option-schema.test.ts b/tests/unit/combo-quota-exhaustion-option-schema.test.ts new file mode 100644 index 0000000000..f2a4cb8ca4 --- /dev/null +++ b/tests/unit/combo-quota-exhaustion-option-schema.test.ts @@ -0,0 +1,113 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { + createComboSchema, + requiresQuotaOnlyComboRefExecute, + updateComboSchema, +} from "../../src/shared/validation/schemas/combo.ts"; +import { normalizeComboStep } from "../../src/lib/combos/steps.ts"; + +const modelStep = { + kind: "model" as const, + model: "openai/gpt-4o", + fallbackOnlyOnQuotaExhaustion: true, +}; +const comboRefStep = { + kind: "combo-ref" as const, + comboName: "child", + fallbackOnlyOnQuotaExhaustion: true, +}; + +test("create/update accept and preserve strict quota-only booleans on model and combo-ref steps", () => { + const payload = { + name: "quota-only", + strategy: "priority" as const, + models: [modelStep, comboRefStep], + config: { nestedComboMode: "execute" as const }, + }; + const created = createComboSchema.parse(payload); + assert.equal(created.models[0].fallbackOnlyOnQuotaExhaustion, true); + assert.equal(created.models[1].fallbackOnlyOnQuotaExhaustion, true); + + const updated = updateComboSchema.parse(payload); + assert.equal(updated.models?.[0].fallbackOnlyOnQuotaExhaustion, true); + assert.equal(updated.models?.[1].fallbackOnlyOnQuotaExhaustion, true); + + for (const bad of ["true", 1, null]) { + assert.equal( + createComboSchema.safeParse({ + ...payload, + models: [{ ...modelStep, fallbackOnlyOnQuotaExhaustion: bad }], + }).success, + false + ); + } +}); + +test("non-priority combos preserve the dormant option", () => { + const parsed = createComboSchema.parse({ + name: "dormant", + strategy: "weighted", + models: [modelStep], + }); + assert.equal(parsed.models[0].fallbackOnlyOnQuotaExhaustion, true); + assert.equal(normalizeComboStep(parsed.models[0])?.fallbackOnlyOnQuotaExhaustion, true); +}); + +test("only active priority combo refs require execute mode", () => { + assert.equal( + createComboSchema.safeParse({ name: "invalid-ref", models: [comboRefStep] }).success, + false + ); + assert.equal( + createComboSchema.safeParse({ + name: "valid-ref", + models: [comboRefStep], + config: { nestedComboMode: "execute" }, + }).success, + true + ); + assert.equal( + createComboSchema.safeParse({ + name: "dormant-ref", + strategy: "weighted", + models: [comboRefStep], + }).success, + true + ); + assert.equal( + createComboSchema.safeParse({ + name: "ordinary-ref", + models: [{ kind: "combo-ref", comboName: "child" }], + }).success, + true + ); +}); + +test("partial updates defer active combo-ref validation to the merged route state", () => { + assert.equal(updateComboSchema.safeParse({ models: [comboRefStep] }).success, true); + assert.equal(updateComboSchema.safeParse({ strategy: "priority" }).success, true); + assert.equal( + requiresQuotaOnlyComboRefExecute({ + strategy: "priority", + models: [comboRefStep], + config: {}, + }), + true + ); + assert.equal( + requiresQuotaOnlyComboRefExecute({ + strategy: "weighted", + models: [comboRefStep], + config: {}, + }), + false + ); +}); + +test("normalization omits false and preserves true", () => { + const disabled = normalizeComboStep({ ...modelStep, fallbackOnlyOnQuotaExhaustion: false }); + const enabled = normalizeComboStep(modelStep); + assert.equal("fallbackOnlyOnQuotaExhaustion" in (disabled || {}), false); + assert.equal(enabled?.fallbackOnlyOnQuotaExhaustion, true); +}); diff --git a/tests/unit/combo-runtime-unit-concurrency.test.ts b/tests/unit/combo-runtime-unit-concurrency.test.ts index d496bb7c2a..36a14a59ce 100644 --- a/tests/unit/combo-runtime-unit-concurrency.test.ts +++ b/tests/unit/combo-runtime-unit-concurrency.test.ts @@ -10,20 +10,28 @@ import assert from "node:assert/strict"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { - acquire, - buildAccountSemaphoreKey, - resetAll, -} from "../../open-sse/services/accountSemaphore.ts"; -import { isRuntimeUnitAtConcurrencyCap } from "../../open-sse/services/combo/runtimeUnitCapacity.ts"; -import type { ResolvedComboUnit } from "../../open-sse/services/combo/types.ts"; const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-runtime-unit-cap-")); const ORIGINAL_DATA_DIR = process.env.DATA_DIR; process.env.DATA_DIR = TEST_DATA_DIR; +const { acquire, buildAccountSemaphoreKey, resetAll } = + await import("../../open-sse/services/accountSemaphore.ts"); +const { isRuntimeUnitAtConcurrencyCap } = + await import("../../open-sse/services/combo/runtimeUnitCapacity.ts"); const { handleComboChat } = await import("../../open-sse/services/combo.ts"); const { getDbInstance, resetDbInstance } = await import("../../src/lib/db/core.ts"); +type ResolvedComboUnit = import("../../open-sse/services/combo/types.ts").ResolvedComboUnit; + +const db = getDbInstance(); +const databases = db.pragma("database_list") as Array<{ file?: string; name?: string }>; +const activeDbPath = databases.find((database) => database.name === "main")?.file; +assert.ok(activeDbPath, "test requires a file-backed main SQLite database"); +assert.equal( + path.dirname(path.resolve(activeDbPath)), + path.resolve(TEST_DATA_DIR), + `active test database must be under TEST_DATA_DIR before inserts: ${activeDbPath}` +); function createLog() { return { @@ -44,7 +52,7 @@ function okResponse() { function seedConnection(id: string, provider: string, maxConcurrent: number) { const db = getDbInstance(); db.prepare( - `INSERT INTO provider_connections + `INSERT OR REPLACE INTO provider_connections (id, provider, auth_type, is_active, max_concurrent, created_at, updated_at) VALUES (?, ?, 'apikey', 1, ?, datetime('now'), datetime('now'))` ).run(id, provider, maxConcurrent); @@ -62,7 +70,7 @@ test("isRuntimeUnitAtConcurrencyCap returns true for a model unit at cap", async const connectionId = "conn-feather-cap"; const provider = "featherless-ai"; const key = buildAccountSemaphoreKey({ provider, accountKey: connectionId }); - const release1 = await acquire(key, { maxConcurrency: 2 }); + let release1 = await acquire(key, { maxConcurrency: 2 }); const release2 = await acquire(key, { maxConcurrency: 2 }); const unit: ResolvedComboUnit = { @@ -77,19 +85,409 @@ test("isRuntimeUnitAtConcurrencyCap returns true for a model unit at cap", async label: null, }; - assert.equal( - await isRuntimeUnitAtConcurrencyCap(unit, [], async () => 2), - true, - "unit should be at cap when two slots are in use" - ); + try { + assert.equal( + await isRuntimeUnitAtConcurrencyCap(unit, [], async () => 2), + true, + "unit should be at cap when two slots are in use" + ); - release1(); - assert.equal( - await isRuntimeUnitAtConcurrencyCap(unit, [], async () => 2), - false, - "unit should have headroom after one slot frees" + release1(); + release1 = () => {}; + assert.equal( + await isRuntimeUnitAtConcurrencyCap(unit, [], async () => 2), + false, + "unit should have headroom after one slot frees" + ); + } finally { + release1(); + release2(); + } +}); + +test("protected priority model stops at local capacity", async () => { + const primaryConnectionId = "conn-protected-capacity"; + const provider = "featherless-ai"; + seedConnection(primaryConnectionId, provider, 1); + const release = await acquire( + buildAccountSemaphoreKey({ provider, accountKey: primaryConnectionId }), + { + maxConcurrency: 1, + } ); - release2(); + try { + const calls: string[] = []; + const combo = { + name: "protected-model-capacity", + strategy: "priority", + models: [ + { + kind: "model", + model: "featherless-ai/deepseek-ai/DeepSeek-V4-Pro", + providerId: provider, + connectionId: primaryConnectionId, + fallbackOnlyOnQuotaExhaustion: true, + }, + { kind: "model", model: "alibaba/backup", providerId: "alibaba" }, + ], + config: { nestedComboMode: "execute", maxRetries: 0 }, + }; + const result = await handleComboChat({ + body: {}, + combo, + allCombos: [combo], + log: createLog(), + settings: null, + isModelAvailable: async () => true, + handleSingleModel: async (_body, modelStr) => { + calls.push(modelStr); + return okResponse(); + }, + }); + assert.equal(result.status, 503); + assert.deepEqual(calls, []); + } finally { + release(); + } +}); + +test("protected priority combo-ref stops when its child is at local capacity", async () => { + const connectionId = "conn-protected-child-capacity"; + const provider = "featherless-ai"; + seedConnection(connectionId, provider, 1); + const release = await acquire(buildAccountSemaphoreKey({ provider, accountKey: connectionId }), { + maxConcurrency: 1, + }); + try { + const child = { + name: "protected-capacity-child", + strategy: "priority", + models: [ + { kind: "model", model: "featherless-ai/child", providerId: provider, connectionId }, + ], + config: { maxRetries: 0 }, + }; + const outer = { + name: "protected-capacity-outer", + strategy: "priority", + models: [ + { kind: "combo-ref", comboName: child.name, fallbackOnlyOnQuotaExhaustion: true }, + { kind: "model", model: "alibaba/backup", providerId: "alibaba" }, + ], + config: { nestedComboMode: "execute", maxRetries: 0 }, + }; + const calls: string[] = []; + const result = await handleComboChat({ + body: {}, + combo: outer, + allCombos: [outer, child], + log: createLog(), + settings: null, + isModelAvailable: async () => true, + handleSingleModel: async (_body, modelStr) => { + calls.push(modelStr); + return okResponse(); + }, + }); + assert.equal(result.status, 503); + assert.deepEqual(calls, []); + } finally { + release(); + } +}); + +test("protected priority unit keeps non-quota failure trust across retries", async () => { + const primary = "runtime-retry-provider/primary"; + const paidBackup = "anthropic/paid-backup"; + const paidCombo = { + name: "runtime-retry-paid-combo", + strategy: "priority", + models: [{ kind: "model", model: paidBackup, providerId: "anthropic" }], + config: { maxRetries: 0, retryDelayMs: 0 }, + }; + const outer = { + name: "runtime-retry-protected-outer", + strategy: "priority", + models: [ + { + kind: "model", + model: primary, + providerId: "runtime-retry-provider", + fallbackOnlyOnQuotaExhaustion: true, + }, + { kind: "combo-ref", comboName: paidCombo.name }, + ], + config: { nestedComboMode: "execute", maxRetries: 1, retryDelayMs: 0 }, + }; + const calls: string[] = []; + const result = await handleComboChat({ + body: {}, + combo: outer, + allCombos: [outer, paidCombo], + log: createLog(), + settings: null, + isModelAvailable: async () => true, + handleSingleModel: async (_body, modelStr) => { + calls.push(modelStr); + if (modelStr === paidBackup) return okResponse(); + if (calls.length === 1) + return Response.json({ error: "Service unavailable" }, { status: 503 }); + return Response.json( + { error: { message: "Payment required", code: "insufficient_quota" } }, + { status: 429 } + ); + }, + }); + + assert.equal(result.status, 429); + assert.deepEqual(calls, [primary, primary]); +}); + +test("protected priority unit returns a successful later retry", async () => { + const primary = "runtime-success-retry-provider/primary"; + const paidCombo = { + name: "runtime-success-retry-paid-combo", + strategy: "priority", + models: [{ kind: "model", model: "anthropic/unused", providerId: "anthropic" }], + config: { maxRetries: 0, retryDelayMs: 0 }, + }; + const outer = { + name: "runtime-success-retry-protected-outer", + strategy: "priority", + models: [ + { + kind: "model", + model: primary, + providerId: "runtime-success-retry-provider", + fallbackOnlyOnQuotaExhaustion: true, + }, + { kind: "combo-ref", comboName: paidCombo.name }, + ], + config: { nestedComboMode: "execute", maxRetries: 1, retryDelayMs: 0 }, + }; + const calls: string[] = []; + const result = await handleComboChat({ + body: {}, + combo: outer, + allCombos: [outer, paidCombo], + log: createLog(), + settings: null, + isModelAvailable: async () => true, + handleSingleModel: async (_body, modelStr) => { + calls.push(modelStr); + return calls.length === 1 + ? Response.json({ error: "Service unavailable" }, { status: 503 }) + : okResponse(); + }, + }); + + assert.equal(result.ok, true); + assert.deepEqual(calls, [primary, primary]); +}); + +test("protected priority unit advances when every retry is explicit quota exhaustion", async () => { + const primary = "runtime-quota-retry-provider/primary"; + const paidBackup = "anthropic/quota-paid-backup"; + const paidCombo = { + name: "runtime-quota-retry-paid-combo", + strategy: "priority", + models: [{ kind: "model", model: paidBackup, providerId: "anthropic" }], + config: { maxRetries: 0, retryDelayMs: 0 }, + }; + const outer = { + name: "runtime-quota-retry-protected-outer", + strategy: "priority", + models: [ + { + kind: "model", + model: primary, + providerId: "runtime-quota-retry-provider", + fallbackOnlyOnQuotaExhaustion: true, + }, + { kind: "combo-ref", comboName: paidCombo.name }, + ], + config: { nestedComboMode: "execute", maxRetries: 1, retryDelayMs: 0 }, + }; + const calls: string[] = []; + const result = await handleComboChat({ + body: {}, + combo: outer, + allCombos: [outer, paidCombo], + log: createLog(), + settings: null, + isModelAvailable: async () => true, + handleSingleModel: async (_body, modelStr) => { + calls.push(modelStr); + if (modelStr === paidBackup) return okResponse(); + return Response.json( + { error: { message: "Payment required", code: "insufficient_quota" } }, + { status: 429 } + ); + }, + }); + + assert.equal(result.ok, true); + assert.deepEqual(calls, [primary, primary, paidBackup]); +}); + +test("nested child aggregate prevents protected parent fallback after any non-quota failure", async () => { + const transientModel = "runtime-nested-transient/primary"; + const quotaModel = "runtime-nested-quota/primary"; + const paidBackup = "anthropic/nested-paid-backup"; + const quotaLeaf = { + name: "runtime-nested-quota-leaf", + strategy: "priority", + models: [{ kind: "model", model: quotaModel, providerId: "runtime-nested-quota" }], + config: { maxRetries: 0, retryDelayMs: 0 }, + }; + const child = { + name: "runtime-nested-mixed-child", + strategy: "priority", + models: [ + { kind: "model", model: transientModel, providerId: "runtime-nested-transient" }, + { kind: "combo-ref", comboName: quotaLeaf.name }, + ], + config: { nestedComboMode: "execute", maxRetries: 0, retryDelayMs: 0 }, + }; + const parent = { + name: "runtime-nested-mixed-parent", + strategy: "priority", + models: [ + { kind: "combo-ref", comboName: child.name, fallbackOnlyOnQuotaExhaustion: true }, + { kind: "model", model: paidBackup, providerId: "anthropic" }, + ], + config: { nestedComboMode: "execute", maxRetries: 0, retryDelayMs: 0 }, + }; + const calls: string[] = []; + const result = await handleComboChat({ + body: {}, + combo: parent, + allCombos: [parent, child, quotaLeaf], + log: createLog(), + settings: null, + isModelAvailable: async () => true, + handleSingleModel: async (_body, modelStr) => { + calls.push(modelStr); + if (modelStr === transientModel) { + return Response.json({ error: "Service unavailable" }, { status: 503 }); + } + if (modelStr === quotaModel) { + return Response.json( + { error: { message: "Payment required", code: "insufficient_quota" } }, + { status: 429 } + ); + } + return okResponse(); + }, + }); + + assert.equal(result.status, 429); + assert.deepEqual(calls, [transientModel, quotaModel]); +}); + +test("nested child aggregate treats local quality rejection as non-quota evidence", async () => { + const invalidModel = "runtime-nested-quality/primary"; + const quotaModel = "runtime-nested-quality-quota/primary"; + const paidBackup = "anthropic/nested-quality-paid-backup"; + const quotaLeaf = { + name: "runtime-nested-quality-quota-leaf", + strategy: "priority", + models: [{ kind: "model", model: quotaModel, providerId: "runtime-nested-quality-quota" }], + config: { maxRetries: 0, retryDelayMs: 0 }, + }; + const child = { + name: "runtime-nested-quality-child", + strategy: "priority", + models: [ + { kind: "model", model: invalidModel, providerId: "runtime-nested-quality" }, + { kind: "combo-ref", comboName: quotaLeaf.name }, + ], + config: { nestedComboMode: "execute", maxRetries: 0, retryDelayMs: 0 }, + }; + const parent = { + name: "runtime-nested-quality-parent", + strategy: "priority", + models: [ + { kind: "combo-ref", comboName: child.name, fallbackOnlyOnQuotaExhaustion: true }, + { kind: "model", model: paidBackup, providerId: "anthropic" }, + ], + config: { nestedComboMode: "execute", maxRetries: 0, retryDelayMs: 0 }, + }; + const calls: string[] = []; + const result = await handleComboChat({ + body: {}, + combo: parent, + allCombos: [parent, child, quotaLeaf], + log: createLog(), + settings: null, + isModelAvailable: async () => true, + handleSingleModel: async (_body, modelStr) => { + calls.push(modelStr); + if (modelStr === invalidModel) { + return Response.json({ choices: [{ message: {} }] }, { status: 200 }); + } + if (modelStr === quotaModel) { + return Response.json( + { error: { message: "Payment required", code: "insufficient_quota" } }, + { status: 429 } + ); + } + return okResponse(); + }, + }); + + assert.equal(result.status, 429); + assert.deepEqual(calls, [invalidModel, quotaModel]); +}); + +test("nested child aggregate allows protected parent fallback when every failure is quota", async () => { + const firstQuotaModel = "runtime-nested-quota/first"; + const secondQuotaModel = "runtime-nested-quota/second"; + const paidBackup = "anthropic/nested-all-quota-backup"; + const quotaLeaf = { + name: "runtime-nested-all-quota-leaf", + strategy: "priority", + models: [{ kind: "model", model: secondQuotaModel, providerId: "runtime-nested-quota" }], + config: { maxRetries: 0, retryDelayMs: 0 }, + }; + const child = { + name: "runtime-nested-all-quota-child", + strategy: "priority", + models: [ + { kind: "model", model: firstQuotaModel, providerId: "runtime-nested-quota" }, + { kind: "combo-ref", comboName: quotaLeaf.name }, + ], + config: { nestedComboMode: "execute", maxRetries: 0, retryDelayMs: 0 }, + }; + const parent = { + name: "runtime-nested-all-quota-parent", + strategy: "priority", + models: [ + { kind: "combo-ref", comboName: child.name, fallbackOnlyOnQuotaExhaustion: true }, + { kind: "model", model: paidBackup, providerId: "anthropic" }, + ], + config: { nestedComboMode: "execute", maxRetries: 0, retryDelayMs: 0 }, + }; + const calls: string[] = []; + const result = await handleComboChat({ + body: {}, + combo: parent, + allCombos: [parent, child, quotaLeaf], + log: createLog(), + settings: null, + isModelAvailable: async () => true, + handleSingleModel: async (_body, modelStr) => { + calls.push(modelStr); + if (modelStr === paidBackup) return okResponse(); + return Response.json( + { error: { message: "Payment required", code: "insufficient_quota" } }, + { status: 429 } + ); + }, + }); + + assert.equal(result.ok, true); + assert.deepEqual(calls, [firstQuotaModel, secondQuotaModel, paidBackup]); }); test("execute fill-first overflows to the next unit when the first connection is at cap", async () => { @@ -103,43 +501,45 @@ test("execute fill-first overflows to the next unit when the first connection is const release1 = await acquire(key, { maxConcurrency: 2 }); const release2 = await acquire(key, { maxConcurrency: 2 }); - const calls: string[] = []; - const combo = { - name: "overflow-execute", - strategy: "fill-first", - models: [ - { - kind: "model", - model: "featherless-ai/deepseek-ai/DeepSeek-V4-Pro", - providerId: provider, - connectionId: primaryConnectionId, + try { + const calls: string[] = []; + const combo = { + name: "overflow-execute", + strategy: "fill-first", + models: [ + { + kind: "model", + model: "featherless-ai/deepseek-ai/DeepSeek-V4-Pro", + providerId: provider, + connectionId: primaryConnectionId, + }, + { + kind: "model", + model: "alibaba/qwen3.7-max-preview", + providerId: "alibaba", + connectionId: backupConnectionId, + }, + ], + config: { nestedComboMode: "execute", maxRetries: 0, retryDelayMs: 0 }, + }; + + const result = await handleComboChat({ + body: {}, + combo, + handleSingleModel: async (_body, modelStr) => { + calls.push(modelStr); + return okResponse(); }, - { - kind: "model", - model: "alibaba/qwen3.7-max-preview", - providerId: "alibaba", - connectionId: backupConnectionId, - }, - ], - config: { nestedComboMode: "execute", maxRetries: 0, retryDelayMs: 0 }, - }; + isModelAvailable: async () => true, + log: createLog(), + settings: null, + allCombos: [combo], + }); - const result = await handleComboChat({ - body: {}, - combo, - handleSingleModel: async (_body, modelStr) => { - calls.push(modelStr); - return okResponse(); - }, - isModelAvailable: async () => true, - log: createLog(), - settings: null, - allCombos: [combo], - }); - - assert.equal(result.ok, true); - assert.deepEqual(calls, ["alibaba/qwen3.7-max-preview"]); - - release1(); - release2(); + assert.equal(result.ok, true); + assert.deepEqual(calls, ["alibaba/qwen3.7-max-preview"]); + } finally { + release1(); + release2(); + } }); diff --git a/tests/unit/combos-quota-protected.test.ts b/tests/unit/combos-quota-protected.test.ts index 6c1776a86d..cf2c4ad6ce 100644 --- a/tests/unit/combos-quota-protected.test.ts +++ b/tests/unit/combos-quota-protected.test.ts @@ -90,7 +90,11 @@ test("PUT /api/combos/[id] returns 409 for a qtSd/* combo and does NOT mutate it // Verify the combo was NOT mutated const unchanged = await combosDb.getComboById(combo.id); - assert.equal(unchanged?.strategy, "priority", "Strategy must remain unchanged after rejected PUT"); + assert.equal( + unchanged?.strategy, + "priority", + "Strategy must remain unchanged after rejected PUT" + ); }); // ---- non-quota combos still work ---- @@ -116,6 +120,43 @@ test("DELETE /api/combos/[id] succeeds for a regular (non-quota) combo", async ( assert.equal(gone, null, "Regular combo must be gone after DELETE"); }); +test("PUT merged state rejects partial updates that leave protected priority refs in flatten mode", async () => { + const combo = await combosDb.createCombo({ + name: "protected-ref-update", + strategy: "priority", + models: [{ kind: "combo-ref", comboName: "child", fallbackOnlyOnQuotaExhaustion: true }], + config: { nestedComboMode: "execute" }, + }); + + for (const update of [ + { config: { nestedComboMode: "flatten" } }, + { + models: [{ kind: "combo-ref", comboName: "child", fallbackOnlyOnQuotaExhaustion: true }], + config: {}, + }, + { strategy: "priority", config: {} }, + ]) { + const response = await comboRoute.PUT(makePutRequest(combo.id, update), { + params: Promise.resolve({ id: combo.id }), + }); + assert.equal(response.status, 400); + } +}); + +test("PUT merged state accepts dormant weighted protected refs", async () => { + const combo = await combosDb.createCombo({ + name: "dormant-protected-ref-update", + strategy: "priority", + models: [{ kind: "combo-ref", comboName: "child", fallbackOnlyOnQuotaExhaustion: true }], + config: { nestedComboMode: "execute" }, + }); + const response = await comboRoute.PUT( + makePutRequest(combo.id, { strategy: "weighted", config: { nestedComboMode: "flatten" } }), + { params: Promise.resolve({ id: combo.id }) } + ); + assert.equal(response.status, 200); +}); + test("PUT /api/combos/[id] succeeds for a regular (non-quota) combo", async () => { const combo = await combosDb.createCombo({ name: "regular-editable-combo", @@ -152,10 +193,7 @@ test("DELETE /api/combos/[id] returns 404 when combo does not exist", async () = test("combos page source filters isHidden from rendered list", async () => { const pageSource = fs.readFileSync( - new URL( - "../../src/app/(dashboard)/dashboard/combos/page.tsx", - import.meta.url - ).pathname, + new URL("../../src/app/(dashboard)/dashboard/combos/page.tsx", import.meta.url).pathname, "utf8" ); assert.ok( diff --git a/tests/unit/ui/combo-quota-exhaustion-option.test.tsx b/tests/unit/ui/combo-quota-exhaustion-option.test.tsx new file mode 100644 index 0000000000..3103bfa78b --- /dev/null +++ b/tests/unit/ui/combo-quota-exhaustion-option.test.tsx @@ -0,0 +1,91 @@ +// @vitest-environment jsdom +import React, { act } from "react"; +import { createRoot } from "react-dom/client"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { ComboTargetOptions } from "@/app/(dashboard)/dashboard/combos/ComboQuotaOnlyFallbackToggle"; +import { + applyQuotaOnlyFallbackConfig, + setQuotaOnlyFallback, +} from "@/app/(dashboard)/dashboard/combos/comboQuotaOnlyFallback"; +import type { ComboStep } from "@/lib/combos/steps"; + +const model = (name: string): ComboStep => ({ kind: "model", model: name }); +const comboRef = (): ComboStep => ({ kind: "combo-ref", comboName: "child" }); +const containers: Array<{ root: ReturnType; element: HTMLDivElement }> = []; +const label = "Only advance on quota exhaustion"; +const shortLabel = "quota-only fallback"; + +function renderOptions(strategy: string, entry: ComboStep) { + const element = document.createElement("div"); + document.body.appendChild(element); + const root = createRoot(element); + act(() => { + root.render( + + key === "fallbackOnlyOnQuotaExhaustion" + ? label + : key === "fallbackOnlyOnQuotaExhaustionShort" + ? shortLabel + : fallback + } + onCheckedChange={vi.fn()} + /> + ); + }); + containers.push({ root, element }); + return element; +} + +afterEach(() => { + for (const { root, element } of containers.splice(0)) { + act(() => root.unmount()); + element.remove(); + } +}); + +describe("combo quota-only fallback state", () => { + it("updates the selected model and combo-ref without mutating other steps", () => { + const entries = [model("openai/first"), comboRef(), model("openai/last")]; + const modelEnabled = setQuotaOnlyFallback(entries, 0, true); + const refEnabled = setQuotaOnlyFallback(modelEnabled, 1, true); + + expect(entries[0].fallbackOnlyOnQuotaExhaustion).toBeUndefined(); + expect(refEnabled[0].fallbackOnlyOnQuotaExhaustion).toBe(true); + expect(refEnabled[1].fallbackOnlyOnQuotaExhaustion).toBe(true); + expect(refEnabled[2]).toBe(entries[2]); + expect(setQuotaOnlyFallback(refEnabled, 1, false)[1]).not.toHaveProperty( + "fallbackOnlyOnQuotaExhaustion" + ); + }); + + it("ignores an invalid index instead of manufacturing a step", () => { + const entries = [model("openai/first")]; + expect(setQuotaOnlyFallback(entries, 5, true)).toEqual(entries); + }); + + it("renders accessible human labels for priority models and combo refs only", () => { + for (const entry of [model("openai/first"), comboRef()]) { + const element = renderOptions("priority", entry); + const checkbox = element.querySelector(`input[aria-label="${label}"]`); + expect(checkbox).not.toBeNull(); + expect(checkbox?.checked).toBe(false); + expect(element.textContent).toContain(shortLabel); + } + expect(renderOptions("weighted", model("openai/first")).querySelector("input")).toBeNull(); + }); + + it("forces execute only for active priority combo refs and preserves dormant config", () => { + const protectedRef = setQuotaOnlyFallback([comboRef()], 0, true); + expect(applyQuotaOnlyFallbackConfig("priority", protectedRef, {})).toEqual({ + nestedComboMode: "execute", + }); + expect(applyQuotaOnlyFallbackConfig("weighted", protectedRef, {})).toEqual({}); + expect(applyQuotaOnlyFallbackConfig("priority", [model("openai/first")], {})).toEqual({}); + }); +});