mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-12 10:12:11 +03:00
feat(combo): add quota-only priority fallback (#9983)
Add a per-target priority option that advances only after trusted quota exhaustion while preserving retry, nested Combo, quality, and Global Fallback semantics.
This commit is contained in:
@@ -220,6 +220,11 @@ import {
|
||||
} from "./combo/quotaExhaustionCutoff.ts";
|
||||
import { expandTargetsByFingerprints } from "./combo/fingerprintExpansion.ts";
|
||||
import { resolveComboTargetPipeline } from "./combo/targetResolution.ts";
|
||||
import {
|
||||
isQuotaExhaustionResponse,
|
||||
recordQuotaExhaustionClassification,
|
||||
withQuotaExhaustionClassification,
|
||||
} from "./combo/quotaExhaustion.ts";
|
||||
|
||||
export { RESET_WINDOW_NAMES, QUOTA_SOFT_DEPRIORITIZE_FACTOR, setCandidateQuotaSoftPenalty };
|
||||
export { scoreAutoTargets, expandAutoComboCandidatePool };
|
||||
@@ -817,6 +822,26 @@ export async function handleComboChat({
|
||||
// Accumulator for per-model error details across targets in the current set try.
|
||||
// Reset at the start of each set retry (same lifecycle as lastError/recordedAttempts).
|
||||
let comboErrors: Array<{ model: string; status: number; error: string }> = [];
|
||||
// Quota trust spans set retries and recursive cooldown re-dispatches. Once any
|
||||
// failure is non-quota, a nested caller must never treat this dispatch as quota-only.
|
||||
let observedFailure = false;
|
||||
let allObservedFailuresQuota = true;
|
||||
const targetFailureTrust = new Map<
|
||||
string,
|
||||
{ observedFailure: boolean; allObservedFailuresQuota: boolean }
|
||||
>();
|
||||
const observeFailure = (quotaExhausted: boolean, targetExecutionKey?: string) => {
|
||||
observedFailure = true;
|
||||
allObservedFailuresQuota &&= quotaExhausted;
|
||||
if (!targetExecutionKey) return;
|
||||
const trust = targetFailureTrust.get(targetExecutionKey) ?? {
|
||||
observedFailure: false,
|
||||
allObservedFailuresQuota: true,
|
||||
};
|
||||
trust.observedFailure = true;
|
||||
trust.allObservedFailuresQuota &&= quotaExhausted;
|
||||
targetFailureTrust.set(targetExecutionKey, trust);
|
||||
};
|
||||
|
||||
// FASE 2.1: per-connection concurrency limit for quota-share. The gating in
|
||||
// selectQuotaShareTarget is fail-open and cannot hard-limit a single-connection
|
||||
@@ -904,6 +929,9 @@ export async function handleComboChat({
|
||||
let anySuccess = false;
|
||||
const abortControllers = new Map<number, AbortController>();
|
||||
const zeroLatencyOptimizationsEnabled = config.zeroLatencyOptimizationsEnabled === true;
|
||||
const hasProtectedPriorityTarget =
|
||||
strategy === "priority" &&
|
||||
orderedTargets.some((target) => target.fallbackOnlyOnQuotaExhaustion === true);
|
||||
|
||||
const executeTarget = async (
|
||||
i: number
|
||||
@@ -912,12 +940,20 @@ export async function handleComboChat({
|
||||
const modelStr = target.modelStr;
|
||||
const rawModel = parseModel(modelStr).model || modelStr;
|
||||
const provider = target.provider;
|
||||
const protectedPriorityTarget =
|
||||
strategy === "priority" && target.fallbackOnlyOnQuotaExhaustion === true;
|
||||
const stopProtectedPriorityTarget = (message: string) => {
|
||||
observeFailure(false, target.executionKey);
|
||||
return protectedPriorityTarget
|
||||
? { ok: false, response: errorResponse(503, message) }
|
||||
: null;
|
||||
};
|
||||
|
||||
const cb = getCircuitBreaker(provider);
|
||||
if (cb.getStatus().state === "OPEN") {
|
||||
log.info("COMBO", `Skipping ${modelStr} — circuit breaker OPEN for ${provider}`);
|
||||
if (i > 0) fallbackCount++;
|
||||
return null;
|
||||
return stopProtectedPriorityTarget(`Provider ${provider} circuit breaker is open`);
|
||||
}
|
||||
|
||||
if (
|
||||
@@ -927,7 +963,7 @@ export async function handleComboChat({
|
||||
) {
|
||||
log.info("COMBO", `Skipping ${modelStr} — provider ${provider} in global cooldown`);
|
||||
if (i > 0) fallbackCount++;
|
||||
return null;
|
||||
return stopProtectedPriorityTarget(`Provider ${provider} is in cooldown`);
|
||||
}
|
||||
|
||||
// Use pre-screened profile if available, otherwise fetch on demand
|
||||
@@ -954,14 +990,14 @@ export async function handleComboChat({
|
||||
if (exhaustedSkip) {
|
||||
log.info("COMBO", exhaustedSkip);
|
||||
if (i > 0) fallbackCount++;
|
||||
return null;
|
||||
return stopProtectedPriorityTarget(`Target ${modelStr} is unavailable`);
|
||||
}
|
||||
|
||||
// Pre-check: skip models locked by the resilience system (model-level lockout)
|
||||
if (provider && rawModel && isModelLocked(provider, target.connectionId || "", rawModel)) {
|
||||
log.info("COMBO", `Skipping ${modelStr} — model locked by resilience (cooldown active)`);
|
||||
if (i > 0) fallbackCount++;
|
||||
return null;
|
||||
return stopProtectedPriorityTarget(`Model ${modelStr} is locked`);
|
||||
}
|
||||
|
||||
// #5923 (Finding #4) — honor the same opt-in quota-exhaustion cutoff the
|
||||
@@ -987,6 +1023,16 @@ export async function handleComboChat({
|
||||
`Skipping ${modelStr} — quota exhaustion cutoff (${quotaCutoff.reason || "quota_exhausted"})`
|
||||
);
|
||||
if (i > 0) fallbackCount++;
|
||||
observeFailure(true, target.executionKey);
|
||||
if (protectedPriorityTarget) {
|
||||
const protectedTargetTrust = targetFailureTrust.get(target.executionKey);
|
||||
if (!protectedTargetTrust?.allObservedFailuresQuota) {
|
||||
return {
|
||||
ok: false,
|
||||
response: errorResponse(503, `Target ${modelStr} is unavailable`),
|
||||
};
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -1004,7 +1050,7 @@ export async function handleComboChat({
|
||||
`Skipping ${modelStr} — no credentials available or model excluded`
|
||||
);
|
||||
if (i > 0) fallbackCount++;
|
||||
return null;
|
||||
return stopProtectedPriorityTarget(`Model ${modelStr} is unavailable`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1015,7 +1061,7 @@ export async function handleComboChat({
|
||||
if (gateResult.allowed === false) {
|
||||
logCredentialSkip(log, modelStr, gateResult.reason || "Credential gate blocked");
|
||||
if (i > 0) fallbackCount++;
|
||||
return null;
|
||||
return stopProtectedPriorityTarget(`Credential gate blocked ${modelStr}`);
|
||||
}
|
||||
|
||||
// Concurrency gate: fail-fast skip when connection is at max_concurrent capacity (e.g. Featherless 1/1)
|
||||
@@ -1029,7 +1075,7 @@ export async function handleComboChat({
|
||||
`Skipping ${modelStr} — connection ${connectionId} is at max concurrency cap (${maxConcurrentCap})`
|
||||
);
|
||||
if (i > 0) fallbackCount++;
|
||||
return null;
|
||||
return stopProtectedPriorityTarget(`Connection capacity reached for ${modelStr}`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1084,7 +1130,7 @@ export async function handleComboChat({
|
||||
"COMBO",
|
||||
`Predictive TTFT Circuit Breaker: skipping ${modelStr} (avg ${m.avgLatencyMs}ms > max ${config.predictiveTtftMs}ms)`
|
||||
);
|
||||
return null;
|
||||
return stopProtectedPriorityTarget(`Predictive latency check rejected ${modelStr}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1294,7 +1340,13 @@ export async function handleComboChat({
|
||||
error: `Quality: ${quality.reason}`,
|
||||
latencyMs: Date.now() - startTime,
|
||||
});
|
||||
return null;
|
||||
observeFailure(false, target.executionKey);
|
||||
return protectedPriorityTarget
|
||||
? {
|
||||
ok: false,
|
||||
response: errorResponse(502, "Upstream response failed quality validation"),
|
||||
}
|
||||
: null;
|
||||
}
|
||||
|
||||
// Success decay: a healthy response walks the model's lockout failure
|
||||
@@ -1645,7 +1697,7 @@ export async function handleComboChat({
|
||||
result.status,
|
||||
errorText,
|
||||
0,
|
||||
null,
|
||||
protectedPriorityTarget ? rawModel : null,
|
||||
provider,
|
||||
result.headers,
|
||||
profile,
|
||||
@@ -1776,6 +1828,15 @@ export async function handleComboChat({
|
||||
recordProviderFailure(provider, log, targetWithConnection.connectionId, profile);
|
||||
}
|
||||
|
||||
const quotaExhausted = await isQuotaExhaustionResponse(
|
||||
result,
|
||||
provider,
|
||||
rawModel,
|
||||
profile
|
||||
);
|
||||
recordQuotaExhaustionClassification(result, quotaExhausted);
|
||||
observeFailure(quotaExhausted, target.executionKey);
|
||||
|
||||
// Check if this is a transient error worth retrying on same model.
|
||||
// A token-limit 429 is terminal for the client — never retry it.
|
||||
const isTransient =
|
||||
@@ -1785,6 +1846,7 @@ export async function handleComboChat({
|
||||
[408, 429, 500, 502, 503, 504].includes(result.status);
|
||||
if (retry < maxRetries && isTransient && !providerExhausted) {
|
||||
if (
|
||||
!protectedPriorityTarget &&
|
||||
provider &&
|
||||
rawModel &&
|
||||
isModelLocked(provider, targetWithConnection.connectionId || "", rawModel)
|
||||
@@ -1807,7 +1869,7 @@ export async function handleComboChat({
|
||||
// once the model is cooling down, retrying it would waste an upstream
|
||||
// call and extend the cooldown via exponential backoff.
|
||||
let lockoutRecorded = false;
|
||||
if (provider && rawModel && retry === 0 && !scopedFailure) {
|
||||
if (!protectedPriorityTarget && provider && rawModel && retry === 0 && !scopedFailure) {
|
||||
const mlSettings = resolveModelLockoutSettings(settings);
|
||||
if (mlSettings.enabled && mlSettings.errorCodes.includes(result.status)) {
|
||||
recordModelLockoutFailure(
|
||||
@@ -1847,6 +1909,22 @@ export async function handleComboChat({
|
||||
}
|
||||
|
||||
// Done retrying this model
|
||||
const protectedTargetTrust = targetFailureTrust.get(target.executionKey);
|
||||
if (
|
||||
protectedPriorityTarget &&
|
||||
(!protectedTargetTrust?.observedFailure ||
|
||||
!protectedTargetTrust.allObservedFailuresQuota)
|
||||
) {
|
||||
recordComboRequest(combo.name, modelStr, {
|
||||
success: false,
|
||||
latencyMs: Date.now() - startTime,
|
||||
fallbackCount,
|
||||
strategy,
|
||||
target: toRecordedTarget(target),
|
||||
});
|
||||
recordedAttempts++;
|
||||
return { ok: false, response: result };
|
||||
}
|
||||
recordComboRequest(combo.name, modelStr, {
|
||||
success: false,
|
||||
latencyMs: Date.now() - startTime,
|
||||
@@ -1975,7 +2053,12 @@ export async function handleComboChat({
|
||||
runningTasks.add(task);
|
||||
task.finally(() => runningTasks.delete(task));
|
||||
|
||||
if (zeroLatencyOptimizationsEnabled && config.hedging && i + 1 < orderedTargets.length) {
|
||||
if (
|
||||
zeroLatencyOptimizationsEnabled &&
|
||||
config.hedging &&
|
||||
!hasProtectedPriorityTarget &&
|
||||
i + 1 < orderedTargets.length
|
||||
) {
|
||||
const hedgeDelay = resolveDelayMs(config.hedgeDelayMs, 500);
|
||||
let timeoutResolve: () => void;
|
||||
const timeoutPromise = new Promise<void>((r) => {
|
||||
@@ -2062,11 +2145,14 @@ export async function handleComboChat({
|
||||
latencyMs,
|
||||
fallbackCount,
|
||||
});
|
||||
return errorResponseWithComboDiagnostics(
|
||||
503,
|
||||
"Service temporarily unavailable: all targets were skipped by pre-dispatch filters",
|
||||
buildComboDiag("all_targets_skipped"),
|
||||
{ code: "ALL_TARGETS_SKIPPED", type: "service_unavailable" }
|
||||
return withQuotaExhaustionClassification(
|
||||
errorResponseWithComboDiagnostics(
|
||||
503,
|
||||
"Service temporarily unavailable: all targets were skipped by pre-dispatch filters",
|
||||
buildComboDiag("all_targets_skipped"),
|
||||
{ code: "ALL_TARGETS_SKIPPED", type: "service_unavailable" }
|
||||
),
|
||||
observedFailure ? allObservedFailuresQuota : null
|
||||
);
|
||||
}
|
||||
notifyWebhookEvent("request.failed", {
|
||||
@@ -2157,7 +2243,10 @@ export async function handleComboChat({
|
||||
if (earliestRetryAfter && isRetryAfterEligibleStatus(status)) {
|
||||
const retryHuman = formatRetryAfter(toRetryAfterDisplayValue(earliestRetryAfter));
|
||||
log.warn("COMBO", `All models failed | ${msg} (${retryHuman})`);
|
||||
return unavailableResponse(status, msg, earliestRetryAfter, retryHuman);
|
||||
return withQuotaExhaustionClassification(
|
||||
unavailableResponse(status, msg, earliestRetryAfter, retryHuman),
|
||||
observedFailure ? allObservedFailuresQuota : null
|
||||
);
|
||||
}
|
||||
|
||||
// Silent-stop fix: bump the failure counter (pin clears on 3rd consecutive) and emit
|
||||
@@ -2173,10 +2262,13 @@ export async function handleComboChat({
|
||||
);
|
||||
}
|
||||
const retryAfterSeconds = undefined;
|
||||
return errorResponseWithComboDiagnostics(
|
||||
status,
|
||||
msg,
|
||||
buildComboDiag(lastError ?? "all_models_failed", retryAfterSeconds)
|
||||
return withQuotaExhaustionClassification(
|
||||
errorResponseWithComboDiagnostics(
|
||||
status,
|
||||
msg,
|
||||
buildComboDiag(lastError ?? "all_models_failed", retryAfterSeconds)
|
||||
),
|
||||
observedFailure ? allObservedFailuresQuota : null
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -115,6 +115,7 @@ function normalizeRuntimeStep(
|
||||
comboName: step.comboName,
|
||||
weight,
|
||||
label,
|
||||
...(step.fallbackOnlyOnQuotaExhaustion ? { fallbackOnlyOnQuotaExhaustion: true } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -138,7 +139,12 @@ function normalizeRuntimeStep(
|
||||
: {}),
|
||||
weight,
|
||||
label,
|
||||
prompt: step.kind === "model" ? step.prompt || null : null,
|
||||
// `prompt` is a per-step pipeline input and only exists on a model step —
|
||||
// #8894 widened the union with ComboProviderWildcardStep, which has no prompt.
|
||||
prompt: (step.kind === "model" ? step.prompt : null) || null,
|
||||
...(step.kind === "model" && step.fallbackOnlyOnQuotaExhaustion
|
||||
? { fallbackOnlyOnQuotaExhaustion: true }
|
||||
: {}),
|
||||
} satisfies ResolvedComboTarget;
|
||||
}
|
||||
|
||||
|
||||
107
open-sse/services/combo/quotaExhaustion.ts
Normal file
107
open-sse/services/combo/quotaExhaustion.ts
Normal file
@@ -0,0 +1,107 @@
|
||||
import { checkFallbackError, type ProviderProfile } from "../accountFallback.ts";
|
||||
import { classifyGeminiQuotaMetricFromText } from "../geminiRateLimitTracker.ts";
|
||||
|
||||
const TERMINAL_QUOTA_CODES = new Set([
|
||||
"billing_hard_limit_reached",
|
||||
"credits_exhausted",
|
||||
"insufficient_quota",
|
||||
"quota_exhausted",
|
||||
]);
|
||||
|
||||
const trustedClassifications = new WeakMap<Response, boolean>();
|
||||
|
||||
type ParsedError = {
|
||||
text: string;
|
||||
structuredError: { code?: string; type?: string } | null;
|
||||
};
|
||||
|
||||
async function parseError(response: Response): Promise<ParsedError> {
|
||||
let text = response.statusText;
|
||||
let structuredError: ParsedError["structuredError"] = null;
|
||||
try {
|
||||
const body = (await response.clone().json()) as {
|
||||
error?: string | { message?: unknown; code?: unknown; type?: unknown };
|
||||
message?: unknown;
|
||||
};
|
||||
if (typeof body.error === "string") text = body.error;
|
||||
else if (body.error && typeof body.error === "object") {
|
||||
if (typeof body.error.message === "string") text = body.error.message;
|
||||
structuredError = {
|
||||
...(body.error.code == null ? {} : { code: String(body.error.code) }),
|
||||
...(body.error.type == null ? {} : { type: String(body.error.type) }),
|
||||
};
|
||||
} else if (typeof body.message === "string") text = body.message;
|
||||
} catch {
|
||||
try {
|
||||
text = await response.clone().text();
|
||||
} catch {
|
||||
// The status and trusted in-process classification remain available.
|
||||
}
|
||||
}
|
||||
return { text, structuredError };
|
||||
}
|
||||
|
||||
export function recordQuotaExhaustionClassification(response: Response, exhausted: boolean): void {
|
||||
trustedClassifications.set(response, exhausted);
|
||||
}
|
||||
|
||||
export function withQuotaExhaustionClassification(
|
||||
response: Response,
|
||||
exhausted: boolean | null
|
||||
): Response {
|
||||
if (exhausted !== null) recordQuotaExhaustionClassification(response, exhausted);
|
||||
return response;
|
||||
}
|
||||
|
||||
export async function isQuotaExhaustionResponse(
|
||||
response: Response,
|
||||
provider: string | null,
|
||||
model: string | null,
|
||||
profile: ProviderProfile | null = null
|
||||
): Promise<boolean> {
|
||||
const trusted = trustedClassifications.get(response);
|
||||
if (trusted !== undefined) return trusted;
|
||||
|
||||
if (response.status !== 402 && response.status !== 429) return false;
|
||||
|
||||
const { text, structuredError } = await parseError(response);
|
||||
if (provider === "gemini" && response.status === 429) {
|
||||
const metric = classifyGeminiQuotaMetricFromText(text);
|
||||
if (metric === "rpm" || metric === "tpm") return false;
|
||||
if (metric === "rpd") return true;
|
||||
}
|
||||
const normalizedCode = structuredError?.code?.toLowerCase();
|
||||
const normalizedType = structuredError?.type?.toLowerCase();
|
||||
if (
|
||||
(normalizedCode && TERMINAL_QUOTA_CODES.has(normalizedCode)) ||
|
||||
(normalizedType && TERMINAL_QUOTA_CODES.has(normalizedType))
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (
|
||||
/\b(?:billing hard limit reached|credits? exhausted|subscription quota exhausted)\b/i.test(text)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (
|
||||
provider?.startsWith("openai-compatible-") ||
|
||||
provider?.startsWith("openai-compatible-chat-")
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return (
|
||||
checkFallbackError(
|
||||
response.status,
|
||||
text,
|
||||
0,
|
||||
model,
|
||||
provider,
|
||||
response.headers,
|
||||
profile,
|
||||
structuredError
|
||||
).reason === "quota_exhausted"
|
||||
);
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import { errorResponse } from "../../utils/error.ts";
|
||||
import { recordComboRequest } from "../comboMetrics.ts";
|
||||
import { resolveDelayMs } from "./comboPredicates.ts";
|
||||
import { isRuntimeUnitAtConcurrencyCap } from "./runtimeUnitCapacity.ts";
|
||||
import { isQuotaExhaustionResponse, withQuotaExhaustionClassification } from "./quotaExhaustion.ts";
|
||||
import { validateResponseQuality, releaseQualityClone } from "./validateQuality.ts";
|
||||
import type { ResponseValidationConfig } from "./responseValidation.ts";
|
||||
import type {
|
||||
@@ -197,8 +198,28 @@ export async function executeRuntimeUnitCombo(args: {
|
||||
const effectiveStrategy = args.effectiveComboStrategy ?? args.strategy;
|
||||
let lastResponse: Response | null = null;
|
||||
let fallbackCount = 0;
|
||||
let observedFailure = false;
|
||||
let allObservedFailuresQuota = true;
|
||||
const targetFailureTrust = new Map<
|
||||
string,
|
||||
{ observedFailure: boolean; allObservedFailuresQuota: boolean }
|
||||
>();
|
||||
const observeFailure = async (response: Response, unit: ResolvedComboUnit): Promise<boolean> => {
|
||||
const quotaExhausted = await isQuotaExhaustionResponse(
|
||||
response,
|
||||
unit.kind === "model" ? unit.provider : null,
|
||||
unit.kind === "model" ? unit.modelStr : null
|
||||
);
|
||||
observedFailure = true;
|
||||
allObservedFailuresQuota &&= quotaExhausted;
|
||||
return quotaExhausted;
|
||||
};
|
||||
const finalFailure = (response: Response): Response =>
|
||||
withQuotaExhaustionClassification(response, observedFailure ? allObservedFailuresQuota : null);
|
||||
|
||||
for (const unit of orderedUnits) {
|
||||
const protectedPriorityUnit =
|
||||
effectiveStrategy === "priority" && unit.fallbackOnlyOnQuotaExhaustion === true;
|
||||
if (
|
||||
await isRuntimeUnitAtConcurrencyCap(
|
||||
unit,
|
||||
@@ -211,16 +232,24 @@ export async function executeRuntimeUnitCombo(args: {
|
||||
"COMBO",
|
||||
`Skipping ${unit.kind} ${unitDisplayName(unit)} — concurrency cap reached`
|
||||
);
|
||||
lastResponse = errorResponse(503, `${unitDisplayName(unit)} is at concurrency capacity`);
|
||||
await observeFailure(lastResponse, unit);
|
||||
if (protectedPriorityUnit) return { response: finalFailure(lastResponse), unit };
|
||||
fallbackCount += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
for (let retry = 0; retry <= maxRetries; retry += 1) {
|
||||
if (args.signal?.aborted)
|
||||
return { response: errorResponse(499, "Client disconnected"), unit };
|
||||
if (args.signal?.aborted) {
|
||||
lastResponse = errorResponse(499, "Client disconnected");
|
||||
await observeFailure(lastResponse, unit);
|
||||
return { response: finalFailure(lastResponse), unit };
|
||||
}
|
||||
args.nesting.attemptBudget.count += 1;
|
||||
if (args.nesting.attemptBudget.count > args.nesting.attemptBudget.limit) {
|
||||
return { response: errorResponse(503, "Maximum combo retry limit reached"), unit };
|
||||
lastResponse = errorResponse(503, "Maximum combo retry limit reached");
|
||||
await observeFailure(lastResponse, unit);
|
||||
return { response: finalFailure(lastResponse), unit };
|
||||
}
|
||||
if (retry > 0) {
|
||||
await new Promise((resolve) => setTimeout(resolve, retryDelayMs));
|
||||
@@ -276,9 +305,31 @@ export async function executeRuntimeUnitCombo(args: {
|
||||
});
|
||||
return { response, unit };
|
||||
}
|
||||
lastResponse = errorResponse(502, "Upstream response failed quality validation");
|
||||
}
|
||||
if (lastResponse) {
|
||||
const quotaExhausted = await observeFailure(lastResponse, unit);
|
||||
if (protectedPriorityUnit) {
|
||||
const trust = targetFailureTrust.get(unit.executionKey) ?? {
|
||||
observedFailure: false,
|
||||
allObservedFailuresQuota: true,
|
||||
};
|
||||
trust.observedFailure = true;
|
||||
trust.allObservedFailuresQuota &&= quotaExhausted;
|
||||
targetFailureTrust.set(unit.executionKey, trust);
|
||||
}
|
||||
}
|
||||
if (![408, 429, 500, 502, 503, 504].includes(response.status)) break;
|
||||
}
|
||||
const protectedTargetTrust = targetFailureTrust.get(unit.executionKey);
|
||||
if (
|
||||
protectedPriorityUnit &&
|
||||
protectedTargetTrust?.observedFailure &&
|
||||
!protectedTargetTrust.allObservedFailuresQuota &&
|
||||
lastResponse
|
||||
) {
|
||||
return { response: finalFailure(lastResponse), unit };
|
||||
}
|
||||
fallbackCount += 1;
|
||||
}
|
||||
recordComboRequest(args.combo.name, null, {
|
||||
@@ -288,7 +339,9 @@ export async function executeRuntimeUnitCombo(args: {
|
||||
strategy: effectiveStrategy,
|
||||
});
|
||||
return {
|
||||
response: lastResponse || errorResponse(503, "All nested combo units unavailable"),
|
||||
response: finalFailure(
|
||||
lastResponse || errorResponse(503, "All nested combo units unavailable")
|
||||
),
|
||||
unit: null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -174,6 +174,7 @@ export type ResolvedComboTarget = {
|
||||
label: string | null;
|
||||
prompt?: string | null;
|
||||
failoverBeforeRetry?: unknown;
|
||||
fallbackOnlyOnQuotaExhaustion?: boolean;
|
||||
trafficType?: "production" | "shadow";
|
||||
/**
|
||||
* Fingerprint-based account pin resolved from a combo builder composite
|
||||
@@ -200,6 +201,7 @@ export type ResolvedComboRefTarget = {
|
||||
comboName: string;
|
||||
weight: number;
|
||||
label: string | null;
|
||||
fallbackOnlyOnQuotaExhaustion?: boolean;
|
||||
};
|
||||
|
||||
export type ResolvedComboUnit = ResolvedComboTarget | ResolvedComboRefTarget;
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
import type { ComboStep } from "@/lib/combos/steps";
|
||||
|
||||
type Translate = (key: string, fallback: string) => string;
|
||||
|
||||
type Props = {
|
||||
strategy: string;
|
||||
entry: ComboStep;
|
||||
index: number;
|
||||
hasPricing: boolean;
|
||||
translate: Translate;
|
||||
onCheckedChange: (index: number, enabled: boolean) => void;
|
||||
};
|
||||
|
||||
export function ComboTargetOptions({
|
||||
strategy,
|
||||
entry,
|
||||
index,
|
||||
hasPricing,
|
||||
translate,
|
||||
onCheckedChange,
|
||||
}: Props) {
|
||||
return (
|
||||
<>
|
||||
{strategy === "cost-optimized" && (
|
||||
<span
|
||||
className={`text-[9px] px-1.5 py-0.5 rounded-full uppercase font-semibold ${
|
||||
hasPricing
|
||||
? "bg-emerald-500/15 text-emerald-600 dark:text-emerald-400"
|
||||
: "bg-amber-500/15 text-amber-600 dark:text-amber-400"
|
||||
}`}
|
||||
title={translate(
|
||||
hasPricing ? "pricingAvailable" : "pricingMissing",
|
||||
hasPricing ? "Pricing available" : "No pricing"
|
||||
)}
|
||||
>
|
||||
{translate(
|
||||
hasPricing ? "pricingAvailableShort" : "pricingMissingShort",
|
||||
hasPricing ? "priced" : "no-price"
|
||||
)}
|
||||
</span>
|
||||
)}
|
||||
{strategy === "priority" && (entry.kind === "model" || entry.kind === "combo-ref") && (
|
||||
<label className="flex items-center gap-1 text-[10px] text-text-muted shrink-0">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={entry.fallbackOnlyOnQuotaExhaustion === true}
|
||||
onChange={(event) => onCheckedChange(index, event.target.checked)}
|
||||
aria-label={translate(
|
||||
"fallbackOnlyOnQuotaExhaustion",
|
||||
"Only advance on quota exhaustion"
|
||||
)}
|
||||
/>
|
||||
<span className="hidden lg:inline">
|
||||
{translate("fallbackOnlyOnQuotaExhaustionShort", "quota-only fallback")}
|
||||
</span>
|
||||
</label>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import type { ComboStep } from "@/lib/combos/steps";
|
||||
|
||||
export function setQuotaOnlyFallback(
|
||||
entries: ComboStep[],
|
||||
index: number,
|
||||
enabled: boolean
|
||||
): ComboStep[] {
|
||||
const selected = entries[index];
|
||||
if (!selected || (selected.kind !== "model" && selected.kind !== "combo-ref")) return entries;
|
||||
const next = [...entries];
|
||||
const { fallbackOnlyOnQuotaExhaustion: _removed, ...entry } = selected;
|
||||
next[index] = enabled ? { ...entry, fallbackOnlyOnQuotaExhaustion: true } : entry;
|
||||
return next;
|
||||
}
|
||||
|
||||
export function requiresExecuteMode(strategy: string, entries: ComboStep[]): boolean {
|
||||
return (
|
||||
strategy === "priority" &&
|
||||
entries.some(
|
||||
(entry) => entry.kind === "combo-ref" && entry.fallbackOnlyOnQuotaExhaustion === true
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
export function applyQuotaOnlyFallbackConfig(
|
||||
strategy: string,
|
||||
entries: ComboStep[],
|
||||
config: Record<string, unknown>
|
||||
): Record<string, unknown> {
|
||||
return requiresExecuteMode(strategy, entries)
|
||||
? { ...config, nestedComboMode: "execute" }
|
||||
: config;
|
||||
}
|
||||
@@ -16,6 +16,8 @@ import Tooltip from "@/shared/components/Tooltip";
|
||||
import { ComboCompressionModeSelect } from "@/shared/components/compression/ComboCompressionModeSelect";
|
||||
import { useCopyToClipboard } from "@/shared/hooks/useCopyToClipboard";
|
||||
import { FieldLabelWithHelp, WeightTotalBar } from "./parts";
|
||||
import { ComboTargetOptions } from "./ComboQuotaOnlyFallbackToggle";
|
||||
import { applyQuotaOnlyFallbackConfig, setQuotaOnlyFallback } from "./comboQuotaOnlyFallback";
|
||||
import { useComboProxyAssignments } from "./useComboProxyAssignments";
|
||||
import { ResponseValidationEditor, type ResponseValidationValue } from "./ResponseValidationEditor";
|
||||
import ReasoningTokenBufferToggle from "./ReasoningTokenBufferToggle";
|
||||
@@ -2793,7 +2795,11 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders, combo
|
||||
saveData.description = null;
|
||||
}
|
||||
|
||||
const configToSave = sanitizeComboRuntimeConfig(config);
|
||||
const configToSave = applyQuotaOnlyFallbackConfig(
|
||||
strategy,
|
||||
models,
|
||||
sanitizeComboRuntimeConfig(config)
|
||||
);
|
||||
if (strategy === "round-robin") {
|
||||
if (config.concurrencyPerModel !== undefined)
|
||||
configToSave.concurrencyPerModel = config.concurrencyPerModel;
|
||||
@@ -3477,24 +3483,16 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders, combo
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{strategy === "cost-optimized" && (
|
||||
<span
|
||||
className={`text-[9px] px-1.5 py-0.5 rounded-full uppercase font-semibold ${
|
||||
hasPricingForModel(entry.model)
|
||||
? "bg-emerald-500/15 text-emerald-600 dark:text-emerald-400"
|
||||
: "bg-amber-500/15 text-amber-600 dark:text-amber-400"
|
||||
}`}
|
||||
title={
|
||||
hasPricingForModel(entry.model)
|
||||
? getI18nOrFallback(t, "pricingAvailable", "Pricing available")
|
||||
: getI18nOrFallback(t, "pricingMissing", "No pricing")
|
||||
}
|
||||
>
|
||||
{hasPricingForModel(entry.model)
|
||||
? getI18nOrFallback(t, "pricingAvailableShort", "priced")
|
||||
: getI18nOrFallback(t, "pricingMissingShort", "no-price")}
|
||||
</span>
|
||||
)}
|
||||
<ComboTargetOptions
|
||||
strategy={strategy}
|
||||
entry={entry}
|
||||
index={index}
|
||||
onCheckedChange={(stepIndex, enabled) =>
|
||||
setModels(setQuotaOnlyFallback(models, stepIndex, enabled))
|
||||
}
|
||||
hasPricing={hasPricingForModel(entry.model)}
|
||||
translate={(key, fallback) => getI18nOrFallback(t, key, fallback)}
|
||||
/>
|
||||
|
||||
{/* Weight input (weighted mode only) */}
|
||||
{strategy === "weighted" && (
|
||||
|
||||
@@ -1,18 +1,13 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import {
|
||||
getComboById,
|
||||
updateCombo,
|
||||
deleteCombo,
|
||||
getComboByName,
|
||||
getCombos,
|
||||
isCloudEnabled,
|
||||
} from "@/lib/localDb";
|
||||
import { getComboById, updateCombo, deleteCombo, getComboByName, getCombos } from "@/lib/db/combos";
|
||||
import { isCloudEnabled } from "@/lib/db/settings";
|
||||
import { getConsistentMachineId } from "@/shared/utils/machineId";
|
||||
import { syncToCloud } from "@/lib/cloudSync";
|
||||
import { validateCompositeTiersConfig } from "@/lib/combos/compositeTiers";
|
||||
import { normalizeComboModels } from "@/lib/combos/steps";
|
||||
import { validateComboDAG, clampComboDepth } from "@omniroute/open-sse/services/combo.ts";
|
||||
import { updateComboSchema } from "@/shared/validation/schemas";
|
||||
import { requiresQuotaOnlyComboRefExecute } from "@/shared/validation/schemas/combo";
|
||||
import { isValidationFailure, validateBody } from "@/shared/validation/helpers";
|
||||
import { requireManagementAuth } from "@/lib/api/requireManagementAuth";
|
||||
import { QUOTA_MODEL_PREFIX } from "@/lib/quota/quotaModelNaming";
|
||||
@@ -151,17 +146,17 @@ export async function PUT(request, { params }) {
|
||||
const normalizedUpdate = { ...validation.data };
|
||||
if (normalizedUpdate.compressionOverride !== undefined) {
|
||||
const legacyCompressionOverride = normalizedUpdate.compressionOverride;
|
||||
const nextConfig: Record<string, unknown> =
|
||||
currentCombo.config &&
|
||||
typeof currentCombo.config === "object" &&
|
||||
!Array.isArray(currentCombo.config)
|
||||
? { ...(currentCombo.config as Record<string, unknown>) }
|
||||
: {};
|
||||
if (legacyCompressionOverride) {
|
||||
nextConfig.compressionMode = legacyCompressionOverride;
|
||||
} else {
|
||||
delete nextConfig.compressionMode;
|
||||
}
|
||||
const nextConfig: Record<string, unknown> =
|
||||
currentCombo.config &&
|
||||
typeof currentCombo.config === "object" &&
|
||||
!Array.isArray(currentCombo.config)
|
||||
? { ...(currentCombo.config as Record<string, unknown>) }
|
||||
: {};
|
||||
if (legacyCompressionOverride) {
|
||||
nextConfig.compressionMode = legacyCompressionOverride;
|
||||
} else {
|
||||
delete nextConfig.compressionMode;
|
||||
}
|
||||
normalizedUpdate.config = nextConfig;
|
||||
delete normalizedUpdate.compressionOverride;
|
||||
}
|
||||
@@ -187,6 +182,17 @@ export async function PUT(request, { params }) {
|
||||
...body,
|
||||
name: comboName,
|
||||
};
|
||||
if (requiresQuotaOnlyComboRefExecute(nextComboState as never)) {
|
||||
return comboErrorResponse(
|
||||
"COMBO_002",
|
||||
400,
|
||||
{
|
||||
firstField: "config.nestedComboMode",
|
||||
firstMessage: "Quota-only combo references require nestedComboMode execute",
|
||||
},
|
||||
request
|
||||
);
|
||||
}
|
||||
const compositeValidation = validateCompositeTiersConfig(nextComboState);
|
||||
if (compositeValidation.success === false) {
|
||||
const failure = compositeValidation as {
|
||||
@@ -235,12 +241,7 @@ export async function PUT(request, { params }) {
|
||||
: dagError instanceof Error && /depth/i.test(dagError.message)
|
||||
? "max-depth-exceeded"
|
||||
: "invalid-graph";
|
||||
return comboErrorResponse(
|
||||
"COMBO_005",
|
||||
400,
|
||||
{ comboName, reason },
|
||||
request
|
||||
);
|
||||
return comboErrorResponse("COMBO_005", 400, { comboName, reason }, request);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -253,9 +254,7 @@ export async function PUT(request, { params }) {
|
||||
// #8530: a combo renamed to a real model id is a supported pattern
|
||||
// (#6940 — bare-model-id provider fallback), so it is never rejected.
|
||||
// Surface it as a non-blocking warning instead of silently shadowing it.
|
||||
const warning = comboName
|
||||
? buildComboNameCollisionWarning(String(comboName))
|
||||
: null;
|
||||
const warning = comboName ? buildComboNameCollisionWarning(String(comboName)) : null;
|
||||
return NextResponse.json(warning ? { ...combo, warning } : combo);
|
||||
} catch (error) {
|
||||
if (error instanceof ComboInvariantError) {
|
||||
|
||||
@@ -3248,6 +3248,8 @@
|
||||
"pricingMissing": "No pricing",
|
||||
"pricingAvailableShort": "priced",
|
||||
"pricingMissingShort": "no-price",
|
||||
"fallbackOnlyOnQuotaExhaustion": "Only advance on quota exhaustion",
|
||||
"fallbackOnlyOnQuotaExhaustionShort": "quota-only fallback",
|
||||
"warningRoundRobinSingleModel": "Round-robin is most useful with at least 2 models.",
|
||||
"warningCostOptimizedPartialPricing": "Only {priced} of {total} models have pricing. Routing may be partially cost-aware.",
|
||||
"warningCostOptimizedNoPricing": "No pricing data found for this combo. Cost-optimized may route unexpectedly.",
|
||||
|
||||
@@ -3273,6 +3273,8 @@
|
||||
"pricingMissing": "无定价",
|
||||
"pricingAvailableShort": "已定价",
|
||||
"pricingMissingShort": "无定价",
|
||||
"fallbackOnlyOnQuotaExhaustion": "仅配额/额度耗尽时进入下一目标",
|
||||
"fallbackOnlyOnQuotaExhaustionShort": "仅配额耗尽回退",
|
||||
"warningRoundRobinSingleModel": "轮询策略至少有 2 个模型时才最有价值。",
|
||||
"warningCostOptimizedPartialPricing": "在 {total} 个模型中,只有 {priced} 个有定价信息。路由可能只具备部分成本感知能力。",
|
||||
"warningCostOptimizedNoPricing": "该组合未找到任何定价数据。成本优化策略的路由结果可能不符合预期。",
|
||||
|
||||
@@ -25,6 +25,7 @@ export interface ComboModelStep {
|
||||
label?: string;
|
||||
prompt?: string | null;
|
||||
tags?: string[];
|
||||
fallbackOnlyOnQuotaExhaustion?: boolean;
|
||||
}
|
||||
|
||||
export interface ComboRefStep {
|
||||
@@ -33,6 +34,7 @@ export interface ComboRefStep {
|
||||
comboName: string;
|
||||
weight: number;
|
||||
label?: string;
|
||||
fallbackOnlyOnQuotaExhaustion?: boolean;
|
||||
}
|
||||
|
||||
export interface ComboProviderWildcardStep {
|
||||
@@ -287,6 +289,7 @@ export function normalizeComboStep(
|
||||
const weight = toWeight(value.weight);
|
||||
const label = toTrimmedString(value.label);
|
||||
const prompt = toTrimmedString(value.prompt);
|
||||
const fallbackOnlyOnQuotaExhaustion = value.fallbackOnlyOnQuotaExhaustion === true;
|
||||
|
||||
if (value.kind === "combo-ref") {
|
||||
const comboRefName = toTrimmedString(value.comboName);
|
||||
@@ -297,6 +300,7 @@ export function normalizeComboStep(
|
||||
comboName: comboRefName,
|
||||
weight,
|
||||
...(label ? { label } : {}),
|
||||
...(fallbackOnlyOnQuotaExhaustion ? { fallbackOnlyOnQuotaExhaustion: true } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -409,6 +413,7 @@ export function normalizeComboStep(
|
||||
...(prompt ? { prompt } : {}),
|
||||
...(tags && tags.length > 0 ? { tags } : {}),
|
||||
...(allowedConnectionIds && allowedConnectionIds.length > 0 ? { allowedConnectionIds } : {}),
|
||||
...(fallbackOnlyOnQuotaExhaustion ? { fallbackOnlyOnQuotaExhaustion: true } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -20,6 +20,7 @@ export const comboStepMetaSchema = {
|
||||
id: z.string().trim().min(1).max(200).optional(),
|
||||
weight: z.number().min(0).max(100).optional().default(0),
|
||||
label: z.string().trim().min(1).max(200).optional(),
|
||||
fallbackOnlyOnQuotaExhaustion: z.boolean().optional(),
|
||||
};
|
||||
|
||||
export const comboModelStepInputSchema = z.object({
|
||||
@@ -284,29 +285,61 @@ export const comboNameSchema = z
|
||||
"Name can only contain letters, numbers, spaces, -, _, /, ., [ and ]."
|
||||
);
|
||||
|
||||
export const createComboSchema = z.object({
|
||||
name: comboNameSchema,
|
||||
description: z.string().max(2000).optional(),
|
||||
models: z.array(comboModelEntry).optional().default([]),
|
||||
strategy: comboStrategySchema.optional().default("priority"),
|
||||
config: comboRuntimeConfigSchema.optional(),
|
||||
allowedProviders: z.array(z.string().trim().min(1).max(200)).max(100).optional(),
|
||||
allowedModelFamilies: z.array(z.string().trim().min(1).max(100)).max(100).optional(),
|
||||
system_message: z.string().max(50000).optional(),
|
||||
tool_filter_regex: z.string().max(1000).optional(),
|
||||
context_cache_protection: z.boolean().optional(),
|
||||
context_length: z.number().int().min(1000).max(2000000).optional(),
|
||||
// Optional embedding dimensions override for embedding combos.
|
||||
// When set, the value is injected into every upstream embedding request as
|
||||
// the `dimensions` field (and translated to `outputDimensionality` for Gemini).
|
||||
// Stored as a string to match the OpenAI API convention; coerced to number
|
||||
// by the embedding handler. Leave unset to use each model's default.
|
||||
dimensions: z
|
||||
.string()
|
||||
.regex(/^\d+$/, "dimensions must be a positive integer string")
|
||||
.optional()
|
||||
.nullable(),
|
||||
});
|
||||
type QuotaOnlyComboRefState = {
|
||||
models?: Array<z.infer<typeof comboModelEntry>>;
|
||||
strategy?: string;
|
||||
config?: z.infer<typeof comboRuntimeConfigSchema>;
|
||||
};
|
||||
|
||||
export function requiresQuotaOnlyComboRefExecute(value: QuotaOnlyComboRefState): boolean {
|
||||
const hasProtectedComboRef = value.models?.some(
|
||||
(step) =>
|
||||
typeof step === "object" &&
|
||||
step.kind === "combo-ref" &&
|
||||
step.fallbackOnlyOnQuotaExhaustion === true
|
||||
);
|
||||
return (
|
||||
(value.strategy === undefined || value.strategy === "priority") &&
|
||||
hasProtectedComboRef === true &&
|
||||
value.config?.nestedComboMode !== "execute"
|
||||
);
|
||||
}
|
||||
|
||||
function validateQuotaOnlyComboRefs(value: QuotaOnlyComboRefState, ctx: z.RefinementCtx): void {
|
||||
if (requiresQuotaOnlyComboRefExecute(value)) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: "Quota-only combo references require nestedComboMode execute",
|
||||
path: ["config", "nestedComboMode"],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export const createComboSchema = z
|
||||
.object({
|
||||
name: comboNameSchema,
|
||||
description: z.string().max(2000).optional(),
|
||||
models: z.array(comboModelEntry).optional().default([]),
|
||||
strategy: comboStrategySchema.optional().default("priority"),
|
||||
config: comboRuntimeConfigSchema.optional(),
|
||||
allowedProviders: z.array(z.string().trim().min(1).max(200)).max(100).optional(),
|
||||
allowedModelFamilies: z.array(z.string().trim().min(1).max(100)).max(100).optional(),
|
||||
system_message: z.string().max(50000).optional(),
|
||||
tool_filter_regex: z.string().max(1000).optional(),
|
||||
context_cache_protection: z.boolean().optional(),
|
||||
context_length: z.number().int().min(1000).max(2000000).optional(),
|
||||
// Optional embedding dimensions override for embedding combos.
|
||||
// When set, the value is injected into every upstream embedding request as
|
||||
// the `dimensions` field (and translated to `outputDimensionality` for Gemini).
|
||||
// Stored as a string to match the OpenAI API convention; coerced to number
|
||||
// by the embedding handler. Leave unset to use each model's default.
|
||||
dimensions: z
|
||||
.string()
|
||||
.regex(/^\d+$/, "dimensions must be a positive integer string")
|
||||
.optional()
|
||||
.nullable(),
|
||||
})
|
||||
.superRefine(validateQuotaOnlyComboRefs);
|
||||
|
||||
export const updateComboDefaultsSchema = z
|
||||
.object({
|
||||
|
||||
@@ -233,34 +233,48 @@ test("handleChat applies task-aware routing when a semantic override is enabled"
|
||||
assert.equal(json.choices[0].message.content, "Task-routed response");
|
||||
});
|
||||
|
||||
test("handleChat routes exact combo names and can recover via global fallback", async () => {
|
||||
test("handleChat keeps protected combo fallback separate from Global Fallback Model", async () => {
|
||||
await seedConnection("openai", { apiKey: "sk-openai-combo-route" });
|
||||
const ordinaryTarget = await seedConnection("deepseek", {
|
||||
apiKey: "sk-deepseek-combo-backup",
|
||||
});
|
||||
await seedConnection("claude", { apiKey: "sk-claude-global-fallback" });
|
||||
await combosDb.createCombo({
|
||||
name: "router-global-fallback",
|
||||
strategy: "priority",
|
||||
config: { maxRetries: 0, retryDelayMs: 0 },
|
||||
models: ["openai/gpt-4.1"],
|
||||
models: [
|
||||
{
|
||||
kind: "model",
|
||||
model: "openai/gpt-4.1",
|
||||
fallbackOnlyOnQuotaExhaustion: true,
|
||||
},
|
||||
{
|
||||
kind: "model",
|
||||
model: "deepseek/deepseek-v4-flash",
|
||||
connectionId: ordinaryTarget.id,
|
||||
},
|
||||
],
|
||||
});
|
||||
await settingsDb.updateSettings({
|
||||
globalFallbackModel: "claude/claude-3-5-sonnet-20241022",
|
||||
});
|
||||
|
||||
let attempts = 0;
|
||||
globalThis.fetch = async (_url, init = {}) => {
|
||||
attempts += 1;
|
||||
const headers = toPlainHeaders(init.headers);
|
||||
if (attempts === 1) {
|
||||
assert.equal(headers.Authorization ?? headers.authorization, "Bearer sk-openai-combo-route");
|
||||
const attemptedKeys: string[] = [];
|
||||
globalThis.fetch = async (_url, init) => {
|
||||
const headers = toPlainHeaders(init?.headers);
|
||||
const key = headers["x-api-key"] ?? headers.Authorization ?? headers.authorization ?? "";
|
||||
attemptedKeys.push(key);
|
||||
if (key === "sk-deepseek-combo-backup") {
|
||||
assert.fail("protected primary must not invoke the ordinary combo target");
|
||||
}
|
||||
if (key === "Bearer sk-openai-combo-route") {
|
||||
return new Response(JSON.stringify({ error: { message: "primary combo failed" } }), {
|
||||
status: 503,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
}
|
||||
assert.equal(
|
||||
headers["x-api-key"] ?? headers.Authorization ?? headers.authorization,
|
||||
"sk-claude-global-fallback"
|
||||
);
|
||||
assert.equal(key, "sk-claude-global-fallback");
|
||||
return buildClaudeResponse("Global fallback answered");
|
||||
};
|
||||
|
||||
@@ -276,7 +290,7 @@ test("handleChat routes exact combo names and can recover via global fallback",
|
||||
const json = (await response.json()) as any;
|
||||
|
||||
assert.equal(response.status, 200);
|
||||
assert.equal(attempts, 2);
|
||||
assert.deepEqual(attemptedKeys, ["Bearer sk-openai-combo-route", "sk-claude-global-fallback"]);
|
||||
assert.equal(json.choices[0].message.content, "Global fallback answered");
|
||||
});
|
||||
|
||||
|
||||
@@ -606,6 +606,82 @@ test("tryRuntimeUnitDispatch: round-robin rotates across successive dispatches",
|
||||
);
|
||||
});
|
||||
|
||||
test("tryRuntimeUnitDispatch: sticky weighted keeps quota-only fallback dormant", async () => {
|
||||
const combo = {
|
||||
...twoRefCombo("weighted-quota-dormant", "weighted", { stickyWeightedLimit: 5 }),
|
||||
models: [
|
||||
{ kind: "combo-ref", comboName: "leafA", fallbackOnlyOnQuotaExhaustion: true },
|
||||
{ kind: "combo-ref", comboName: "leafB" },
|
||||
],
|
||||
};
|
||||
const ctx = setup(combo);
|
||||
const units = resolveComboRuntimeUnits(ctx.combo, [ctx.combo, LEAF_A, LEAF_B], "execute", 3);
|
||||
const protectedUnit = units.find(
|
||||
(unit) => unit.kind === "combo-ref" && unit.fallbackOnlyOnQuotaExhaustion === true
|
||||
);
|
||||
assert.ok(protectedUnit);
|
||||
recordStickyWeightedSuccess(combo.name, protectedUnit.executionKey, 5);
|
||||
|
||||
const recursedInto: string[] = [];
|
||||
const result = await tryRuntimeUnitDispatch({
|
||||
body: ctx.body,
|
||||
combo: ctx.combo,
|
||||
config: ctx.config,
|
||||
strategy: "weighted",
|
||||
allCombos: [ctx.combo, LEAF_A, LEAF_B],
|
||||
handleSingleModel: async () => okResponse("raw"),
|
||||
handleSingleModelWithTimeout: async () => okResponse("wrapped"),
|
||||
log: ctx.log,
|
||||
settings: {},
|
||||
runCombo: async (options) => {
|
||||
recursedInto.push(options.combo.name);
|
||||
return options.combo.name === "leafA"
|
||||
? new Response("unavailable", { status: 503 })
|
||||
: okResponse("backup");
|
||||
},
|
||||
});
|
||||
assert.equal(result?.status, 200);
|
||||
assert.deepEqual(recursedInto, [protectedUnit.comboName, protectedUnit.comboName, "leafB"]);
|
||||
});
|
||||
|
||||
test("tryRuntimeUnitDispatch: protected execute response validation returns local 502", async () => {
|
||||
const combo = {
|
||||
name: "protected-quality-execute",
|
||||
strategy: "priority",
|
||||
models: [
|
||||
{ kind: "model", model: "p/invalid", fallbackOnlyOnQuotaExhaustion: true },
|
||||
COMBO_REF_STEP,
|
||||
],
|
||||
config: {
|
||||
nestedComboMode: "execute",
|
||||
maxRetries: 0,
|
||||
responseValidation: { minContentLength: 100 },
|
||||
},
|
||||
};
|
||||
const ctx = setup(combo);
|
||||
const calls: string[] = [];
|
||||
const result = await tryRuntimeUnitDispatch({
|
||||
body: ctx.body,
|
||||
combo: ctx.combo,
|
||||
config: ctx.config,
|
||||
strategy: "priority",
|
||||
allCombos: [ctx.combo, LEAF_COMBO],
|
||||
handleSingleModel: async () => okResponse("raw"),
|
||||
handleSingleModelWithTimeout: async (_body, modelStr) => {
|
||||
calls.push(modelStr);
|
||||
return okResponse("short");
|
||||
},
|
||||
log: ctx.log,
|
||||
settings: {},
|
||||
runCombo: async () => {
|
||||
calls.push("backup");
|
||||
return okResponse("backup");
|
||||
},
|
||||
});
|
||||
assert.equal(result?.status, 502);
|
||||
assert.deepEqual(calls, ["p/invalid"]);
|
||||
});
|
||||
|
||||
test("tryRuntimeUnitDispatch: weighted honors a previously recorded sticky unit", async () => {
|
||||
const combo = twoRefCombo("weighted-sticky", "weighted", { stickyWeightedLimit: 5 });
|
||||
const ctx = setup(combo);
|
||||
|
||||
696
tests/unit/combo-quota-exhaustion-only-fallback.test.ts
Normal file
696
tests/unit/combo-quota-exhaustion-only-fallback.test.ts
Normal file
@@ -0,0 +1,696 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-quota-only-"));
|
||||
const ORIGINAL_DATA_DIR = process.env.DATA_DIR;
|
||||
process.env.DATA_DIR = TEST_DATA_DIR;
|
||||
const { handleComboChat } = await import("../../open-sse/services/combo.ts");
|
||||
const { isQuotaExhaustionResponse } =
|
||||
await import("../../open-sse/services/combo/quotaExhaustion.ts");
|
||||
const { clearAllModelLockouts, lockModel } =
|
||||
await import("../../open-sse/services/accountFallback.ts");
|
||||
const { recordComboRequest, resetComboMetrics } =
|
||||
await import("../../open-sse/services/comboMetrics.ts");
|
||||
const { resetDbInstance } = await import("../../src/lib/db/core.ts");
|
||||
const { resetAllCircuitBreakers } = await import("../../src/shared/utils/circuitBreaker.ts");
|
||||
|
||||
test.afterEach(() => {
|
||||
clearAllModelLockouts();
|
||||
resetAllCircuitBreakers();
|
||||
});
|
||||
|
||||
test.after(() => {
|
||||
resetDbInstance();
|
||||
if (ORIGINAL_DATA_DIR === undefined) delete process.env.DATA_DIR;
|
||||
else process.env.DATA_DIR = ORIGINAL_DATA_DIR;
|
||||
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function log() {
|
||||
return { info() {}, warn() {}, debug() {}, error() {} };
|
||||
}
|
||||
|
||||
function response(
|
||||
status: number,
|
||||
message: string,
|
||||
options: { code?: string; type?: string; headers?: Record<string, string> } = {}
|
||||
) {
|
||||
return Response.json(
|
||||
{
|
||||
error: {
|
||||
message,
|
||||
...(options.code ? { code: options.code } : {}),
|
||||
...(options.type ? { type: options.type } : {}),
|
||||
},
|
||||
},
|
||||
{ status, headers: options.headers }
|
||||
);
|
||||
}
|
||||
|
||||
function ok(model: string) {
|
||||
return Response.json({ model });
|
||||
}
|
||||
|
||||
function model(model: string, fallbackOnlyOnQuotaExhaustion?: boolean) {
|
||||
return {
|
||||
kind: "model" as const,
|
||||
model,
|
||||
providerId: model.split("/")[0],
|
||||
...(fallbackOnlyOnQuotaExhaustion === undefined ? {} : { fallbackOnlyOnQuotaExhaustion }),
|
||||
};
|
||||
}
|
||||
|
||||
async function run(
|
||||
first: Response | Response[],
|
||||
options: {
|
||||
enabled?: boolean;
|
||||
maxRetries?: number;
|
||||
strategy?: string;
|
||||
primary?: string;
|
||||
connectionId?: string;
|
||||
config?: Record<string, unknown>;
|
||||
settings?: Record<string, unknown> | null;
|
||||
available?: (modelStr: string) => Promise<boolean> | boolean;
|
||||
} = {}
|
||||
) {
|
||||
const calls: string[] = [];
|
||||
const primaryResponses = Array.isArray(first) ? first : [first];
|
||||
let primaryAttempt = 0;
|
||||
const combo = {
|
||||
name: `quota-only-${Math.random()}`,
|
||||
strategy: options.strategy ?? "priority",
|
||||
models: [
|
||||
{
|
||||
...model(options.primary ?? "openai/primary", options.enabled),
|
||||
...(options.connectionId ? { connectionId: options.connectionId } : {}),
|
||||
},
|
||||
model("anthropic/backup"),
|
||||
],
|
||||
config: { maxRetries: options.maxRetries ?? 0, retryDelayMs: 0, ...options.config },
|
||||
};
|
||||
const result = await handleComboChat({
|
||||
body: {},
|
||||
combo,
|
||||
allCombos: [combo],
|
||||
settings: options.settings ?? null,
|
||||
log: log(),
|
||||
isModelAvailable: options.available ?? (async () => true),
|
||||
handleSingleModel: async (_body, modelStr) => {
|
||||
calls.push(modelStr);
|
||||
if (modelStr !== (options.primary ?? "openai/primary")) return ok(modelStr);
|
||||
const result = primaryResponses[Math.min(primaryAttempt, primaryResponses.length - 1)];
|
||||
primaryAttempt += 1;
|
||||
return result.clone();
|
||||
},
|
||||
});
|
||||
return { result, calls };
|
||||
}
|
||||
|
||||
test("quota classifier rejects terminal-looking evidence on ineligible statuses", async () => {
|
||||
for (const status of [400, 401, 403, 404, 408, 409, 422, 500, 502, 503, 504]) {
|
||||
for (const terminal of ["insufficient_quota", "quota_exhausted", "credits_exhausted"]) {
|
||||
assert.equal(
|
||||
await isQuotaExhaustionResponse(
|
||||
response(status, `${terminal}: billing hard limit reached`, {
|
||||
code: terminal,
|
||||
type: terminal,
|
||||
}),
|
||||
"openai",
|
||||
"gpt-4o"
|
||||
),
|
||||
false,
|
||||
`${status} with ${terminal} must not classify as terminal depletion`
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test("quota classifier accepts explicit terminal depletion only on eligible statuses", async () => {
|
||||
for (const status of [402, 429]) {
|
||||
assert.equal(
|
||||
await isQuotaExhaustionResponse(
|
||||
response(status, "Payment required", { code: "insufficient_quota" }),
|
||||
"openai",
|
||||
"gpt-4o"
|
||||
),
|
||||
true
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("priority target advances on explicit quota exhaustion", async () => {
|
||||
const { result, calls } = await run(response(429, "You have exceeded your current quota"), {
|
||||
enabled: true,
|
||||
});
|
||||
assert.equal(result.ok, true);
|
||||
assert.deepEqual(calls, ["openai/primary", "anthropic/backup"]);
|
||||
});
|
||||
|
||||
test("priority target retries plain 429 then returns the last response without advancing", async () => {
|
||||
const { result, calls } = await run(response(429, "Rate limit exceeded; retry later"), {
|
||||
enabled: true,
|
||||
maxRetries: 1,
|
||||
});
|
||||
assert.equal(result.status, 429);
|
||||
assert.deepEqual(calls, ["openai/primary", "openai/primary"]);
|
||||
});
|
||||
|
||||
test("priority target keeps a non-quota retry monotonic through terminal quota exhaustion", async () => {
|
||||
const { result, calls } = await run(
|
||||
[
|
||||
response(503, "Service unavailable"),
|
||||
response(429, "Payment required", { code: "insufficient_quota" }),
|
||||
],
|
||||
{ enabled: true, maxRetries: 1 }
|
||||
);
|
||||
assert.equal(result.status, 429);
|
||||
assert.deepEqual(calls, ["openai/primary", "openai/primary"]);
|
||||
});
|
||||
|
||||
test("priority target returns a later successful retry", async () => {
|
||||
const { result, calls } = await run(
|
||||
[response(503, "Service unavailable"), ok("openai/primary")],
|
||||
{ enabled: true, maxRetries: 1 }
|
||||
);
|
||||
assert.equal(result.ok, true);
|
||||
assert.deepEqual(calls, ["openai/primary", "openai/primary"]);
|
||||
});
|
||||
|
||||
test("priority target advances when every retry is quota exhaustion", async () => {
|
||||
const { result, calls } = await run(
|
||||
[
|
||||
response(429, "Payment required", { code: "quota_exhausted" }),
|
||||
response(429, "Billing hard limit reached", { code: "quota_exhausted" }),
|
||||
],
|
||||
{ enabled: true, maxRetries: 1 }
|
||||
);
|
||||
assert.equal(result.ok, true);
|
||||
assert.deepEqual(calls, ["openai/primary", "openai/primary", "anthropic/backup"]);
|
||||
});
|
||||
|
||||
test("failures of another target do not contaminate protected target trust", async () => {
|
||||
const calls: string[] = [];
|
||||
const combo = {
|
||||
name: `target-local-${Math.random()}`,
|
||||
strategy: "priority",
|
||||
models: [model("openai/unprotected"), model("anthropic/protected", true), model("paid/backup")],
|
||||
config: { maxRetries: 0, retryDelayMs: 0 },
|
||||
};
|
||||
const result = await handleComboChat({
|
||||
body: {},
|
||||
combo,
|
||||
allCombos: [combo],
|
||||
settings: null,
|
||||
log: log(),
|
||||
isModelAvailable: async () => true,
|
||||
handleSingleModel: async (_body, modelStr) => {
|
||||
calls.push(modelStr);
|
||||
return modelStr === "openai/unprotected"
|
||||
? response(503, "Service unavailable")
|
||||
: modelStr === "anthropic/protected"
|
||||
? response(429, "Payment required", { code: "quota_exhausted" })
|
||||
: ok(modelStr);
|
||||
},
|
||||
});
|
||||
assert.equal(result.ok, true);
|
||||
assert.deepEqual(calls, ["openai/unprotected", "anthropic/protected", "paid/backup"]);
|
||||
});
|
||||
|
||||
for (const [status, message] of [
|
||||
[408, "Request timeout"],
|
||||
[502, "Bad gateway"],
|
||||
[503, "Service unavailable"],
|
||||
[503, "Local concurrency capacity reached"],
|
||||
[504, "Gateway timeout"],
|
||||
[401, "Invalid API key"],
|
||||
] as const) {
|
||||
test(`priority target does not advance on non-quota ${status} (${message})`, async () => {
|
||||
const { result, calls } = await run(response(status, message), { enabled: true });
|
||||
assert.equal(result.status, status);
|
||||
assert.deepEqual(calls, ["openai/primary"]);
|
||||
});
|
||||
}
|
||||
|
||||
test("protected non-quota response preserves status, body, and headers", async () => {
|
||||
const first = response(502, "upstream detail", { headers: { "x-upstream": "kept" } });
|
||||
const { result } = await run(first, { enabled: true });
|
||||
assert.equal(result.status, 502);
|
||||
assert.equal(result.headers.get("x-upstream"), "kept");
|
||||
assert.deepEqual(await result.json(), { error: { message: "upstream detail" } });
|
||||
});
|
||||
|
||||
for (const [metric, advances] of [
|
||||
["generate_content_free_tier_requests", false],
|
||||
["generate_content_free_tier_input_token_count", false],
|
||||
["generate_content_free_tier_requests_per_day", true],
|
||||
] as const) {
|
||||
test(`Gemini ${metric} classification ${advances ? "advances" : "stops"} direct routing`, async () => {
|
||||
const message = `You exceeded your current quota for metric generativelanguage.googleapis.com/${metric}`;
|
||||
const { result, calls } = await run(response(429, message), {
|
||||
enabled: true,
|
||||
primary: "gemini/gemini-2.5-flash",
|
||||
});
|
||||
assert.equal(result.ok, advances);
|
||||
assert.deepEqual(
|
||||
calls,
|
||||
advances ? ["gemini/gemini-2.5-flash", "anthropic/backup"] : ["gemini/gemini-2.5-flash"]
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
test("compatible provider advances only on explicit structured terminal depletion", async () => {
|
||||
const plain = await run(response(402, "billing service unavailable"), {
|
||||
enabled: true,
|
||||
primary: "openai-compatible-chat-test/primary",
|
||||
});
|
||||
assert.equal(plain.result.status, 402);
|
||||
assert.deepEqual(plain.calls, ["openai-compatible-chat-test/primary"]);
|
||||
|
||||
const terminal = await run(response(402, "Payment required", { code: "insufficient_quota" }), {
|
||||
enabled: true,
|
||||
primary: "openai-compatible-chat-test/primary",
|
||||
});
|
||||
assert.equal(terminal.result.ok, true);
|
||||
assert.deepEqual(terminal.calls, ["openai-compatible-chat-test/primary", "anthropic/backup"]);
|
||||
});
|
||||
|
||||
test("unset Gemini RPM response retains baseline fallback", async () => {
|
||||
const { result, calls } = await run(
|
||||
response(
|
||||
429,
|
||||
"You exceeded your current quota for metric generativelanguage.googleapis.com/generate_content_free_tier_requests"
|
||||
),
|
||||
{ primary: "gemini/gemini-2.5-flash" }
|
||||
);
|
||||
assert.equal(result.ok, true);
|
||||
assert.deepEqual(calls, ["gemini/gemini-2.5-flash", "anthropic/backup"]);
|
||||
});
|
||||
|
||||
test("unset target retains generic fallback", async () => {
|
||||
const { result, calls } = await run(response(503, "Service unavailable"));
|
||||
assert.equal(result.ok, true);
|
||||
assert.deepEqual(calls, ["openai/primary", "anthropic/backup"]);
|
||||
});
|
||||
|
||||
test("option is dormant outside priority", async () => {
|
||||
const { result, calls } = await run(response(503, "Service unavailable"), {
|
||||
enabled: true,
|
||||
strategy: "fill-first",
|
||||
});
|
||||
assert.equal(result.ok, true);
|
||||
assert.deepEqual(calls, ["openai/primary", "anthropic/backup"]);
|
||||
});
|
||||
|
||||
test("authoritative local quota cutoff advances an opted-in priority target", async () => {
|
||||
const calls: string[] = [];
|
||||
const combo = {
|
||||
name: `local-quota-${Math.random()}`,
|
||||
strategy: "priority",
|
||||
models: [
|
||||
{ ...model("openai/primary", true), connectionId: "quota-empty" },
|
||||
model("anthropic/backup"),
|
||||
],
|
||||
config: { maxRetries: 0, retryDelayMs: 0 },
|
||||
};
|
||||
const quotaPreflight = await import("../../open-sse/services/quotaPreflight.ts");
|
||||
quotaPreflight.registerQuotaFetcher("openai", async () => ({
|
||||
used: 100,
|
||||
total: 100,
|
||||
percentUsed: 1,
|
||||
}));
|
||||
const result = await handleComboChat({
|
||||
body: {},
|
||||
combo,
|
||||
allCombos: [combo],
|
||||
log: log(),
|
||||
settings: {
|
||||
resilienceSettings: {
|
||||
quotaPreflight: { enabled: true, defaultThresholdPercent: 2, warnThresholdPercent: 20 },
|
||||
},
|
||||
},
|
||||
isModelAvailable: async () => true,
|
||||
handleSingleModel: async (_body, modelStr) => {
|
||||
calls.push(modelStr);
|
||||
return ok(modelStr);
|
||||
},
|
||||
});
|
||||
assert.equal(result.ok, true);
|
||||
assert.deepEqual(calls, ["anthropic/backup"]);
|
||||
});
|
||||
|
||||
test("protected pre-existing model lockout stops routing with a local 503", async () => {
|
||||
lockModel("openai", "locked-connection", "primary", "rate_limited", 60_000);
|
||||
const { result, calls } = await run(response(503, "unused"), {
|
||||
enabled: true,
|
||||
connectionId: "locked-connection",
|
||||
});
|
||||
assert.equal(result.status, 503);
|
||||
assert.deepEqual(calls, []);
|
||||
});
|
||||
|
||||
test("protected target keeps configured retries after recording a transient model lockout", async () => {
|
||||
const { result, calls } = await run(response(503, "Service unavailable"), {
|
||||
enabled: true,
|
||||
maxRetries: 1,
|
||||
connectionId: "retry-connection",
|
||||
settings: {
|
||||
resilienceSettings: {
|
||||
modelLockout: { enabled: true, errorCodes: [503], baseCooldownMs: 60_000 },
|
||||
},
|
||||
},
|
||||
});
|
||||
assert.equal(result.status, 503);
|
||||
assert.deepEqual(calls, ["openai/primary", "openai/primary"]);
|
||||
});
|
||||
|
||||
test("protected response-quality rejection returns a sanitized 502 without advancing", async () => {
|
||||
const { result, calls } = await run(ok("openai/primary"), {
|
||||
enabled: true,
|
||||
config: { responseValidation: { minContentLength: 100 } },
|
||||
});
|
||||
assert.equal(result.status, 502);
|
||||
assert.deepEqual(await result.json(), {
|
||||
error: {
|
||||
message: "Upstream response failed quality validation",
|
||||
type: "server_error",
|
||||
code: "bad_gateway",
|
||||
},
|
||||
});
|
||||
assert.deepEqual(calls, ["openai/primary"]);
|
||||
});
|
||||
|
||||
test("protected predictive TTFT skip returns local 503 without advancing", async () => {
|
||||
const name = `predictive-${Math.random()}`;
|
||||
resetComboMetrics(name);
|
||||
for (let index = 0; index < 5; index += 1) {
|
||||
recordComboRequest(name, "openai/primary", { success: true, latencyMs: 10_000 });
|
||||
}
|
||||
const calls: string[] = [];
|
||||
const combo = {
|
||||
name,
|
||||
strategy: "priority",
|
||||
models: [model("openai/primary", true), model("anthropic/backup")],
|
||||
config: { maxRetries: 0, zeroLatencyOptimizationsEnabled: true, predictiveTtftMs: 1 },
|
||||
};
|
||||
const result = await handleComboChat({
|
||||
body: {},
|
||||
combo,
|
||||
allCombos: [combo],
|
||||
log: log(),
|
||||
settings: null,
|
||||
isModelAvailable: async () => true,
|
||||
handleSingleModel: async (_body, modelStr) => {
|
||||
calls.push(modelStr);
|
||||
return ok(modelStr);
|
||||
},
|
||||
});
|
||||
assert.equal(result.status, 503);
|
||||
assert.deepEqual(calls, []);
|
||||
});
|
||||
|
||||
test("opted-in combo ref remains a black box and only quota exhaustion advances parent", async () => {
|
||||
for (const childError of [
|
||||
response(429, "Billing hard limit reached"),
|
||||
response(503, "Service unavailable"),
|
||||
]) {
|
||||
const calls: string[] = [];
|
||||
const child = {
|
||||
name: `child-${childError.status}-${Math.random()}`,
|
||||
strategy: "priority",
|
||||
models: [model("openai/child")],
|
||||
config: { maxRetries: 0, retryDelayMs: 0 },
|
||||
};
|
||||
const outer = {
|
||||
name: `outer-${Math.random()}`,
|
||||
strategy: "priority",
|
||||
models: [
|
||||
{
|
||||
kind: "combo-ref" as const,
|
||||
comboName: child.name,
|
||||
fallbackOnlyOnQuotaExhaustion: true,
|
||||
},
|
||||
model("anthropic/backup"),
|
||||
],
|
||||
config: { nestedComboMode: "execute", maxRetries: 0, retryDelayMs: 0 },
|
||||
};
|
||||
const result = await handleComboChat({
|
||||
body: {},
|
||||
combo: outer,
|
||||
allCombos: [outer, child],
|
||||
settings: null,
|
||||
log: log(),
|
||||
isModelAvailable: async () => true,
|
||||
handleSingleModel: async (_body, modelStr) => {
|
||||
calls.push(modelStr);
|
||||
return modelStr === "openai/child" ? childError.clone() : ok(modelStr);
|
||||
},
|
||||
});
|
||||
if (childError.status === 429) {
|
||||
assert.equal(result.ok, true);
|
||||
assert.deepEqual(calls, ["openai/child", "anthropic/backup"]);
|
||||
} else {
|
||||
assert.equal(result.status, 503);
|
||||
assert.deepEqual(calls, ["openai/child"]);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test("protected parent combo-ref stops after normal child quality rejection then quota exhaustion", async () => {
|
||||
const calls: string[] = [];
|
||||
const child = {
|
||||
name: `quality-child-${Math.random()}`,
|
||||
strategy: "priority",
|
||||
models: [model("openai/quality-invalid"), model("anthropic/quota")],
|
||||
config: {
|
||||
maxRetries: 0,
|
||||
retryDelayMs: 0,
|
||||
flattenSingleTargetNested: true,
|
||||
responseValidation: { minContentLength: 100 },
|
||||
},
|
||||
};
|
||||
const outer = {
|
||||
name: `quality-outer-${Math.random()}`,
|
||||
strategy: "priority",
|
||||
models: [
|
||||
{ kind: "combo-ref" as const, comboName: child.name, fallbackOnlyOnQuotaExhaustion: true },
|
||||
model("paid/backup"),
|
||||
],
|
||||
config: { nestedComboMode: "execute", maxRetries: 0, retryDelayMs: 0 },
|
||||
};
|
||||
const result = await handleComboChat({
|
||||
body: {},
|
||||
combo: outer,
|
||||
allCombos: [outer, child],
|
||||
settings: null,
|
||||
log: log(),
|
||||
isModelAvailable: async () => true,
|
||||
handleSingleModel: async (_body, modelStr) => {
|
||||
calls.push(modelStr);
|
||||
return modelStr === "openai/quality-invalid"
|
||||
? Response.json({ choices: [{ message: {} }] })
|
||||
: modelStr === "anthropic/quota"
|
||||
? response(429, "Payment required", { code: "insufficient_quota" })
|
||||
: ok(modelStr);
|
||||
},
|
||||
});
|
||||
assert.equal(result.status, 429);
|
||||
assert.deepEqual(calls, ["openai/quality-invalid", "anthropic/quota"]);
|
||||
});
|
||||
|
||||
test("protected parent combo-ref stops when a child has mixed quota and non-quota failures", async () => {
|
||||
const calls: string[] = [];
|
||||
const child = {
|
||||
name: `mixed-child-${Math.random()}`,
|
||||
strategy: "priority",
|
||||
models: [model("openai/quota"), model("anthropic/unavailable")],
|
||||
config: { maxRetries: 0, retryDelayMs: 0 },
|
||||
};
|
||||
const outer = {
|
||||
name: `mixed-outer-${Math.random()}`,
|
||||
strategy: "priority",
|
||||
models: [
|
||||
{ kind: "combo-ref" as const, comboName: child.name, fallbackOnlyOnQuotaExhaustion: true },
|
||||
model("anthropic/paid-backup"),
|
||||
],
|
||||
config: { nestedComboMode: "execute", maxRetries: 0, retryDelayMs: 0 },
|
||||
};
|
||||
const result = await handleComboChat({
|
||||
body: {},
|
||||
combo: outer,
|
||||
allCombos: [outer, child],
|
||||
settings: null,
|
||||
log: log(),
|
||||
isModelAvailable: async (modelStr) => modelStr !== "anthropic/unavailable",
|
||||
handleSingleModel: async (_body, modelStr) => {
|
||||
calls.push(modelStr);
|
||||
return modelStr === "openai/quota"
|
||||
? response(429, "Billing hard limit reached")
|
||||
: ok(modelStr);
|
||||
},
|
||||
});
|
||||
assert.equal(result.status, 429);
|
||||
assert.deepEqual(calls, ["openai/quota"]);
|
||||
});
|
||||
|
||||
test("protected parent combo-ref keeps a child's non-quota target retry monotonic", async () => {
|
||||
const calls: string[] = [];
|
||||
const child = {
|
||||
name: `target-retry-child-${Math.random()}`,
|
||||
strategy: "priority",
|
||||
models: [model("retry-provider/child")],
|
||||
config: { maxRetries: 1, retryDelayMs: 0 },
|
||||
};
|
||||
const outer = {
|
||||
name: `target-retry-outer-${Math.random()}`,
|
||||
strategy: "priority",
|
||||
models: [
|
||||
{ kind: "combo-ref" as const, comboName: child.name, fallbackOnlyOnQuotaExhaustion: true },
|
||||
model("anthropic/paid-backup"),
|
||||
],
|
||||
config: { nestedComboMode: "execute", maxRetries: 0, retryDelayMs: 0 },
|
||||
};
|
||||
const result = await handleComboChat({
|
||||
body: {},
|
||||
combo: outer,
|
||||
allCombos: [outer, child],
|
||||
settings: null,
|
||||
log: log(),
|
||||
isModelAvailable: async () => true,
|
||||
handleSingleModel: async (_body, modelStr) => {
|
||||
calls.push(modelStr);
|
||||
return calls.length === 1
|
||||
? response(503, "Service unavailable")
|
||||
: response(429, "Payment required", { code: "insufficient_quota" });
|
||||
},
|
||||
});
|
||||
assert.equal(result.status, 429);
|
||||
assert.deepEqual(calls, ["retry-provider/child", "retry-provider/child"]);
|
||||
});
|
||||
|
||||
test("protected parent combo-ref keeps a child's non-quota set failure monotonic", async () => {
|
||||
const calls: string[] = [];
|
||||
const child = {
|
||||
name: `set-retry-child-${Math.random()}`,
|
||||
strategy: "priority",
|
||||
models: [model("openai/child")],
|
||||
config: { maxRetries: 0, retryDelayMs: 0, maxSetRetries: 1, setRetryDelayMs: 0 },
|
||||
};
|
||||
const outer = {
|
||||
name: `set-retry-outer-${Math.random()}`,
|
||||
strategy: "priority",
|
||||
models: [
|
||||
{ kind: "combo-ref" as const, comboName: child.name, fallbackOnlyOnQuotaExhaustion: true },
|
||||
model("anthropic/paid-backup"),
|
||||
],
|
||||
config: { nestedComboMode: "execute", maxRetries: 0, retryDelayMs: 0 },
|
||||
};
|
||||
const result = await handleComboChat({
|
||||
body: {},
|
||||
combo: outer,
|
||||
allCombos: [outer, child],
|
||||
settings: null,
|
||||
log: log(),
|
||||
isModelAvailable: async () => true,
|
||||
handleSingleModel: async (_body, modelStr) => {
|
||||
calls.push(modelStr);
|
||||
return calls.length === 1
|
||||
? response(503, "Service unavailable")
|
||||
: response(429, "Payment required", { code: "insufficient_quota" });
|
||||
},
|
||||
});
|
||||
assert.equal(result.status, 429);
|
||||
assert.deepEqual(calls, ["openai/child", "openai/child"]);
|
||||
});
|
||||
|
||||
test("protected parent combo-ref advances on a single authoritative child quota cutoff", async () => {
|
||||
const calls: string[] = [];
|
||||
const child = {
|
||||
name: `cutoff-child-${Math.random()}`,
|
||||
strategy: "priority",
|
||||
models: [{ ...model("openai/cutoff"), connectionId: "nested-quota-empty" }],
|
||||
config: { maxRetries: 0, retryDelayMs: 0 },
|
||||
};
|
||||
const outer = {
|
||||
name: `cutoff-outer-${Math.random()}`,
|
||||
strategy: "priority",
|
||||
models: [
|
||||
{ kind: "combo-ref" as const, comboName: child.name, fallbackOnlyOnQuotaExhaustion: true },
|
||||
model("anthropic/paid-backup"),
|
||||
],
|
||||
config: { nestedComboMode: "execute", maxRetries: 0, retryDelayMs: 0 },
|
||||
};
|
||||
const quotaPreflight = await import("../../open-sse/services/quotaPreflight.ts");
|
||||
quotaPreflight.registerQuotaFetcher("openai", async () => ({
|
||||
used: 100,
|
||||
total: 100,
|
||||
percentUsed: 1,
|
||||
}));
|
||||
const result = await handleComboChat({
|
||||
body: {},
|
||||
combo: outer,
|
||||
allCombos: [outer, child],
|
||||
settings: {
|
||||
resilienceSettings: {
|
||||
quotaPreflight: { enabled: true, defaultThresholdPercent: 2, warnThresholdPercent: 20 },
|
||||
},
|
||||
},
|
||||
log: log(),
|
||||
isModelAvailable: async () => true,
|
||||
handleSingleModel: async (_body, modelStr) => {
|
||||
calls.push(modelStr);
|
||||
return ok(modelStr);
|
||||
},
|
||||
});
|
||||
assert.equal(result.ok, true);
|
||||
assert.deepEqual(calls, ["anthropic/paid-backup"]);
|
||||
});
|
||||
|
||||
for (const [metric, advances] of [
|
||||
["generate_content_free_tier_requests", false],
|
||||
["generate_content_free_tier_input_token_count", false],
|
||||
["generate_content_free_tier_requests_per_day", true],
|
||||
] as const) {
|
||||
test(`nested Gemini ${metric} ${advances ? "advances" : "stops"} parent routing`, async () => {
|
||||
const calls: string[] = [];
|
||||
const child = {
|
||||
name: `gemini-child-${Math.random()}`,
|
||||
strategy: "priority",
|
||||
models: [model("gemini/gemini-2.5-flash")],
|
||||
config: { maxRetries: 0, retryDelayMs: 0 },
|
||||
};
|
||||
const outer = {
|
||||
name: `gemini-outer-${Math.random()}`,
|
||||
strategy: "priority",
|
||||
models: [
|
||||
{ kind: "combo-ref" as const, comboName: child.name, fallbackOnlyOnQuotaExhaustion: true },
|
||||
model("anthropic/backup"),
|
||||
],
|
||||
config: { nestedComboMode: "execute", maxRetries: 0, retryDelayMs: 0 },
|
||||
};
|
||||
const result = await handleComboChat({
|
||||
body: {},
|
||||
combo: outer,
|
||||
allCombos: [outer, child],
|
||||
settings: null,
|
||||
log: log(),
|
||||
isModelAvailable: async () => true,
|
||||
handleSingleModel: async (_body, modelStr) => {
|
||||
calls.push(modelStr);
|
||||
return modelStr.startsWith("gemini/")
|
||||
? response(
|
||||
429,
|
||||
`You exceeded your current quota for metric generativelanguage.googleapis.com/${metric}`
|
||||
)
|
||||
: ok(modelStr);
|
||||
},
|
||||
});
|
||||
assert.equal(result.ok, advances);
|
||||
assert.deepEqual(
|
||||
calls,
|
||||
advances ? ["gemini/gemini-2.5-flash", "anthropic/backup"] : ["gemini/gemini-2.5-flash"]
|
||||
);
|
||||
});
|
||||
}
|
||||
113
tests/unit/combo-quota-exhaustion-option-schema.test.ts
Normal file
113
tests/unit/combo-quota-exhaustion-option-schema.test.ts
Normal file
@@ -0,0 +1,113 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
createComboSchema,
|
||||
requiresQuotaOnlyComboRefExecute,
|
||||
updateComboSchema,
|
||||
} from "../../src/shared/validation/schemas/combo.ts";
|
||||
import { normalizeComboStep } from "../../src/lib/combos/steps.ts";
|
||||
|
||||
const modelStep = {
|
||||
kind: "model" as const,
|
||||
model: "openai/gpt-4o",
|
||||
fallbackOnlyOnQuotaExhaustion: true,
|
||||
};
|
||||
const comboRefStep = {
|
||||
kind: "combo-ref" as const,
|
||||
comboName: "child",
|
||||
fallbackOnlyOnQuotaExhaustion: true,
|
||||
};
|
||||
|
||||
test("create/update accept and preserve strict quota-only booleans on model and combo-ref steps", () => {
|
||||
const payload = {
|
||||
name: "quota-only",
|
||||
strategy: "priority" as const,
|
||||
models: [modelStep, comboRefStep],
|
||||
config: { nestedComboMode: "execute" as const },
|
||||
};
|
||||
const created = createComboSchema.parse(payload);
|
||||
assert.equal(created.models[0].fallbackOnlyOnQuotaExhaustion, true);
|
||||
assert.equal(created.models[1].fallbackOnlyOnQuotaExhaustion, true);
|
||||
|
||||
const updated = updateComboSchema.parse(payload);
|
||||
assert.equal(updated.models?.[0].fallbackOnlyOnQuotaExhaustion, true);
|
||||
assert.equal(updated.models?.[1].fallbackOnlyOnQuotaExhaustion, true);
|
||||
|
||||
for (const bad of ["true", 1, null]) {
|
||||
assert.equal(
|
||||
createComboSchema.safeParse({
|
||||
...payload,
|
||||
models: [{ ...modelStep, fallbackOnlyOnQuotaExhaustion: bad }],
|
||||
}).success,
|
||||
false
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("non-priority combos preserve the dormant option", () => {
|
||||
const parsed = createComboSchema.parse({
|
||||
name: "dormant",
|
||||
strategy: "weighted",
|
||||
models: [modelStep],
|
||||
});
|
||||
assert.equal(parsed.models[0].fallbackOnlyOnQuotaExhaustion, true);
|
||||
assert.equal(normalizeComboStep(parsed.models[0])?.fallbackOnlyOnQuotaExhaustion, true);
|
||||
});
|
||||
|
||||
test("only active priority combo refs require execute mode", () => {
|
||||
assert.equal(
|
||||
createComboSchema.safeParse({ name: "invalid-ref", models: [comboRefStep] }).success,
|
||||
false
|
||||
);
|
||||
assert.equal(
|
||||
createComboSchema.safeParse({
|
||||
name: "valid-ref",
|
||||
models: [comboRefStep],
|
||||
config: { nestedComboMode: "execute" },
|
||||
}).success,
|
||||
true
|
||||
);
|
||||
assert.equal(
|
||||
createComboSchema.safeParse({
|
||||
name: "dormant-ref",
|
||||
strategy: "weighted",
|
||||
models: [comboRefStep],
|
||||
}).success,
|
||||
true
|
||||
);
|
||||
assert.equal(
|
||||
createComboSchema.safeParse({
|
||||
name: "ordinary-ref",
|
||||
models: [{ kind: "combo-ref", comboName: "child" }],
|
||||
}).success,
|
||||
true
|
||||
);
|
||||
});
|
||||
|
||||
test("partial updates defer active combo-ref validation to the merged route state", () => {
|
||||
assert.equal(updateComboSchema.safeParse({ models: [comboRefStep] }).success, true);
|
||||
assert.equal(updateComboSchema.safeParse({ strategy: "priority" }).success, true);
|
||||
assert.equal(
|
||||
requiresQuotaOnlyComboRefExecute({
|
||||
strategy: "priority",
|
||||
models: [comboRefStep],
|
||||
config: {},
|
||||
}),
|
||||
true
|
||||
);
|
||||
assert.equal(
|
||||
requiresQuotaOnlyComboRefExecute({
|
||||
strategy: "weighted",
|
||||
models: [comboRefStep],
|
||||
config: {},
|
||||
}),
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
test("normalization omits false and preserves true", () => {
|
||||
const disabled = normalizeComboStep({ ...modelStep, fallbackOnlyOnQuotaExhaustion: false });
|
||||
const enabled = normalizeComboStep(modelStep);
|
||||
assert.equal("fallbackOnlyOnQuotaExhaustion" in (disabled || {}), false);
|
||||
assert.equal(enabled?.fallbackOnlyOnQuotaExhaustion, true);
|
||||
});
|
||||
@@ -10,20 +10,28 @@ import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import {
|
||||
acquire,
|
||||
buildAccountSemaphoreKey,
|
||||
resetAll,
|
||||
} from "../../open-sse/services/accountSemaphore.ts";
|
||||
import { isRuntimeUnitAtConcurrencyCap } from "../../open-sse/services/combo/runtimeUnitCapacity.ts";
|
||||
import type { ResolvedComboUnit } from "../../open-sse/services/combo/types.ts";
|
||||
|
||||
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-runtime-unit-cap-"));
|
||||
const ORIGINAL_DATA_DIR = process.env.DATA_DIR;
|
||||
process.env.DATA_DIR = TEST_DATA_DIR;
|
||||
|
||||
const { acquire, buildAccountSemaphoreKey, resetAll } =
|
||||
await import("../../open-sse/services/accountSemaphore.ts");
|
||||
const { isRuntimeUnitAtConcurrencyCap } =
|
||||
await import("../../open-sse/services/combo/runtimeUnitCapacity.ts");
|
||||
const { handleComboChat } = await import("../../open-sse/services/combo.ts");
|
||||
const { getDbInstance, resetDbInstance } = await import("../../src/lib/db/core.ts");
|
||||
type ResolvedComboUnit = import("../../open-sse/services/combo/types.ts").ResolvedComboUnit;
|
||||
|
||||
const db = getDbInstance();
|
||||
const databases = db.pragma("database_list") as Array<{ file?: string; name?: string }>;
|
||||
const activeDbPath = databases.find((database) => database.name === "main")?.file;
|
||||
assert.ok(activeDbPath, "test requires a file-backed main SQLite database");
|
||||
assert.equal(
|
||||
path.dirname(path.resolve(activeDbPath)),
|
||||
path.resolve(TEST_DATA_DIR),
|
||||
`active test database must be under TEST_DATA_DIR before inserts: ${activeDbPath}`
|
||||
);
|
||||
|
||||
function createLog() {
|
||||
return {
|
||||
@@ -44,7 +52,7 @@ function okResponse() {
|
||||
function seedConnection(id: string, provider: string, maxConcurrent: number) {
|
||||
const db = getDbInstance();
|
||||
db.prepare(
|
||||
`INSERT INTO provider_connections
|
||||
`INSERT OR REPLACE INTO provider_connections
|
||||
(id, provider, auth_type, is_active, max_concurrent, created_at, updated_at)
|
||||
VALUES (?, ?, 'apikey', 1, ?, datetime('now'), datetime('now'))`
|
||||
).run(id, provider, maxConcurrent);
|
||||
@@ -62,7 +70,7 @@ test("isRuntimeUnitAtConcurrencyCap returns true for a model unit at cap", async
|
||||
const connectionId = "conn-feather-cap";
|
||||
const provider = "featherless-ai";
|
||||
const key = buildAccountSemaphoreKey({ provider, accountKey: connectionId });
|
||||
const release1 = await acquire(key, { maxConcurrency: 2 });
|
||||
let release1 = await acquire(key, { maxConcurrency: 2 });
|
||||
const release2 = await acquire(key, { maxConcurrency: 2 });
|
||||
|
||||
const unit: ResolvedComboUnit = {
|
||||
@@ -77,19 +85,409 @@ test("isRuntimeUnitAtConcurrencyCap returns true for a model unit at cap", async
|
||||
label: null,
|
||||
};
|
||||
|
||||
assert.equal(
|
||||
await isRuntimeUnitAtConcurrencyCap(unit, [], async () => 2),
|
||||
true,
|
||||
"unit should be at cap when two slots are in use"
|
||||
);
|
||||
try {
|
||||
assert.equal(
|
||||
await isRuntimeUnitAtConcurrencyCap(unit, [], async () => 2),
|
||||
true,
|
||||
"unit should be at cap when two slots are in use"
|
||||
);
|
||||
|
||||
release1();
|
||||
assert.equal(
|
||||
await isRuntimeUnitAtConcurrencyCap(unit, [], async () => 2),
|
||||
false,
|
||||
"unit should have headroom after one slot frees"
|
||||
release1();
|
||||
release1 = () => {};
|
||||
assert.equal(
|
||||
await isRuntimeUnitAtConcurrencyCap(unit, [], async () => 2),
|
||||
false,
|
||||
"unit should have headroom after one slot frees"
|
||||
);
|
||||
} finally {
|
||||
release1();
|
||||
release2();
|
||||
}
|
||||
});
|
||||
|
||||
test("protected priority model stops at local capacity", async () => {
|
||||
const primaryConnectionId = "conn-protected-capacity";
|
||||
const provider = "featherless-ai";
|
||||
seedConnection(primaryConnectionId, provider, 1);
|
||||
const release = await acquire(
|
||||
buildAccountSemaphoreKey({ provider, accountKey: primaryConnectionId }),
|
||||
{
|
||||
maxConcurrency: 1,
|
||||
}
|
||||
);
|
||||
release2();
|
||||
try {
|
||||
const calls: string[] = [];
|
||||
const combo = {
|
||||
name: "protected-model-capacity",
|
||||
strategy: "priority",
|
||||
models: [
|
||||
{
|
||||
kind: "model",
|
||||
model: "featherless-ai/deepseek-ai/DeepSeek-V4-Pro",
|
||||
providerId: provider,
|
||||
connectionId: primaryConnectionId,
|
||||
fallbackOnlyOnQuotaExhaustion: true,
|
||||
},
|
||||
{ kind: "model", model: "alibaba/backup", providerId: "alibaba" },
|
||||
],
|
||||
config: { nestedComboMode: "execute", maxRetries: 0 },
|
||||
};
|
||||
const result = await handleComboChat({
|
||||
body: {},
|
||||
combo,
|
||||
allCombos: [combo],
|
||||
log: createLog(),
|
||||
settings: null,
|
||||
isModelAvailable: async () => true,
|
||||
handleSingleModel: async (_body, modelStr) => {
|
||||
calls.push(modelStr);
|
||||
return okResponse();
|
||||
},
|
||||
});
|
||||
assert.equal(result.status, 503);
|
||||
assert.deepEqual(calls, []);
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
});
|
||||
|
||||
test("protected priority combo-ref stops when its child is at local capacity", async () => {
|
||||
const connectionId = "conn-protected-child-capacity";
|
||||
const provider = "featherless-ai";
|
||||
seedConnection(connectionId, provider, 1);
|
||||
const release = await acquire(buildAccountSemaphoreKey({ provider, accountKey: connectionId }), {
|
||||
maxConcurrency: 1,
|
||||
});
|
||||
try {
|
||||
const child = {
|
||||
name: "protected-capacity-child",
|
||||
strategy: "priority",
|
||||
models: [
|
||||
{ kind: "model", model: "featherless-ai/child", providerId: provider, connectionId },
|
||||
],
|
||||
config: { maxRetries: 0 },
|
||||
};
|
||||
const outer = {
|
||||
name: "protected-capacity-outer",
|
||||
strategy: "priority",
|
||||
models: [
|
||||
{ kind: "combo-ref", comboName: child.name, fallbackOnlyOnQuotaExhaustion: true },
|
||||
{ kind: "model", model: "alibaba/backup", providerId: "alibaba" },
|
||||
],
|
||||
config: { nestedComboMode: "execute", maxRetries: 0 },
|
||||
};
|
||||
const calls: string[] = [];
|
||||
const result = await handleComboChat({
|
||||
body: {},
|
||||
combo: outer,
|
||||
allCombos: [outer, child],
|
||||
log: createLog(),
|
||||
settings: null,
|
||||
isModelAvailable: async () => true,
|
||||
handleSingleModel: async (_body, modelStr) => {
|
||||
calls.push(modelStr);
|
||||
return okResponse();
|
||||
},
|
||||
});
|
||||
assert.equal(result.status, 503);
|
||||
assert.deepEqual(calls, []);
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
});
|
||||
|
||||
test("protected priority unit keeps non-quota failure trust across retries", async () => {
|
||||
const primary = "runtime-retry-provider/primary";
|
||||
const paidBackup = "anthropic/paid-backup";
|
||||
const paidCombo = {
|
||||
name: "runtime-retry-paid-combo",
|
||||
strategy: "priority",
|
||||
models: [{ kind: "model", model: paidBackup, providerId: "anthropic" }],
|
||||
config: { maxRetries: 0, retryDelayMs: 0 },
|
||||
};
|
||||
const outer = {
|
||||
name: "runtime-retry-protected-outer",
|
||||
strategy: "priority",
|
||||
models: [
|
||||
{
|
||||
kind: "model",
|
||||
model: primary,
|
||||
providerId: "runtime-retry-provider",
|
||||
fallbackOnlyOnQuotaExhaustion: true,
|
||||
},
|
||||
{ kind: "combo-ref", comboName: paidCombo.name },
|
||||
],
|
||||
config: { nestedComboMode: "execute", maxRetries: 1, retryDelayMs: 0 },
|
||||
};
|
||||
const calls: string[] = [];
|
||||
const result = await handleComboChat({
|
||||
body: {},
|
||||
combo: outer,
|
||||
allCombos: [outer, paidCombo],
|
||||
log: createLog(),
|
||||
settings: null,
|
||||
isModelAvailable: async () => true,
|
||||
handleSingleModel: async (_body, modelStr) => {
|
||||
calls.push(modelStr);
|
||||
if (modelStr === paidBackup) return okResponse();
|
||||
if (calls.length === 1)
|
||||
return Response.json({ error: "Service unavailable" }, { status: 503 });
|
||||
return Response.json(
|
||||
{ error: { message: "Payment required", code: "insufficient_quota" } },
|
||||
{ status: 429 }
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(result.status, 429);
|
||||
assert.deepEqual(calls, [primary, primary]);
|
||||
});
|
||||
|
||||
test("protected priority unit returns a successful later retry", async () => {
|
||||
const primary = "runtime-success-retry-provider/primary";
|
||||
const paidCombo = {
|
||||
name: "runtime-success-retry-paid-combo",
|
||||
strategy: "priority",
|
||||
models: [{ kind: "model", model: "anthropic/unused", providerId: "anthropic" }],
|
||||
config: { maxRetries: 0, retryDelayMs: 0 },
|
||||
};
|
||||
const outer = {
|
||||
name: "runtime-success-retry-protected-outer",
|
||||
strategy: "priority",
|
||||
models: [
|
||||
{
|
||||
kind: "model",
|
||||
model: primary,
|
||||
providerId: "runtime-success-retry-provider",
|
||||
fallbackOnlyOnQuotaExhaustion: true,
|
||||
},
|
||||
{ kind: "combo-ref", comboName: paidCombo.name },
|
||||
],
|
||||
config: { nestedComboMode: "execute", maxRetries: 1, retryDelayMs: 0 },
|
||||
};
|
||||
const calls: string[] = [];
|
||||
const result = await handleComboChat({
|
||||
body: {},
|
||||
combo: outer,
|
||||
allCombos: [outer, paidCombo],
|
||||
log: createLog(),
|
||||
settings: null,
|
||||
isModelAvailable: async () => true,
|
||||
handleSingleModel: async (_body, modelStr) => {
|
||||
calls.push(modelStr);
|
||||
return calls.length === 1
|
||||
? Response.json({ error: "Service unavailable" }, { status: 503 })
|
||||
: okResponse();
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(result.ok, true);
|
||||
assert.deepEqual(calls, [primary, primary]);
|
||||
});
|
||||
|
||||
test("protected priority unit advances when every retry is explicit quota exhaustion", async () => {
|
||||
const primary = "runtime-quota-retry-provider/primary";
|
||||
const paidBackup = "anthropic/quota-paid-backup";
|
||||
const paidCombo = {
|
||||
name: "runtime-quota-retry-paid-combo",
|
||||
strategy: "priority",
|
||||
models: [{ kind: "model", model: paidBackup, providerId: "anthropic" }],
|
||||
config: { maxRetries: 0, retryDelayMs: 0 },
|
||||
};
|
||||
const outer = {
|
||||
name: "runtime-quota-retry-protected-outer",
|
||||
strategy: "priority",
|
||||
models: [
|
||||
{
|
||||
kind: "model",
|
||||
model: primary,
|
||||
providerId: "runtime-quota-retry-provider",
|
||||
fallbackOnlyOnQuotaExhaustion: true,
|
||||
},
|
||||
{ kind: "combo-ref", comboName: paidCombo.name },
|
||||
],
|
||||
config: { nestedComboMode: "execute", maxRetries: 1, retryDelayMs: 0 },
|
||||
};
|
||||
const calls: string[] = [];
|
||||
const result = await handleComboChat({
|
||||
body: {},
|
||||
combo: outer,
|
||||
allCombos: [outer, paidCombo],
|
||||
log: createLog(),
|
||||
settings: null,
|
||||
isModelAvailable: async () => true,
|
||||
handleSingleModel: async (_body, modelStr) => {
|
||||
calls.push(modelStr);
|
||||
if (modelStr === paidBackup) return okResponse();
|
||||
return Response.json(
|
||||
{ error: { message: "Payment required", code: "insufficient_quota" } },
|
||||
{ status: 429 }
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(result.ok, true);
|
||||
assert.deepEqual(calls, [primary, primary, paidBackup]);
|
||||
});
|
||||
|
||||
test("nested child aggregate prevents protected parent fallback after any non-quota failure", async () => {
|
||||
const transientModel = "runtime-nested-transient/primary";
|
||||
const quotaModel = "runtime-nested-quota/primary";
|
||||
const paidBackup = "anthropic/nested-paid-backup";
|
||||
const quotaLeaf = {
|
||||
name: "runtime-nested-quota-leaf",
|
||||
strategy: "priority",
|
||||
models: [{ kind: "model", model: quotaModel, providerId: "runtime-nested-quota" }],
|
||||
config: { maxRetries: 0, retryDelayMs: 0 },
|
||||
};
|
||||
const child = {
|
||||
name: "runtime-nested-mixed-child",
|
||||
strategy: "priority",
|
||||
models: [
|
||||
{ kind: "model", model: transientModel, providerId: "runtime-nested-transient" },
|
||||
{ kind: "combo-ref", comboName: quotaLeaf.name },
|
||||
],
|
||||
config: { nestedComboMode: "execute", maxRetries: 0, retryDelayMs: 0 },
|
||||
};
|
||||
const parent = {
|
||||
name: "runtime-nested-mixed-parent",
|
||||
strategy: "priority",
|
||||
models: [
|
||||
{ kind: "combo-ref", comboName: child.name, fallbackOnlyOnQuotaExhaustion: true },
|
||||
{ kind: "model", model: paidBackup, providerId: "anthropic" },
|
||||
],
|
||||
config: { nestedComboMode: "execute", maxRetries: 0, retryDelayMs: 0 },
|
||||
};
|
||||
const calls: string[] = [];
|
||||
const result = await handleComboChat({
|
||||
body: {},
|
||||
combo: parent,
|
||||
allCombos: [parent, child, quotaLeaf],
|
||||
log: createLog(),
|
||||
settings: null,
|
||||
isModelAvailable: async () => true,
|
||||
handleSingleModel: async (_body, modelStr) => {
|
||||
calls.push(modelStr);
|
||||
if (modelStr === transientModel) {
|
||||
return Response.json({ error: "Service unavailable" }, { status: 503 });
|
||||
}
|
||||
if (modelStr === quotaModel) {
|
||||
return Response.json(
|
||||
{ error: { message: "Payment required", code: "insufficient_quota" } },
|
||||
{ status: 429 }
|
||||
);
|
||||
}
|
||||
return okResponse();
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(result.status, 429);
|
||||
assert.deepEqual(calls, [transientModel, quotaModel]);
|
||||
});
|
||||
|
||||
test("nested child aggregate treats local quality rejection as non-quota evidence", async () => {
|
||||
const invalidModel = "runtime-nested-quality/primary";
|
||||
const quotaModel = "runtime-nested-quality-quota/primary";
|
||||
const paidBackup = "anthropic/nested-quality-paid-backup";
|
||||
const quotaLeaf = {
|
||||
name: "runtime-nested-quality-quota-leaf",
|
||||
strategy: "priority",
|
||||
models: [{ kind: "model", model: quotaModel, providerId: "runtime-nested-quality-quota" }],
|
||||
config: { maxRetries: 0, retryDelayMs: 0 },
|
||||
};
|
||||
const child = {
|
||||
name: "runtime-nested-quality-child",
|
||||
strategy: "priority",
|
||||
models: [
|
||||
{ kind: "model", model: invalidModel, providerId: "runtime-nested-quality" },
|
||||
{ kind: "combo-ref", comboName: quotaLeaf.name },
|
||||
],
|
||||
config: { nestedComboMode: "execute", maxRetries: 0, retryDelayMs: 0 },
|
||||
};
|
||||
const parent = {
|
||||
name: "runtime-nested-quality-parent",
|
||||
strategy: "priority",
|
||||
models: [
|
||||
{ kind: "combo-ref", comboName: child.name, fallbackOnlyOnQuotaExhaustion: true },
|
||||
{ kind: "model", model: paidBackup, providerId: "anthropic" },
|
||||
],
|
||||
config: { nestedComboMode: "execute", maxRetries: 0, retryDelayMs: 0 },
|
||||
};
|
||||
const calls: string[] = [];
|
||||
const result = await handleComboChat({
|
||||
body: {},
|
||||
combo: parent,
|
||||
allCombos: [parent, child, quotaLeaf],
|
||||
log: createLog(),
|
||||
settings: null,
|
||||
isModelAvailable: async () => true,
|
||||
handleSingleModel: async (_body, modelStr) => {
|
||||
calls.push(modelStr);
|
||||
if (modelStr === invalidModel) {
|
||||
return Response.json({ choices: [{ message: {} }] }, { status: 200 });
|
||||
}
|
||||
if (modelStr === quotaModel) {
|
||||
return Response.json(
|
||||
{ error: { message: "Payment required", code: "insufficient_quota" } },
|
||||
{ status: 429 }
|
||||
);
|
||||
}
|
||||
return okResponse();
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(result.status, 429);
|
||||
assert.deepEqual(calls, [invalidModel, quotaModel]);
|
||||
});
|
||||
|
||||
test("nested child aggregate allows protected parent fallback when every failure is quota", async () => {
|
||||
const firstQuotaModel = "runtime-nested-quota/first";
|
||||
const secondQuotaModel = "runtime-nested-quota/second";
|
||||
const paidBackup = "anthropic/nested-all-quota-backup";
|
||||
const quotaLeaf = {
|
||||
name: "runtime-nested-all-quota-leaf",
|
||||
strategy: "priority",
|
||||
models: [{ kind: "model", model: secondQuotaModel, providerId: "runtime-nested-quota" }],
|
||||
config: { maxRetries: 0, retryDelayMs: 0 },
|
||||
};
|
||||
const child = {
|
||||
name: "runtime-nested-all-quota-child",
|
||||
strategy: "priority",
|
||||
models: [
|
||||
{ kind: "model", model: firstQuotaModel, providerId: "runtime-nested-quota" },
|
||||
{ kind: "combo-ref", comboName: quotaLeaf.name },
|
||||
],
|
||||
config: { nestedComboMode: "execute", maxRetries: 0, retryDelayMs: 0 },
|
||||
};
|
||||
const parent = {
|
||||
name: "runtime-nested-all-quota-parent",
|
||||
strategy: "priority",
|
||||
models: [
|
||||
{ kind: "combo-ref", comboName: child.name, fallbackOnlyOnQuotaExhaustion: true },
|
||||
{ kind: "model", model: paidBackup, providerId: "anthropic" },
|
||||
],
|
||||
config: { nestedComboMode: "execute", maxRetries: 0, retryDelayMs: 0 },
|
||||
};
|
||||
const calls: string[] = [];
|
||||
const result = await handleComboChat({
|
||||
body: {},
|
||||
combo: parent,
|
||||
allCombos: [parent, child, quotaLeaf],
|
||||
log: createLog(),
|
||||
settings: null,
|
||||
isModelAvailable: async () => true,
|
||||
handleSingleModel: async (_body, modelStr) => {
|
||||
calls.push(modelStr);
|
||||
if (modelStr === paidBackup) return okResponse();
|
||||
return Response.json(
|
||||
{ error: { message: "Payment required", code: "insufficient_quota" } },
|
||||
{ status: 429 }
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(result.ok, true);
|
||||
assert.deepEqual(calls, [firstQuotaModel, secondQuotaModel, paidBackup]);
|
||||
});
|
||||
|
||||
test("execute fill-first overflows to the next unit when the first connection is at cap", async () => {
|
||||
@@ -103,43 +501,45 @@ test("execute fill-first overflows to the next unit when the first connection is
|
||||
const release1 = await acquire(key, { maxConcurrency: 2 });
|
||||
const release2 = await acquire(key, { maxConcurrency: 2 });
|
||||
|
||||
const calls: string[] = [];
|
||||
const combo = {
|
||||
name: "overflow-execute",
|
||||
strategy: "fill-first",
|
||||
models: [
|
||||
{
|
||||
kind: "model",
|
||||
model: "featherless-ai/deepseek-ai/DeepSeek-V4-Pro",
|
||||
providerId: provider,
|
||||
connectionId: primaryConnectionId,
|
||||
try {
|
||||
const calls: string[] = [];
|
||||
const combo = {
|
||||
name: "overflow-execute",
|
||||
strategy: "fill-first",
|
||||
models: [
|
||||
{
|
||||
kind: "model",
|
||||
model: "featherless-ai/deepseek-ai/DeepSeek-V4-Pro",
|
||||
providerId: provider,
|
||||
connectionId: primaryConnectionId,
|
||||
},
|
||||
{
|
||||
kind: "model",
|
||||
model: "alibaba/qwen3.7-max-preview",
|
||||
providerId: "alibaba",
|
||||
connectionId: backupConnectionId,
|
||||
},
|
||||
],
|
||||
config: { nestedComboMode: "execute", maxRetries: 0, retryDelayMs: 0 },
|
||||
};
|
||||
|
||||
const result = await handleComboChat({
|
||||
body: {},
|
||||
combo,
|
||||
handleSingleModel: async (_body, modelStr) => {
|
||||
calls.push(modelStr);
|
||||
return okResponse();
|
||||
},
|
||||
{
|
||||
kind: "model",
|
||||
model: "alibaba/qwen3.7-max-preview",
|
||||
providerId: "alibaba",
|
||||
connectionId: backupConnectionId,
|
||||
},
|
||||
],
|
||||
config: { nestedComboMode: "execute", maxRetries: 0, retryDelayMs: 0 },
|
||||
};
|
||||
isModelAvailable: async () => true,
|
||||
log: createLog(),
|
||||
settings: null,
|
||||
allCombos: [combo],
|
||||
});
|
||||
|
||||
const result = await handleComboChat({
|
||||
body: {},
|
||||
combo,
|
||||
handleSingleModel: async (_body, modelStr) => {
|
||||
calls.push(modelStr);
|
||||
return okResponse();
|
||||
},
|
||||
isModelAvailable: async () => true,
|
||||
log: createLog(),
|
||||
settings: null,
|
||||
allCombos: [combo],
|
||||
});
|
||||
|
||||
assert.equal(result.ok, true);
|
||||
assert.deepEqual(calls, ["alibaba/qwen3.7-max-preview"]);
|
||||
|
||||
release1();
|
||||
release2();
|
||||
assert.equal(result.ok, true);
|
||||
assert.deepEqual(calls, ["alibaba/qwen3.7-max-preview"]);
|
||||
} finally {
|
||||
release1();
|
||||
release2();
|
||||
}
|
||||
});
|
||||
|
||||
@@ -90,7 +90,11 @@ test("PUT /api/combos/[id] returns 409 for a qtSd/* combo and does NOT mutate it
|
||||
|
||||
// Verify the combo was NOT mutated
|
||||
const unchanged = await combosDb.getComboById(combo.id);
|
||||
assert.equal(unchanged?.strategy, "priority", "Strategy must remain unchanged after rejected PUT");
|
||||
assert.equal(
|
||||
unchanged?.strategy,
|
||||
"priority",
|
||||
"Strategy must remain unchanged after rejected PUT"
|
||||
);
|
||||
});
|
||||
|
||||
// ---- non-quota combos still work ----
|
||||
@@ -116,6 +120,43 @@ test("DELETE /api/combos/[id] succeeds for a regular (non-quota) combo", async (
|
||||
assert.equal(gone, null, "Regular combo must be gone after DELETE");
|
||||
});
|
||||
|
||||
test("PUT merged state rejects partial updates that leave protected priority refs in flatten mode", async () => {
|
||||
const combo = await combosDb.createCombo({
|
||||
name: "protected-ref-update",
|
||||
strategy: "priority",
|
||||
models: [{ kind: "combo-ref", comboName: "child", fallbackOnlyOnQuotaExhaustion: true }],
|
||||
config: { nestedComboMode: "execute" },
|
||||
});
|
||||
|
||||
for (const update of [
|
||||
{ config: { nestedComboMode: "flatten" } },
|
||||
{
|
||||
models: [{ kind: "combo-ref", comboName: "child", fallbackOnlyOnQuotaExhaustion: true }],
|
||||
config: {},
|
||||
},
|
||||
{ strategy: "priority", config: {} },
|
||||
]) {
|
||||
const response = await comboRoute.PUT(makePutRequest(combo.id, update), {
|
||||
params: Promise.resolve({ id: combo.id }),
|
||||
});
|
||||
assert.equal(response.status, 400);
|
||||
}
|
||||
});
|
||||
|
||||
test("PUT merged state accepts dormant weighted protected refs", async () => {
|
||||
const combo = await combosDb.createCombo({
|
||||
name: "dormant-protected-ref-update",
|
||||
strategy: "priority",
|
||||
models: [{ kind: "combo-ref", comboName: "child", fallbackOnlyOnQuotaExhaustion: true }],
|
||||
config: { nestedComboMode: "execute" },
|
||||
});
|
||||
const response = await comboRoute.PUT(
|
||||
makePutRequest(combo.id, { strategy: "weighted", config: { nestedComboMode: "flatten" } }),
|
||||
{ params: Promise.resolve({ id: combo.id }) }
|
||||
);
|
||||
assert.equal(response.status, 200);
|
||||
});
|
||||
|
||||
test("PUT /api/combos/[id] succeeds for a regular (non-quota) combo", async () => {
|
||||
const combo = await combosDb.createCombo({
|
||||
name: "regular-editable-combo",
|
||||
@@ -152,10 +193,7 @@ test("DELETE /api/combos/[id] returns 404 when combo does not exist", async () =
|
||||
|
||||
test("combos page source filters isHidden from rendered list", async () => {
|
||||
const pageSource = fs.readFileSync(
|
||||
new URL(
|
||||
"../../src/app/(dashboard)/dashboard/combos/page.tsx",
|
||||
import.meta.url
|
||||
).pathname,
|
||||
new URL("../../src/app/(dashboard)/dashboard/combos/page.tsx", import.meta.url).pathname,
|
||||
"utf8"
|
||||
);
|
||||
assert.ok(
|
||||
|
||||
91
tests/unit/ui/combo-quota-exhaustion-option.test.tsx
Normal file
91
tests/unit/ui/combo-quota-exhaustion-option.test.tsx
Normal file
@@ -0,0 +1,91 @@
|
||||
// @vitest-environment jsdom
|
||||
import React, { act } from "react";
|
||||
import { createRoot } from "react-dom/client";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { ComboTargetOptions } from "@/app/(dashboard)/dashboard/combos/ComboQuotaOnlyFallbackToggle";
|
||||
import {
|
||||
applyQuotaOnlyFallbackConfig,
|
||||
setQuotaOnlyFallback,
|
||||
} from "@/app/(dashboard)/dashboard/combos/comboQuotaOnlyFallback";
|
||||
import type { ComboStep } from "@/lib/combos/steps";
|
||||
|
||||
const model = (name: string): ComboStep => ({ kind: "model", model: name });
|
||||
const comboRef = (): ComboStep => ({ kind: "combo-ref", comboName: "child" });
|
||||
const containers: Array<{ root: ReturnType<typeof createRoot>; element: HTMLDivElement }> = [];
|
||||
const label = "Only advance on quota exhaustion";
|
||||
const shortLabel = "quota-only fallback";
|
||||
|
||||
function renderOptions(strategy: string, entry: ComboStep) {
|
||||
const element = document.createElement("div");
|
||||
document.body.appendChild(element);
|
||||
const root = createRoot(element);
|
||||
act(() => {
|
||||
root.render(
|
||||
<ComboTargetOptions
|
||||
strategy={strategy}
|
||||
entry={entry}
|
||||
index={0}
|
||||
hasPricing={false}
|
||||
translate={(key, fallback) =>
|
||||
key === "fallbackOnlyOnQuotaExhaustion"
|
||||
? label
|
||||
: key === "fallbackOnlyOnQuotaExhaustionShort"
|
||||
? shortLabel
|
||||
: fallback
|
||||
}
|
||||
onCheckedChange={vi.fn()}
|
||||
/>
|
||||
);
|
||||
});
|
||||
containers.push({ root, element });
|
||||
return element;
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
for (const { root, element } of containers.splice(0)) {
|
||||
act(() => root.unmount());
|
||||
element.remove();
|
||||
}
|
||||
});
|
||||
|
||||
describe("combo quota-only fallback state", () => {
|
||||
it("updates the selected model and combo-ref without mutating other steps", () => {
|
||||
const entries = [model("openai/first"), comboRef(), model("openai/last")];
|
||||
const modelEnabled = setQuotaOnlyFallback(entries, 0, true);
|
||||
const refEnabled = setQuotaOnlyFallback(modelEnabled, 1, true);
|
||||
|
||||
expect(entries[0].fallbackOnlyOnQuotaExhaustion).toBeUndefined();
|
||||
expect(refEnabled[0].fallbackOnlyOnQuotaExhaustion).toBe(true);
|
||||
expect(refEnabled[1].fallbackOnlyOnQuotaExhaustion).toBe(true);
|
||||
expect(refEnabled[2]).toBe(entries[2]);
|
||||
expect(setQuotaOnlyFallback(refEnabled, 1, false)[1]).not.toHaveProperty(
|
||||
"fallbackOnlyOnQuotaExhaustion"
|
||||
);
|
||||
});
|
||||
|
||||
it("ignores an invalid index instead of manufacturing a step", () => {
|
||||
const entries = [model("openai/first")];
|
||||
expect(setQuotaOnlyFallback(entries, 5, true)).toEqual(entries);
|
||||
});
|
||||
|
||||
it("renders accessible human labels for priority models and combo refs only", () => {
|
||||
for (const entry of [model("openai/first"), comboRef()]) {
|
||||
const element = renderOptions("priority", entry);
|
||||
const checkbox = element.querySelector<HTMLInputElement>(`input[aria-label="${label}"]`);
|
||||
expect(checkbox).not.toBeNull();
|
||||
expect(checkbox?.checked).toBe(false);
|
||||
expect(element.textContent).toContain(shortLabel);
|
||||
}
|
||||
expect(renderOptions("weighted", model("openai/first")).querySelector("input")).toBeNull();
|
||||
});
|
||||
|
||||
it("forces execute only for active priority combo refs and preserves dormant config", () => {
|
||||
const protectedRef = setQuotaOnlyFallback([comboRef()], 0, true);
|
||||
expect(applyQuotaOnlyFallbackConfig("priority", protectedRef, {})).toEqual({
|
||||
nestedComboMode: "execute",
|
||||
});
|
||||
expect(applyQuotaOnlyFallbackConfig("weighted", protectedRef, {})).toEqual({});
|
||||
expect(applyQuotaOnlyFallbackConfig("priority", [model("openai/first")], {})).toEqual({});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user