pjandro/OmniRoute
1
0
Fork 0
mirror of https://github.com/diegosouzapw/OmniRoute.git synced 2026-08-06 23:32:12 +03:00
Code Issues Packages Projects Releases Wiki Activity

feat(models): treat quota-exhausted errors as non-hideable in Test All (#9511) (#9537)

Browse Source 
Validated in local merge-train (diegosouzapw batch)
This commit is contained in:
Diego Rodrigues de Sa e Souza
2026-08-06 10:40:00 -03:00
committed by GitHub
parent 2d617325e7
commit ce6faa44e5
14 changed files with 223 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
changelog.d/features/9511-quota-exhausted-transient-test-all.md Normal file
View File

@@ -0,0 +1 @@
- **feat(models):** Test All's "Auto-hide failed models" no longer hides quota errors — daily-quota-exhausted and credits-exhausted responses are now classified via the routing path's existing quota detectors, so an evening Test All on a free-tier provider no longer silently wipes the catalog. Quota results stay visible with a distinct amber badge ([#9511](https://github.com/diegosouzapw/OmniRoute/issues/9511))

2
src/app/(dashboard)/dashboard/providers/[id]/components/CompatibleModelsSection.tsx
View File

@@ -67,7 +67,7 @@ export interface CompatibleModelsSectionProps {
bulkTogglePending?: boolean;
togglingModelId?: string | null;
onTestModel?: (modelId: string, fullModel: string) => Promise<void>;
modelTestStatus?: Record<string, "ok" | "error" | null>;
modelTestStatus?: Record<string, "ok" | "error" | "quota" | null>;
testingModelId?: string | null;
onTestAll?: (targets: Array<{ modelId: string; fullModel: string }>) => Promise<void>;
testingAll?: boolean;

14
src/app/(dashboard)/dashboard/providers/[id]/components/ModelRow.tsx
View File

@@ -278,7 +278,7 @@ export interface ModelRowProps {
onToggleHidden?: (modelId: string, hidden: boolean) => Promise<void>;
togglingHidden?: boolean;
onTestModel?: (modelId: string, fullModel: string) => Promise<void>;
testStatus?: "ok" | "error" | null;
testStatus?: "ok" | "error" | "quota" | null;
testingModel?: boolean;
}
@@ -404,15 +404,17 @@ export default function ModelRow({
<button
onClick={() => onTestModel(model.id, fullModel)}
disabled={testingModel}
className={`rounded p-0.5 hover:bg-sidebar transition-colors disabled:opacity-40 disabled:cursor-not-allowed ${testStatus === "ok" ? "text-green-500" : testStatus === "error" ? "text-red-500" : "text-text-muted hover:text-primary"}`}
className={`rounded p-0.5 hover:bg-sidebar transition-colors disabled:opacity-40 disabled:cursor-not-allowed ${testStatus === "ok" ? "text-green-500" : testStatus === "quota" ? "text-amber-500" : testStatus === "error" ? "text-red-500" : "text-text-muted hover:text-primary"}`}
title={
testingModel
? t("testingModel")
: testStatus === "ok"
? "OK"
: testStatus === "error"
? "Error"
: t("testModel")
: testStatus === "quota"
? t("modelTestQuotaTooltip")
: testStatus === "error"
? "Error"
: t("testModel")
}
>
{testingModel ? (
@@ -421,6 +423,8 @@ export default function ModelRow({
</span>
) : testStatus === "ok" ? (
<span className="material-symbols-outlined text-sm">check_circle</span>
) : testStatus === "quota" ? (
<span className="material-symbols-outlined text-sm">warning</span>
) : testStatus === "error" ? (
<span className="material-symbols-outlined text-sm">error</span>
) : (

14
src/app/(dashboard)/dashboard/providers/[id]/components/PassthroughModelRow.tsx
View File

@@ -44,7 +44,7 @@ export interface PassthroughModelRowProps {
onToggleHidden?: (modelId: string, hidden: boolean) => Promise<void>;
togglingHidden?: boolean;
onTestModel?: (modelId: string, fullModel: string) => Promise<void>;
testStatus?: "ok" | "error" | null;
testStatus?: "ok" | "error" | "quota" | null;
testingModel?: boolean;
}
@@ -195,15 +195,17 @@ export default function PassthroughModelRow({
<button
onClick={() => onTestModel(modelId, fullModel)}
disabled={testingModel}
className={`rounded p-0.5 hover:bg-sidebar transition-colors disabled:opacity-40 disabled:cursor-not-allowed ${testStatus === "ok" ? "text-green-500" : testStatus === "error" ? "text-red-500" : "text-text-muted hover:text-primary"}`}
className={`rounded p-0.5 hover:bg-sidebar transition-colors disabled:opacity-40 disabled:cursor-not-allowed ${testStatus === "ok" ? "text-green-500" : testStatus === "quota" ? "text-amber-500" : testStatus === "error" ? "text-red-500" : "text-text-muted hover:text-primary"}`}
title={
testingModel
? t("testingModel")
: testStatus === "ok"
? "OK"
: testStatus === "error"
? "Error"
: t("testModel")
: testStatus === "quota"
? t("modelTestQuotaTooltip")
: testStatus === "error"
? "Error"
: t("testModel")
}
>
{testingModel ? (
@@ -212,6 +214,8 @@ export default function PassthroughModelRow({
</span>
) : testStatus === "ok" ? (
<span className="material-symbols-outlined text-sm">check_circle</span>
) : testStatus === "quota" ? (
<span className="material-symbols-outlined text-sm">warning</span>
) : testStatus === "error" ? (
<span className="material-symbols-outlined text-sm">error</span>
) : (

9
src/app/(dashboard)/dashboard/providers/[id]/components/PassthroughModelsSection.tsx
View File

@@ -70,9 +70,9 @@ export interface PassthroughModelsSectionProps {
bulkTogglePending?: boolean;
togglingModelId?: string | null;
onTestModel?: (modelId: string, fullModel: string) => Promise<void>;
modelTestStatus?: Record<string, "ok" | "error" | null>;
modelTestStatus?: Record<string, "ok" | "error" | "quota" | null>;
/** Report a model's test-all result so the parent updates the green/red icon. */
onModelTestStatusChange?: (modelId: string, status: "ok" | "error") => void;
onModelTestStatusChange?: (modelId: string, status: "ok" | "error" | "quota") => void;
testingModelId?: string | null;
providerId: string;
connectionId: string;
@@ -181,8 +181,9 @@ export default function PassthroughModelsSection({
const entry = result.results?.[model.modelId];
const outcome = evaluateTestAllEntry(entry, autoHideFailed);
// Paint the per-model icon green/red, same as the single-model ▶ test.
onModelTestStatusChange?.(model.modelId, outcome.status);
// #9511: paint "quota" status for quota-exhausted models (amber badge),
// "ok" for healthy, "error" for genuine failures.
onModelTestStatusChange?.(model.modelId, outcome.isQuota ? "quota" : outcome.status);
if (outcome.status === "ok") {
ok++;
} else {

2
src/app/(dashboard)/dashboard/providers/[id]/components/ProviderModelsSection.tsx
View File

@@ -80,7 +80,7 @@ export interface ProviderModelsSectionProps {
clearingModels: boolean;
modelFilter: string;
testingModelId: string | null;
modelTestStatus: Record<string, "ok" | "error">;
modelTestStatus: Record<string, "ok" | "error" | "quota">;
onModelTestStatusChange: (modelId: string, status: "ok" | "error") => void;
testingAll: boolean;
testProgress: { done: number; total: number } | null;

16
src/app/(dashboard)/dashboard/providers/[id]/hooks/useModelVisibilityHandlers.ts
View File

@@ -70,7 +70,7 @@ export interface UseModelVisibilityHandlersReturn {
clearingModels: boolean;
modelFilter: string;
testingModelId: string | null;
modelTestStatus: Record<string, "ok" | "error">;
modelTestStatus: Record<string, "ok" | "error" | "quota">;
testingAll: boolean;
testProgress: { done: number; total: number } | null;
autoHideFailed: boolean;
@@ -91,7 +91,7 @@ export interface UseModelVisibilityHandlersReturn {
handleTestAll: (targets: Array<{ modelId: string; fullModel: string }>) => Promise<void>;
/** Apply a model's test-all result to the per-row status icon (used by the
* passthrough section, which runs its own test-all loop). */
onModelTestStatusChange: (modelId: string, status: "ok" | "error") => void;
onModelTestStatusChange: (modelId: string, status: "ok" | "error" | "quota") => void;
}
// ──── hook ───────────────────────────────────────────────────────────────────
@@ -116,7 +116,7 @@ export function useModelVisibilityHandlers({
const [clearingModels, setClearingModels] = useState(false);
const [modelFilter, setModelFilter] = useState("");
const [testingModelId, setTestingModelId] = useState<string | null>(null);
const [modelTestStatus, setModelTestStatus] = useState<Record<string, "ok" | "error">>({});
const [modelTestStatus, setModelTestStatus] = useState<Record<string, "ok" | "error" | "quota">>({});
const [testingAll, setTestingAll] = useState(false);
const [testProgress, setTestProgress] = useState<{ done: number; total: number } | null>(null);
const [autoHideFailed, setAutoHideFailed] = useState(false);
@@ -370,8 +370,12 @@ export function useModelVisibilityHandlers({
const entry = result.results?.[fullModel];
const outcome = evaluateTestAllEntry(entry, autoHideFailed);
// Paint the per-model icon green/red, same as the single-model ▶ test.
setModelTestStatus((prev) => ({ ...prev, [modelId]: outcome.status }));
// #9511: paint "quota" status for quota-exhausted models (amber badge),
// "ok" for healthy, "error" for genuine failures.
setModelTestStatus((prev) => ({
...prev,
[modelId]: outcome.isQuota ? "quota" : outcome.status,
}));
if (outcome.status === "ok") {
ok++;
} else {
@@ -431,7 +435,7 @@ export function useModelVisibilityHandlers({
handleClearAllModels,
onTestModel,
handleTestAll,
onModelTestStatusChange: (modelId: string, status: "ok" | "error") =>
onModelTestStatusChange: (modelId: string, status: "ok" | "error" | "quota") =>
setModelTestStatus((prev) => ({ ...prev, [modelId]: status })),
};
}

10
src/app/(dashboard)/dashboard/providers/[id]/providerPageHelpers.ts
View File

@@ -130,6 +130,7 @@ export function validationBadgeProps(result: string): {
/** A single model's outcome from a `/api/models/test-all` response. */
export interface TestAllModelOutcome {
status: "ok" | "error";
isQuota?: boolean;
shouldHide: boolean;
}
type TestAllEntryStatus = "ok" | "error" | "slow";
@@ -159,16 +160,21 @@ export function evaluateTestAllEntry(
rateLimited?: boolean;
isTimeout?: boolean;
isTransient?: boolean;
isQuota?: boolean;
}
| null
| undefined,
autoHideFailed: boolean
): TestAllModelOutcome {
const ok = entry?.status === "ok";
const transient = [entry?.rateLimited, entry?.isTimeout, entry?.isTransient].some(Boolean);
const transient = [entry?.rateLimited, entry?.isTimeout, entry?.isTransient, entry?.isQuota].some(Boolean);
return {
status: ok ? "ok" : "error",
// Hide only persistent failures. Transient (rate-limited, timeout) are
// #9511: quota errors (isQuota) are surfaced on the icon but kept visible
// so an evening Test All on a free-tier provider doesn't silently wipe the
// catalog for the next day.
...(entry?.isQuota ? { isQuota: true } : {}),
// Hide only persistent failures. Transient (rate-limited, timeout, quota) are
// surfaced on the icon but kept visible so a single throttled batch test
// does not silently wipe the catalog.
shouldHide: !ok && autoHideFailed && !transient,

14
src/app/api/models/test-all/route.ts
View File

@@ -47,6 +47,7 @@ export interface BatchTestResultEntry {
statusCode?: number;
rateLimited?: boolean;
isTransient?: boolean;
isQuota?: boolean;
hidden?: boolean;
isTimeout?: boolean;
}
@@ -63,6 +64,7 @@ function toBatchEntry(
if (result.statusCode !== undefined) entry.statusCode = result.statusCode;
if (result.rateLimited === true) entry.rateLimited = true;
if (result.isTransient === true) entry.isTransient = true;
if (result.isQuota === true) entry.isQuota = true;
if (result.isTimeout === true) entry.isTimeout = true;
return entry;
}
@@ -176,10 +178,13 @@ export async function POST(request: Request) {
consecutiveRateLimits = 0;
}
// #9511: quota entries (403 with "insufficient balance" etc.) are NOT
// bot-blocks — they should not count toward the bot-block stop threshold.
const botBlocked =
entry.statusCode === 403 ||
(typeof entry.error === "string" &&
/cloudflare|bot management|recaptcha|cf-chl|just a moment/i.test(entry.error));
!entry.isQuota &&
(entry.statusCode === 403 ||
(typeof entry.error === "string" &&
/cloudflare|bot management|recaptcha|cf-chl|just a moment/i.test(entry.error)));
if (botBlocked) {
consecutiveBotBlocks += 1;
} else if (entry.status === "ok") {
@@ -191,7 +196,8 @@ export async function POST(request: Request) {
entry.status === "error" &&
!entry.rateLimited &&
!entry.isTimeout &&
!entry.isTransient
!entry.isTransient &&
!entry.isQuota
) {
try {
await setModelIsHidden(providerId, modelId, true);

1
src/i18n/messages/en.json
View File

@@ -5602,6 +5602,7 @@
"tagGroupPlaceholder": "Tag Group Placeholder",
"testModel": "Test Model",
"testingModel": "Testing Model",
"modelTestQuotaTooltip": "Quota exhausted — resets tomorrow or needs a top-up",
"toggleOffShort": "Off",
"toggleOnShort": "On",
"tokenExpiredBadge": "Token Expired Badge",

1
src/i18n/messages/pt-BR.json
View File

@@ -5590,6 +5590,7 @@
"tagGroupPlaceholder": "ex.: personal, work, team-a",
"testModel": "Test Model",
"testingModel": "Testing Model",
"modelTestQuotaTooltip": "Cota esgotada — reinicia amanhã ou precisa de recarga",
"toggleOffShort": "OFF",
"toggleOnShort": "ON",
"tokenExpiredBadge": "Expirado",

58
src/lib/api/modelTestRunner.ts
View File

@@ -12,6 +12,11 @@ import { getCustomModels } from "@/lib/localDb";
import { getProviderNodeById } from "@/lib/db/providers";
import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/error";
import { withRateLimit } from "@omniroute/open-sse/services/rateLimitManager";
import {
isCreditsExhausted,
isDailyQuotaExhausted,
} from "@omniroute/open-sse/services/accountFallback";
import { looksLikeQuotaExhausted } from "@/shared/utils/classify429";
const INTERNAL_ORIGIN = "http://omniroute.internal";
export const DEFAULT_MODEL_TEST_TIMEOUT_MS = 30_000;
@@ -294,6 +299,7 @@ export interface SingleModelTestResult {
error?: string;
rateLimited?: boolean;
isTransient?: boolean;
isQuota?: boolean;
isTimeout?: boolean;
retryAfter?: number;
}
@@ -322,6 +328,43 @@ function isBotBlockMessage(message: string): boolean {
return /cloudflare|bot management|recaptcha|cf-chl|just a moment/i.test(message);
}
/**
* Classify an error message for quota signals (#9511).
*
* Distinguishes three outcomes:
* 1. Daily-quota exhausted → isQuota + isTransient (resets tomorrow)
* 2. Credits/balance exhausted → isQuota only (needs top-up, not transient)
* 3. Other errors → no quota flags (still auto-hidable)
*
* Reuses the routing path's existing quota vocabulary from accountFallback.ts
* and classify429.ts instead of inventing a new vocabulary.
*/
export function classifyTestErrorQuota(
errorText: string
): { isQuota?: boolean; isTransient?: boolean } {
const trimmed = typeof errorText === "string" ? errorText.trim() : "";
if (!trimmed) return {};
// Check daily-quota FIRST — it's the more specific (transient) classification
// and should win over credits-exhausted if both match.
if (isDailyQuotaExhausted(trimmed)) {
return { isQuota: true, isTransient: true };
}
// Credits-exhausted is terminal — isQuota but NOT isTransient.
if (isCreditsExhausted(trimmed)) {
return { isQuota: true };
}
// Broad quota wording from classify429 (catches patterns not in the
// accountFallback signals, e.g. "quota exceeded", "billing cap").
if (looksLikeQuotaExhausted(trimmed)) {
return { isQuota: true };
}
return {};
}
/**
* Run a single model test. When `connectionId` is provided, wraps the
* upstream call with `withRateLimit` (Bottleneck). Returns a plain
@@ -506,7 +549,11 @@ export async function runSingleModelTest(
if (streamError) {
const error = sanitizeErrorMessage(streamError.message) || "Upstream stream failed";
const rateLimited = streamError.statusCode === 429 || isRateLimitMessage(error);
const isBotBlock = streamError.statusCode === 403 || isBotBlockMessage(error);
// #9511: Check quota BEFORE bot-block — 403 with quota wording is a quota
// error, not a bot-block. A bare 403 status without quota/bot wording still
// falls through to the generic error branch.
const quotaFlags = classifyTestErrorQuota(error);
const isBotBlock = !quotaFlags.isQuota && (streamError.statusCode === 403 || isBotBlockMessage(error));
return {
modelId: fullModelStr,
status: rateLimited ? "rate_limited" : "error",
@@ -515,7 +562,8 @@ export async function runSingleModelTest(
httpStatus: streamError.statusCode ?? 502,
error,
...(rateLimited ? { rateLimited: true } : {}),
...(rateLimited || isBotBlock ? { isTransient: true } : {}),
...(rateLimited || isBotBlock || quotaFlags.isTransient ? { isTransient: true } : {}),
...(quotaFlags.isQuota ? { isQuota: true } : {}),
};
}
if (timedOut && !responseText) {
@@ -565,6 +613,10 @@ export async function runSingleModelTest(
} finally {
clearTimeout(timeoutHandle);
}
// #9511: classify quota signals on the generic error branch so that
// 401/402/403 "insufficient balance" / "quota exhausted" errors are
// NOT auto-hidden by Test All.
const quotaFlags = classifyTestErrorQuota(errorMsg);
return {
modelId: fullModelStr,
status: "error",
@@ -572,5 +624,7 @@ export async function runSingleModelTest(
statusCode: res.status,
httpStatus: res.status,
error: errorMsg,
...(quotaFlags.isTransient ? { isTransient: true } : {}),
...(quotaFlags.isQuota ? { isQuota: true } : {}),
};
}

73
tests/unit/model-test-runner.test.ts
View File

@@ -8,6 +8,7 @@ import {
extractModelTestResponseText,
runSingleModelTest,
resolveModelTestTimeoutMs,
classifyTestErrorQuota,
} from "@/lib/api/modelTestRunner.ts";
// ---------------------------------------------------------------------------
@@ -302,3 +303,75 @@ test("resolveModelTestTimeoutMs gives GitHub Phi-4 Reasoning up to 60 seconds",
90_000
);
});
// ---------------------------------------------------------------------------
// classifyTestErrorQuota — #9511 quota classification for Test All auto-hide.
// Distinguishes three outcomes:
// 1. Daily-quota exhausted → isQuota + isTransient (resets tomorrow)
// 2. Credits/balance exhausted → isQuota only (needs top-up, not transient)
// 3. Other errors → no quota flags (still auto-hidable)
// ---------------------------------------------------------------------------
test("classifyTestErrorQuota: credits-exhausted signals produce isQuota without isTransient", () => {
const creditsSignals = [
"insufficient_balance",
"insufficient balance",
"insufficient_quota",
"insufficient account balance",
"credits exhausted",
"out of credits",
"credit_balance_too_low",
"your credit balance is too low",
"payment required",
"billing_hard_limit_reached",
"exceeded your current quota",
"free tier of the model has been exhausted",
];
for (const signal of creditsSignals) {
const result = classifyTestErrorQuota(signal);
assert.equal(result.isQuota, true, `signal="${signal}" should be isQuota`);
assert.equal(result.isTransient, undefined, `signal="${signal}" should NOT be isTransient`);
}
});
test("classifyTestErrorQuota: daily-quota signals produce isQuota + isTransient", () => {
const dailySignals = [
"today's quota has been exceeded",
"daily quota exhausted",
"Resource exhausted. Try again tomorrow.",
];
for (const signal of dailySignals) {
const result = classifyTestErrorQuota(signal);
assert.equal(result.isQuota, true, `signal="${signal}" should be isQuota`);
assert.equal(result.isTransient, true, `signal="${signal}" should be isTransient`);
}
});
test("classifyTestErrorQuota: generic errors produce no quota flags", () => {
const genericErrors = [
"invalid model",
"model not found",
"unauthorized",
"forbidden",
"bad request",
"internal server error",
];
for (const msg of genericErrors) {
const result = classifyTestErrorQuota(msg);
assert.equal(result.isQuota, undefined, `msg="${msg}" should NOT be isQuota`);
assert.equal(result.isTransient, undefined, `msg="${msg}" should NOT be isTransient`);
}
});
test("classifyTestErrorQuota: empty/null input produces no quota flags", () => {
assert.deepEqual(classifyTestErrorQuota(""), {});
assert.deepEqual(classifyTestErrorQuota(" "), {});
});
test("classifyTestErrorQuota: daily-quota wins over credits-exhausted (isTransient=true)", () => {
// If an error text matches both daily-quota and credits-exhausted signals,
// daily-quota wins — it's the more specific (transient) classification.
const result = classifyTestErrorQuota("daily quota exhausted, insufficient balance");
assert.equal(result.isQuota, true);
assert.equal(result.isTransient, true);
});

38
tests/unit/test-all-model-status.test.ts
View File

@@ -72,3 +72,41 @@ test("missing / null / empty entry is treated as a failure", () => {
assert.equal(out.shouldHide, true);
}
});
// ---------------------------------------------------------------------------
// #9511 — isQuota errors are NOT auto-hidden (quota-exhausted / insufficient_balance)
// ---------------------------------------------------------------------------
test("quota-exhausted entry is NOT auto-hidden even when autoHideFailed is on", () => {
// Credits-exhausted (terminal) — isQuota but NOT isTransient
assert.deepEqual(evaluateTestAllEntry({ status: "error", isQuota: true }, true), {
status: "error",
isQuota: true,
shouldHide: false,
});
// Daily-quota (transient) — isQuota + isTransient
assert.deepEqual(
evaluateTestAllEntry({ status: "error", isQuota: true, isTransient: true }, true),
{
status: "error",
isQuota: true,
shouldHide: false,
}
);
});
test("quota-exhausted entry with autoHideFailed off is also not hidden", () => {
assert.deepEqual(evaluateTestAllEntry({ status: "error", isQuota: true }, false), {
status: "error",
isQuota: true,
shouldHide: false,
});
});
test("regression: non-quota error is still auto-hidden when autoHideFailed is on", () => {
// A plain error without isQuota/isTransient/rateLimited/isTimeout should still hide
assert.deepEqual(evaluateTestAllEntry({ status: "error" }, true), {
status: "error",
shouldHide: true,
});
});