pjandro/OmniRoute
1
0
Fork 0
mirror of https://github.com/diegosouzapw/OmniRoute.git synced 2026-07-26 09:52:11 +03:00
Code Issues Packages Projects Releases Wiki Activity

fix(models): don't auto-hide transient (rate-limited/timeout) failures on Test All (#3849)

Browse Source 
With Auto-hide failed models on (default), a Test All sweep across 10+ models in
parallel reliably trips per-account rate limits on subscription-tier providers, and
the 429'd/timed-out models were auto-hidden — silently removing working models from
/v1/models with no easy recovery. evaluateTestAllEntry now surfaces transient failures
(rateLimited/isTimeout) as an 'error' icon but keeps them visible; only genuine
(non-transient) failures are still auto-hidden.

Integrated into release/v3.8.25.

Co-authored-by: diegosouzapw <diegosouza.pw@gmail.com>
This commit is contained in:
lukmanc405
2026-06-14 20:25:55 +07:00
committed by GitHub
parent bcb7ed00c7
commit 058946bd04
3 changed files with 34 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
file-size-baseline.json
View File

@@ -56,7 +56,7 @@
"src/app/(dashboard)/dashboard/providers/[id]/hooks/useProviderConnections.ts": 954,
"src/app/(dashboard)/dashboard/providers/[id]/hooks/useProviderModels.ts": 155,
"src/app/(dashboard)/dashboard/providers/[id]/hooks/useProviderSettings.ts": 264,
"src/app/(dashboard)/dashboard/providers/[id]/providerPageHelpers.ts": 939,
"src/app/(dashboard)/dashboard/providers/[id]/providerPageHelpers.ts": 955,
"src/app/(dashboard)/dashboard/providers/components/onboarding/ProviderOnboardingWizard.tsx": 906,
"src/app/(dashboard)/dashboard/providers/page.tsx": 1925,
"src/app/(dashboard)/dashboard/runtime/RuntimePageClient.tsx": 1198,
@@ -124,5 +124,6 @@
"_rebaseline_2026_06_13_v3825_prettier_reconcile": "Reconciliação tardia: o prettier do pre-commit reformatou 3 arquivos DEPOIS da medição de file-size dos PRs, inflando linhas além do baseline setado — OAuthModal.tsx 956→960 e providers.ts 3146→3147 (#3324), combo.ts 5162→5164 (#2743d). Bumps de reformatação automática (sem lógica nova). LIÇÃO: medir file-size pós-commit (pós-prettier), não antes.",
"_rebaseline_2026_06_14_3826_release_drift": "Re-baseline release/v3.8.25 drift already documented from #3809 owner changes: ProxyRegistryManager.tsx 1072→1089, sidebarVisibility.ts 990→1006, schemas.ts 2519→2522. This PR does not touch those source files; updating the frozen values restores Fast Quality Gates on the current release branch.",
"_rebaseline_2026_06_14_3825_combo_stickiness": "Re-baseline #3825 (sessionless combo stickiness + reasoning-aware readiness): combo.ts 5164→5198 (+34, pós-prettier). Crescimento = deriveComboSessionKey() + effectiveSessionId threading nos sites de read/write do pin server-side. streamReadinessPolicy.ts não-frozen (sob cap). Lógica coesa no handler de combo; não-extraível.",
"_rebaseline_2026_06_14_r3_3835_kiro_pricing": "PR #3835 own growth: pricing.ts 1470→1508 (+38 = missing Kiro pricing rows, claude-sonnet-4.6 etc., pure data). Also carries inherited release/v3.8.25 drift not yet frozen by prior r2 merges: RequestLoggerV2.tsx 1276→1282 (#3820 resizable log table) and combo.ts 5198→5203 (#3811 round-robin replay-response fix). This PR does not touch RequestLoggerV2/combo.ts source; updating the frozen values restores Fast Quality Gates on the current release branch."
"_rebaseline_2026_06_14_r3_3835_kiro_pricing": "PR #3835 own growth: pricing.ts 1470→1508 (+38 = missing Kiro pricing rows, claude-sonnet-4.6 etc., pure data). Also carries inherited release/v3.8.25 drift not yet frozen by prior r2 merges: RequestLoggerV2.tsx 1276→1282 (#3820 resizable log table) and combo.ts 5198→5203 (#3811 round-robin replay-response fix). This PR does not touch RequestLoggerV2/combo.ts source; updating the frozen values restores Fast Quality Gates on the current release branch.",
"_rebaseline_2026_06_14_r3_3849_transient_hide": "PR #3849 own growth: providerPageHelpers.ts 939→955 (+16 = expanded JSDoc on the auto-hide policy + transient-failure guard in evaluateTestAllEntry). Cohesive helper logic; not extractable."
}

28
src/app/(dashboard)/dashboard/providers/[id]/providerPageHelpers.ts
View File

@@ -120,20 +120,36 @@ export interface TestAllModelOutcome {
* PassthroughModelsSection) derive — and then apply — the same per-model status.
* Previously test-all only counted ok/error for a toast and never updated
* `modelTestStatus`, so the icons stayed blank and users could not tell which
* model failed (unlike the single-model ▶ test). When `autoHideFailed` is on,
* ANY non-ok result is auto-hidden — including rate-limited / timed-out failures
* (the user opted for "hide every failure").
* model failed (unlike the single-model ▶ test).
*
* Auto-hide policy: when `autoHideFailed` is on, only NON-TRANSIENT failures are
* hidden. Transient failures (rate-limited, timeout) are surfaced as 'error' on
* the row icon but NOT hidden, because:
* - The provider may have been temporarily throttled during a parallel batch
* (a single Test All across 10+ models routinely trips per-account rate
* limits on subscription-tier APIs).
* - The model itself is not broken — a retry seconds later would succeed.
* - Hidden state persists across server restarts and silently removes the
* model from `/v1/models`, so a transient blip turns into a permanent
* catalog gap that the user can only recover from by editing the DB or
* hand-toggling each row.
*
* Genuine failures (`status:"error"` without a transient flag — e.g. upstream
* 400 "invalid model", schema mismatch, auth failure) ARE still auto-hidden,
* which is the intended use of the toggle.
*/
export function evaluateTestAllEntry(
entry: { status?: "ok" | "error"; rateLimited?: boolean; isTimeout?: boolean } | null | undefined,
autoHideFailed: boolean
): TestAllModelOutcome {
const ok = entry?.status === "ok";
const transient = Boolean(entry?.rateLimited || entry?.isTimeout);
return {
status: ok ? "ok" : "error",
// User opted for "hide every failure": any non-ok result is auto-hidden when
// the toggle is on, including rate-limited / timed-out failures.
shouldHide: !ok && autoHideFailed,
// Hide only persistent failures. Transient (rate-limited, timeout) are
// surfaced on the icon but kept visible so a single throttled batch test
// does not silently wipe the catalog.
shouldHide: !ok && autoHideFailed && !transient,
};
}

13
tests/unit/test-all-model-status.test.ts
View File

@@ -33,16 +33,21 @@ test("failed entry maps to status 'error' and hides only when autoHide is on", (
});
});
test("rate-limited / timeout failures show 'error' and ARE auto-hidden (hide every failure)", () => {
test("rate-limited / timeout failures show 'error' but are NOT auto-hidden", () => {
// Rate-limited and timeout are TRANSIENT — the model itself is fine, the
// provider was just throttled during a parallel Test All. Hiding it would
// silently remove a working model from /v1/models with no recovery path
// short of manual DB edit or per-row eye-toggle. We surface the failure on
// the row icon (status: 'error') but keep the model visible.
assert.deepEqual(evaluateTestAllEntry({ status: "error", rateLimited: true }, true), {
status: "error",
shouldHide: true,
shouldHide: false,
});
assert.deepEqual(evaluateTestAllEntry({ status: "error", isTimeout: true }, true), {
status: "error",
shouldHide: true,
shouldHide: false,
});
// ...but only when the toggle is on
// Toggle off → still not hidden, of course.
assert.deepEqual(evaluateTestAllEntry({ status: "error", rateLimited: true }, false), {
status: "error",
shouldHide: false,