From 058946bd0493ca0c48d9a9a7397b32f9ca28e227 Mon Sep 17 00:00:00 2001 From: lukmanc405 Date: Sun, 14 Jun 2026 20:25:55 +0700 Subject: [PATCH] fix(models): don't auto-hide transient (rate-limited/timeout) failures on Test All (#3849) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With Auto-hide failed models on (default), a Test All sweep across 10+ models in parallel reliably trips per-account rate limits on subscription-tier providers, and the 429'd/timed-out models were auto-hidden — silently removing working models from /v1/models with no easy recovery. evaluateTestAllEntry now surfaces transient failures (rateLimited/isTimeout) as an 'error' icon but keeps them visible; only genuine (non-transient) failures are still auto-hidden. Integrated into release/v3.8.25. Co-authored-by: diegosouzapw --- file-size-baseline.json | 5 ++-- .../providers/[id]/providerPageHelpers.ts | 28 +++++++++++++++---- tests/unit/test-all-model-status.test.ts | 13 ++++++--- 3 files changed, 34 insertions(+), 12 deletions(-) diff --git a/file-size-baseline.json b/file-size-baseline.json index 9f1468ce33..853191f92c 100644 --- a/file-size-baseline.json +++ b/file-size-baseline.json @@ -56,7 +56,7 @@ "src/app/(dashboard)/dashboard/providers/[id]/hooks/useProviderConnections.ts": 954, "src/app/(dashboard)/dashboard/providers/[id]/hooks/useProviderModels.ts": 155, "src/app/(dashboard)/dashboard/providers/[id]/hooks/useProviderSettings.ts": 264, - "src/app/(dashboard)/dashboard/providers/[id]/providerPageHelpers.ts": 939, + "src/app/(dashboard)/dashboard/providers/[id]/providerPageHelpers.ts": 955, "src/app/(dashboard)/dashboard/providers/components/onboarding/ProviderOnboardingWizard.tsx": 906, "src/app/(dashboard)/dashboard/providers/page.tsx": 1925, "src/app/(dashboard)/dashboard/runtime/RuntimePageClient.tsx": 1198, @@ -124,5 +124,6 @@ "_rebaseline_2026_06_13_v3825_prettier_reconcile": "Reconciliação tardia: o prettier do pre-commit reformatou 3 arquivos DEPOIS da medição de file-size dos PRs, inflando linhas além do baseline setado — OAuthModal.tsx 956→960 e providers.ts 3146→3147 (#3324), combo.ts 5162→5164 (#2743d). Bumps de reformatação automática (sem lógica nova). LIÇÃO: medir file-size pós-commit (pós-prettier), não antes.", "_rebaseline_2026_06_14_3826_release_drift": "Re-baseline release/v3.8.25 drift already documented from #3809 owner changes: ProxyRegistryManager.tsx 1072→1089, sidebarVisibility.ts 990→1006, schemas.ts 2519→2522. This PR does not touch those source files; updating the frozen values restores Fast Quality Gates on the current release branch.", "_rebaseline_2026_06_14_3825_combo_stickiness": "Re-baseline #3825 (sessionless combo stickiness + reasoning-aware readiness): combo.ts 5164→5198 (+34, pós-prettier). Crescimento = deriveComboSessionKey() + effectiveSessionId threading nos sites de read/write do pin server-side. streamReadinessPolicy.ts não-frozen (sob cap). Lógica coesa no handler de combo; não-extraível.", - "_rebaseline_2026_06_14_r3_3835_kiro_pricing": "PR #3835 own growth: pricing.ts 1470→1508 (+38 = missing Kiro pricing rows, claude-sonnet-4.6 etc., pure data). Also carries inherited release/v3.8.25 drift not yet frozen by prior r2 merges: RequestLoggerV2.tsx 1276→1282 (#3820 resizable log table) and combo.ts 5198→5203 (#3811 round-robin replay-response fix). This PR does not touch RequestLoggerV2/combo.ts source; updating the frozen values restores Fast Quality Gates on the current release branch." + "_rebaseline_2026_06_14_r3_3835_kiro_pricing": "PR #3835 own growth: pricing.ts 1470→1508 (+38 = missing Kiro pricing rows, claude-sonnet-4.6 etc., pure data). Also carries inherited release/v3.8.25 drift not yet frozen by prior r2 merges: RequestLoggerV2.tsx 1276→1282 (#3820 resizable log table) and combo.ts 5198→5203 (#3811 round-robin replay-response fix). This PR does not touch RequestLoggerV2/combo.ts source; updating the frozen values restores Fast Quality Gates on the current release branch.", + "_rebaseline_2026_06_14_r3_3849_transient_hide": "PR #3849 own growth: providerPageHelpers.ts 939→955 (+16 = expanded JSDoc on the auto-hide policy + transient-failure guard in evaluateTestAllEntry). Cohesive helper logic; not extractable." } diff --git a/src/app/(dashboard)/dashboard/providers/[id]/providerPageHelpers.ts b/src/app/(dashboard)/dashboard/providers/[id]/providerPageHelpers.ts index 51caf7c31c..4790dbfbee 100644 --- a/src/app/(dashboard)/dashboard/providers/[id]/providerPageHelpers.ts +++ b/src/app/(dashboard)/dashboard/providers/[id]/providerPageHelpers.ts @@ -120,20 +120,36 @@ export interface TestAllModelOutcome { * PassthroughModelsSection) derive — and then apply — the same per-model status. * Previously test-all only counted ok/error for a toast and never updated * `modelTestStatus`, so the icons stayed blank and users could not tell which - * model failed (unlike the single-model ▶ test). When `autoHideFailed` is on, - * ANY non-ok result is auto-hidden — including rate-limited / timed-out failures - * (the user opted for "hide every failure"). + * model failed (unlike the single-model ▶ test). + * + * Auto-hide policy: when `autoHideFailed` is on, only NON-TRANSIENT failures are + * hidden. Transient failures (rate-limited, timeout) are surfaced as 'error' on + * the row icon but NOT hidden, because: + * - The provider may have been temporarily throttled during a parallel batch + * (a single Test All across 10+ models routinely trips per-account rate + * limits on subscription-tier APIs). + * - The model itself is not broken — a retry seconds later would succeed. + * - Hidden state persists across server restarts and silently removes the + * model from `/v1/models`, so a transient blip turns into a permanent + * catalog gap that the user can only recover from by editing the DB or + * hand-toggling each row. + * + * Genuine failures (`status:"error"` without a transient flag — e.g. upstream + * 400 "invalid model", schema mismatch, auth failure) ARE still auto-hidden, + * which is the intended use of the toggle. */ export function evaluateTestAllEntry( entry: { status?: "ok" | "error"; rateLimited?: boolean; isTimeout?: boolean } | null | undefined, autoHideFailed: boolean ): TestAllModelOutcome { const ok = entry?.status === "ok"; + const transient = Boolean(entry?.rateLimited || entry?.isTimeout); return { status: ok ? "ok" : "error", - // User opted for "hide every failure": any non-ok result is auto-hidden when - // the toggle is on, including rate-limited / timed-out failures. - shouldHide: !ok && autoHideFailed, + // Hide only persistent failures. Transient (rate-limited, timeout) are + // surfaced on the icon but kept visible so a single throttled batch test + // does not silently wipe the catalog. + shouldHide: !ok && autoHideFailed && !transient, }; } diff --git a/tests/unit/test-all-model-status.test.ts b/tests/unit/test-all-model-status.test.ts index 0bf1318bba..8ec63149bf 100644 --- a/tests/unit/test-all-model-status.test.ts +++ b/tests/unit/test-all-model-status.test.ts @@ -33,16 +33,21 @@ test("failed entry maps to status 'error' and hides only when autoHide is on", ( }); }); -test("rate-limited / timeout failures show 'error' and ARE auto-hidden (hide every failure)", () => { +test("rate-limited / timeout failures show 'error' but are NOT auto-hidden", () => { + // Rate-limited and timeout are TRANSIENT — the model itself is fine, the + // provider was just throttled during a parallel Test All. Hiding it would + // silently remove a working model from /v1/models with no recovery path + // short of manual DB edit or per-row eye-toggle. We surface the failure on + // the row icon (status: 'error') but keep the model visible. assert.deepEqual(evaluateTestAllEntry({ status: "error", rateLimited: true }, true), { status: "error", - shouldHide: true, + shouldHide: false, }); assert.deepEqual(evaluateTestAllEntry({ status: "error", isTimeout: true }, true), { status: "error", - shouldHide: true, + shouldHide: false, }); - // ...but only when the toggle is on + // Toggle off → still not hidden, of course. assert.deepEqual(evaluateTestAllEntry({ status: "error", rateLimited: true }, false), { status: "error", shouldHide: false,