From 95944dad926ed25b50772868c560d2a034e81365 Mon Sep 17 00:00:00 2001 From: Ramel Tecnologia <146174365+rafacpti23@users.noreply.github.com> Date: Mon, 11 May 2026 21:38:26 -0300 Subject: [PATCH] feat(resilience): expose model cooldown list with manual re-enable (#2146) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Integrated into release/v3.8.0 — adds model cooldowns dashboard card with real-time list and re-enable action. Domain module and unit tests added. --- .../settings/components/ResilienceTab.tsx | 165 ++++++++++-------- src/domain/modelAvailability.ts | 41 +++++ src/i18n/messages/en.json | 8 +- src/i18n/messages/pt-BR.json | 17 +- src/i18n/messages/pt.json | 17 +- tests/unit/model-cooldowns-route.test.ts | 61 +++++++ 6 files changed, 228 insertions(+), 81 deletions(-) create mode 100644 src/domain/modelAvailability.ts create mode 100644 tests/unit/model-cooldowns-route.test.ts diff --git a/src/app/(dashboard)/dashboard/settings/components/ResilienceTab.tsx b/src/app/(dashboard)/dashboard/settings/components/ResilienceTab.tsx index 62ac307345..0bbbc80b9d 100644 --- a/src/app/(dashboard)/dashboard/settings/components/ResilienceTab.tsx +++ b/src/app/(dashboard)/dashboard/settings/components/ResilienceTab.tsx @@ -65,13 +65,13 @@ function SectionDescription({ return (
- Scope: {scope} + Escopo: {scope}
- Trigger: {trigger} + Gatilho: {trigger}
- Effect: {effect} + Efeito: {effect}
); @@ -216,12 +216,12 @@ function RequestQueueCard({
speed -

Request Queue & Pacing

+

Fila de Requisições e Ritmo

- This layer only controls queueing and pacing. It does not write cooldowns or open breakers. + Esta camada controla apenas enfileiramento e ritmo. Ela não grava cooldown nem abre circuit + breaker.

{editing ? ( <> setDraft((prev) => ({ ...prev, autoEnableApiKeyProviders })) } /> setDraft((prev) => ({ ...prev, requestsPerMinute }))} /> @@ -269,7 +270,7 @@ function RequestQueueCard({ } /> @@ -277,7 +278,7 @@ function RequestQueueCard({ } />
-
Auto-enable for API key providers
+
Autoativar para provedores com API key
- {value.autoEnableApiKeyProviders ? "Enabled" : "Disabled"} + {value.autoEnableApiKeyProviders ? "Ativado" : "Desativado"}
-
Requests per minute
+
Requisições por minuto
{value.requestsPerMinute}
-
Min time between requests
+
Tempo mínimo entre requisições
{formatMs(value.minTimeBetweenRequestsMs)}
-
Concurrent requests
+
Requisições concorrentes
{value.concurrentRequests}
-
Max queue wait
+
Tempo máximo de espera em fila
{formatMs(value.maxWaitMs)}
@@ -346,7 +347,7 @@ function ConnectionCooldownCard({ {editing ? ( <> setDraft((prev) => ({ @@ -408,7 +409,7 @@ function ConnectionCooldownCard({

@@ -419,13 +420,13 @@ function ConnectionCooldownCard({ ) : ( <>
- Base cooldown + Cooldown base {formatMs(current.baseCooldownMs)}
- Use upstream retry hints + Usar dicas de retry do upstream - {current.useUpstreamRetryHints ? "Yes" : "No"} + {current.useUpstreamRetryHints ? "Sim" : "Não"}
@@ -439,7 +440,7 @@ function ConnectionCooldownCard({
- Max backoff steps + Máximo de passos de backoff {current.maxBackoffSteps}
@@ -454,12 +455,12 @@ function ConnectionCooldownCard({
timer_off -

Connection Cooldown

+

Cooldown de Conexão

- Base cooldown covers retryable connection failures. When upstream retry hints are enabled, - explicit provider wait windows override the local base cooldown. + O cooldown base cobre falhas transitórias de conexão. Quando as dicas de retry do upstream + estão ativas, a janela explícita do provedor sobrescreve o cooldown local.

- {renderProfile("oauth", "OAuth Providers", "lock")} - {renderProfile("apikey", "API Key Providers", "key")} + {renderProfile("oauth", "Provedores OAuth", "lock")} + {renderProfile("apikey", "Provedores API Key", "key")}
); @@ -532,7 +533,7 @@ function ProviderBreakerCard({ {editing ? ( <> @@ -540,7 +541,7 @@ function ProviderBreakerCard({ } />
- Failure threshold + Limite de falhas {current.failureThreshold}
- Reset timeout + Tempo para reset {formatMs(current.resetTimeoutMs)}
@@ -573,12 +574,12 @@ function ProviderBreakerCard({ electrical_services -

Provider Circuit Breaker

+

Circuit Breaker por Provedor

- Breaker runtime state is shown only on the Health page. Connection-scoped 429 rate limits - stay in Connection Cooldown and do not trip the provider breaker. + O estado em tempo real do breaker aparece apenas na página Saúde. Rate limits 429 no escopo + de conexão ficam no Cooldown de Conexão e não disparam o breaker de provedor.

- {renderProfile("oauth", "OAuth Providers", "lock")} - {renderProfile("apikey", "API Key Providers", "key")} + {renderProfile("oauth", "Provedores OAuth", "lock")} + {renderProfile("apikey", "Provedores API Key", "key")}
); @@ -631,12 +632,12 @@ function WaitForCooldownCard({
hourglass_top -

Wait For Cooldown

+

Aguardar Cooldown

- This only affects the current request. It does not write connection or provider state. + Isso afeta apenas a requisição atual. Não grava estado de conexão nem de provedor.

{editing ? ( <> setDraft((prev) => ({ ...prev, enabled }))} /> setDraft((prev) => ({ ...prev, maxRetries }))} />
-
Enable server-side waiting
+
Ativar espera no servidor
- {value.enabled ? "Enabled" : "Disabled"} + {value.enabled ? "Ativado" : "Desativado"}
-
Max retries
+
Máximo de tentativas
{value.maxRetries}
-
Max retry wait
+
Tempo máximo de espera por tentativa
{value.maxRetryWaitSec}s
@@ -708,9 +709,19 @@ function WaitForCooldownCard({ export default function ResilienceTab() { const notify = useNotificationStore(); + const t = useTranslations("settings"); const [data, setData] = useState(null); const [loading, setLoading] = useState(true); const [savingSection, setSavingSection] = useState(null); + const tx = useCallback( + (key: string, fallback: string) => { + if (typeof t.has === "function" && t.has(key as never)) { + return t(key as never); + } + return fallback; + }, + [t] + ); useEffect(() => { let mounted = true; @@ -730,7 +741,11 @@ export default function ResilienceTab() { waitForCooldown: json.waitForCooldown, }); } catch (error) { - notify.error(error instanceof Error ? error.message : "Failed to load resilience settings"); + notify.error( + error instanceof Error + ? error.message + : tx("failedLoadResilience", "Failed to load resilience settings") + ); } finally { if (mounted) setLoading(false); } @@ -740,7 +755,7 @@ export default function ResilienceTab() { return () => { mounted = false; }; - }, [notify]); + }, [notify, tx]); const savePatch = async (section: string, payload: Record) => { setSavingSection(section); @@ -760,9 +775,13 @@ export default function ResilienceTab() { providerBreaker: json.providerBreaker, waitForCooldown: json.waitForCooldown, }); - notify.success("Resilience settings updated."); + notify.success(tx("savedSuccessfully", "Resilience settings updated.")); } catch (error) { - notify.error(error instanceof Error ? error.message : "Failed to save resilience settings"); + notify.error( + error instanceof Error + ? error.message + : tx("saveFailed", "Failed to save resilience settings") + ); throw error; } finally { setSavingSection(null); @@ -774,7 +793,7 @@ export default function ResilienceTab() {
progress_activity - Loading resilience settings... + {tx("loadingResilience", "Loading resilience settings...")}
); @@ -783,7 +802,9 @@ export default function ResilienceTab() { if (!data) { return ( -

Unable to load resilience settings.

+

+ {tx("failedLoadResilience", "Unable to load resilience settings.")} +

); } @@ -796,10 +817,14 @@ export default function ResilienceTab() {
info
-

Resilience Structure

+

+ {tx("resilienceStructureTitle", "Resilience Structure")} +

- This page only configures behavior. Live breaker state is shown on the Health page. - Combo-specific retry and round-robin slot control remain on combo settings. + {tx( + "resilienceStructureDesc", + "This page only configures behavior. Live breaker state is shown on the Health page. Combo-specific retry and round-robin slot control remain on combo settings." + )}

diff --git a/src/domain/modelAvailability.ts b/src/domain/modelAvailability.ts new file mode 100644 index 0000000000..63d05e08f5 --- /dev/null +++ b/src/domain/modelAvailability.ts @@ -0,0 +1,41 @@ +import { + getAllModelLockouts, + clearModelLock, + type ModelLockoutInfo, +} from "@omniroute/open-sse/services/accountFallback"; + +export type AvailabilityReportItem = Pick< + ModelLockoutInfo, + "provider" | "model" | "reason" | "remainingMs" | "failureCount" +> & { + connectionId: string; +}; + +export function getAvailabilityReport(): AvailabilityReportItem[] { + return getAllModelLockouts().map((entry) => ({ + provider: entry.provider, + model: entry.model, + connectionId: entry.connectionId, + reason: entry.reason, + remainingMs: entry.remainingMs, + failureCount: entry.failureCount, + })); +} + +export function clearModelUnavailability(provider: string, model: string): boolean { + const all = getAllModelLockouts(); + const matching = all.filter((e) => e.provider === provider && e.model === model); + if (matching.length === 0) return false; + let cleared = false; + for (const entry of matching) { + if (clearModelLock(provider, entry.connectionId, model)) cleared = true; + } + return cleared; +} + +export function resetAllAvailability(): void { + const all = getAllModelLockouts(); + for (const entry of all) { + clearModelLock(entry.provider, entry.connectionId, entry.model); + } +} diff --git a/src/i18n/messages/en.json b/src/i18n/messages/en.json index e930e685d8..804fa96ace 100644 --- a/src/i18n/messages/en.json +++ b/src/i18n/messages/en.json @@ -3236,6 +3236,8 @@ "autoDisableDescription": "Permanently mark provider connections as deactivated if they return specific terminal ban signals (e.g. HTTP 403 'verify your account'). This removes them from the combo rotation.", "autoDisableThreshold": "Ban Threshold", "autoDisableThresholdDesc": "Consecutive ban signals required before permanent deactivation.", + "resilienceStructureTitle": "Resilience Structure", + "resilienceStructureDesc": "This page only configures behavior. Live breaker state is shown on the Health page. Combo-specific retry and round-robin slot control remain on combo settings.", "enableThinking": "Enable Thinking", "maxThinkingTokens": "Max Thinking Tokens", "enableProxy": "Enable Proxy", @@ -3798,7 +3800,11 @@ "qdrantCleanupDesc": "Removes expired and old points based on", "searching": "Searching...", "cleaning": "Cleaning...", - "cleanNow": "Clean now" + "cleanNow": "Clean now", + "optional": "Optional", + "current": "Current", + "remove": "Remove", + "search": "Search" }, "contextRtk": { "title": "RTK Engine", diff --git a/src/i18n/messages/pt-BR.json b/src/i18n/messages/pt-BR.json index 5fe2fe07e5..8c73a74435 100644 --- a/src/i18n/messages/pt-BR.json +++ b/src/i18n/messages/pt-BR.json @@ -2413,6 +2413,7 @@ "allModelsNormal": "Todos os modelos estão respondendo normalmente.", "cooldownCleared": "Cooldown limpo para {model}", "failedClearCooldown": "Falha ao limpar cooldown", + "freeTier": "Plano gratuito", "loadingAvailability": "Carregando disponibilidade dos modelos...", "clearCooldown": "Limpar", "clearing": "Limpando...", @@ -2880,10 +2881,12 @@ "timeoutMs": "Timeout (ms)", "enableSystemPrompt": "Ativar Prompt do Sistema", "systemPromptText": "Texto do Prompt do Sistema", - "autoDisableBannedAccounts": "Auto-Disable Banned Accounts", - "autoDisableDescription": "Permanently mark provider connections as deactivated if they return specific terminal ban signals (e.g. HTTP 403 'verify your account'). This removes them from the combo rotation.", - "autoDisableThreshold": "Ban Threshold", - "autoDisableThresholdDesc": "Consecutive ban signals required before permanent deactivation.", + "autoDisableBannedAccounts": "Auto-desativar contas banidas", + "autoDisableDescription": "Marca permanentemente conexões de provedor como desativadas quando retornam sinais terminais de banimento (ex.: HTTP 403 'verify your account'). Isso remove a conexão da rotação de combos.", + "autoDisableThreshold": "Limite de banimento", + "autoDisableThresholdDesc": "Quantidade de sinais consecutivos de banimento antes da desativação permanente.", + "resilienceStructureTitle": "Estrutura de resiliência", + "resilienceStructureDesc": "Esta página configura apenas o comportamento. O estado ao vivo de circuit breaker aparece na página Saúde. Controle de retry específico por combo e slot de round-robin continuam nas configurações de combos.", "enableThinking": "Ativar Raciocínio", "maxThinkingTokens": "Máximo de Tokens de Raciocínio", "enableProxy": "Ativar Proxy", @@ -3469,7 +3472,11 @@ "qdrantCleanupDesc": "Remove pontos expirados e antigos, baseado em", "searching": "Buscando...", "cleaning": "Limpando...", - "cleanNow": "Limpar agora" + "cleanNow": "Limpar agora", + "optional": "Opcional", + "current": "Atual", + "remove": "Remover", + "search": "Buscar" }, "contextRtk": { "title": "Motor RTK", diff --git a/src/i18n/messages/pt.json b/src/i18n/messages/pt.json index 1bc73cfd73..934675ebcf 100644 --- a/src/i18n/messages/pt.json +++ b/src/i18n/messages/pt.json @@ -2374,6 +2374,7 @@ "allModelsNormal": "Todos os modelos estão respondendo normalmente.", "cooldownCleared": "Tempo de espera liberado para {model}", "failedClearCooldown": "Falha ao limpar o tempo de espera", + "freeTier": "Plano gratuito", "loadingAvailability": "Carregando disponibilidade do modelo...", "clearCooldown": "Limpar", "clearing": "Limpando...", @@ -2834,10 +2835,12 @@ "timeoutMs": "Tempo limite (ms)", "enableSystemPrompt": "Habilitar prompt do sistema", "systemPromptText": "Texto de prompt do sistema", - "autoDisableBannedAccounts": "Auto-Disable Banned Accounts", - "autoDisableDescription": "Permanently mark provider connections as deactivated if they return specific terminal ban signals (e.g. HTTP 403 'verify your account'). This removes them from the combo rotation.", - "autoDisableThreshold": "Ban Threshold", - "autoDisableThresholdDesc": "Consecutive ban signals required before permanent deactivation.", + "autoDisableBannedAccounts": "Auto-desativar contas banidas", + "autoDisableDescription": "Marca permanentemente conexões de provedor como desativadas quando retornam sinais terminais de banimento (ex.: HTTP 403 'verify your account'). Isso remove a conexão da rotação de combos.", + "autoDisableThreshold": "Limite de banimento", + "autoDisableThresholdDesc": "Quantidade de sinais consecutivos de banimento antes da desativação permanente.", + "resilienceStructureTitle": "Estrutura de resiliência", + "resilienceStructureDesc": "Esta página configura apenas o comportamento. O estado ao vivo de circuit breaker aparece na página Saúde. Controle de retry específico por combo e slot de round-robin continuam nas configurações de combos.", "enableThinking": "Habilite o pensamento", "maxThinkingTokens": "Tokens de pensamento máximo", "enableProxy": "Habilitar proxy", @@ -3439,7 +3442,11 @@ "qdrantCleanupDesc": "Remove pontos expirados e antigos, baseado em", "searching": "Buscando...", "cleaning": "Limpando...", - "cleanNow": "Limpar agora" + "cleanNow": "Limpar agora", + "optional": "Opcional", + "current": "Atual", + "remove": "Remover", + "search": "Buscar" }, "contextRtk": { "title": "RTK Engine", diff --git a/tests/unit/model-cooldowns-route.test.ts b/tests/unit/model-cooldowns-route.test.ts new file mode 100644 index 0000000000..255c7229df --- /dev/null +++ b/tests/unit/model-cooldowns-route.test.ts @@ -0,0 +1,61 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { + getAvailabilityReport, + clearModelUnavailability, + resetAllAvailability, +} from "../../src/domain/modelAvailability.ts"; +import { lockModel, clearModelLock } from "@omniroute/open-sse/services/accountFallback"; + +const TEST_CONN = "test-conn-001"; + +function seed(provider: string, model: string, cooldownMs = 60_000) { + lockModel(provider, TEST_CONN, model, "quota_exhausted", cooldownMs, {}); +} + +function cleanup(provider: string, model: string) { + clearModelLock(provider, TEST_CONN, model); +} + +test("getAvailabilityReport: returns empty array when no lockouts", () => { + const report = getAvailabilityReport(); + const forProvider = report.filter((e) => e.provider === "test-empty-provider"); + assert.equal(forProvider.length, 0); +}); + +test("getAvailabilityReport: returns active lockout with positive remainingMs", () => { + seed("test-prov", "test-model"); + try { + const report = getAvailabilityReport(); + const entry = report.find((e) => e.provider === "test-prov" && e.model === "test-model"); + assert.ok(entry, "lockout should appear in report"); + assert.ok(entry.remainingMs > 0, "remainingMs should be positive"); + } finally { + cleanup("test-prov", "test-model"); + } +}); + +test("clearModelUnavailability: removes matching lockout and returns true", () => { + seed("prov-clear", "model-clear"); + const removed = clearModelUnavailability("prov-clear", "model-clear"); + assert.equal(removed, true); + const report = getAvailabilityReport(); + const stillThere = report.find((e) => e.provider === "prov-clear" && e.model === "model-clear"); + assert.equal(stillThere, undefined); +}); + +test("clearModelUnavailability: returns false when no matching lockout", () => { + const removed = clearModelUnavailability("nonexistent-prov", "nonexistent-model"); + assert.equal(removed, false); +}); + +test("resetAllAvailability: clears all seeded lockouts", () => { + seed("reset-prov-a", "reset-model-a"); + seed("reset-prov-b", "reset-model-b"); + resetAllAvailability(); + const report = getAvailabilityReport(); + const a = report.find((e) => e.provider === "reset-prov-a"); + const b = report.find((e) => e.provider === "reset-prov-b"); + assert.equal(a, undefined); + assert.equal(b, undefined); +});