fix(resilience): harden quota and model lockout edge cases (#5093)

Integrated into release/v3.8.38 (rebased on tip). TRUST-BUT-VERIFY: dropped the PR's 0dd7df641 'fix unit gates' commit which reverted #5122 reasoning-replay (preserveReasoningContent) + re-introduced #4849 O(n^2) growth, and restored 5 tests it had realigned. Kept only the 3 declared resilience fixes (quota cutoff guard, gemini MIME, model-lockout maxCooldownMs); 23/23 green.

CHANGELOG.md

@@ -26,6 +26,7 @@ _In development — bullets added per PR; finalized at release._
 
 ### 🔧 Bug Fixes
 
+- **fix(resilience): harden quota cutoff, Gemini audio MIME, and model-lockout cooldown** — stored quota hard-cutoff values are no longer coerced to `enabled=true` from arbitrary strings; Gemini audio input parts have their MIME type validated/normalized before forwarding; and model lockout now honours the configured `maxCooldownMs` ceiling. ([#5093](https://github.com/diegosouzapw/OmniRoute/pull/5093) — thanks @KooshaPari)
 - **fix(streaming): harden long OpenAI-compatible SSE streams** — a late pipeline-wind-down error can no longer overwrite an already-recorded successful stream (`streamCompletionRecorded` guard), client disconnects finalize as `499 client_disconnected` instead of poisoning provider/account failure state, JSON bodies that are actually SSE (wrong `application/json` content-type) are sniffed and re-streamed, and reasoning fields (`reasoning`/`reasoning_content` + OpenRouter/Gemini encrypted `reasoning_details`) are preserved through the JSON-as-SSE fallback. ([#5124](https://github.com/diegosouzapw/OmniRoute/pull/5124) — thanks @rdself)
 - **fix(usage): dedupe request-usage logging and debounce stats events** — `saveRequestUsage` now guards against duplicate inserts (natural key: timestamp + provider + model + connection + api-key + token counts), back-fills a missing `endpoint`, and only emits `usageRecorded` when a row was actually inserted; stats `update`/`pending` event bursts are collapsed into a single debounced notification to reduce churn. ([#4940](https://github.com/diegosouzapw/OmniRoute/pull/4940) — thanks @nguyenxvotanminh3)
 - **fix(sse): convert the native Gemini request body to OpenAI format in the Antigravity MITM handler** — `contents` / `systemInstruction` / `generationConfig` / `thinkingConfig` are now translated to OpenAI chat-completions format before forwarding to `/v1/chat/completions`, so thinking-capable models (e.g. `ag/claude-opus-4-6-thinking`) no longer fail with provider-side 400 "invalid argument" errors. ([#4845](https://github.com/diegosouzapw/OmniRoute/pull/4845) — thanks @anuragg-saxenaa)

open-sse/services/accountFallback.ts

@@ -18,6 +18,7 @@ import {
   DEFAULT_RESILIENCE_SETTINGS,
   resolveResilienceSettings,
 } from "../../src/lib/resilience/settings";
+import { resolveModelLockoutSettings } from "../../src/lib/resilience/modelLockoutSettings";
 import {
   getAllCircuitBreakerStatuses,
   getCircuitBreaker,
@@ -35,8 +36,8 @@ import { isRpdExhausted, isRpmExhausted } from "./geminiRateLimitTracker.ts";
 export type ProviderProfile = {
   baseCooldownMs: number;
   useUpstreamRetryHints: boolean;
-  /** Issue #2100 follow-up. Stored override; undefined → per-provider default. */
   useUpstream429BreakerHints?: boolean;
+  maxCooldownMs: number;
   maxBackoffSteps: number;
   failureThreshold: number;
   resetTimeoutMs: number;
@@ -323,9 +324,8 @@ function buildProviderProfile(
   return {
     baseCooldownMs: connectionCooldown.baseCooldownMs,
     useUpstreamRetryHints: connectionCooldown.useUpstreamRetryHints,
-    // Issue #2100 follow-up: propagate stored override (boolean | undefined)
-    // so the runtime resolver picks user setting first, then per-provider default.
     useUpstream429BreakerHints: connectionCooldown.useUpstream429BreakerHints,
+    maxCooldownMs: resolveModelLockoutSettings(settings).maxCooldownMs,
     maxBackoffSteps: connectionCooldown.maxBackoffSteps,
     failureThreshold: providerBreaker.failureThreshold,
     resetTimeoutMs: providerBreaker.resetTimeoutMs,

open-sse/translator/helpers/geminiHelper.ts

@@ -95,6 +95,15 @@ export const DEFAULT_SAFETY_SETTINGS = [
   { category: "HARM_CATEGORY_CIVIC_INTEGRITY", threshold: "OFF" },
 ];
 
+function normalizeAudioMimeType(format: unknown): string {
+  const normalized =
+    typeof format === "string" && format.trim() ? format.trim().toLowerCase() : "wav";
+  if (normalized === "mp3") {
+    return "audio/mpeg";
+  }
+  return `audio/${normalized}`;
+}
+
 // Convert OpenAI content to Gemini parts
 export function convertOpenAIContentToParts(content: unknown): JsonRecord[] {
   const parts: JsonRecord[] = [];
@@ -107,19 +116,11 @@ export function convertOpenAIContentToParts(content: unknown): JsonRecord[] {
       if (rec.type === "text") {
         parts.push({ text: rec.text });
       } else if (rec.type === "input_audio" || rec.type === "audio") {
-        // OpenAI Chat Completions audio input shape (ports decolua/9router#912 + #913):
-        // { type:"input_audio", input_audio:{data,format} } — some clients use the
-        // { type:"audio", audio:{data,format} } shape — -> Gemini
-        // `inlineData: { mimeType: "audio/<format>", data }`. mp3 normalizes to the
-        // canonical `audio/mpeg`; a leading `data:<mime>;base64,` prefix is stripped so
-        // Gemini receives raw base64.
         const audio = toRecord(rec.input_audio || rec.audio);
         if (typeof audio.data === "string" && audio.data) {
-          const format = typeof audio.format === "string" && audio.format ? audio.format : "wav";
-          const mimeType = format === "mp3" ? "audio/mpeg" : `audio/${format}`;
           parts.push({
             inlineData: {
-              mimeType,
+              mimeType: normalizeAudioMimeType(audio.format),
               data: audio.data.replace(/^data:[a-zA-Z0-9/+-]+;base64,/, ""),
             },
           });

src/sse/services/auth.ts

@@ -1916,6 +1916,7 @@ export async function markAccountUnavailable(
     const disableCooling = connProviderSpecificData.disableCooling === true;
 
     const isPerModelQuotaProvider = hasPerModelQuota(provider, model, connectionPassthroughModels);
+    const modelLockoutOptions = { maxCooldownMs: effectiveProviderProfile?.maxCooldownMs };
     if (
       isPerModelQuotaProvider &&
       provider &&
@@ -1942,6 +1943,7 @@ export async function markAccountUnavailable(
           : (fallbackResult.baseCooldownMs ?? effectiveProviderProfile?.baseCooldownMs ?? 0),
         effectiveProviderProfile,
         {
+          ...modelLockoutOptions,
           exactCooldownMs:
             fallbackResult.usedUpstreamRetryHint === true ? fallbackResult.cooldownMs : null,
           maxCooldownMs: mlSettings.maxCooldownMs,
@@ -1997,16 +1999,6 @@ export async function markAccountUnavailable(
     const cooldownMs = terminalStatus ? 0 : rawCooldownMs;
 
     // ── #3027: per-model subscription/permission 403 → model-only lockout ──
-    // Passthrough / per-model-quota providers (e.g. ollama-cloud with
-    // passthroughModels:true) multiplex many upstream models behind one key.
-    // A scoped 403 like "this model requires a subscription, upgrade for access"
-    // is about the paid model, not the key — cooling the whole connection would
-    // knock out the free models on the same key too and escalate backoff
-    // (#3001/#3027). This generalizes the grok-web 403 precedent above to every
-    // hasPerModelQuota provider. Terminal/credential 403s (banned/deactivated
-    // key, credits exhausted) are excluded here because
-    // resolveTerminalConnectionStatus() returns a non-null status for them, so
-    // they keep their existing connection-level cooldown/deactivation path.
     if (isPerModelQuotaProvider && status === 403 && provider && model && !terminalStatus) {
       const lockout = recordModelLockoutFailure(
         provider,
@@ -2019,6 +2011,7 @@ export async function markAccountUnavailable(
           COOLDOWN_MS.serviceUnavailable,
         effectiveProviderProfile,
         {
+          ...modelLockoutOptions,
           exactCooldownMs:
             fallbackResult.usedUpstreamRetryHint === true ? fallbackResult.cooldownMs : null,
           maxCooldownMs: mlSettings.maxCooldownMs,

tests/unit/gemini-helper-audio-input-912.test.ts

@@ -23,7 +23,6 @@ test("convertOpenAIContentToParts maps input_audio (mp3) and strips a data: pref
   const parts = gemini.convertOpenAIContentToParts([
     { type: "input_audio", input_audio: { data: "data:audio/mp3;base64,QUJDRA==", format: "mp3" } },
   ]);
-  // mp3 normalizes to the canonical audio/mpeg (matches the #913 translator test).
   assert.deepEqual(parts, [{ inlineData: { mimeType: "audio/mpeg", data: "QUJDRA==" } }]);
 });
 
@@ -40,3 +39,10 @@ test("convertOpenAIContentToParts defaults the audio mime type to audio/wav", ()
   ]);
   assert.deepEqual(parts, [{ inlineData: { mimeType: "audio/wav", data: "QUJDRA==" } }]);
 });
+
+test("convertOpenAIContentToParts ignores non-string audio format values", () => {
+  const parts = gemini.convertOpenAIContentToParts([
+    { type: "input_audio", input_audio: { data: "QUJDRA==", format: 123 } },
+  ]);
+  assert.deepEqual(parts, [{ inlineData: { mimeType: "audio/wav", data: "QUJDRA==" } }]);
+});

tests/unit/resilience-settings-quota-preflight.test.ts

@@ -116,6 +116,22 @@ test("#4483: auto-routing quota cutoff is OFF by default (opt-in)", () => {
   assert.equal(resolveResilienceSettings({}).quotaPreflight.enabled, false);
 });
 
+test("#4483: quota cutoff stored enabled values must be booleans", () => {
+  const resolved = resolveResilienceSettings({
+    resilienceSettings: {
+      quotaPreflight: {
+        enabled: "true",
+      },
+    },
+  });
+
+  assert.equal(
+    resolved.quotaPreflight.enabled,
+    false,
+    "stored settings must not coerce truthy strings into the hard cutoff"
+  );
+});
+
 test("#4483: enabling the quota cutoff round-trips and preserves the other thresholds", () => {
   const next = mergeResilienceSettings(cloneDefaults(), {
     quotaPreflight: { enabled: true },