OmniRoute/tests/unit/account-fallback-service.test.ts

import test from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";

// #10460: DATA_DIR must be assigned BEFORE any transitive DB import. The
// accountFallback.ts import below statically imports `@/lib/db/providers`, which
// imports `src/lib/db/core.ts`, whose `DATA_DIR` is a top-level
// `export const DATA_DIR = resolveWritableDataDir(...)` captured once at module-load
// time. Setting `process.env.DATA_DIR` after that first import is a no-op — the DB
// singleton keeps whatever DATA_DIR it resolved at import time, so an isolated test
// directory assigned later is silently never used and the #10460 tests below would
// actually read/write the shared default DATA_DIR instead.
const TEST_DATA_DIR_10460 = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-10460-"));
process.env.DATA_DIR = TEST_DATA_DIR_10460;
process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "10460-test-secret";

const accountFallback = await import("../../open-sse/services/accountFallback.ts");
const accountSelector = await import("../../open-sse/services/accountSelector.ts");
const { RateLimitReason, COOLDOWN_MS, PROVIDER_PROFILES } =
  await import("../../open-sse/config/constants.ts");
const { getCircuitBreaker } = await import("../../src/shared/utils/circuitBreaker.ts");
const core = await import("../../src/lib/db/core.ts");
const providersDb = await import("../../src/lib/db/providers.ts");
const auth = await import("../../src/sse/services/auth.ts");

const {
  isOAuthInvalidToken,
  parseRetryFromErrorText,
  checkFallbackError,
  filterAvailableAccounts,
  getEarliestRateLimitedUntil,
  formatRetryAfter,
  applyErrorState,
  lockModelIfPerModelQuota,
  isModelLocked,
  getModelLockoutInfo,
  hasPerModelQuota,
  getProviderProfile,
  recordModelLockoutFailure,
  clearModelLock,
  shouldMarkAccountExhaustedFrom429,
  recordProviderFailure,
  isProviderInCooldown,
  getProviderCooldownRemainingMs,
  clearProviderFailure,
  isProviderFailureCode,
  getProvidersInCooldown,
  getProviderBreakerState,
  isCreditsExhausted,
  CREDITS_EXHAUSTED_SIGNALS,
} = accountFallback;

const { selectAccount } = accountSelector;

/** Build a full ProviderProfile from partial overrides (test helper). */
function makeProfile(overrides: Record<string, unknown> = {}): any {
  return {
    baseCooldownMs: 125,
    useUpstreamRetryHints: false,
    maxBackoffSteps: 3,
    failureThreshold: 60,
    resetTimeoutMs: 5000,
    transientCooldown: 125,
    rateLimitCooldown: 125,
    maxBackoffLevel: 3,
    circuitBreakerThreshold: 60,
    circuitBreakerReset: 5000,
    providerFailureThreshold: 5,
    providerFailureWindowMs: 300000,
    providerCooldownMs: 60000,
    ...overrides,
  };
}

function withMockedNow(now, fn) {
  const originalNow = Date.now;
  Date.now = () => now;
  try {
    return fn();
  } finally {
    Date.now = originalNow;
  }
}

test("isOAuthInvalidToken detects refreshable oauth failures", () => {
  assert.equal(
    isOAuthInvalidToken("Invalid authentication credentials for this OAuth 2 session"),
    true
  );
  assert.equal(isOAuthInvalidToken("plain rate limit"), false);
});

test("parseRetryFromErrorText parses both compact reset formats", () => {
  assert.equal(parseRetryFromErrorText("Your quota will reset after 2h30m14s"), 9_014_000);
  assert.equal(parseRetryFromErrorText("The pool will reset after 45m"), 2_700_000);
  assert.equal(parseRetryFromErrorText("This will reset after 30s"), 30_000);
  assert.equal(parseRetryFromErrorText("No reset metadata"), null);
});

test("parseRetryFromErrorText parses Antigravity 'Resets in XhYmZs' phrasing", () => {
  assert.equal(
    parseRetryFromErrorText(
      "Individual quota reached. Contact your administrator to enable overages. " +
        "Resets in 164h27m24s."
    ),
    (164 * 3600 + 27 * 60 + 24) * 1000
  );
  assert.equal(parseRetryFromErrorText("Resets in 2h7m23s"), 7_643_000);
  assert.equal(parseRetryFromErrorText("Reset in 45m"), 2_700_000);
});

test("parseRetryFromErrorText caps extreme reset windows at 30 days", () => {
  const thirtyDaysMs = 30 * 24 * 60 * 60 * 1000;
  // 100 days → capped to 30
  assert.equal(parseRetryFromErrorText("Resets in 2400h"), thirtyDaysMs);
  // Absurd value → capped
  assert.equal(parseRetryFromErrorText("Resets in 999999h"), thirtyDaysMs);
});

test("checkFallbackError locks Antigravity quota-reached 429 for the full reset window", () => {
  const message =
    "Individual quota reached. Contact your administrator to enable overages. " +
    "Resets in 164h27m24s.";
  const result = checkFallbackError(
    429,
    message,
    0,
    "gemini-3-flash-agent",
    "antigravity",
    null,
    makeProfile({ useUpstreamRetryHints: true })
  );

  assert.equal(result.shouldFallback, true);
  assert.equal(result.reason, RateLimitReason.QUOTA_EXHAUSTED);
  assert.equal(result.usedUpstreamRetryHint, true);
  // Full parsed window (≈164.46h), under the 30-day cap — not the generic ~5s rate-limit backoff.
  assert.equal(result.cooldownMs, (164 * 3600 + 27 * 60 + 24) * 1000);
});

test("recordModelLockoutFailure honors a multi-day exactCooldownMs (under 30-day cap)", () => {
  const provider = "antigravity";
  const connectionId = "conn-quota-window";
  const model = "gemini-3-flash-agent";
  const exactCooldownMs = (164 * 3600 + 27 * 60 + 24) * 1000;

  clearModelLock(provider, connectionId, model);
  const lockout = recordModelLockoutFailure(
    provider,
    connectionId,
    model,
    "quota_exhausted",
    429,
    0,
    makeProfile(),
    { exactCooldownMs, maxCooldownMs: exactCooldownMs + 1 }
  );

  assert.equal(lockout.cooldownMs, exactCooldownMs);
  assert.equal(isModelLocked(provider, connectionId, model), true);
  clearModelLock(provider, connectionId, model);
});

test("checkFallbackError marks deactivated accounts as permanent auth failures", () => {
  const result = checkFallbackError(401, "This account has been deactivated");
  assert.equal(result.shouldFallback, true);
  assert.equal(result.reason, RateLimitReason.AUTH_ERROR);
  assert.equal(result.permanent, true);
  assert.ok(result.cooldownMs >= 300 * 24 * 60 * 60 * 1000);
});

test("checkFallbackError classifies 'free tier of the model has been exhausted' as quota exhausted", () => {
  const result = checkFallbackError(429, "free tier of the model has been exhausted");
  assert.equal(result.shouldFallback, true);
  assert.equal(result.reason, RateLimitReason.QUOTA_EXHAUSTED);
  assert.equal(result.creditsExhausted, true);
});

test("checkFallbackError treats non-429 exhausted credits as long quota cooldowns", () => {
  const result = checkFallbackError(402, "credit_balance_too_low");
  assert.equal(result.shouldFallback, true);
  assert.equal(result.reason, RateLimitReason.QUOTA_EXHAUSTED);
  assert.equal(result.creditsExhausted, true);
  assert.equal(result.cooldownMs, COOLDOWN_MS.paymentRequired ?? 3600 * 1000);
});

test("checkFallbackError keeps API-key 429 exhausted-credit text on the resilience cooldown path", () => {
  const result = checkFallbackError(
    429,
    "credit_balance_too_low",
    0,
    null,
    "openai",
    null,
    makeProfile()
  );

  assert.equal(result.shouldFallback, true);
  assert.equal(result.reason, RateLimitReason.RATE_LIMIT_EXCEEDED);
  assert.equal(result.creditsExhausted, undefined);
  assert.equal(result.cooldownMs, 125);
});

test("checkFallbackError preserves OAuth 429 exhausted-credit semantics", () => {
  const result = checkFallbackError(
    429,
    "credit_balance_too_low",
    0,
    null,
    "codex",
    null,
    makeProfile()
  );

  assert.equal(result.shouldFallback, true);
  assert.equal(result.reason, RateLimitReason.QUOTA_EXHAUSTED);
  assert.equal(result.creditsExhausted, true);
  assert.equal(result.cooldownMs, COOLDOWN_MS.paymentRequired ?? 3600 * 1000);
});

test("#6638: checkFallbackError classifies API-key 429 explicit quota text as quota_exhausted", () => {
  const result = checkFallbackError(429, "quota exceeded", 0, null, "openai", null, makeProfile());

  assert.equal(result.shouldFallback, true);
  assert.equal(result.reason, RateLimitReason.QUOTA_EXHAUSTED);
  assert.equal(result.cooldownMs, 125);
});

test("checkFallbackError honors Retry-After header for rate limits", () => {
  withMockedNow(1_700_000_000_000, () => {
    const headers = new Headers({ "retry-after": "120" });
    const result = checkFallbackError(429, "Rate limit hit", 3, null, "openai", headers);

    assert.equal(result.shouldFallback, true);
    assert.equal(result.reason, RateLimitReason.RATE_LIMIT_EXCEEDED);
    assert.equal(result.newBackoffLevel, 0);
    assert.equal(result.cooldownMs, 120_000);
  });
});

test("checkFallbackError honors x-ratelimit-reset for transient 5xx errors", () => {
  withMockedNow(1_700_000_000_000, () => {
    const resetSeconds = Math.floor((Date.now() + 90_000) / 1000);
    const headers = new Headers({ "x-ratelimit-reset": String(resetSeconds) });
    const result = checkFallbackError(503, "upstream unavailable", 1, null, "openai", headers);

    assert.equal(result.shouldFallback, true);
    assert.equal(result.reason, RateLimitReason.SERVER_ERROR);
    assert.equal(result.newBackoffLevel, 0);
    assert.ok(result.cooldownMs >= 89_000);
    assert.ok(result.cooldownMs <= 90_000);
  });
});

test("checkFallbackError keeps generic 400 client errors terminal", () => {
  const result = checkFallbackError(400, "bad request payload");
  assert.deepEqual(result, {
    shouldFallback: false,
    cooldownMs: 0,
    reason: RateLimitReason.UNKNOWN,
  });
});

test("checkFallbackError treats a genuine 400 model-access error as combo fallback", () => {
  const result = checkFallbackError(400, "The model `foo` does not exist or is not available");
  assert.equal(result.shouldFallback, true);
  assert.equal(result.reason, RateLimitReason.MODEL_CAPACITY);
});

test("checkFallbackError does NOT treat a bad-credential 400 as model-access fallback", () => {
  // Phrased so it would otherwise match MODEL_ACCESS_DENIED_PATTERNS ("...api key
  // ... model"), but the bad-credential signal must keep it terminal so the real
  // auth error surfaces instead of silently exhausting every combo target.
  const result = checkFallbackError(400, "Invalid API key provided for model gpt-4o");
  assert.deepEqual(result, {
    shouldFallback: false,
    cooldownMs: 0,
    reason: RateLimitReason.UNKNOWN,
  });
});

test("checkFallbackError still honors structured model_not_found even with credential-like text", () => {
  // Structured codes are authoritative and unaffected by the credential guard.
  const result = checkFallbackError(400, "unauthorized-ish blob", 0, null, "openai", null, null, {
    code: "model_not_found",
  });
  assert.equal(result.shouldFallback, true);
  assert.equal(result.reason, RateLimitReason.MODEL_CAPACITY);
});

test("filterAvailableAccounts skips exclusion and active cooldowns but keeps recovered ones", () => {
  withMockedNow(1_700_000_000_000, () => {
    const accounts = [
      { id: "exclude-me" },
      { id: "cooling", rateLimitedUntil: new Date(Date.now() + 60_000).toISOString() },
      { id: "recovered", rateLimitedUntil: new Date(Date.now() - 1_000).toISOString() },
      { id: "healthy" },
    ];

    const available = filterAvailableAccounts(accounts, "exclude-me");
    assert.deepEqual(
      available.map((account) => account.id),
      ["recovered", "healthy"]
    );
  });
});

test("getEarliestRateLimitedUntil returns the shortest future cooldown and formatRetryAfter humanizes it", () => {
  withMockedNow(1_700_000_000_000, () => {
    const earliest = getEarliestRateLimitedUntil([
      { rateLimitedUntil: new Date(Date.now() - 5_000).toISOString() },
      { rateLimitedUntil: new Date(Date.now() + 90_000).toISOString() },
      { rateLimitedUntil: new Date(Date.now() + 30_000).toISOString() },
    ]);

    assert.equal(earliest, new Date(Date.now() + 30_000).toISOString());
    assert.equal(formatRetryAfter(earliest), "reset after 30s");
  });
});

test("applyErrorState and selectAccount advance to the next account after an auth failure", () => {
  withMockedNow(1_700_000_000_000, () => {
    const accounts = [
      { id: "conn-a", backoffLevel: 0 },
      { id: "conn-b", backoffLevel: 0 },
    ];

    const firstSelection = selectAccount(accounts, "fill-first");
    assert.equal(firstSelection.account.id, "conn-a");

    const failedFirst = applyErrorState(firstSelection.account, 401, "Unauthorized", "claude");
    assert.equal(failedFirst.status, "error");
    assert.equal(failedFirst.lastError.reason, RateLimitReason.AUTH_ERROR);

    const candidates = filterAvailableAccounts([failedFirst, accounts[1]], failedFirst.id);
    const nextSelection = selectAccount(candidates, "fill-first");
    assert.equal(nextSelection.account.id, "conn-b");
  });
});

test("lockModelIfPerModelQuota only locks supported providers and real models", () => {
  const geminiConnectionId = `gemini-${Date.now()}`;
  const openAiConnectionId = `openai-${Date.now()}`;
  const compatibleConnectionId = `compatible-${Date.now()}`;
  const compatibleProvider = "openai-compatible-custom-node";
  const compatibleModel = "custom-model-a";

  assert.equal(hasPerModelQuota("gemini"), true);
  assert.equal(hasPerModelQuota("openai"), false);
  assert.equal(hasPerModelQuota(compatibleProvider, compatibleModel), true);

  assert.equal(
    lockModelIfPerModelQuota(
      "gemini",
      geminiConnectionId,
      "gemini-2.5-pro",
      RateLimitReason.RATE_LIMIT_EXCEEDED,
      30_000
    ),
    true
  );
  assert.equal(isModelLocked("gemini", geminiConnectionId, "gemini-2.5-pro"), true);

  assert.equal(
    lockModelIfPerModelQuota(
      "openai",
      openAiConnectionId,
      "gpt-5-mini",
      RateLimitReason.RATE_LIMIT_EXCEEDED,
      30_000
    ),
    false
  );
  assert.equal(isModelLocked("openai", openAiConnectionId, "gpt-5-mini"), false);

  assert.equal(
    lockModelIfPerModelQuota(
      compatibleProvider,
      compatibleConnectionId,
      compatibleModel,
      RateLimitReason.RATE_LIMIT_EXCEEDED,
      30_000
    ),
    true
  );
  assert.equal(isModelLocked(compatibleProvider, compatibleConnectionId, compatibleModel), true);
});

test("getProviderProfile differentiates oauth and api-key providers", () => {
  const oauthProfile = getProviderProfile("claude");
  assert.equal(oauthProfile.transientCooldown, PROVIDER_PROFILES.oauth.transientCooldown);
  assert.equal(
    oauthProfile.rateLimitCooldown,
    oauthProfile.useUpstreamRetryHints ? 0 : oauthProfile.baseCooldownMs
  );
  assert.equal(oauthProfile.maxBackoffLevel, PROVIDER_PROFILES.oauth.maxBackoffLevel);
  assert.equal(
    oauthProfile.circuitBreakerThreshold,
    PROVIDER_PROFILES.oauth.circuitBreakerThreshold
  );
  assert.equal(oauthProfile.circuitBreakerReset, PROVIDER_PROFILES.oauth.circuitBreakerReset);
  assert.equal(oauthProfile.baseCooldownMs, PROVIDER_PROFILES.oauth.transientCooldown);
  assert.equal(oauthProfile.failureThreshold, PROVIDER_PROFILES.oauth.circuitBreakerThreshold);
  assert.equal(oauthProfile.resetTimeoutMs, PROVIDER_PROFILES.oauth.circuitBreakerReset);

  const apiKeyProfile = getProviderProfile("openai");
  assert.equal(apiKeyProfile.transientCooldown, PROVIDER_PROFILES.apikey.transientCooldown);
  assert.equal(
    apiKeyProfile.rateLimitCooldown,
    apiKeyProfile.useUpstreamRetryHints ? 0 : apiKeyProfile.baseCooldownMs
  );
  assert.equal(apiKeyProfile.maxBackoffLevel, PROVIDER_PROFILES.apikey.maxBackoffLevel);
  assert.equal(
    apiKeyProfile.circuitBreakerThreshold,
    PROVIDER_PROFILES.apikey.circuitBreakerThreshold
  );
  assert.equal(apiKeyProfile.circuitBreakerReset, PROVIDER_PROFILES.apikey.circuitBreakerReset);
  assert.equal(apiKeyProfile.baseCooldownMs, PROVIDER_PROFILES.apikey.transientCooldown);
  assert.equal(apiKeyProfile.failureThreshold, PROVIDER_PROFILES.apikey.circuitBreakerThreshold);
  assert.equal(apiKeyProfile.resetTimeoutMs, PROVIDER_PROFILES.apikey.circuitBreakerReset);
});

test("shouldMarkAccountExhaustedFrom429 skips connection poisoning for compatible providers", () => {
  assert.equal(shouldMarkAccountExhaustedFrom429("gemini", "gemini-2.5-pro"), false);
  assert.equal(
    shouldMarkAccountExhaustedFrom429("openai-compatible-custom-node", "any-model"),
    false
  );
  assert.equal(shouldMarkAccountExhaustedFrom429("openai", "gpt-4o-mini"), false);
  assert.equal(shouldMarkAccountExhaustedFrom429("claude", "claude-sonnet-4-6"), true);
});

test("shouldMarkAccountExhaustedFrom429 does not poison quota cache for transient 429s", () => {
  assert.equal(
    shouldMarkAccountExhaustedFrom429("kiro", "claude-opus-4.7", undefined, "rate_limit"),
    false
  );
  assert.equal(
    shouldMarkAccountExhaustedFrom429("kiro", "claude-opus-4.7", undefined, "transient"),
    false
  );
  assert.equal(
    shouldMarkAccountExhaustedFrom429("kiro", "claude-opus-4.7", undefined, "quota_exhausted"),
    true
  );
});

test("hasPerModelQuota returns true for GitHub Copilot provider (#1624)", () => {
  assert.equal(hasPerModelQuota("github"), true);
  assert.equal(hasPerModelQuota("github", "gpt-5.1-codex-max"), true);
  assert.equal(hasPerModelQuota("github", "gpt-5-mini"), true);
});

test("Codex Spark 429s are scoped away from normal Codex models", () => {
  const connectionId = `codex-${Date.now()}`;
  clearModelLock("codex", connectionId, "gpt-5.3-codex-spark");
  clearModelLock("codex", connectionId, "gpt-5.3-codex");

  assert.equal(hasPerModelQuota("codex", "gpt-5.3-codex-spark"), true);
  assert.equal(shouldMarkAccountExhaustedFrom429("codex", "gpt-5.3-codex-spark"), false);
  assert.equal(
    lockModelIfPerModelQuota(
      "codex",
      connectionId,
      "gpt-5.3-codex-spark",
      RateLimitReason.RATE_LIMIT_EXCEEDED,
      30_000
    ),
    true
  );
  assert.equal(isModelLocked("codex", connectionId, "gpt-5.3-codex-spark"), true);
  assert.equal(isModelLocked("codex", connectionId, "codex-spark-mini"), true);
  assert.equal(isModelLocked("codex", connectionId, "gpt-5.3-codex"), false);

  clearModelLock("codex", connectionId, "gpt-5.3-codex-spark");
  clearModelLock("codex", connectionId, "gpt-5.3-codex");
});

test("shouldMarkAccountExhaustedFrom429 skips connection-wide lockout for GitHub (#1624)", () => {
  assert.equal(shouldMarkAccountExhaustedFrom429("github", "gpt-5.1-codex-max"), false);
  assert.equal(shouldMarkAccountExhaustedFrom429("github", "gpt-5-mini"), false);
  assert.equal(shouldMarkAccountExhaustedFrom429("github", "claude-haiku-4.5"), false);
});

test("lockModelIfPerModelQuota locks individual GitHub models without poisoning the connection (#1624)", () => {
  const connectionId = `github-${Date.now()}`;

  // A 429 on a high-PRU model should lock ONLY that model
  assert.equal(
    lockModelIfPerModelQuota(
      "github",
      connectionId,
      "gpt-5.1-codex-max",
      RateLimitReason.RATE_LIMIT_EXCEEDED,
      30_000
    ),
    true
  );
  assert.equal(isModelLocked("github", connectionId, "gpt-5.1-codex-max"), true);

  // Other models on the same connection should remain unlocked
  assert.equal(isModelLocked("github", connectionId, "gpt-5-mini"), false);
  assert.equal(isModelLocked("github", connectionId, "claude-haiku-4.5"), false);
});

test("recordModelLockoutFailure uses provider profile cooldowns, backoff, and reset window", () => {
  const originalNow = Date.now;
  let now = 1_700_000_000_000;
  Date.now = () => now;

  try {
    const compatibleProvider = "openai-compatible-custom-node";
    const compatibleModel = "custom-model-a";
    const profile = makeProfile({
      maxBackoffSteps: 2,
      maxBackoffLevel: 2,
      resetTimeoutMs: 500,
    });

    const first = recordModelLockoutFailure(
      compatibleProvider,
      "conn-compatible",
      compatibleModel,
      "rate_limited",
      429,
      0,
      profile
    );
    now += 50;
    const second = recordModelLockoutFailure(
      compatibleProvider,
      "conn-compatible",
      compatibleModel,
      "rate_limited",
      429,
      0,
      profile
    );
    now += 50;
    const third = recordModelLockoutFailure(
      compatibleProvider,
      "conn-compatible",
      compatibleModel,
      "rate_limited",
      429,
      0,
      profile
    );

    const info = getModelLockoutInfo(compatibleProvider, "conn-compatible", compatibleModel);

    assert.equal(first.failureCount, 1);
    assert.equal(first.cooldownMs, 125);
    assert.equal(second.failureCount, 2);
    assert.equal(second.cooldownMs, 250);
    assert.equal(third.failureCount, 3);
    assert.equal(third.cooldownMs, 500);
    assert.equal(info.failureCount, 3);

    clearModelLock(compatibleProvider, "conn-compatible", compatibleModel);
    now += 600;

    const afterReset = recordModelLockoutFailure(
      compatibleProvider,
      "conn-compatible",
      compatibleModel,
      "rate_limited",
      429,
      0,
      profile
    );

    assert.equal(afterReset.failureCount, 1);
    assert.equal(afterReset.cooldownMs, 125);
  } finally {
    Date.now = originalNow;
    clearModelLock("openai-compatible-custom-node", "conn-compatible", "custom-model-a");
  }
});

// Provider-level failure circuit breaker tests
test("isProviderFailureCode correctly identifies provider-wide transient error codes", () => {
  assert.equal(isProviderFailureCode(429), true);
  assert.equal(isProviderFailureCode(408), true);
  assert.equal(isProviderFailureCode(500), true);
  assert.equal(isProviderFailureCode(502), true);
  assert.equal(isProviderFailureCode(503), true);
  assert.equal(isProviderFailureCode(504), true);
  assert.equal(isProviderFailureCode(401), false);
  assert.equal(isProviderFailureCode(403), false);
  assert.equal(isProviderFailureCode(400), false);
  assert.equal(isProviderFailureCode(404), false);
  assert.equal(isProviderFailureCode(200), false);
});

test("recordProviderFailure tracks failures and triggers cooldown after threshold", () => {
  const originalNow = Date.now;
  let now = 1_700_000_000_000;
  Date.now = () => now;

  try {
    const provider = "test-provider";

    // Clear any existing state
    clearProviderFailure(provider);
    assert.equal(isProviderInCooldown(provider), false);
    assert.equal(getProviderCooldownRemainingMs(provider), null);

    const threshold = PROVIDER_PROFILES.apikey.circuitBreakerThreshold;

    // Record failures up to threshold - 1
    for (let i = 0; i < threshold - 1; i++) {
      recordProviderFailure(provider);
      now += 1000; // 1 second between failures
    }
    assert.equal(isProviderInCooldown(provider), false);

    // Final failure to trigger threshold
    recordProviderFailure(provider);
    assert.equal(isProviderInCooldown(provider), true);

    const remaining = getProviderCooldownRemainingMs(provider);
    assert.ok(remaining !== null);
    assert.ok(remaining > 0);
    assert.ok(remaining <= 10 * 60 * 1000); // 10 minutes max

    // Check getProvidersInCooldown returns the provider
    const inCooldown = getProvidersInCooldown();
    assert.ok(inCooldown.some((p) => p.provider === provider));
    assert.equal(inCooldown.find((p) => p.provider === provider)?.failureCount, threshold);

    // Simulate cooldown expiration
    now += 11 * 60 * 1000; // 11 minutes later
    assert.equal(isProviderInCooldown(provider), false);
    assert.equal(getProviderCooldownRemainingMs(provider), null);
    assert.equal(
      getProvidersInCooldown().some((p) => p.provider === provider),
      false
    );
  } finally {
    Date.now = originalNow;
    clearProviderFailure("test-provider");
  }
});

test("recordProviderFailure honors runtime provider breaker profile", () => {
  const provider = "test-provider-runtime-profile";
  clearProviderFailure(provider);

  try {
    const runtimeProfile = {
      failureThreshold: PROVIDER_PROFILES.apikey.circuitBreakerThreshold + 7,
      resetTimeoutMs: PROVIDER_PROFILES.apikey.circuitBreakerReset + 45_000,
    };

    recordProviderFailure(provider, undefined, "conn-runtime-profile", runtimeProfile);

    const breaker = getCircuitBreaker(provider);
    assert.equal(breaker.failureThreshold, runtimeProfile.failureThreshold);
    assert.equal(breaker.resetTimeout, runtimeProfile.resetTimeoutMs);
    assert.equal(isProviderInCooldown(provider), false);

    const breakerAfterStatusCheck = getCircuitBreaker(provider);
    assert.equal(breakerAfterStatusCheck.failureThreshold, runtimeProfile.failureThreshold);
    assert.equal(breakerAfterStatusCheck.resetTimeout, runtimeProfile.resetTimeoutMs);
  } finally {
    clearProviderFailure(provider);
  }
});

test("recordProviderFailure preserves provider breaker cooldown while open", () => {
  const originalNow = Date.now;
  let now = 1_700_000_000_000;
  Date.now = () => now;

  try {
    const provider = "test-provider-open-cooldown-stability";
    const profile = { failureThreshold: 1, resetTimeoutMs: 60_000 };
    clearProviderFailure(provider);

    recordProviderFailure(provider, undefined, "conn-open-cooldown", profile);
    assert.equal(isProviderInCooldown(provider), true);

    const openedAt = getProviderBreakerState(provider)?.lastFailureTime;
    const initialRemaining = getProviderCooldownRemainingMs(provider);
    assert.equal(openedAt, now);
    assert.equal(initialRemaining, 60_000);

    now += 10_000;
    recordProviderFailure(provider, undefined, "conn-open-cooldown-later", profile);

    assert.equal(getProviderBreakerState(provider)?.lastFailureTime, openedAt);
    assert.equal(getProviderCooldownRemainingMs(provider), 50_000);
  } finally {
    Date.now = originalNow;
    clearProviderFailure("test-provider-open-cooldown-stability");
  }
});

test("recordProviderFailure keeps recent connection dedupe entries when pruning", () => {
  const originalNow = Date.now;
  const now = 1_700_000_000_000;
  Date.now = () => now;

  try {
    const provider = "test-provider-dedupe-prune";
    const profile = { failureThreshold: 20_000, resetTimeoutMs: 60_000 };
    clearProviderFailure(provider);

    for (let i = 0; i <= 10_000; i++) {
      recordProviderFailure(provider, undefined, `conn-${i}`, profile);
    }

    const beforeDuplicate = getProviderBreakerState(provider)?.failureCount;
    recordProviderFailure(provider, undefined, "conn-10000", profile);
    const afterDuplicate = getProviderBreakerState(provider)?.failureCount;

    assert.equal(afterDuplicate, beforeDuplicate);
    assert.equal(isProviderInCooldown(provider), false);
  } finally {
    Date.now = originalNow;
    clearProviderFailure("test-provider-dedupe-prune");
  }
});

test("recordProviderFailure refreshes insertion order for existing dedupe keys", () => {
  const originalNow = Date.now;
  let now = 1_700_000_000_000;
  Date.now = () => now;

  try {
    const provider = "test-provider-dedupe-lru";
    const profile = { failureThreshold: 20_000, resetTimeoutMs: 60_000 };
    clearProviderFailure(provider);

    for (let i = 0; i < 9_999; i++) {
      recordProviderFailure(provider, undefined, `conn-${i}`, profile);
    }

    now += 10_000;
    recordProviderFailure(provider, undefined, "conn-0", profile);

    for (let i = 10_000; i < 10_050; i++) {
      recordProviderFailure(provider, undefined, `conn-${i}`, profile);
    }

    const breakerState = getProviderBreakerState(provider);
    assert.equal(breakerState?.failureCount !== undefined, true);
    assert.equal(isProviderInCooldown(provider), false);
  } finally {
    Date.now = originalNow;
    clearProviderFailure("test-provider-dedupe-lru");
  }
});

test("checkFallbackError no longer mutates provider breaker state on per-connection failures", () => {
  const provider = "test-provider-check";
  clearProviderFailure(provider);

  for (let i = 0; i < 5; i++) {
    checkFallbackError(429, "rate limited", 0, null, provider);
  }

  assert.equal(isProviderInCooldown(provider), false);
  clearProviderFailure(provider);
});

test("checkFallbackError does not record provider failure for non-transient errors", () => {
  const originalNow = Date.now;
  let now = 1_700_000_000_000;
  Date.now = () => now;

  try {
    const provider = "test-provider-no-record";
    clearProviderFailure(provider);

    // Simulate 5 auth errors (401) - should NOT trigger provider cooldown
    for (let i = 0; i < 5; i++) {
      checkFallbackError(401, "unauthorized", 0, null, provider);
      now += 1000;
    }

    // Provider should NOT be in cooldown
    assert.equal(isProviderInCooldown(provider), false);
  } finally {
    Date.now = originalNow;
    clearProviderFailure("test-provider-no-record");
  }
});

test("clearProviderFailure removes provider from cooldown", () => {
  const originalNow = Date.now;
  let now = 1_700_000_000_000;
  Date.now = () => now;

  try {
    const provider = "test-provider-clear";
    clearProviderFailure(provider);

    // Trigger cooldown
    const threshold = PROVIDER_PROFILES.apikey.circuitBreakerThreshold;
    for (let i = 0; i < threshold; i++) {
      recordProviderFailure(provider);
      now += 1000;
    }
    assert.equal(isProviderInCooldown(provider), true);

    // Clear the failure state
    clearProviderFailure(provider);
    assert.equal(isProviderInCooldown(provider), false);
    assert.equal(getProviderCooldownRemainingMs(provider), null);
  } finally {
    Date.now = originalNow;
    clearProviderFailure("test-provider-clear");
  }
});

// Daily quota exhausted detection tests
test("isDailyQuotaExhausted detects today's quota errors", () => {
  const { isDailyQuotaExhausted } = accountFallback;
  assert.equal(isDailyQuotaExhausted("You have exceeded today's quota for model X"), true);
  assert.equal(isDailyQuotaExhausted("exceeded your daily quota"), true);
  assert.equal(isDailyQuotaExhausted("Please try again tomorrow"), true);
  assert.equal(isDailyQuotaExhausted("rate limit exceeded"), false);
  assert.equal(isDailyQuotaExhausted(""), false);
  assert.equal(isDailyQuotaExhausted(null), false);
});

test("getMsUntilTomorrow returns positive value less than 24 hours", () => {
  const { getMsUntilTomorrow } = accountFallback;
  const ms = getMsUntilTomorrow();
  assert.ok(ms > 0, "should be positive");
  assert.ok(ms <= 24 * 60 * 60 * 1000, "should be <= 24 hours");
});

test("checkFallbackError locks model until tomorrow for non-429 daily quota exhaustion", () => {
  const result = checkFallbackError(
    402,
    "You have exceeded today's quota for model moonshotai/Kimi-K2.5, please try again tomorrow"
  );
  assert.equal(result.shouldFallback, true);
  assert.equal(result.reason, RateLimitReason.QUOTA_EXHAUSTED);
  assert.equal(result.dailyQuotaExhausted, true);
  assert.ok(result.cooldownMs > 0, "cooldown should be positive");
  assert.ok(result.cooldownMs <= 24 * 60 * 60 * 1000, "cooldown should be <= 24 hours");
});

test("checkFallbackError routes API-key 429 'try again tomorrow' through resilience cooldown", () => {
  const result = checkFallbackError(
    429,
    "Please try again tomorrow",
    0,
    null,
    "openai",
    null,
    makeProfile()
  );
  assert.equal(result.shouldFallback, true);
  assert.equal(result.dailyQuotaExhausted, undefined);
  assert.equal(result.cooldownMs, 125);
});

test("#6638: checkFallbackError routes API-key 429 'daily quota' text as quota_exhausted", () => {
  const result = checkFallbackError(
    429,
    "You have exceeded your daily quota",
    0,
    null,
    "openai",
    null,
    makeProfile()
  );
  assert.equal(result.shouldFallback, true);
  assert.equal(result.dailyQuotaExhausted, true);
  assert.equal(result.reason, RateLimitReason.QUOTA_EXHAUSTED);
});

test("checkFallbackError preserves OAuth 429 daily quota semantics", () => {
  const result = checkFallbackError(
    429,
    "You have exceeded your daily quota",
    0,
    null,
    "codex",
    null,
    makeProfile()
  );

  assert.equal(result.shouldFallback, true);
  assert.equal(result.reason, RateLimitReason.QUOTA_EXHAUSTED);
  assert.equal(result.dailyQuotaExhausted, true);
  assert.ok(result.cooldownMs > 0);
});

// ModelScope daily quota lockout tests (commit 0456a1f5)
test("recordModelLockoutFailure sets cooldown until tomorrow 0:00 for quota_exhausted reason", () => {
  const originalNow = Date.now;
  // Use a fixed local time (noon) to ensure predictable results
  const testDate = new Date();
  testDate.setHours(12, 0, 0, 0); // Set to noon today
  const now = testDate.getTime();
  Date.now = () => now;

  try {
    const provider = "modelscope";
    const connectionId = "test-conn-modelscope-1";
    const model = "qwen/Qwen2.5-Coder-32B-Instruct";

    // Clear any existing state
    clearModelLock(provider, connectionId, model);

    const profile = makeProfile();

    // Calculate milliseconds until tomorrow 00:00 local time
    const tomorrow = new Date(now);
    tomorrow.setDate(tomorrow.getDate() + 1);
    tomorrow.setHours(0, 0, 0, 0);
    const expectedMsUntilTomorrow = tomorrow.getTime() - now;

    // Record failure with quota_exhausted reason
    const result = recordModelLockoutFailure(
      provider,
      connectionId,
      model,
      "quota_exhausted",
      429,
      0,
      profile
    );

    // Verify the cooldown is set to ms until tomorrow 0:00 (with tolerance)
    const diff = Math.abs(result.cooldownMs - expectedMsUntilTomorrow);
    assert.ok(
      diff <= 300_000,
      `cooldown should be ms until tomorrow 0:00 (expected ${expectedMsUntilTomorrow}ms, got ${result.cooldownMs}ms, diff ${diff}ms)`
    );

    // Verify model is locked
    assert.equal(isModelLocked(provider, connectionId, model), true);

    const lockInfo = getModelLockoutInfo(provider, connectionId, model);
    assert.ok(lockInfo !== null, "lockInfo should not be null");
    assert.ok(lockInfo.remainingMs > 0, "remaining time should be positive");

    clearModelLock(provider, connectionId, model);
  } finally {
    Date.now = originalNow;
    clearModelLock("modelscope", "test-conn-modelscope-1", "qwen/Qwen2.5-Coder-32B-Instruct");
  }
});

test("recordModelLockoutFailure uses regular backoff for non-quota reasons", () => {
  const originalNow = Date.now;
  const now = 1_700_000_000_000;
  Date.now = () => now;

  try {
    const provider = "modelscope";
    const connectionId = "test-conn-modelscope-2";
    const model = "qwen/Qwen2.5-Coder-32B-Instruct";

    clearModelLock(provider, connectionId, model);

    const profile = makeProfile({
      baseCooldownMs: 5000,
      transientCooldown: 5000,
      rateLimitCooldown: 5000,
    });

    // Record failure with rate_limited reason (not quota_exhausted)
    const result = recordModelLockoutFailure(
      provider,
      connectionId,
      model,
      "rate_limited",
      429,
      0,
      profile
    );

    // Verify the cooldown uses regular profile baseCooldownMs (5000ms)
    assert.ok(
      result.cooldownMs < 24 * 60 * 60 * 1000,
      "cooldown should be less than 24h for non-quota reasons"
    );
    assert.equal(result.cooldownMs, 5000, "cooldown should use profile baseCooldownMs");

    clearModelLock(provider, connectionId, model);
  } finally {
    Date.now = originalNow;
    clearModelLock("modelscope", "test-conn-modelscope-2", "qwen/Qwen2.5-Coder-32B-Instruct");
  }
});

// Test for hour quota related error messages
test("checkFallbackError classifies hour quota errors correctly", () => {
  // For OAuth providers (e.g., codex), hour quota errors should be QUOTA_EXHAUSTED
  const result1 = checkFallbackError(
    429,
    "Coding Plan hour quota has been exceeded",
    0,
    null,
    "codex"
  );
  assert.equal(result1.shouldFallback, true);
  assert.equal(result1.reason, RateLimitReason.QUOTA_EXHAUSTED);

  const result2 = checkFallbackError(429, "hour quota exceeded", 0, null, "codex");
  assert.equal(result2.shouldFallback, true);
  assert.equal(result2.reason, RateLimitReason.QUOTA_EXHAUSTED);

  const result3 = checkFallbackError(429, "Your hour quota is exceeded", 0, null, "codex");
  assert.equal(result3.shouldFallback, true);
  assert.equal(result3.reason, RateLimitReason.QUOTA_EXHAUSTED);

  const result4 = checkFallbackError(429, "hour quota depleted", 0, null, "codex");
  assert.equal(result4.shouldFallback, true);
  assert.equal(result4.reason, RateLimitReason.QUOTA_EXHAUSTED);

  // For API-key providers with 402 status, hour quota errors should be QUOTA_EXHAUSTED
  const result5 = checkFallbackError(402, "hour quota has been exceeded", 0, null, "openai");
  assert.equal(result5.shouldFallback, true);
  assert.equal(result5.reason, RateLimitReason.QUOTA_EXHAUSTED);

  const result6 = checkFallbackError(
    403,
    "Coding Plan hour quota has been exceeded",
    0,
    null,
    "openai"
  );
  assert.equal(result6.shouldFallback, true);
  assert.equal(result6.reason, RateLimitReason.QUOTA_EXHAUSTED);
});

// Test for classifyErrorText function with hour quota
test("classifyErrorText handles hour quota messages", () => {
  const { classifyErrorText } = accountFallback;

  assert.equal(
    classifyErrorText("Coding Plan hour quota has been exceeded"),
    RateLimitReason.QUOTA_EXHAUSTED
  );
  assert.equal(classifyErrorText("hour quota exceeded"), RateLimitReason.QUOTA_EXHAUSTED);
  assert.equal(classifyErrorText("Your hour quota is exceeded"), RateLimitReason.QUOTA_EXHAUSTED);
  assert.equal(classifyErrorText("hour quota has been exceeded"), RateLimitReason.QUOTA_EXHAUSTED);
  assert.equal(classifyErrorText("quota has been exceeded"), RateLimitReason.QUOTA_EXHAUSTED);
});

// ─── Model Access Denied (structured error codes + regex fallback) ─────

test("checkFallbackError detects model access denied via structured error code (OpenAI)", () => {
  const result = checkFallbackError(
    400,
    "The model `gpt-5` does not exist",
    0,
    null,
    "openai",
    null,
    null,
    { code: "model_not_found", type: null }
  );
  assert.equal(result.shouldFallback, true);
  assert.equal(result.cooldownMs, 0);
  assert.equal(result.reason, RateLimitReason.MODEL_CAPACITY);
});

test("checkFallbackError detects model access denied via structured error type (Anthropic not_found_error)", () => {
  const result = checkFallbackError(
    400,
    "model: claude-sonnet-4-7-20260515",
    0,
    null,
    "anthropic",
    null,
    null,
    { code: null, type: "not_found_error" }
  );
  assert.equal(result.shouldFallback, true);
  assert.equal(result.cooldownMs, 0);
  assert.equal(result.reason, RateLimitReason.MODEL_CAPACITY);
});

test("checkFallbackError detects model access denied via structured error type (Anthropic permission_error) when the message confirms the model", () => {
  const result = checkFallbackError(
    400,
    "you do not have access to the requested model",
    0,
    null,
    "anthropic",
    null,
    null,
    { code: null, type: "permission_error" }
  );
  assert.equal(result.shouldFallback, true);
  assert.equal(result.cooldownMs, 0);
  assert.equal(result.reason, RateLimitReason.MODEL_CAPACITY);
});

test("checkFallbackError does NOT fallback on a permission_error that is a key/feature scope issue (not model access)", () => {
  // permission_error is ambiguous on Anthropic — also raised for API-key scope,
  // org restrictions and feature gating. Without a model-related message it must
  // surface the real error instead of silently exhausting every combo target.
  const result = checkFallbackError(
    400,
    "Your API key does not have permission to use the Message Batches API",
    0,
    null,
    "anthropic",
    null,
    null,
    { code: null, type: "permission_error" }
  );
  assert.equal(result.shouldFallback, false);
});

test("checkFallbackError detects model access denied via regex fallback (invalid model)", () => {
  const result = checkFallbackError(
    400,
    "Invalid model: gpt-5-turbo",
    0,
    null,
    "some-provider",
    null,
    null
  );
  assert.equal(result.shouldFallback, true);
  assert.equal(result.cooldownMs, 0);
  assert.equal(result.reason, RateLimitReason.MODEL_CAPACITY);
});

test("checkFallbackError does NOT fallback on generic 400 without model access denied", () => {
  const result = checkFallbackError(400, "bad request payload", 0, null, "openai", null, null);
  assert.equal(result.shouldFallback, false);
});

test("checkFallbackError ignores structured error with unrelated code on 400", () => {
  const result = checkFallbackError(400, "something went wrong", 0, null, "openai", null, null, {
    code: "invalid_api_key",
    type: null,
  });
  // "invalid_api_key" is not in MODEL_ACCESS_DENIED_CODES,
  // no MODEL_ACCESS_DENIED_PATTERNS match either → shouldFallback: false
  assert.equal(result.shouldFallback, false);
});

// ─── Gemini RPM 429 Classification (CREDITS_EXHAUSTED_SIGNALS fix) ─────

test("isCreditsExhausted returns false for Gemini RPM 429 body text", () => {
  const geminiRpmText = "Resource has been exhausted (e.g. check quota).";
  assert.equal(isCreditsExhausted(geminiRpmText), false);
});

test("isCreditsExhausted returns true for actual credits-exhausted signals", () => {
  assert.equal(isCreditsExhausted("insufficient_quota"), true);
  assert.equal(isCreditsExhausted("credits exhausted"), true);
  assert.equal(isCreditsExhausted("payment required"), true);
  assert.equal(isCreditsExhausted("free tier of the model has been exhausted"), true);
  assert.equal(isCreditsExhausted("exceeded your current usage quota"), true);
  // #5239: "Insufficient account balance" out-of-credit bodies
  assert.equal(isCreditsExhausted("Insufficient account balance"), true);
  assert.equal(isCreditsExhausted("insufficient_balance"), true);
  assert.equal(isCreditsExhausted("Insufficient credit balance"), true);
  // Command Code returns 400 "You have insufficient credits to make this
  // request. Please purchase more credits to continue using the service."
  // when the account's billing credits run out. Without this signal the
  // error is unclassified (errorType=null), so the connection is never
  // marked credits_exhausted and keeps being re-selected on every request.
  assert.equal(isCreditsExhausted("insufficient credits"), true);
  assert.equal(
    isCreditsExhausted(
      "You have insufficient credits to make this request. Please purchase more credits to continue using the service."
    ),
    true
  );
});

test("CREDITS_EXHAUSTED_SIGNALS no longer contains generic gRPC resource-exhausted patterns", () => {
  // These patterns were removed because they falsely matched Gemini RPM 429 errors
  assert.equal(CREDITS_EXHAUSTED_SIGNALS.includes("resource has been exhausted"), false);
  assert.equal(CREDITS_EXHAUSTED_SIGNALS.includes("resource_exhausted"), false);
  assert.equal(CREDITS_EXHAUSTED_SIGNALS.includes("check quota"), false);
});

test("checkFallbackError classifies Gemini RPM 429 as RATE_LIMIT_EXCEEDED (not QUOTA_EXHAUSTED)", () => {
  // provider=null → preserveQuota429=true → text quota checks run
  // isCreditsExhausted must NOT match Gemini's "Resource has been exhausted"
  const result = checkFallbackError(
    429,
    "Resource has been exhausted (e.g. check quota).",
    0,
    null,
    null,
    null,
    makeProfile()
  );
  assert.equal(result.shouldFallback, true);
  assert.equal(result.reason, RateLimitReason.RATE_LIMIT_EXCEEDED);
  assert.equal(result.creditsExhausted, undefined);
  assert.equal(result.dailyQuotaExhausted, undefined);
  assert.ok(result.cooldownMs > 0, "cooldownMs should be positive");
});

test("checkFallbackError classifies Gemini RPM 429 as RATE_LIMIT_EXCEEDED for API-key provider", () => {
  // provider="gemini" → preserveQuota429=false → status-based rule applies
  const result = checkFallbackError(
    429,
    "Resource has been exhausted (e.g. check quota).",
    0,
    null,
    "gemini",
    null,
    makeProfile()
  );
  assert.equal(result.shouldFallback, true);
  assert.equal(result.reason, RateLimitReason.RATE_LIMIT_EXCEEDED);
  assert.equal(result.cooldownMs, 125); // makeProfile().baseCooldownMs
});

test("checkFallbackError still classifies genuine OAuth quota-exhausted text as QUOTA_EXHAUSTED", () => {
  // Regression: OAuth providers must still get QUOTA_EXHAUSTED for actual quota messages
  const result = checkFallbackError(
    429,
    "Coding Plan hour quota has been exceeded",
    0,
    null,
    "codex",
    null,
    makeProfile()
  );
  assert.equal(result.shouldFallback, true);
  assert.equal(result.reason, RateLimitReason.QUOTA_EXHAUSTED);
});

test("checkFallbackError preserves daily-quota exhaustion for non-429 status codes", () => {
  // Non-429 status codes with daily quota text must still be QUOTA_EXHAUSTED
  const result = checkFallbackError(
    402,
    "You have exceeded today's quota, please try again tomorrow"
  );
  assert.equal(result.shouldFallback, true);
  assert.equal(result.reason, RateLimitReason.QUOTA_EXHAUSTED);
  assert.equal(result.dailyQuotaExhausted, true);
});

// ─── Gemini 429 → Model Lockout: rate_limited (not quota_exhausted) ────

test("Gemini RPM 429: recordModelLockoutFailure uses exponential backoff for rate_limited reason", () => {
  const originalNow = Date.now;
  const now = 1_700_000_000_000;
  Date.now = () => now;
  const provider = "gemini";
  const connectionId = "test-conn-gemini-rpm";
  const model = "gemini/gemma-4-31b-it";

  try {
    clearModelLock(provider, connectionId, model);

    const profile = makeProfile({
      baseCooldownMs: 5000,
      transientCooldown: 5000,
      rateLimitCooldown: 5000,
    });

    // auth.ts flow: 429 + fallbackResult.reason=RATE_LIMIT_EXCEEDED
    // → reason="rate_limited" → recordModelLockoutFailure
    const first = recordModelLockoutFailure(
      provider,
      connectionId,
      model,
      "rate_limited",
      429,
      0,
      profile
    );
    assert.equal(first.failureCount, 1);
    assert.equal(first.cooldownMs, 5000, "first failure: 5s base cooldown");

    const second = recordModelLockoutFailure(
      provider,
      connectionId,
      model,
      "rate_limited",
      429,
      0,
      profile
    );
    assert.equal(second.failureCount, 2);
    assert.equal(second.cooldownMs, 10000, "second failure: 10s exponential backoff");

    assert.equal(isModelLocked(provider, connectionId, model), true);
    clearModelLock(provider, connectionId, model);
  } finally {
    Date.now = originalNow;
    clearModelLock("gemini", "test-conn-gemini-rpm", "gemini/gemma-4-31b-it");
  }
});

test("Gemini RPD (quota_exhausted) still triggers midnight lockout in recordModelLockoutFailure", () => {
  const originalNow = Date.now;
  const testDate = new Date();
  testDate.setHours(12, 0, 0, 0);
  const now = testDate.getTime();
  Date.now = () => now;
  const provider = "gemini";
  const connectionId = "test-conn-gemini-rpd";
  const model = "gemini/gemma-4-31b-it";

  try {
    clearModelLock(provider, connectionId, model);
    const profile = makeProfile();
    const result = recordModelLockoutFailure(
      provider,
      connectionId,
      model,
      "quota_exhausted",
      429,
      0,
      profile
    );

    // Must lock until midnight, NOT exponential backoff
    const tomorrow = new Date(now);
    tomorrow.setDate(tomorrow.getDate() + 1);
    tomorrow.setHours(0, 0, 0, 0);
    const expected = tomorrow.getTime() - now;
    assert.ok(
      Math.abs(result.cooldownMs - expected) <= 300_000,
      `cooldown should be until tomorrow (expected ~${expected}, got ${result.cooldownMs})`
    );
    clearModelLock(provider, connectionId, model);
  } finally {
    Date.now = originalNow;
    clearModelLock("gemini", "test-conn-gemini-rpd", "gemini/gemma-4-31b-it");
  }
});

// ─── G-02: X-Omni-Fallback-Hint: connection_cooldown ─────────────────────────
// When 9router executor signals a supervisor-not-running 503, checkFallbackError
// must return 5s cooldown with skipProviderBreaker:true — not trip the circuit breaker.

test("G-02: X-Omni-Fallback-Hint connection_cooldown on 503 returns 5s cooldown + skipProviderBreaker", () => {
  const headers = new Headers({ "X-Omni-Fallback-Hint": "connection_cooldown" });
  const result = checkFallbackError(
    503,
    "9router is not running (state: stopped)",
    0,
    null,
    "9router",
    headers
  );
  assert.equal(result.shouldFallback, true);
  assert.equal(result.cooldownMs, 5_000);
  assert.equal(result.skipProviderBreaker, true);
  assert.equal(result.newBackoffLevel, 0);
  assert.equal(result.reason, "service_not_running");
});

test("G-02: X-Omni-Fallback-Hint connection_cooldown header lookup is case-insensitive (lowercase header key)", () => {
  // Headers object normalises keys to lowercase — test the plain-object path
  const headers: Record<string, string> = { "x-omni-fallback-hint": "connection_cooldown" };
  const result = checkFallbackError(
    503,
    "9router is not running (state: stopped)",
    0,
    null,
    "9router",
    headers
  );
  assert.equal(result.skipProviderBreaker, true);
  assert.equal(result.cooldownMs, 5_000);
});

test("G-02: hint header is ignored for non-503 status codes", () => {
  const headers = new Headers({ "X-Omni-Fallback-Hint": "connection_cooldown" });
  // 502 should NOT trigger the hint path even if the header is present
  const result = checkFallbackError(502, "bad gateway", 0, null, "9router", headers);
  assert.equal(result.skipProviderBreaker, undefined); // normal path, no skip flag
});

test("G-02: 503 without hint header follows normal circuit-breaker path", () => {
  // A plain 503 from a real upstream must still feed the circuit breaker
  const result = checkFallbackError(503, "service unavailable", 0, null, "openai", null);
  assert.equal(result.skipProviderBreaker, undefined);
  assert.ok(result.cooldownMs > 0);
});

test("G-02: five consecutive 503 service_not_running do NOT trip provider circuit breaker (flag)", () => {
  // Verify that every call returns skipProviderBreaker:true so callers can skip recordProviderFailure
  const headers = new Headers({ "X-Omni-Fallback-Hint": "connection_cooldown" });
  for (let i = 0; i < 5; i++) {
    const result = checkFallbackError(
      503,
      "9router is not running (state: stopped)",
      0,
      null,
      "9router",
      headers
    );
    assert.equal(
      result.skipProviderBreaker,
      true,
      `call ${i + 1} should have skipProviderBreaker:true`
    );
  }
  // Verify the circuit breaker for 9router is NOT open after those 5 calls
  const { isProviderInCooldown, clearProviderFailure } = accountFallback;
  assert.equal(
    isProviderInCooldown("9router"),
    false,
    "9router circuit breaker must remain closed"
  );
  clearProviderFailure("9router"); // cleanup
});

test("recordModelLockoutFailure caps cooldown at BACKOFF_CONFIG.max to prevent absurdly long lockouts", () => {
  const originalNow = Date.now;
  let now = 1_700_000_000_000;
  Date.now = () => now;

  try {
    const provider = "openai";
    const connectionId = "conn-capped";
    const model = "gpt-5-trillium";

    clearModelLock(provider, connectionId, model);

    // Fire 9 consecutive failures so the backoff exceeds the 120s cap
    // baseCooldownMs=1000 (getQuotaCooldown(0)), failure 9: 1000*2^8=256000 > 120000
    let lastResult;
    for (let i = 0; i < 9; i++) {
      lastResult = recordModelLockoutFailure(
        provider,
        connectionId,
        model,
        "rate_limited",
        429,
        0,
        null
      );
      now += 50; // each failure within the reset window
    }

    assert.ok(
      lastResult.cooldownMs <= 120_000,
      `cooldown ${lastResult.cooldownMs}ms should not exceed BACKOFF_CONFIG.max (120000ms)`
    );
    assert.equal(lastResult.cooldownMs, 120_000);
    assert.equal(lastResult.failureCount, 9);

    clearModelLock(provider, connectionId, model);
  } finally {
    Date.now = originalNow;
  }
});

test("recordModelLockoutFailure groups provider aliases under canonical provider", () => {
  const providerAlias = "cx";
  const providerCanonical = "codex";
  const connectionId = "conn-alias-test";
  const model = "gpt-5.5";

  clearModelLock(providerCanonical, connectionId, model);
  clearModelLock(providerAlias, connectionId, model);

  const result1 = recordModelLockoutFailure(
    providerAlias,
    connectionId,
    model,
    "rate_limited",
    429,
    1000,
    null
  );

  assert.equal(isModelLocked(providerAlias, connectionId, model), true);
  assert.equal(isModelLocked(providerCanonical, connectionId, model), true);

  clearModelLock(providerCanonical, connectionId, model);
});

test("recordModelLockoutFailure escalates backoff correctly after cooldown expiration (long interval)", () => {
  const originalNow = Date.now;
  let now = 1_700_000_000_000;
  Date.now = () => now;

  try {
    const provider = "openai";
    const connectionId = "conn-long-interval";
    const model = "gpt-5-escalate";

    clearModelLock(provider, connectionId, model);

    const profile = makeProfile({
      baseCooldownMs: 120000,
      resetTimeoutMs: 30000,
    });

    const first = recordModelLockoutFailure(
      provider,
      connectionId,
      model,
      "rate_limited",
      429,
      120000,
      profile,
      { maxCooldownMs: 1800000 }
    );
    assert.equal(first.failureCount, 1);
    assert.equal(first.cooldownMs, 120000);

    now += 130000;

    const second = recordModelLockoutFailure(
      provider,
      connectionId,
      model,
      "rate_limited",
      429,
      120000,
      profile,
      { maxCooldownMs: 1800000 }
    );
    assert.equal(second.failureCount, 2);
    assert.equal(second.cooldownMs, 240000);

    clearModelLock(provider, connectionId, model);
  } finally {
    Date.now = originalNow;
  }
});
// ── Custom banned signals (PR #3454) ──────────────────────────────────────────
// Operators can extend ACCOUNT_DEACTIVATED_SIGNALS with provider-specific
// permanent-ban phrasing via Settings → Security. These persist in the
// key_value settings store and are applied at boot + on hot-reload through
// setCustomBannedSignals(). Regression guard for the merge/detection behavior.

const {
  setCustomBannedSignals,
  getMergedBannedSignals,
  isAccountDeactivated,
  ACCOUNT_DEACTIVATED_SIGNALS,
} = accountFallback;

test("getMergedBannedSignals returns built-in list unchanged when no custom signals", () => {
  setCustomBannedSignals([]);
  const merged = getMergedBannedSignals();
  assert.deepEqual(merged, ACCOUNT_DEACTIVATED_SIGNALS);
});

test("getMergedBannedSignals appends custom signals to the built-in list", () => {
  setCustomBannedSignals(["api key revoked", "tenant suspended"]);
  const merged = getMergedBannedSignals();
  // Built-ins still present
  for (const sig of ACCOUNT_DEACTIVATED_SIGNALS) {
    assert.ok(merged.includes(sig), `built-in signal "${sig}" must survive merge`);
  }
  // Custom appended
  assert.ok(merged.includes("api key revoked"));
  assert.ok(merged.includes("tenant suspended"));
  setCustomBannedSignals([]); // cleanup
});

test("isAccountDeactivated still matches built-in signals when custom list is empty", () => {
  setCustomBannedSignals([]);
  assert.equal(isAccountDeactivated("Your account has been suspended"), true);
  assert.equal(isAccountDeactivated("rate limit exceeded, retry later"), false);
});

test("isAccountDeactivated matches a custom signal after setCustomBannedSignals", () => {
  setCustomBannedSignals([]);
  // Before registration the custom phrase is not a ban signal
  assert.equal(
    isAccountDeactivated("Error: API key revoked by administrator"),
    false,
    "custom phrase must not match before it is registered"
  );

  setCustomBannedSignals(["api key revoked"]);
  // Case-insensitive substring match against the merged list
  assert.equal(
    isAccountDeactivated("Error: API key revoked by administrator"),
    true,
    "custom phrase must match once registered (case-insensitive substring)"
  );

  // Built-ins remain matchable alongside custom signals
  assert.equal(isAccountDeactivated("account_deactivated"), true);

  setCustomBannedSignals([]); // cleanup — restore module state for other tests
});

// ─── #10460: model-unsupported 400 skips account rotation ────────────────────
// TEST_DATA_DIR_10460 / DATA_DIR / core / providersDb / auth are set up at the top
// of this file, BEFORE the accountFallback.ts import — see the comment there.

async function resetStorage10460() {
  core.resetDbInstance();
  fs.rmSync(TEST_DATA_DIR_10460, { recursive: true, force: true });
  fs.mkdirSync(TEST_DATA_DIR_10460, { recursive: true });
}

async function seedConn10460(provider: string): Promise<string> {
  const conn = await providersDb.createProviderConnection({
    provider,
    authType: "apikey",
    apiKey: `${provider}-key-10460`,
    isActive: true,
    testStatus: "active",
  });
  return (conn as Record<string, unknown>).id as string;
}

test.after(() => {
  core.resetDbInstance();
  fs.rmSync(TEST_DATA_DIR_10460, { recursive: true, force: true });
});

test("#10460: model-unsupported 400 returns shouldFallback:false (no account cooldown)", async () => {
  await resetStorage10460();
  const connId = await seedConn10460("github");

  const result = await auth.markAccountUnavailable(
    connId,
    400,
    "The requested model is not supported",
    "github",
    "claude-fable-5"
  );

  // The guard must prevent account cooldown — the error belongs to the combo layer
  assert.strictEqual(result.shouldFallback, false, "must not trigger account rotation");
  assert.strictEqual(result.cooldownMs, 0, "must not cool down the account");

  // Verify the connection was NOT marked unavailable
  const after = await providersDb.getProviderConnectionById(connId);
  assert.ok(!after.rateLimitedUntil, "connection must not be rate-limited");
  assert.notStrictEqual(after.testStatus, "unavailable", "connection must stay active");
});

test("#10460: model-unsupported 400 handles various phrasings", async () => {
  await resetStorage10460();
  const connId = await seedConn10460("github");

  const phrasings = [
    "The requested model is not supported",
    "model claude-fable-5 is not supported",
    "invalid_request_error: model is not supported",
    "unsupported model: gpt-9",
  ];

  for (const errorText of phrasings) {
    await resetStorage10460();
    const id = await seedConn10460("github");
    const result = await auth.markAccountUnavailable(id, 400, errorText, "github", "test-model");
    assert.strictEqual(result.shouldFallback, false, `phrasing "${errorText}" must not rotate`);
    // Verify connection stays healthy after each iteration
    const conn = await providersDb.getProviderConnectionById(id);
    assert.ok(!conn.rateLimitedUntil, `"${errorText}" must not rate-limit connection`);
    assert.notStrictEqual(conn.testStatus, "unavailable", `"${errorText}" must not mark unavailable`);
  }
});

test("#10460: body-specific 400 still goes through normal path (not blocked by model guard)", async () => {
  await resetStorage10460();
  const connId = await seedConn10460("openai");

  const result = await auth.markAccountUnavailable(
    connId,
    400,
    "Invalid message format: the request body is malformed",
    "openai",
    "gpt-4"
  );

  // Body-specific 400 does NOT match MODEL_ACCESS_DENIED_PATTERNS — normal path applies
  assert.strictEqual(result.shouldFallback, true, "body-specific 400 must still allow fallback");
});

test("#10460: non-400 status with model-unsupported text does NOT trigger guard", async () => {
  await resetStorage10460();
  const connId = await seedConn10460("github");

  // 429 + model-unsupported text should go through normal path (guard only fires for status===400)
  const result = await auth.markAccountUnavailable(
    connId,
    429,
    "The requested model is not supported",
    "github",
    "test-model"
  );

  assert.strictEqual(result.shouldFallback, true, "non-400 must not be short-circuited by model guard");
  // The key assertion: guard returns shouldFallback:false. If we get here with
  // shouldFallback:true, the guard did NOT fire (correct behavior).
});

test("#10460: empty errorText with status 400 does NOT trigger guard", async () => {
  await resetStorage10460();
  const connId = await seedConn10460("github");

  const result = await auth.markAccountUnavailable(connId, 400, "", "github", "test-model");

  // Empty string matches no patterns — normal path applies
  assert.strictEqual(result.shouldFallback, false, "empty errorText is generic 400 → no fallback");
});

test("#10460: auth-credential 400 text does NOT match model-unsupported guard", async () => {
  await resetStorage10460();
  const connId = await seedConn10460("github");

  // "Invalid API key provided for model gpt-4o" contains "model" but is NOT a
  // model-unsupported error — it's a credential issue. The guard must not fire.
  const result = await auth.markAccountUnavailable(
    connId,
    400,
    "Invalid API key provided for model gpt-4o",
    "github",
    "gpt-4o"
  );

  // This text does NOT match MODEL_ACCESS_DENIED_PATTERNS (verified by regex test)
  // so it falls through to checkFallbackError which returns shouldFallback:false for generic 400
  assert.strictEqual(result.shouldFallback, false, "auth-credential 400 must not be caught by model guard");
  // The generic 400 path returns cooldownMs:0 — same as the guard, but the
  // connection was NOT touched (no rateLimitedUntil set). This distinguishes
  // it from the normal fallback path which would set a cooldown.
  const conn = await providersDb.getProviderConnectionById(connId);
  assert.ok(!conn.rateLimitedUntil, "generic 400 must not rate-limit connection");
});

test("#10460: guard early return does not touch DB (distinguishes from normal path)", async () => {
  await resetStorage10460();
  const connId = await seedConn10460("github");

  // Guard path: model-unsupported 400 → shouldFallback:false, cooldownMs:0, no DB change
  const guardResult = await auth.markAccountUnavailable(
    connId, 400, "The requested model is not supported", "github", "test-model"
  );
  assert.strictEqual(guardResult.shouldFallback, false);
  assert.strictEqual(guardResult.cooldownMs, 0);
  const guardConn = await providersDb.getProviderConnectionById(connId);
  assert.ok(!guardConn.rateLimitedUntil, "guard path must not touch DB");
  assert.strictEqual(guardConn.testStatus, "active", "guard path must keep connection active");
});

test("#10460: returned result exposes a sanitized provider_model_unsupported reason", async () => {
  await resetStorage10460();
  const connId = await seedConn10460("github");

  const result = await auth.markAccountUnavailable(
    connId,
    400,
    "The requested model is not supported",
    "github",
    "claude-fable-5"
  );

  assert.strictEqual(result.shouldFallback, false);
  assert.strictEqual(
    (result as { reason?: string }).reason,
    "provider_model_unsupported",
    "the canonical sanitized reason must be exposed on the returned result, not only in logs"
  );
});

test("#10460: account-scoped permission/entitlement 400 keeps rotating (not misclassified as provider-wide unsupported)", async () => {
  await resetStorage10460();
  const connIds: string[] = [];
  for (let i = 0; i < 3; i++) {
    connIds.push(await seedConn10460("github"));
  }
  let calls = 0;

  // Phrased so it matches the broader/ambiguous MODEL_ACCESS_DENIED_PATTERNS
  // ("permission" ... "model") but is NOT an unambiguous provider-wide "model not
  // supported" response — it reads as an account/key entitlement gap (e.g. this
  // key's plan doesn't include this model), where a DIFFERENT account of the same
  // provider may still have access. Rotation through all 3 accounts must continue,
  // unlike the unambiguous "model is not supported" case above.
  for (const connId of connIds) {
    calls += 1;
    const result = await auth.markAccountUnavailable(
      connId,
      400,
      "Your API key does not have permission to use model gpt-4o",
      "github",
      "gpt-4o"
    );
    if (!result.shouldFallback) break;
  }

  assert.equal(
    calls,
    3,
    "an account-scoped permission/entitlement 400 must keep rotating through all accounts, " +
      "not be short-circuited by the provider-wide model-unsupported guard"
  );
});

test("#10460: account-scoped 401 keeps rotating through all 3 accounts (not short-circuited)", async () => {
  await resetStorage10460();
  const connIds: string[] = [];
  for (let i = 0; i < 3; i++) {
    connIds.push(await seedConn10460("github"));
  }
  let calls = 0;

  for (const connId of connIds) {
    calls += 1;
    const result = await auth.markAccountUnavailable(
      connId,
      401,
      "Unauthorized: invalid credentials",
      "github",
      "claude-fable-5"
    );
    if (!result.shouldFallback) break;
  }

  assert.equal(
    calls,
    3,
    "401 account-scoped errors must keep rotating through every account, unlike model-unsupported 400"
  );
});

test("#10460: account-scoped 403 keeps rotating through all 3 accounts", async () => {
  await resetStorage10460();
  const connIds: string[] = [];
  for (let i = 0; i < 3; i++) {
    connIds.push(await seedConn10460("github"));
  }
  let calls = 0;

  for (const connId of connIds) {
    calls += 1;
    const result = await auth.markAccountUnavailable(
      connId,
      403,
      "Forbidden: access denied for this account",
      "github",
      "claude-fable-5"
    );
    if (!result.shouldFallback) break;
  }

  assert.equal(calls, 3, "403 account-scoped errors must keep rotating through every account");
});

test("#10460: 429 rate limit keeps rotating through all 3 accounts", async () => {
  await resetStorage10460();
  const connIds: string[] = [];
  for (let i = 0; i < 3; i++) {
    connIds.push(await seedConn10460("github"));
  }
  let calls = 0;

  for (const connId of connIds) {
    calls += 1;
    const result = await auth.markAccountUnavailable(
      connId,
      429,
      "Rate limit exceeded",
      "github",
      "claude-fable-5"
    );
    if (!result.shouldFallback) break;
  }

  assert.equal(calls, 3, "429 rate-limit errors must keep rotating through every account");
});

// ─── #10460 acceptance criteria: 3-account rotation + combo target advancement ─
//
// Reproduces the exact regression from issue #10460: a combo with a
// (github/model, 3 accounts) target followed by a sibling target. When GitHub
// returns an unambiguous "model not supported" 400, the account-rotation loop
// must make exactly ONE upstream call (not one per account) and the combo must
// advance to the next target — not just that markAccountUnavailable() in
// isolation returns shouldFallback:false (covered above), but that a realistic
// rotation loop wired to the REAL markAccountUnavailable()/
// isProviderModelUnsupported400() gating actually stops after account 1 and lets
// handleComboChat() move on.
//
// The inner loop below is a faithful, minimal reproduction of the account-rotation
// contract in src/sse/handlers/chat.ts::handleSingleModelChat step 8 ("Fallback to
// next account", ~line 1936): call markAccountUnavailable(); continue to the next
// connection only while shouldFallback is true, otherwise stop immediately and
// surface the failure. Reimplemented at this scope (rather than driving the full
// handleChat()/route stack) so the test can assert on upstream-call counts and
// per-connection DB state directly, while still exercising the real gating logic
// that decides whether rotation continues.
test("#10460 acceptance: unambiguous model-unsupported 400 makes exactly ONE upstream call across 3 accounts, then the combo advances to the next target", async () => {
  await resetStorage10460();
  const { handleComboChat } = await import("../../open-sse/services/combo.ts");

  const githubConnIds: string[] = [];
  for (let i = 0; i < 3; i++) {
    githubConnIds.push(await seedConn10460("github"));
  }

  let githubUpstreamCalls = 0;
  const triedGithubConnections: string[] = [];
  const noopLog = { info: () => {}, warn: () => {}, debug: () => {}, error: () => {} };

  const handleSingleModel = async (_body: unknown, modelStr: string) => {
    if (modelStr.startsWith("github/")) {
      for (const connId of githubConnIds) {
        triedGithubConnections.push(connId);
        githubUpstreamCalls += 1;
        const result = await auth.markAccountUnavailable(
          connId,
          400,
          "The requested model is not supported",
          "github",
          "claude-fable-5"
        );
        if (result.shouldFallback) continue;
        return new Response(
          JSON.stringify({ error: { message: "The requested model is not supported" } }),
          { status: 400, headers: { "Content-Type": "application/json" } }
        );
      }
      // A test-fixture bug (guard not firing) would otherwise silently exhaust
      // every account and mask the regression this test exists to catch.
      throw new Error("all 3 github accounts were tried — the guard did not fire");
    }
    // Second combo target (a different provider) — succeeds immediately.
    return new Response(JSON.stringify({ id: "ok", choices: [] }), {
      status: 200,
      headers: { "Content-Type": "application/json" },
    });
  };

  const result = await handleComboChat({
    body: { model: "test", messages: [{ role: "user", content: "hi" }] },
    combo: {
      name: "test-combo-10460",
      strategy: "priority",
      models: [{ model: "github/claude-fable-5" }, { model: "openai/gpt-4o-mini" }],
    },
    handleSingleModel,
    log: noopLog,
    settings: {},
    allCombos: [],
  });

  assert.equal(
    githubUpstreamCalls,
    1,
    `expected exactly ONE upstream call for github/claude-fable-5 across 3 accounts, got ` +
      `${githubUpstreamCalls} (tried: ${triedGithubConnections.join(", ")})`
  );
  assert.equal(triedGithubConnections.length, 1, "only the first account should have been tried");
  assert.equal(result.status, 200, "the combo must advance to the next target and succeed");

  // The two untouched accounts must remain completely unaffected — proves the
  // guard did not just avoid an upstream call but also never cooled them down,
  // so they stay immediately eligible for the next unrelated request.
  for (const connId of githubConnIds.slice(1)) {
    const conn = await providersDb.getProviderConnectionById(connId);
    assert.ok(!conn.rateLimitedUntil, `untried account ${connId} must not be rate-limited`);
    assert.notStrictEqual(
      conn.testStatus,
      "unavailable",
      `untried account ${connId} must stay active`
    );
  }
});