OmniRoute/tests/unit/t23-t24-fallback-resilience.test.ts

import test from "node:test";
import assert from "node:assert/strict";

const { checkFallbackError } = await import("../../open-sse/services/accountFallback.ts");
const { handleComboChat } = await import("../../open-sse/services/combo.ts");
const { resetAllCircuitBreakers } = await import("../../src/shared/utils/circuitBreaker.ts");

test.beforeEach(() => {
  resetAllCircuitBreakers();
});

function createLog() {
  const entries = [];
  return {
    info: (tag: string, msg: string) => entries.push({ level: "info", tag, msg }),
    warn: (tag: string, msg: string) => entries.push({ level: "warn", tag, msg }),
    error: (tag: string, msg: string) => entries.push({ level: "error", tag, msg }),
    debug: (tag: string, msg: string) => entries.push({ level: "debug", tag, msg }),
    entries,
  };
}

function createStatusSequenceHandler(sequence) {
  let idx = 0;
  return async () => {
    const step = sequence[idx++] || { status: 200 };
    if (step.status === 200) {
      return new Response(JSON.stringify({ ok: true }), { status: 200 });
    }
    return new Response(
      JSON.stringify({
        error: { message: step.message || `Error ${step.status}` },
      }),
      {
        status: step.status,
        headers: step.headers || { "content-type": "application/json" },
      }
    );
  };
}

test("T23: 429 with long Retry-After uses real reset cooldown instead of short exponential backoff", () => {
  const headers = new Headers({ "retry-after": "3600" });
  const result = checkFallbackError(429, "Rate limit exceeded", 2, null, "groq", headers);

  assert.equal(result.shouldFallback, true);
  assert.equal(result.reason, "rate_limit_exceeded");
  assert.equal(result.newBackoffLevel, 0);
  assert.ok(result.cooldownMs > 3_590_000);
});

test("T24: combo awaits short 503 cooldown before falling through to next model", async () => {
  const log = createLog();

  const result = await handleComboChat({
    body: {},
    combo: {
      name: "t24-short-cooldown",
      strategy: "priority",
      // Cross-provider targets: a 503 marks the failing provider's remaining same-provider
      // targets for skip (#1731v2), so the fallthrough target must be a DIFFERENT provider
      // for this cooldown-wait test to exercise the fall-through-to-next-model path.
      models: [
        { model: "groq/model-a", weight: 0 },
        { model: "openai/model-b", weight: 0 },
      ],
      config: { fallbackDelayMs: 2000, maxRetries: 1 },
    },
    // Two transient failures on first model, then success on fallback model.
    handleSingleModel: createStatusSequenceHandler([
      { status: 503 },
      { status: 503 },
      { status: 200 },
    ]),
    isModelAvailable: () => true,
    log,
    settings: null,
    allCombos: null,
  });

  assert.equal(result.ok, true);
  // checkFallbackError returns COOLDOWN_MS.transient (5000ms) for a plain 503.
  // fallbackDelayMs=2000, cooldownMs=5000 ≤ MAX_FALLBACK_WAIT_MS(5000) → fallbackWaitMs=2000ms.
  // The combo MUST emit a debug log before waiting, proving the wait behavior is wired.
  const waitLog = log.entries.find((e) => e.msg.includes("Waiting") && e.msg.includes("fallback"));
  assert.ok(waitLog, "combo must emit a debug wait-before-fallback log for short 503 cooldowns");
});

test("T24: combo skips wait when 503 cooldown is long (>5s)", async () => {
  const log = createLog();

  const result = await handleComboChat({
    body: {},
    combo: {
      name: "t24-long-cooldown",
      strategy: "priority",
      // Cross-provider targets (see t24-short-cooldown): the fall-through target must be a
      // different provider so the #1731v2 same-provider skip doesn't short-circuit it.
      models: [
        { model: "groq/model-a", weight: 0 },
        { model: "openai/model-b", weight: 0 },
      ],
      config: { fallbackDelayMs: 2000, maxRetries: 1 },
    },
    handleSingleModel: createStatusSequenceHandler([
      {
        status: 503,
        message: "rate limit exceeded",
        headers: { "content-type": "application/json", "retry-after": "120" },
      },
      {
        status: 503,
        message: "rate limit exceeded",
        headers: { "content-type": "application/json", "retry-after": "120" },
      },
      { status: 200 },
    ]),
    isModelAvailable: () => true,
    log,
    settings: null,
    allCombos: null,
  });

  assert.equal(result.ok, true);
  const waitLog = log.entries.find((e) => e.msg.includes("Waiting") && e.msg.includes("fallback"));
  assert.equal(waitLog, undefined);
});

test("T24: all inactive accounts return 503 service_unavailable (not 406)", async () => {
  const result = await handleComboChat({
    body: {},
    combo: {
      name: "t24-all-inactive",
      strategy: "priority",
      models: [
        { model: "groq/model-a", weight: 0 },
        { model: "groq/model-b", weight: 0 },
      ],
    },
    handleSingleModel: async () => {
      throw new Error("handleSingleModel should not be called when all models are unavailable");
    },
    isModelAvailable: () => false,
    log: createLog(),
    settings: null,
    allCombos: null,
  });

  assert.equal(result.status, 503);
  const body = (await result.json()) as any;
  assert.equal(body.error?.code, "ALL_TARGETS_SKIPPED");
});

test("combo falls through 400s and reaches the next model", async () => {
  const calls = [];
  const sequence = [
    { status: 429, message: "No capacity available for model gemini-3.1-pro-preview" },
    { status: 400, message: "bad request" },
    { status: 200 },
  ];

  const result = await handleComboChat({
    body: {},
    combo: {
      name: "t24-provider-scoped-400",
      strategy: "priority",
      models: [
        { model: "free/gemini-3.1-pro-preview", weight: 0 },
        { model: "aio/gemini-3.1-pro-preview-thinking-high", weight: 0 },
        { model: "openrouter/google/gemini-3.1-pro-preview", weight: 0 },
      ],
      config: { maxRetries: 0 },
    },
    handleSingleModel: async (_body, modelStr) => {
      calls.push(modelStr);
      const step = sequence[calls.length - 1] || { status: 200 };
      if (step.status === 200) {
        return new Response(JSON.stringify({ ok: true }), { status: 200 });
      }
      return new Response(JSON.stringify({ error: { message: step.message } }), {
        status: step.status,
        headers: { "content-type": "application/json" },
      });
    },
    isModelAvailable: () => true,
    log: createLog(),
    settings: null,
    allCombos: null,
  });

  assert.equal(result.ok, true);
  assert.deepEqual(calls, [
    "free/gemini-3.1-pro-preview",
    "aio/gemini-3.1-pro-preview-thinking-high",
    "openrouter/google/gemini-3.1-pro-preview",
  ]);
});