OmniRoute/tests/unit/api/sync-models-readiness.test.ts

import { test, beforeEach, mock } from "node:test";
import assert from "node:assert/strict";
import {
  selfFetchWithRetry,
  ensureLoopbackServerReady,
  __resetLoopbackReadinessForTests,
} from "../../../src/app/api/providers/[id]/sync-models/route.ts";

// ---------------------------------------------------------------------------
// Test 1: retry succeeds on attempt 3
// ---------------------------------------------------------------------------
test("self-fetch retries with backoff and succeeds on attempt 3", async () => {
  let attempts = 0;
  const fetchMock: typeof fetch = async () => {
    attempts++;
    if (attempts < 3) {
      throw new Error("fetch failed");
    }
    return new Response(JSON.stringify({ models: [{ id: "model-1" }] }), { status: 200 });
  };

  let inProcCalls = 0;
  const inProcMock = async () => {
    inProcCalls++;
    return new Response(JSON.stringify({ models: [] }), { status: 200 });
  };

  const result = await selfFetchWithRetry("http://127.0.0.1:20128/api/providers/conn-1/models", {
    fetch: fetchMock,
    maxRetries: 5,
    backoffMs: 5,
    inProcessFallback: inProcMock,
    skipReadinessGate: true,
  });

  assert.equal(attempts, 3, "should have retried twice before succeeding on attempt 3");
  assert.equal(inProcCalls, 0, "should not have called in-process route");
  assert.equal(result.ok, true, "response should be ok");
});

// ---------------------------------------------------------------------------
// Test 2: falls back to in-process after maxRetries failures
// ---------------------------------------------------------------------------
test("self-fetch falls back to in-process route after maxRetries failures", async () => {
  let attempts = 0;
  const fetchMock: typeof fetch = async () => {
    attempts++;
    throw new Error("fetch failed");
  };

  let inProcCalls = 0;
  const inProcMock = async () => {
    inProcCalls++;
    return new Response(JSON.stringify({ models: [{ id: "in-proc-model" }] }), { status: 200 });
  };

  const result = await selfFetchWithRetry("http://127.0.0.1:20128/api/providers/conn-2/models", {
    fetch: fetchMock,
    maxRetries: 3,
    backoffMs: 5,
    connectionId: "conn-2",
    inProcessFallback: inProcMock,
    skipReadinessGate: true,
  });

  assert.equal(attempts, 3, "should retry exactly maxRetries times");
  assert.equal(inProcCalls, 1, "should fall back to in-process exactly once");
  const body = await result.json();
  assert.equal(body.models[0].id, "in-proc-model");
});

// ---------------------------------------------------------------------------
// Test 3: HTTP error responses are returned as-is (no retry on HTTP errors)
//
// Retry contract: only network-level failures (ECONNREFUSED, "fetch failed")
// are retried. HTTP responses (even 4xx/5xx) mean the server IS up and
// returned an error that should be propagated as-is to the caller.
// ---------------------------------------------------------------------------
test("self-fetch returns HTTP error responses immediately without retrying", async () => {
  // 5xx HTTP response: server is up but returned error
  {
    let attempts = 0;
    const fetchMock: typeof fetch = async () => {
      attempts++;
      return new Response("server error", { status: 503 });
    };
    const inProcMock = async () => new Response(JSON.stringify({ models: [] }), { status: 200 });

    const res = await selfFetchWithRetry("http://127.0.0.1:20128/api/providers/conn-3/models", {
      fetch: fetchMock,
      maxRetries: 5,
      backoffMs: 5,
      inProcessFallback: inProcMock,
      skipReadinessGate: true,
    });

    assert.equal(attempts, 1, "5xx HTTP response should NOT retry (got " + attempts + ")");
    assert.equal(res.status, 503, "should propagate the 503 response as-is");
  }

  // 4xx HTTP response: also returned immediately without retry
  {
    let attempts = 0;
    const fetchMock: typeof fetch = async () => {
      attempts++;
      return new Response("not found", { status: 404 });
    };
    const inProcMock = async () => new Response(JSON.stringify({ models: [] }), { status: 200 });

    const res = await selfFetchWithRetry("http://127.0.0.1:20128/api/providers/conn-4/models", {
      fetch: fetchMock,
      maxRetries: 5,
      backoffMs: 5,
      inProcessFallback: inProcMock,
      skipReadinessGate: true,
    });

    assert.equal(attempts, 1, "4xx HTTP response should NOT retry (got " + attempts + ")");
    assert.equal(res.status, 404, "should propagate the 404 response as-is");
  }
});

// ---------------------------------------------------------------------------
// Readiness gate tests
// ---------------------------------------------------------------------------

test("ensureLoopbackServerReady: 17 concurrent callers trigger exactly ONE probe sequence", async () => {
  __resetLoopbackReadinessForTests();
  let probeCalls = 0;
  let serverIsUp = false;
  // Server becomes ready after 30ms
  setTimeout(() => {
    serverIsUp = true;
  }, 30);
  const mockFetch = async (_url) => {
    probeCalls++;
    if (!serverIsUp) throw new Error("fetch failed");
    return new Response("", { status: 200 });
  };

  await Promise.all(
    Array.from({ length: 17 }, () =>
      ensureLoopbackServerReady({ fetch: mockFetch, pollMs: 5, maxWaitMs: 1000 })
    )
  );

  // Probe may have polled multiple times before server came up -- that is fine.
  // What matters: 17 concurrent callers share ONE probe sequence.
  // Expected: roughly (30ms / 5ms) = ~6 attempts, not 17 x 6 = 102.
  assert.ok(probeCalls <= 15, "single shared probe expected <=15 attempts, got " + probeCalls);
});

test("ensureLoopbackServerReady: rejects after maxWaitMs with consistent network errors", async () => {
  __resetLoopbackReadinessForTests();
  const mockFetch = async (_url) => {
    throw new Error("ECONNREFUSED");
  };
  await assert.rejects(
    () =>
      ensureLoopbackServerReady({
        fetch: mockFetch,
        maxWaitMs: 50,
        pollMs: 10,
      }),
    /loopback server not ready/
  );
});

test("ensureLoopbackServerReady: resolves on 4xx (any HTTP status confirms server is up)", async () => {
  __resetLoopbackReadinessForTests();
  let calls = 0;
  const mockFetch = async (_url) => {
    calls++;
    return new Response("not found", { status: 404 });
  };
  // Should not throw: 404 means the server is dispatching
  await ensureLoopbackServerReady({ fetch: mockFetch, maxWaitMs: 500, pollMs: 10 });
  assert.equal(calls, 1, "resolved after exactly 1 probe (server immediately responded 404)");
});

test("selfFetchWithRetry with gate: 17 concurrent callers produce one probe + one fetch each", async () => {
  __resetLoopbackReadinessForTests();
  let probeCalls = 0;
  let modelFetchCalls = 0;
  let serverIsUp = false;
  setTimeout(() => {
    serverIsUp = true;
  }, 30);

  const mockFetch = async (url) => {
    const isReadinessProbe = url.includes("__readiness_probe__");
    if (isReadinessProbe) {
      probeCalls++;
      if (!serverIsUp) throw new Error("fetch failed");
      return new Response("", { status: 404 });
    }
    modelFetchCalls++;
    if (!serverIsUp) throw new Error("fetch failed");
    return new Response(JSON.stringify({ models: [{ id: "model-x" }] }), { status: 200 });
  };

  await Promise.all(
    Array.from({ length: 17 }, (_, i) =>
      selfFetchWithRetry("http://127.0.0.1:20128/api/providers/conn-" + i + "/models", {
        fetch: mockFetch,
        maxRetries: 3,
        backoffMs: 5,
      })
    )
  );

  // After readiness gate succeeds, each connection makes EXACTLY one model fetch
  assert.equal(modelFetchCalls, 17, "each connection fetches its own models exactly once");
  // Probe may have polled several times but NOT 17 x poll-count
  assert.ok(probeCalls <= 15, "probe should be shared, got " + probeCalls + " attempts");
});

// Sanity check: disabling the gate shows amplification (verifies the gate is doing work)
test("sanity: without readiness gate, 17 callers retry independently (amplification confirmed)", async () => {
  __resetLoopbackReadinessForTests();
  let modelFetchCalls = 0;
  let serverIsUp = false;
  setTimeout(() => {
    serverIsUp = true;
  }, 30);

  const mockFetch = async (_url) => {
    modelFetchCalls++;
    if (!serverIsUp) throw new Error("fetch failed");
    return new Response(JSON.stringify({ models: [{ id: "model-x" }] }), { status: 200 });
  };

  await Promise.all(
    Array.from({ length: 17 }, (_, i) =>
      selfFetchWithRetry("http://127.0.0.1:20128/api/providers/conn-" + i + "/models", {
        fetch: mockFetch,
        maxRetries: 5,
        backoffMs: 5,
        skipReadinessGate: true,
      })
    )
  );

  // Without gate: each of the 17 callers retries independently during boot race.
  // Expect well above 17 total fetch attempts.
  assert.ok(
    modelFetchCalls > 17,
    "without gate, callers retry independently, got " + modelFetchCalls + " (expected >17)"
  );
});

// ---------------------------------------------------------------------------
// Failure-memory tests: a rejected probe must not condemn the process
// ---------------------------------------------------------------------------

test("ensureLoopbackServerReady: a failed probe is re-attempted after the retry window (no permanent failure memory)", async () => {
  __resetLoopbackReadinessForTests();
  let probeCalls = 0;
  let serverIsUp = false;
  const mockFetch = async (_url) => {
    probeCalls++;
    if (!serverIsUp) throw new Error("ECONNREFUSED");
    return new Response("", { status: 200 });
  };

  // First probe: server down -> rejection after maxWaitMs (no prior failure,
  // so the default 30s retry window is inactive)
  await assert.rejects(
    () =>
      ensureLoopbackServerReady({
        fetch: mockFetch,
        maxWaitMs: 50,
        pollMs: 10,
      }),
    /loopback server not ready/
  );
  const callsAfterFirstFailure = probeCalls;

  // Server comes up shortly after the first failure
  setTimeout(() => {
    serverIsUp = true;
  }, 60);

  // Inside the retry window: must reject WITHOUT re-probing (storm bound)
  await assert.rejects(
    () =>
      ensureLoopbackServerReady({
        fetch: mockFetch,
        maxWaitMs: 50,
        pollMs: 10,
        minRetryIntervalMs: 500,
      }),
    /loopback server not ready/
  );
  assert.equal(
    probeCalls,
    callsAfterFirstFailure,
    "callers inside the retry window must not re-probe"
  );

  // After the window: a fresh probe runs and succeeds (server is up)
  await new Promise((r) => setTimeout(r, 15));
  await ensureLoopbackServerReady({
    fetch: mockFetch,
    maxWaitMs: 500,
    pollMs: 10,
    minRetryIntervalMs: 5,
  });
  assert.ok(
    probeCalls > callsAfterFirstFailure,
    "a later caller must re-probe after the retry window"
  );
});

test("ensureLoopbackServerReady: 54 concurrent callers inside the retry window share one rejection (no probe storm)", async () => {
  __resetLoopbackReadinessForTests();
  let probeCalls = 0;
  const mockFetch = async (_url) => {
    probeCalls++;
    throw new Error("ECONNREFUSED");
  };

  await assert.rejects(
    () =>
      ensureLoopbackServerReady({
        fetch: mockFetch,
        maxWaitMs: 50,
        pollMs: 10,
        minRetryIntervalMs: 500,
      }),
    /loopback server not ready/
  );
  const afterFirst = probeCalls;

  const results = await Promise.allSettled(
    Array.from({ length: 54 }, () =>
      ensureLoopbackServerReady({
        fetch: mockFetch,
        maxWaitMs: 50,
        pollMs: 10,
        minRetryIntervalMs: 500,
      })
    )
  );

  assert.equal(probeCalls, afterFirst, "54 callers inside the window must not launch 54 probes");
  assert.ok(
    results.every((r) => r.status === "rejected"),
    "all burst callers reject immediately"
  );
});

test("ensureLoopbackServerReady: readiness failure is logged once per probe, not once per caller", async () => {
  __resetLoopbackReadinessForTests();
  const warns: string[] = [];
  const warnMock = mock.method(console, "warn", (...args: unknown[]) => {
    warns.push(args.map(String).join(" "));
  });
  try {
    const mockFetch = async (_url) => {
      throw new Error("ECONNREFUSED");
    };
    await Promise.allSettled(
      Array.from({ length: 17 }, () =>
        ensureLoopbackServerReady({
          fetch: mockFetch,
          maxWaitMs: 50,
          pollMs: 10,
          minRetryIntervalMs: 500,
        })
      )
    );
  } finally {
    warnMock.mock.restore();
  }
  const readinessWarns = warns.filter((w) => w.includes("readiness probe failed"));
  assert.equal(
    readinessWarns.length,
    1,
    `expected exactly 1 readiness warn, got ${readinessWarns.length}`
  );
});