OmniRoute/tests/unit/github-models-curated-catalog.test.ts

import assert from "node:assert/strict";
import test from "node:test";

import { FREE_MODEL_BUDGETS } from "../../open-sse/config/freeModelCatalog.data.ts";
import { getEmbeddingProvider } from "../../open-sse/config/embeddingRegistry.ts";
import { REGISTRY } from "../../open-sse/config/providerRegistry.ts";
import { deriveConfigFromRegistryModelsUrl } from "../../src/app/api/providers/[id]/models/discoveryConfig.ts";
import { getStaticModelsForProvider } from "../../src/lib/providers/staticModels.ts";

const EXPECTED_CHAT_IDS = [
  "cohere/cohere-command-a",
  "deepseek/deepseek-r1-0528",
  "deepseek/deepseek-v3-0324",
  "meta/llama-4-maverick-17b-128e-instruct-fp8",
  "meta/llama-3.3-70b-instruct",
  "meta/llama-4-scout-17b-16e-instruct",
  "microsoft/phi-4-multimodal-instruct",
  "microsoft/phi-4-reasoning",
  "mistral-ai/codestral-2501",
  "mistral-ai/mistral-medium-2505",
  "openai/gpt-4.1",
  "openai/gpt-4.1-mini",
  "openai/gpt-4o",
  "openai/gpt-4o-mini",
  "openai/gpt-5",
  "openai/gpt-5-chat",
  "openai/gpt-5-mini",
  "openai/o3",
  "openai/o4-mini",
] as const;

const EXPECTED_EMBEDDING_IDS = [
  "openai/text-embedding-3-large",
  "openai/text-embedding-3-small",
] as const;

test("github-models curates the exact chat roster and preserves live discovery", () => {
  const provider = REGISTRY["github-models"];
  const ids = provider.models.map((model) => model.id);
  assert.deepEqual(ids, EXPECTED_CHAT_IDS);
  assert.equal(provider.modelsUrl, "https://models.github.ai/catalog/models");
  assert.ok(!ids.includes("xai/grok-3"));
  assert.ok(!ids.includes("openai/o1"));
  assert.ok(!ids.includes("meta/meta-llama-3.1-405b-instruct"));
  assert.ok(!ids.includes("meta/llama-3.2-11b-vision-instruct"));
  assert.ok(!ids.includes("meta/llama-3.2-90b-vision-instruct"));

  const scout = provider.models.find((model) => model.id === "meta/llama-4-scout-17b-16e-instruct");
  assert.deepEqual(scout, {
    id: "meta/llama-4-scout-17b-16e-instruct",
    name: "Llama 4 Scout 17B 16E Instruct",
    contextLength: 10_000_000,
    maxInputTokens: 10_000_000,
    maxOutputTokens: 4_096,
    supportsVision: true,
    toolCalling: true,
  });

  const gpt5 = provider.models.find((model) => model.id === "openai/gpt-5");
  assert.equal(gpt5?.contextLength, 200_000);
  assert.equal(gpt5?.maxInputTokens, 200_000);
  assert.equal(gpt5?.maxOutputTokens, 100_000);
  assert.equal(gpt5?.supportsVision, true);
  assert.equal(gpt5?.supportsReasoning, true);
  assert.equal(gpt5?.toolCalling, true);
});

test("github-models registers embedding-only models outside the chat roster", () => {
  const chatIds = REGISTRY["github-models"].models.map((model) => model.id);
  const provider = getEmbeddingProvider("github-models");
  assert.ok(provider);
  assert.equal(provider.baseUrl, "https://models.github.ai/inference/embeddings");
  assert.equal(provider.authType, "apikey");
  assert.equal(provider.authHeader, "bearer");
  assert.deepEqual(
    provider.models.map((model) => model.id),
    EXPECTED_EMBEDDING_IDS
  );
  for (const id of EXPECTED_EMBEDDING_IDS) assert.ok(!chatIds.includes(id));

  const specialty = getStaticModelsForProvider("github-models") || [];
  assert.deepEqual(
    specialty.map((model) => ({
      id: model.id,
      apiFormat: model.apiFormat,
      supportedEndpoints: model.supportedEndpoints,
    })),
    EXPECTED_EMBEDDING_IDS.map((id) => ({
      id,
      apiFormat: "embeddings",
      supportedEndpoints: ["embeddings"],
    }))
  );
});

test("github-models free metadata matches the combined curated roster", () => {
  const freeIds = FREE_MODEL_BUDGETS.filter((entry) => entry.provider === "github-models").map(
    (entry) => entry.modelId
  );
  assert.deepEqual(freeIds, [...EXPECTED_CHAT_IDS, ...EXPECTED_EMBEDDING_IDS]);
  assert.equal(new Set(freeIds).size, 21);
});

test("github-models live discovery filters confirmed dead Llama 3.2 vision models", () => {
  const config = deriveConfigFromRegistryModelsUrl("github-models");
  assert.ok(config);
  assert.equal(config.url, "https://models.github.ai/catalog/models");
  assert.deepEqual(
    config.parseResponse([
      { id: "meta/llama-3.2-11b-vision-instruct", name: "dead-11b" },
      { id: "meta/llama-3.2-90b-vision-instruct", name: "dead-90b" },
      { id: "meta/llama-3.3-70b-instruct", name: "live" },
    ]),
    [{ id: "meta/llama-3.3-70b-instruct", name: "live" }]
  );
});