OmniRoute/tests/unit/llm-selector-custom-vision-models.test.ts

/**
 * Custom vision models in the LLM selector / `/v1/models` catalog
 * (port of upstream decolua/9router 5e5e78d3 — "fix: show custom vision
 * models in LLM selector and model list").
 *
 * Upstream stored an `imageToText` kind on user-added custom models and the
 * selector hid them from the default LLM list. OmniRoute already keeps custom
 * chat models in the LLM list, but the equivalent gap is the `supportsVision`
 * flag on a custom model: the catalog used to only set `capabilities.vision`
 * when the model id matched the conservative `isVisionModelId` heuristic
 * (`pixtral`, `llava`, `qwen-vl`, …). A user who registered a custom chat
 * model called e.g. `my-vision-llm` and explicitly checked "vision-capable"
 * still saw no `capabilities.vision` in `/v1/models`, so the LLM selector and
 * downstream routing treated it as text-only.
 *
 * This file pins the OmniRoute-flavoured behaviour:
 *   • a custom chat model with `supportsVision: true` MUST surface
 *     `capabilities: { vision: true }` via the catalog helper, regardless of
 *     whether the model id matches the static vision heuristic.
 *   • the existing id-based heuristic (`isVisionModelId`) still works.
 *   • when neither signal is present, no vision fields are emitted (no false
 *     positives — same conservative guard #4071 / #4072).
 */
import test from "node:test";
import assert from "node:assert/strict";

const catalog = await import("../../src/app/api/v1/models/catalog.ts");

type VisionFields = { capabilities: { vision: true } } | null;
type Helper = (entry: unknown, ...candidateIds: Array<string | null | undefined>) => VisionFields;

const getCustomVisionCapabilityFields = (
  catalog as unknown as { getCustomVisionCapabilityFields?: Helper }
).getCustomVisionCapabilityFields;

test("custom vision helper is exported from the catalog module", () => {
  assert.equal(
    typeof getCustomVisionCapabilityFields,
    "function",
    "expected getCustomVisionCapabilityFields to be exported from src/app/api/v1/models/catalog.ts"
  );
});

test("supportsVision flag on a custom model surfaces capabilities.vision (LLM selector unblock)", () => {
  const helper = getCustomVisionCapabilityFields as Helper;
  const entry = { id: "my-vision-llm", supportsVision: true };
  const fields = helper(entry, "openai-compat/my-vision-llm", "my-vision-llm");
  assert.ok(fields, "supportsVision=true must yield vision fields");
  assert.equal(fields?.capabilities.vision, true);
});

test("id-based isVisionModelId heuristic still works when supportsVision is unset", () => {
  const helper = getCustomVisionCapabilityFields as Helper;
  const entry = { id: "pixtral-12b" };
  const fields = helper(entry, "openai-compat/pixtral-12b", "pixtral-12b");
  assert.ok(fields, "well-known vision model id must yield vision fields");
  assert.equal(fields?.capabilities.vision, true);
});

test("no signal → no vision fields (avoids #4071-style false positives)", () => {
  const helper = getCustomVisionCapabilityFields as Helper;
  const entry = { id: "plain-text-llm" };
  const fields = helper(entry, "openai-compat/plain-text-llm", "plain-text-llm");
  assert.equal(fields, null);
});

test("explicit supportsVision=false is respected (text-only override)", () => {
  const helper = getCustomVisionCapabilityFields as Helper;
  // Even if the id matches the heuristic, an explicit false flag wins so the
  // user can downgrade a mis-classified model.
  const entry = { id: "pixtral-12b", supportsVision: false };
  const fields = helper(entry, "alias/pixtral-12b", "pixtral-12b");
  assert.equal(fields, null);
});

// ─── catalog.ts:1613/1569 vision-branch narrowing (TS2367 fix) ──────────────
// The production expression `!modelType || modelType === "chat"` was reduced to
// `!modelType` because classifyModelSupportedEndpoints() can never return
// "chat". These asserts pin that invariant so a future union widening forces a
// revisit of the call sites instead of silently changing /v1/models output.

test("classifyModelSupportedEndpoints never yields type 'chat'", async () => {
  const { classifyModelSupportedEndpoints } =
    await import("../../src/shared/constants/modelSupportedEndpoints.ts");
  const endpointSets = [
    [],
    ["chat"],
    ["chat", "completions"],
    ["responses"],
    ["embeddings"],
    ["rerank"],
    ["images"],
    ["videos"],
    ["video"],
    ["audio-speech"],
    ["audio-transcriptions"],
    ["audio"],
    ["chat", "images"],
    ["fim"],
    ["totally-unknown-endpoint"],
  ];
  for (const endpoints of endpointSets) {
    const { type } = classifyModelSupportedEndpoints(endpoints);
    assert.notEqual(
      type,
      "chat" as never,
      `type must never be "chat" for [${endpoints.join(", ")}] — the catalog vision branch relies on !modelType covering chat-type models`
    );
  }
});

test("custom chat-type models keep their vision fields after the TS2367 narrowing", async () => {
  const { classifyModelSupportedEndpoints } =
    await import("../../src/shared/constants/modelSupportedEndpoints.ts");
  const helper = getCustomVisionCapabilityFields as Helper;
  // A user-registered custom model with no specialty endpoints classifies as
  // plain chat (modelType === undefined) and checked "vision-capable".
  const classification = classifyModelSupportedEndpoints(["chat"]);
  const modelType = classification.type; // undefined by the invariant above
  const fields = helper({ supportsVision: true }, "my-vision-llm");
  assert.ok(fields, "supportsVision:true custom model must surface vision fields");
  // The exact production expression after the narrowing:
  const visionFields = !modelType ? fields : null;
  assert.ok(visionFields, "chat-type (modelType=undefined) keeps vision fields");
  assert.equal(visionFields?.capabilities.vision, true);
});