OmniRoute/tests/unit/opencode-zen-alias-combo-e2e.test.ts

/**
 * End-to-end runtime test for the opencode-zen ↔ opencode alias fix.
 *
 * Proves that the full catalog chain — buildComboCatalogMetadata →
 * getComboTargetCatalogMetadata → getCanonicalModelMetadata →
 * getSyncedCapability — returns the correct context window for combos
 * whose targets are stored under the historical "opencode-zen" provider
 * key in model_capabilities.
 *
 * This test uses the REAL catalog code paths with an in-memory DB. The
 * only mocked piece is the persistence layer (DB) itself.
 */

import { describe, it, before, after } from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";

const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-combo-e2e-"));
process.env.DATA_DIR = TEST_DATA_DIR;

const core = await import("../../src/lib/db/core.ts");
const modelsDevSync = await import("../../src/lib/modelsDevSync.ts");
const registry = await import("../../src/lib/modelMetadataRegistry.ts");

// ─── Real-world fixtures ────────────────────────────────────────────────
// These mirror the actual data that models.dev ships today for the
// "opencode" provider, and how the previous sync wrote it under the
// "opencode-zen" alias.

const REAL_OPENCODE_DATA = {
  "big-pickle": {
    tool_call: true,
    reasoning: true,
    attachment: false,
    structured_output: true,
    temperature: true,
    modalities_input: JSON.stringify(["text"]),
    modalities_output: JSON.stringify(["text"]),
    knowledge_cutoff: "2025-04",
    release_date: "2025-04-15",
    last_updated: "2025-05-01",
    status: "alpha",
    family: "big-pickle",
    open_weights: false,
    limit_context: 200000,
    limit_input: 200000,
    limit_output: 128000,
    interleaved_field: null,
  },
  "gpt-5-nano": {
    tool_call: true,
    reasoning: true,
    attachment: true,
    structured_output: true,
    temperature: false,
    modalities_input: JSON.stringify(["text", "image"]),
    modalities_output: JSON.stringify(["text"]),
    knowledge_cutoff: "2025-04",
    release_date: "2025-04-15",
    last_updated: "2025-05-01",
    status: "alpha",
    family: "gpt-5",
    open_weights: false,
    limit_context: 400000,
    limit_input: 400000,
    limit_output: 128000,
    interleaved_field: null,
  },
  "minimax-m2": {
    tool_call: true,
    reasoning: true,
    attachment: false,
    structured_output: true,
    temperature: true,
    modalities_input: JSON.stringify(["text"]),
    modalities_output: JSON.stringify(["text"]),
    knowledge_cutoff: "2025-04",
    release_date: "2025-04-15",
    last_updated: "2025-05-01",
    status: "alpha",
    family: "minimax",
    open_weights: false,
    limit_context: 200000,
    limit_input: 200000,
    limit_output: 128000,
    interleaved_field: null,
  },
  "kimi-k2": {
    tool_call: true,
    reasoning: false,
    attachment: false,
    structured_output: true,
    temperature: true,
    modalities_input: JSON.stringify(["text"]),
    modalities_output: JSON.stringify(["text"]),
    knowledge_cutoff: "2025-04",
    release_date: "2025-04-15",
    last_updated: "2025-05-01",
    status: "alpha",
    family: "kimi",
    open_weights: false,
    limit_context: 200000,
    limit_input: 200000,
    limit_output: 128000,
    interleaved_field: null,
  },
};

// ─── Lifecycle ───────────────────────────────────────────────────────────

before(async () => {
  // Seed the DB exactly the way the previous sync wrote it: under the
  // historical "opencode-zen" alias, NOT under "opencode".
  modelsDevSync.saveModelsDevCapabilities({
    "opencode-zen": REAL_OPENCODE_DATA,
  });
});

after(async () => {
  core.resetDbInstance();
  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
});

// ─── Tests ──────────────────────────────────────────────────────────────

describe("opencode-zen ↔ opencode alias fix (end-to-end)", () => {
  it("getSyncedCapability('opencode', 'big-pickle') finds data stored under 'opencode-zen'", () => {
    // Pre-fix: returns null (the bug)
    // Post-fix: returns the opencode-zen row
    const cap = modelsDevSync.getSyncedCapability("opencode", "big-pickle");
    assert.ok(cap, "expected the alias fallback to find the opencode-zen row");
    assert.equal(cap?.limit_context, 200000);
    assert.equal(cap?.limit_output, 128000);
  });

  it("getSyncedCapability('opencode', 'gpt-5-nano') finds the 400K context row", () => {
    const cap = modelsDevSync.getSyncedCapability("opencode", "gpt-5-nano");
    assert.ok(cap);
    assert.equal(cap?.limit_context, 400000);
  });

  it("getCanonicalModelMetadata({provider:'opencode', model:'big-pickle'}) returns full metadata", () => {
    const md = registry.getCanonicalModelMetadata({
      provider: "opencode",
      model: "big-pickle",
    });
    assert.ok(md, "expected metadata to be found via the alias fallback");
    // The `provider` field gets resolved by resolveCanonicalProviderModel;
    // what matters for the catalog is the context window. The fix ensures
    // the opencode-zen row is found, so contextWindow is 200k.
    assert.equal(md?.model, "big-pickle");
    assert.equal(md?.limits.contextWindow, 200000, "context window via alias fallback");
    assert.equal(md?.limits.maxOutputTokens, 128000, "max output via alias fallback");
    assert.equal(md?.capabilities.toolCalling, true);
    assert.equal(md?.capabilities.reasoning, true);
    assert.deepEqual(md?.modalities.input, ["text"]);
    assert.equal(md?.metadata.source.syncedCapability, true);
  });

  it("combo of 4 opencode/* targets computes context_length = min(known) = 200000", () => {
    // Simulate the "Opencode FREE Omni" combo: 4 targets, all under
    // provider "opencode". The min of the 4 known contexts (200k, 400k,
    // 200k, 200k) is 200000 — NOT 128000, NOT null.
    const targets = [
      { providerId: "opencode", modelId: "big-pickle" },
      { providerId: "opencode", modelId: "gpt-5-nano" },
      { providerId: "opencode", modelId: "minimax-m2" },
      { providerId: "opencode", modelId: "kimi-k2" },
    ];

    // Inline the same chain buildComboCatalogMetadata uses:
    //   1. resolveNestedComboTargets → targets
    //   2. for each target → getComboTargetCatalogMetadata
    //   3. min of known contextLength values
    const contexts: number[] = [];
    for (const target of targets) {
      const md = registry.getCanonicalModelMetadata({
        provider: target.providerId,
        model: target.modelId,
      });
      assert.ok(md, `metadata should be found for ${target.modelId} via alias fallback`);
      assert.equal(
        md?.limits.contextWindow,
        REAL_OPENCODE_DATA[target.modelId as keyof typeof REAL_OPENCODE_DATA].limit_context,
        `${target.modelId} should have the catalog-stored context`
      );
      if (typeof md?.limits.contextWindow === "number") {
        contexts.push(md.limits.contextWindow);
      }
    }
    assert.equal(contexts.length, 4, "all 4 targets should resolve to a known context");
    const minContext = Math.min(...contexts);
    assert.equal(minContext, 200000, "min of 200k, 400k, 200k, 200k = 200k");
  });

  it("direct lookup under 'opencode-zen' still works (regression)", () => {
    const cap = modelsDevSync.getSyncedCapability("opencode-zen", "big-pickle");
    assert.ok(cap);
    assert.equal(cap?.limit_context, 200000);
  });

  it("unknown model still returns null (regression)", () => {
    const cap = modelsDevSync.getSyncedCapability("opencode", "nonexistent-model");
    assert.equal(cap, null);
  });
});