OmniRoute/tests/unit/model-capability-resolution-snapshot-9199.test.ts

/**
 * #9199 — build-local ModelCapabilityResolutionSnapshot parity.
 *
 * Snapshot-backed resolution must match ordinary on-demand resolvers for the
 * same pure chain (synced + max_token override + context override + alias
 * fallback + missing data). The snapshot must not flip models.dev's module-
 * global all-row cache.
 */
import test from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";

const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-cap-snapshot-9199-"));
process.env.DATA_DIR = TEST_DATA_DIR;
process.env.API_KEY_SECRET = process.env.API_KEY_SECRET ?? "cap-snapshot-9199-secret";
// Isolate getTokenLimit from host env CONTEXT_LENGTH_* overrides.
const originalContextLengthEnv = new Map(
  Object.entries(process.env).filter(([key]) => key.startsWith("CONTEXT_LENGTH_"))
);
for (const key of originalContextLengthEnv.keys()) delete process.env[key];

const core = await import("../../src/lib/db/core.ts");
const modelsDevSync = await import("../../src/lib/modelsDevSync.ts");
const capabilityOverrides = await import("../../src/lib/db/modelCapabilityOverrides.ts");
const contextOverrides = await import("../../src/lib/db/modelContextOverrides.ts");
const modelCapabilities = await import("../../src/lib/modelCapabilities.ts");
const contextManager = await import("../../open-sse/services/contextManager.ts");
const model = await import("../../open-sse/services/model.ts");
const { MODEL_SPECS } = await import("../../src/shared/constants/modelSpecs.ts");
const { REGISTRY } = await import("../../open-sse/config/providerRegistry.ts");

test.after(() => {
  core.resetDbInstance();
  for (const key of Object.keys(process.env)) {
    if (key.startsWith("CONTEXT_LENGTH_")) delete process.env[key];
  }
  for (const [key, value] of originalContextLengthEnv) process.env[key] = value;
  try {
    fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
  } catch {
    // best-effort cleanup
  }
});

function seedFixture() {
  core.resetDbInstance();
  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
  fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
  core.getDbInstance();

  modelsDevSync.saveModelsDevCapabilities({
    // Direct provider/model hit used by ordinary + snapshot paths.
    "parity-provider": {
      "parity-model": {
        tool_call: true,
        reasoning: true,
        attachment: false,
        structured_output: null,
        temperature: true,
        modalities_input: '["text"]',
        modalities_output: '["text"]',
        knowledge_cutoff: null,
        release_date: null,
        last_updated: null,
        status: null,
        family: null,
        open_weights: null,
        limit_context: 111111,
        limit_input: 100000,
        limit_output: 2222,
        interleaved_field: null,
      },
    },
    // Alias-side storage only — canonical `opencode` must still resolve via fallback.
    "opencode-zen": {
      "zen-only-model": {
        tool_call: true,
        reasoning: false,
        attachment: false,
        structured_output: null,
        temperature: true,
        modalities_input: '["text"]',
        modalities_output: '["text"]',
        knowledge_cutoff: null,
        release_date: null,
        last_updated: null,
        status: null,
        family: null,
        open_weights: null,
        limit_context: 333333,
        limit_input: 300000,
        limit_output: 4444,
        interleaved_field: null,
      },
    },
  });

  assert.equal(
    capabilityOverrides.setModelCapabilityOverride(
      "parity-provider/parity-model",
      "max_output_tokens",
      99999
    ),
    true
  );
  // Malformed capability override row must be filtered from list/bulk maps.
  core
    .getDbInstance()
    .prepare(
      "INSERT OR REPLACE INTO model_capability_overrides " +
        "(provider, model_id, override_key, override_value, refreshed_at) " +
        "VALUES (?, ?, ?, ?, datetime('now'))"
    )
    .run("parity-provider", "parity-model-bad", "max_output_tokens", "not-a-number");

  // Collision pairs that a delimiter-composed key would merge: (a, b\0c) vs (a\0b, c).
  assert.equal(
    capabilityOverrides.setModelCapabilityOverride("a/b\u0000c", "max_output_tokens", 10101),
    true
  );
  assert.equal(
    capabilityOverrides.setModelCapabilityOverride("a\u0000b/c", "max_output_tokens", 20202),
    true
  );
  assert.equal(contextOverrides.setModelContextOverride("a", "b\u0000c", 30303, "manual"), true);
  assert.equal(contextOverrides.setModelContextOverride("a\u0000b", "c", 40404, "manual"), true);

  // Real provider-model alias from open-sse/services/model.ts PROVIDER_MODEL_ALIASES.
  // Override is stored only under the raw alias id, not the canonical model.
  const aliasCanonical = model.resolveCanonicalProviderModel("github", "claude-4.5-opus");
  assert.equal(aliasCanonical.provider, "github");
  assert.equal(aliasCanonical.model, "claude-opus-4-5-20251101");
  assert.notEqual(aliasCanonical.model, "claude-4.5-opus");
  assert.equal(
    capabilityOverrides.setModelCapabilityOverride("github/claude-4.5-opus", "max_output_tokens", 77777),
    true
  );
  assert.equal(
    capabilityOverrides.getModelCapabilityOverride(
      "github",
      "claude-opus-4-5-20251101",
      "max_output_tokens"
    ),
    null,
    "override must exist only under the raw alias model id"
  );

  assert.equal(
    contextOverrides.setModelContextOverride("parity-provider", "parity-model", 555555, "manual"),
    true
  );
}

function insertRawCapability(provider: string, modelId: string): void {
  modelsDevSync.ensureCapabilitiesTable();
  core
    .getDbInstance()
    .prepare(
      "INSERT OR REPLACE INTO model_capabilities " +
        "(provider, model_id, tool_call, reasoning, attachment, structured_output, temperature, " +
        "modalities_input, modalities_output, knowledge_cutoff, release_date, last_updated, " +
        "status, family, open_weights, limit_context, limit_input, limit_output, interleaved_field, last_synced) " +
        "VALUES (?, ?, 1, 1, 0, NULL, 1, '[]', '[]', NULL, NULL, NULL, NULL, NULL, NULL, 111111, 100000, 2222, NULL, datetime('now'))"
    )
    .run(provider, modelId);
}

function pickParityFields(
  resolved: ReturnType<typeof modelCapabilities.getResolvedModelCapabilities>
) {
  return {
    provider: resolved.provider,
    model: resolved.model,
    rawModel: resolved.rawModel,
    toolCalling: resolved.toolCalling,
    reasoning: resolved.reasoning,
    supportsThinking: resolved.supportsThinking,
    supportsTools: resolved.supportsTools,
    supportsVision: resolved.supportsVision,
    contextWindow: resolved.contextWindow,
    maxInputTokens: resolved.maxInputTokens,
    maxOutputTokens: resolved.maxOutputTokens,
    attachment: resolved.attachment,
    modalitiesInput: resolved.modalitiesInput,
    modalitiesOutput: resolved.modalitiesOutput,
  };
}

function assertOrdinarySnapshotParity(
  provider: string,
  modelId: string,
  snapshot: modelCapabilities.ModelCapabilityResolutionSnapshot,
  label: string
) {
  const ordinaryCaps = modelCapabilities.getResolvedModelCapabilities({
    provider,
    model: modelId,
  });
  const snapshotCaps = modelCapabilities.getResolvedModelCapabilities(
    { provider, model: modelId },
    snapshot
  );
  assert.deepEqual(
    pickParityFields(snapshotCaps),
    pickParityFields(ordinaryCaps),
    `${label}: getResolvedModelCapabilities parity`
  );

  const ordinaryLimit = contextManager.getTokenLimit(provider, modelId);
  const snapshotLimit = contextManager.getTokenLimit(provider, modelId, snapshot);
  assert.equal(
    snapshotLimit,
    ordinaryLimit,
    `${label}: getTokenLimit parity (got ${snapshotLimit}, want ${ordinaryLimit})`
  );

  const ordinaryContext = modelCapabilities.getModelContextLimit(provider, modelId);
  const snapshotContext = modelCapabilities.getModelContextLimit(provider, modelId, snapshot);
  assert.equal(snapshotContext, ordinaryContext, `${label}: getModelContextLimit parity`);
}

test("#9199 bulk capability rows treat prototype-shaped keys as data", () => {
  core.resetDbInstance();
  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
  fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
  core.getDbInstance();

  insertRawCapability("__proto__", "polluted-model");
  insertRawCapability("prototype-provider", "__proto__");
  assert.equal(Object.hasOwn(Object.prototype, "polluted-model"), false);

  const ordinary = modelsDevSync.getSyncedCapability("__proto__", "polluted-model");
  assert.equal(ordinary?.limit_context, 111111, "ordinary point lookup must support the row");
  assert.equal(Object.hasOwn(Object.prototype, "polluted-model"), false);

  try {
    const bulk = modelsDevSync.loadAllSyncedCapabilitiesUncached();
    assert.equal(Object.hasOwn(bulk, "__proto__"), true);
    assert.equal(bulk["__proto__"]?.["polluted-model"]?.limit_output, 2222);
    assert.equal(Object.hasOwn(bulk["prototype-provider"], "__proto__"), true);
    assert.equal(bulk["prototype-provider"]?.["__proto__"]?.limit_context, 111111);
    assert.equal(Object.hasOwn(Object.prototype, "polluted-model"), false);
  } finally {
    delete (Object.prototype as Record<string, unknown>)["polluted-model"];
  }
});

test("#9199 uncached bulk load does not mutate models.dev all-row cache", () => {
  seedFixture();
  // clearModelsDevCapabilities leaves cachedCapabilitiesLoadedAll=true with {}.
  modelsDevSync.clearModelsDevCapabilities();
  assert.equal(modelsDevSync.getSyncedCapability("parity-provider", "parity-model"), null);

  // Insert behind the module-global cache via raw SQL.
  const db = core.getDbInstance();
  insertRawCapability("parity-provider", "parity-model");

  // Ordinary path still sees the empty module cache.
  assert.equal(modelsDevSync.getSyncedCapability("parity-provider", "parity-model"), null);

  const bulk = modelsDevSync.loadAllSyncedCapabilitiesUncached();
  assert.equal(bulk["parity-provider"]?.["parity-model"]?.limit_context, 111111);

  // Uncached bulk must not publish into the module-global cache.
  assert.equal(
    modelsDevSync.getSyncedCapability("parity-provider", "parity-model"),
    null,
    "loadAllSyncedCapabilitiesUncached must not flip cachedCapabilitiesLoadedAll data"
  );

  const originalPrepare = db.prepare;
  const callPrepare = originalPrepare.bind(db);
  const counts = { synced: 0, capabilityOverrides: 0, contextOverrides: 0 };
  (db as unknown as { prepare: typeof db.prepare }).prepare = ((sql: string) => {
    const normalized = String(sql).replace(/\s+/g, " ").trim();
    if (normalized.includes("FROM model_capabilities")) counts.synced++;
    if (normalized.includes("FROM model_capability_overrides")) counts.capabilityOverrides++;
    if (normalized.includes("FROM model_context_overrides")) counts.contextOverrides++;
    return callPrepare(sql);
  }) as typeof db.prepare;

  try {
    const snapshot = modelCapabilities.createModelCapabilityResolutionSnapshot();
    assert.equal(snapshot.synced["parity-provider"]?.["parity-model"]?.limit_output, 2222);
  } finally {
    (db as unknown as { prepare: typeof db.prepare }).prepare = originalPrepare;
  }

  assert.deepEqual(counts, { synced: 1, capabilityOverrides: 1, contextOverrides: 1 });
  assert.equal(
    modelsDevSync.getSyncedCapability("parity-provider", "parity-model"),
    null,
    "snapshot creation must remain build-local"
  );
});

test("#9199 nested override maps keep delimiter-colliding pairs distinct", () => {
  seedFixture();
  const snapshot = modelCapabilities.createModelCapabilityResolutionSnapshot();

  // Delimiter-composed keys would merge these pairs; nested maps must not.
  assert.equal(snapshot.maxTokenOverrides.get("a")?.get("b\u0000c"), 10101);
  assert.equal(snapshot.maxTokenOverrides.get("a\u0000b")?.get("c"), 20202);
  assert.notEqual(
    snapshot.maxTokenOverrides.get("a")?.get("b\u0000c"),
    snapshot.maxTokenOverrides.get("a\u0000b")?.get("c")
  );

  assert.equal(snapshot.contextOverrides.get("a")?.get("b\u0000c"), 30303);
  assert.equal(snapshot.contextOverrides.get("a\u0000b")?.get("c"), 40404);
  assert.notEqual(
    snapshot.contextOverrides.get("a")?.get("b\u0000c"),
    snapshot.contextOverrides.get("a\u0000b")?.get("c")
  );

  assert.equal(
    capabilityOverrides.getModelCapabilityOverride(
      "a",
      "b\u0000c",
      "max_output_tokens",
      snapshot.maxTokenOverrides
    ),
    10101
  );
  assert.equal(
    capabilityOverrides.getModelCapabilityOverride(
      "a\u0000b",
      "c",
      "max_output_tokens",
      snapshot.maxTokenOverrides
    ),
    20202
  );
  assert.equal(
    contextOverrides.getModelContextOverride("a", "b\u0000c", snapshot.contextOverrides),
    30303
  );
  assert.equal(
    contextOverrides.getModelContextOverride("a\u0000b", "c", snapshot.contextOverrides),
    40404
  );

  assertOrdinarySnapshotParity("a", "b\u0000c", snapshot, "collision pair (a, b\\0c)");
  assertOrdinarySnapshotParity("a\u0000b", "c", snapshot, "collision pair (a\\0b, c)");

  assert.equal(
    modelCapabilities.getResolvedModelCapabilities({ provider: "a", model: "b\u0000c" }, snapshot)
      .maxOutputTokens,
    10101
  );
  assert.equal(
    modelCapabilities.getResolvedModelCapabilities({ provider: "a\u0000b", model: "c" }, snapshot)
      .maxOutputTokens,
    20202
  );
  assert.equal(modelCapabilities.getModelContextLimit("a", "b\u0000c", snapshot), 30303);
  assert.equal(modelCapabilities.getModelContextLimit("a\u0000b", "c", snapshot), 40404);
});

test("#9199 snapshot-backed resolution matches ordinary resolvers across resolution paths", () => {
  seedFixture();
  const snapshot = modelCapabilities.createModelCapabilityResolutionSnapshot();

  // Static MODEL_SPECS pin (not seeded in models.dev / overrides).
  const staticSpec = MODEL_SPECS["gpt-4o-mini"];
  assert.ok(staticSpec, "gpt-4o-mini must remain a real MODEL_SPECS entry");
  assert.equal(typeof staticSpec.maxOutputTokens, "number");
  assert.equal(typeof staticSpec.contextWindow, "number");
  assert.equal(
    MODEL_SPECS["glm-5-turbo"],
    undefined,
    "glm-5-turbo must stay absent from MODEL_SPECS so its case exercises PROVIDER_MODELS"
  );

  // Provider registry defaultContextLength path: no per-model catalog hit.
  const defaultContextProvider = "gitlab-duo";
  const defaultContextModel = "no-catalog-model-9199";
  assert.equal(
    REGISTRY[defaultContextProvider]?.defaultContextLength,
    128000,
    "gitlab-duo must keep a real provider-registry defaultContextLength"
  );
  assert.equal(
    modelCapabilities.getModelContextLimit(defaultContextProvider, defaultContextModel),
    null,
    "default-context path requires no resolved per-model contextWindow"
  );

  // Model-name heuristic path: unknown provider with a claude-shaped model id.
  const heuristicProvider = "unknown-provider-9199";
  const heuristicModel = "custom-claude-lab";
  assert.equal(REGISTRY[heuristicProvider], undefined);
  assert.equal(
    modelCapabilities.getModelContextLimit(heuristicProvider, heuristicModel),
    null,
    "heuristic path requires no resolved per-model contextWindow"
  );

  const cases: Array<{ provider: string; model: string; label: string }> = [
    { provider: "parity-provider", model: "parity-model", label: "direct synced + both overrides" },
    { provider: "opencode", model: "zen-only-model", label: "alias fallback to opencode-zen" },
    {
      provider: "github",
      model: "claude-4.5-opus",
      label: "canonical model alias with rawModel-only max_token override",
    },
    {
      provider: "glm",
      model: "glm-5-turbo",
      label: "registry-backed model (PROVIDER_MODELS context/output limits)",
    },
    {
      provider: "missing-provider",
      model: "gpt-4o-mini",
      label: "static MODEL_SPECS-backed model",
    },
    {
      provider: defaultContextProvider,
      model: defaultContextModel,
      label: "provider registry defaultContextLength path",
    },
    {
      provider: heuristicProvider,
      model: heuristicModel,
      label: "model-name heuristic/default path",
    },
    { provider: "missing-provider", model: "missing-model", label: "missing synced + overrides" },
  ];

  for (const entry of cases) {
    assertOrdinarySnapshotParity(entry.provider, entry.model, snapshot, entry.label);
  }

  // Explicit override/precedence pins so the test is not only structural.
  assert.equal(
    modelCapabilities.getResolvedModelCapabilities(
      { provider: "parity-provider", model: "parity-model" },
      snapshot
    ).maxOutputTokens,
    99999,
    "max_token override must win over synced limit_output"
  );
  assert.equal(
    modelCapabilities.getModelContextLimit("parity-provider", "parity-model", snapshot),
    555555,
    "context override must win over synced limit_context / resolved contextWindow"
  );
  assert.equal(
    modelCapabilities.getResolvedModelCapabilities(
      { provider: "opencode", model: "zen-only-model" },
      snapshot
    ).contextWindow,
    333333,
    "canonical opencode must resolve capabilities stored under opencode-zen"
  );

  const aliasResolved = modelCapabilities.getResolvedModelCapabilities(
    { provider: "github", model: "claude-4.5-opus" },
    snapshot
  );
  assert.equal(aliasResolved.rawModel, "claude-4.5-opus");
  assert.equal(aliasResolved.model, "claude-opus-4-5-20251101");
  assert.equal(
    aliasResolved.maxOutputTokens,
    77777,
    "max_token override stored only under rawModel must still apply after alias resolution"
  );
  const canonicalOnly = modelCapabilities.getResolvedModelCapabilities(
    { provider: "github", model: "claude-opus-4-5-20251101" },
    snapshot
  );
  assert.equal(canonicalOnly.rawModel, "claude-opus-4-5-20251101");
  assert.equal(canonicalOnly.model, "claude-opus-4-5-20251101");
  assert.notEqual(
    canonicalOnly.maxOutputTokens,
    77777,
    "canonical id alone must not invent the rawModel-only max_token override"
  );

  // Registry-backed pins: glm-5-turbo is absent from MODEL_SPECS.
  const registryResolved = modelCapabilities.getResolvedModelCapabilities(
    { provider: "glm", model: "glm-5-turbo" },
    snapshot
  );
  assert.equal(registryResolved.contextWindow, 200000);
  assert.equal(registryResolved.maxOutputTokens, 131072);

  // Static MODEL_SPECS pin without a provider registry hit for this missing provider.
  assert.equal(
    modelCapabilities.getResolvedModelCapabilities(
      { provider: "missing-provider", model: "gpt-4o-mini" },
      snapshot
    ).maxOutputTokens,
    staticSpec.maxOutputTokens
  );
  assert.equal(
    modelCapabilities.getResolvedModelCapabilities(
      { provider: "missing-provider", model: "gpt-4o-mini" },
      snapshot
    ).contextWindow,
    staticSpec.contextWindow
  );

  // Provider defaultContextLength path (getTokenLimit step 3).
  assert.equal(
    contextManager.getTokenLimit(defaultContextProvider, defaultContextModel, snapshot),
    128000
  );

  // Model-name heuristic path (getTokenLimit step 4 → DEFAULT_LIMITS.claude).
  assert.equal(contextManager.getTokenLimit(heuristicProvider, heuristicModel, snapshot), 200000);

  // Missing path remains a pure default/fallback, not a catalog hit.
  assert.equal(
    modelCapabilities.getResolvedModelCapabilities(
      { provider: "missing-provider", model: "missing-model" },
      snapshot
    ).contextWindow,
    null
  );
  assert.equal(contextManager.getTokenLimit("missing-provider", "missing-model", snapshot), 128000);

  assert.equal(
    snapshot.maxTokenOverrides.get("parity-provider")?.has("parity-model-bad") ?? false,
    false,
    "malformed max_token rows must be filtered from the bulk map"
  );
});