OmniRoute/tests/unit/model-intelligence-db.test.ts

/**
 * Unit tests for src/lib/db/modelIntelligence.ts
 *
 * Uses the project's own DB infrastructure (core.ts getDbInstance)
 * with a temp DATA_DIR. Follows the Node.js native test runner pattern.
 */

import { describe, it, beforeEach } from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";

const TEST_DATA_DIR = fs.mkdtempSync(
  path.join(os.tmpdir(), "omniroute-mi-test-"),
);
process.env.DATA_DIR = TEST_DATA_DIR;

const core = await import("../../src/lib/db/core.ts");
const mi = await import("../../src/lib/db/modelIntelligence.ts");

function resetStorage(): void {
  core.resetDbInstance();
  try {
    if (fs.existsSync(TEST_DATA_DIR)) {
      fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
    }
  } catch { /* EBUSY — ignore */ }
  fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
}

function insertEntry(
  model: string,
  source: string,
  category: string,
  score: number,
  opts: {
    eloRaw?: number | null;
    confidence?: string | null;
    expiresAt?: string | null;
  } = {},
): void {
  const db = core.getDbInstance();
  db.prepare(
    `INSERT OR REPLACE INTO model_intelligence
     (model, source, category, score, elo_raw, confidence, synced_at, expires_at)
     VALUES (?, ?, ?, ?, ?, ?, datetime('now'), ?)`,
  ).run(
    model,
    source,
    category,
    score,
    opts.eloRaw ?? null,
    opts.confidence ?? null,
    opts.expiresAt ?? null,
  );
}

// ─── Tests ───────────────────────────────────────────────

describe("upsertModelIntelligence", () => {
  beforeEach(() => { resetStorage(); });

  it("inserts a new entry", () => {
    mi.upsertModelIntelligence({
      model: "claude-sonnet",
      source: "arena_elo",
      category: "coding",
      score: 0.92,
      eloRaw: 1350,
      confidence: "high",
      expiresAt: null,
    });

    const entry = mi.getModelIntelligenceBySource("claude-sonnet", "arena_elo", "coding");
    assert.ok(entry, "Row should exist after upsert");
    assert.strictEqual(entry.model, "claude-sonnet");
    assert.strictEqual(entry.source, "arena_elo");
    assert.strictEqual(entry.category, "coding");
    assert.strictEqual(entry.score, 0.92);
    assert.strictEqual(entry.eloRaw, 1350);
    assert.strictEqual(entry.confidence, "high");
    assert.ok(entry.syncedAt, "synced_at should be auto-populated");
  });

  it("updates an existing entry (INSERT OR REPLACE)", () => {
    insertEntry("gpt-4o", "arena_elo", "coding", 0.85);

    mi.upsertModelIntelligence({
      model: "gpt-4o",
      source: "arena_elo",
      category: "coding",
      score: 0.90,
      eloRaw: 1400,
      confidence: "high",
      expiresAt: null,
    });

    const entry = mi.getModelIntelligenceBySource("gpt-4o", "arena_elo", "coding");
    assert.ok(entry);
    assert.strictEqual(entry.score, 0.90);
    assert.strictEqual(entry.eloRaw, 1400);
  });
});

describe("getModelIntelligence", () => {
  beforeEach(() => { resetStorage(); });

  it("returns user_override when all three sources exist (highest priority)", () => {
    insertEntry("claude-sonnet", "models_dev_tier", "coding", 0.75);
    insertEntry("claude-sonnet", "arena_elo", "coding", 0.88);
    insertEntry("claude-sonnet", "user_override", "coding", 0.99);

    const entry = mi.getModelIntelligence("claude-sonnet", "coding");
    assert.ok(entry);
    assert.strictEqual(entry.source, "user_override");
    assert.strictEqual(entry.score, 0.99);
  });

  it("returns arena_elo when no user_override exists", () => {
    insertEntry("gpt-4o", "arena_elo", "coding", 0.87);
    insertEntry("gpt-4o", "models_dev_tier", "coding", 0.70);

    const entry = mi.getModelIntelligence("gpt-4o", "coding");
    assert.ok(entry);
    assert.strictEqual(entry.source, "arena_elo");
    assert.strictEqual(entry.score, 0.87);
  });

  it("returns models_dev_tier when only that source exists", () => {
    insertEntry("llama-4", "models_dev_tier", "coding", 0.65);

    const entry = mi.getModelIntelligence("llama-4", "coding");
    assert.ok(entry);
    assert.strictEqual(entry.source, "models_dev_tier");
    assert.strictEqual(entry.score, 0.65);
  });

  it("returns null when no entry exists", () => {
    const entry = mi.getModelIntelligence("nonexistent-model", "coding");
    assert.strictEqual(entry, null);
  });

  it("skips expired entries and falls through to next source", () => {
    insertEntry("gemini-pro", "arena_elo", "coding", 0.82, {
      expiresAt: "2000-01-01T00:00:00Z",
    });
    insertEntry("gemini-pro", "models_dev_tier", "coding", 0.70);

    const entry = mi.getModelIntelligence("gemini-pro", "coding");
    assert.ok(entry);
    assert.strictEqual(entry.source, "models_dev_tier");
  });

  it("returns null when all entries for a model+category are expired", () => {
    insertEntry("expired-model", "arena_elo", "coding", 0.80, {
      expiresAt: "2000-01-01T00:00:00Z",
    });

    const entry = mi.getModelIntelligence("expired-model", "coding");
    assert.strictEqual(entry, null);
  });

  it("model names require exact match (case-sensitive in DB)", () => {
    insertEntry("Claude-Sonnet", "arena_elo", "coding", 0.90);

    const exact = mi.getModelIntelligence("Claude-Sonnet", "coding");
    assert.ok(exact);

    const different = mi.getModelIntelligence("claude-sonnet", "coding");
    assert.strictEqual(different, null);
  });
});

describe("getModelIntelligenceBySource", () => {
  beforeEach(() => { resetStorage(); });

  it("returns a specific source entry", () => {
    insertEntry("claude-sonnet", "arena_elo", "coding", 0.88);
    insertEntry("claude-sonnet", "user_override", "coding", 0.99);

    const entry = mi.getModelIntelligenceBySource("claude-sonnet", "arena_elo", "coding");
    assert.ok(entry);
    assert.strictEqual(entry.source, "arena_elo");
    assert.strictEqual(entry.score, 0.88);
  });

  it("returns null when the specific source does not exist", () => {
    insertEntry("claude-sonnet", "arena_elo", "coding", 0.88);

    const entry = mi.getModelIntelligenceBySource("claude-sonnet", "user_override", "coding");
    assert.strictEqual(entry, null);
  });

  it("returns null when no entries exist at all", () => {
    const entry = mi.getModelIntelligenceBySource("nonexistent", "arena_elo", "coding");
    assert.strictEqual(entry, null);
  });
});

describe("deleteModelIntelligence", () => {
  beforeEach(() => { resetStorage(); });

  it("deletes an entry and returns true", () => {
    insertEntry("gpt-4o", "arena_elo", "coding", 0.87);

    const deleted = mi.deleteModelIntelligence("gpt-4o", "arena_elo", "coding");
    assert.strictEqual(deleted, true);

    const entry = mi.getModelIntelligenceBySource("gpt-4o", "arena_elo", "coding");
    assert.strictEqual(entry, null);
  });

  it("returns false when entry does not exist", () => {
    const deleted = mi.deleteModelIntelligence("nonexistent", "arena_elo", "coding");
    assert.strictEqual(deleted, false);
  });
});

describe("deleteExpiredIntelligence", () => {
  beforeEach(() => { resetStorage(); });

  it("deletes only expired entries leaving valid ones", () => {
    insertEntry("old-model", "arena_elo", "coding", 0.7, {
      expiresAt: "2000-01-01T00:00:00Z",
    });
    insertEntry("fresh-model", "arena_elo", "coding", 0.9, {
      expiresAt: "2099-12-31T23:59:59Z",
    });
    insertEntry("permanent-model", "user_override", "coding", 0.95);

    const deleted = mi.deleteExpiredIntelligence();
    assert.strictEqual(deleted, 1);

    const remaining = mi.listModelIntelligence();
    assert.strictEqual(remaining.length, 2);
  });

  it("deletes expired entries for a specific source only", () => {
    insertEntry("old-arena", "arena_elo", "coding", 0.7, {
      expiresAt: "2000-01-01T00:00:00Z",
    });
    insertEntry("old-tier", "models_dev_tier", "coding", 0.5, {
      expiresAt: "2000-01-01T00:00:00Z",
    });

    const deleted = mi.deleteExpiredIntelligence("arena_elo");
    assert.strictEqual(deleted, 1);

    const remaining = mi.listModelIntelligence();
    assert.strictEqual(remaining.length, 1);
    assert.strictEqual(remaining[0].source, "models_dev_tier");
  });

  it("returns 0 when no expired entries exist", () => {
    insertEntry("fresh-model", "arena_elo", "coding", 0.9, {
      expiresAt: "2099-12-31T23:59:59Z",
    });

    const deleted = mi.deleteExpiredIntelligence();
    assert.strictEqual(deleted, 0);
  });
});

describe("listModelIntelligence", () => {
  beforeEach(() => { resetStorage(); });

  it("lists all entries when no filters provided", () => {
    insertEntry("model-a", "arena_elo", "coding", 0.8);
    insertEntry("model-b", "arena_elo", "review", 0.75);
    insertEntry("model-c", "user_override", "coding", 0.95);

    const entries = mi.listModelIntelligence();
    assert.strictEqual(entries.length, 3);
  });

  it("filters by source", () => {
    insertEntry("model-a", "arena_elo", "coding", 0.8);
    insertEntry("model-b", "user_override", "coding", 0.95);
    insertEntry("model-c", "models_dev_tier", "coding", 0.6);

    const entries = mi.listModelIntelligence({ source: "arena_elo" });
    assert.strictEqual(entries.length, 1);
    assert.strictEqual(entries[0].source, "arena_elo");
  });

  it("filters by category", () => {
    insertEntry("model-a", "arena_elo", "coding", 0.8);
    insertEntry("model-b", "arena_elo", "review", 0.75);
    insertEntry("model-c", "arena_elo", "documentation", 0.7);

    const entries = mi.listModelIntelligence({ category: "review" });
    assert.strictEqual(entries.length, 1);
    assert.strictEqual(entries[0].category, "review");
  });

  it("filters by both source and category", () => {
    insertEntry("model-a", "arena_elo", "coding", 0.8);
    insertEntry("model-b", "arena_elo", "review", 0.75);
    insertEntry("model-c", "user_override", "coding", 0.95);

    const entries = mi.listModelIntelligence({ source: "arena_elo", category: "coding" });
    assert.strictEqual(entries.length, 1);
    assert.strictEqual(entries[0].model, "model-a");
  });

  it("returns empty array for empty table", () => {
    const entries = mi.listModelIntelligence();
    assert.ok(Array.isArray(entries));
    assert.strictEqual(entries.length, 0);
  });
});

describe("bulkUpsertModelIntelligence", () => {
  beforeEach(() => { resetStorage(); });

  it("bulk inserts multiple entries", () => {
    const count = mi.bulkUpsertModelIntelligence([
      { model: "model-a", source: "arena_elo", category: "coding", score: 0.80, eloRaw: 1300, confidence: "high", expiresAt: "2099-12-31T23:59:59Z" },
      { model: "model-b", source: "arena_elo", category: "coding", score: 0.70, eloRaw: 1200, confidence: "medium", expiresAt: "2099-12-31T23:59:59Z" },
      { model: "model-c", source: "arena_elo", category: "review", score: 0.85, eloRaw: 1350, confidence: "high", expiresAt: "2099-12-31T23:59:59Z" },
    ]);

    assert.strictEqual(count, 3);
    const entries = mi.listModelIntelligence();
    assert.strictEqual(entries.length, 3);
  });

  it("returns 0 for empty input array", () => {
    const count = mi.bulkUpsertModelIntelligence([]);
    assert.strictEqual(count, 0);
  });

  it("replaces existing entries on conflict (INSERT OR REPLACE)", () => {
    insertEntry("model-a", "arena_elo", "coding", 0.70);

    mi.bulkUpsertModelIntelligence([
      { model: "model-a", source: "arena_elo", category: "coding", score: 0.90, eloRaw: 1450, confidence: "high", expiresAt: null },
    ]);

    const entry = mi.getModelIntelligenceBySource("model-a", "arena_elo", "coding");
    assert.ok(entry);
    assert.strictEqual(entry.score, 0.90);
    assert.strictEqual(entry.eloRaw, 1450);
  });
});

describe("getResolvedTaskFitness", () => {
  beforeEach(() => { resetStorage(); });

  it("returns user_override score when all sources exist", () => {
    insertEntry("claude-sonnet", "models_dev_tier", "coding", 0.75);
    insertEntry("claude-sonnet", "arena_elo", "coding", 0.88);
    insertEntry("claude-sonnet", "user_override", "coding", 0.99);

    const score = mi.getResolvedTaskFitness("claude-sonnet", "coding");
    assert.strictEqual(score, 0.99);
  });

  it("returns arena_elo score when no user_override exists", () => {
    insertEntry("gpt-4o", "arena_elo", "coding", 0.87);
    insertEntry("gpt-4o", "models_dev_tier", "coding", 0.70);

    const score = mi.getResolvedTaskFitness("gpt-4o", "coding");
    assert.strictEqual(score, 0.87);
  });

  it("returns models_dev_tier score when only that source exists", () => {
    insertEntry("llama-4", "models_dev_tier", "coding", 0.65);

    const score = mi.getResolvedTaskFitness("llama-4", "coding");
    assert.strictEqual(score, 0.65);
  });

  it("returns null when nothing exists", () => {
    const score = mi.getResolvedTaskFitness("nonexistent", "coding");
    assert.strictEqual(score, null);
  });

  it("skips expired entries in the resolution chain", () => {
    insertEntry("gemini-pro", "arena_elo", "coding", 0.82, {
      expiresAt: "2000-01-01T00:00:00Z",
    });
    insertEntry("gemini-pro", "models_dev_tier", "coding", 0.70);

    const score = mi.getResolvedTaskFitness("gemini-pro", "coding");
    assert.strictEqual(score, 0.70);
  });
});

describe("edge cases", () => {
  beforeEach(() => { resetStorage(); });

  it("score values are stored and retrieved with float precision", () => {
    mi.upsertModelIntelligence({
      model: "precise-model",
      source: "arena_elo",
      category: "coding",
      score: 0.1234,
      eloRaw: null,
      confidence: null,
      expiresAt: null,
    });

    const entry = mi.getModelIntelligence("precise-model", "coding");
    assert.ok(entry);
    assert.ok(Math.abs(entry.score - 0.1234) < 0.0001);
  });

  it("null eloRaw and confidence are handled correctly", () => {
    mi.upsertModelIntelligence({
      model: "minimal-model",
      source: "models_dev_tier",
      category: "default",
      score: 0.6,
      eloRaw: null,
      confidence: null,
      expiresAt: null,
    });

    const entry = mi.getModelIntelligence("minimal-model", "default");
    assert.ok(entry);
    assert.strictEqual(entry.eloRaw, null);
    assert.strictEqual(entry.confidence, null);
  });

  it("NULL expires_at means never expires", () => {
    mi.upsertModelIntelligence({
      model: "permanent-model",
      source: "user_override",
      category: "coding",
      score: 0.95,
      eloRaw: null,
      confidence: null,
      expiresAt: null,
    });

    const entry = mi.getModelIntelligence("permanent-model", "coding");
    assert.ok(entry);
    assert.strictEqual(entry.expiresAt, null);
    assert.strictEqual(entry.score, 0.95);
  });
});