OmniRoute/tests/unit/memory-vectorstore-ensure-ready.test.ts

/**
 * tests/unit/memory-vectorstore-ensure-ready.test.ts
 *
 * Plan 21 — Memory Engine Redesign (F4)
 * Tests for VectorStore.ensureReady():
 *   - First call with signature "X" creates vec_memories with correct dim.
 *   - Second call same signature is idempotent (no-op).
 *   - Call with new signature "Y" drops + recreates and marks all memories needs_reindex=1.
 *   - Returns {ready: false} when sqlite-vec is not available.
 */

import test from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { mock } from "node:test";

const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omr-vecstore-ensure-"));
process.env.DATA_DIR = TEST_DATA_DIR;
process.env.DISABLE_SQLITE_AUTO_BACKUP = "true";

const core = await import("../../src/lib/db/core.ts");
const { getMemoryVecMeta } = await import("../../src/lib/db/memoryVec.ts");
const vsModule = await import("../../src/lib/memory/vectorStore.ts");
const { getVectorStore, _resetVectorStoreSingleton } = vsModule;

import type { EmbeddingResolution } from "../../src/lib/memory/embedding/types.ts";

function makeResolution(sig: string, dim: number): EmbeddingResolution {
  return {
    source: "remote",
    model: "openai/text-embedding-3-small",
    dimensions: dim,
    signature: sig,
    reason: "test",
  };
}

function cleanup() {
  mock.restoreAll();
  _resetVectorStoreSingleton();
  core.resetDbInstance();
  if (fs.existsSync(TEST_DATA_DIR)) {
    fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
  }
  fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
}

test.afterEach(() => {
  cleanup();
});

test.after(() => {
  core.resetDbInstance();
  if (fs.existsSync(TEST_DATA_DIR)) {
    fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
  }
});

// Helper: get VectorStore or skip if sqlite-vec is not available.
function getStoreOrSkip(t: { skip: (msg: string) => void }): ReturnType<typeof getVectorStore> {
  _resetVectorStoreSingleton();
  const store = getVectorStore();
  if (store === null) {
    t.skip("sqlite-vec not available in this environment — skipping");
    return null;
  }
  return store;
}

// ──────────────── Tests ────────────────

test("ensureReady: first call creates vec_memories with correct dim", async (t) => {
  const store = getStoreOrSkip(t);
  if (!store) return;

  const db = core.getDbInstance();
  const res = makeResolution("openai:text-embedding-3-small:1536", 1536);

  const result = await store.ensureReady(res);

  assert.equal(result.ready, true, "should be ready after first ensureReady");

  // Verify the virtual table was created.
  const rows = db.prepare("SELECT COUNT(*) AS cnt FROM vec_memories").get() as { cnt: number };
  assert.equal(rows.cnt, 0, "vec_memories should exist (empty after creation)");

  // Verify meta was updated.
  const meta = getMemoryVecMeta();
  assert.equal(meta.embeddingSignature, "openai:text-embedding-3-small:1536");
  assert.equal(meta.activeDim, 1536);
  assert.equal(meta.vecLoaded, true);
});

test("ensureReady: second call with same signature is idempotent", async (t) => {
  const store = getStoreOrSkip(t);
  if (!store) return;

  const res = makeResolution("openai:text-embedding-3-small:1536", 1536);

  await store.ensureReady(res);

  // Read meta after first call.
  const meta1 = getMemoryVecMeta();

  // Second call — should be no-op.
  const result = await store.ensureReady(res);

  assert.equal(result.ready, true);
  const meta2 = getMemoryVecMeta();

  // Meta should not have changed (lastResetAt remains the same).
  assert.equal(meta1.embeddingSignature, meta2.embeddingSignature);
  assert.equal(meta1.activeDim, meta2.activeDim);
  assert.equal(meta1.vecLoaded, meta2.vecLoaded);
});

test("ensureReady: signature change triggers reset + marks memories needs_reindex=1", async (t) => {
  const store = getStoreOrSkip(t);
  if (!store) return;

  const db = core.getDbInstance();

  // Insert a few memories first.
  for (let i = 0; i < 3; i++) {
    db.prepare(
      `INSERT INTO memories (id, api_key_id, type, key, content, created_at)
       VALUES (?, 'key1', 'factual', ?, ?, datetime('now'))`
    ).run(`mem-${i}`, `key-${i}`, `content-${i}`);
  }

  // First ensureReady with signature X.
  const resX = makeResolution("openai:ada-002:1024", 1024);
  await store.ensureReady(resX);

  // Check X is set.
  assert.equal(getMemoryVecMeta().embeddingSignature, "openai:ada-002:1024");
  assert.equal(getMemoryVecMeta().activeDim, 1024);

  // Now switch to signature Y (different model + dim).
  const resY = makeResolution("openai:text-embedding-3-small:1536", 1536);
  const resetResult = await store.ensureReady(resY);

  assert.equal(resetResult.ready, true, "should be ready after signature change");

  // Verify new signature is stored.
  const metaAfter = getMemoryVecMeta();
  assert.equal(metaAfter.embeddingSignature, "openai:text-embedding-3-small:1536");
  assert.equal(metaAfter.activeDim, 1536);
  assert.ok(metaAfter.lastResetAt !== null, "lastResetAt should be set after reset");

  // All 3 memories should have needs_reindex = 1.
  const needsRows = db
    .prepare("SELECT COUNT(*) AS cnt FROM memories WHERE needs_reindex = 1")
    .get() as { cnt: number };
  assert.equal(
    needsRows.cnt,
    3,
    "all memories should be marked needs_reindex=1 after signature change"
  );
});

test("ensureReady: returns {ready: false} when dimensions are null (no probe done yet)", async (t) => {
  const store = getStoreOrSkip(t);
  if (!store) return;

  // Resolution with null dimensions — lazy probe not done yet.
  const resNullDim: EmbeddingResolution = {
    source: "remote",
    model: "openai/text-embedding-3-small",
    dimensions: null,
    signature: "openai:text-embedding-3-small:null",
    reason: "test - dim not probed yet",
  };

  const result = await store.ensureReady(resNullDim);

  // Should not crash, but cannot create table without dim.
  // Either ready (if signature already matches a loaded table) or not ready.
  assert.ok(
    typeof result.ready === "boolean",
    "ensureReady must return {ready: boolean, reason: string}"
  );
  assert.ok(typeof result.reason === "string");
});