OmniRoute/open-sse/services/reasoningCache.ts

/**
 * Reasoning Replay Cache — In-memory + DB hybrid service.
 *
 * Captures reasoning_content from thinking-mode model responses (DeepSeek V4,
 * Kimi K2, Qwen-Thinking, GLM, etc.) and re-injects it on subsequent turns
 * when clients omit it. This prevents the DeepSeek 400 error:
 * "The reasoning_content in the thinking mode must be passed back to the API."
 *
 * Architecture:
 *   Write: memory + DB in the same process
 *   Read:  memory first → DB fallback → miss
 *
 * @see Issue #1628
 */

import {
  clearAllReasoningCache,
  cleanupExpiredReasoning,
  deleteReasoningCache,
  getReasoningCache,
  getReasoningCacheEntries,
  getReasoningCacheStats,
  setReasoningCache,
} from "../../src/lib/db/reasoningCache.ts";

// ──────────────── Provider/Model Detection ────────────────

const REASONING_REPLAY_PROVIDERS = new Set([
  "deepseek",
  "opencode-go",
  "siliconflow",
  "nebius",
  "deepinfra",
  "sambanova",
  "fireworks",
  "together",
  // Xiaomi MiMo enforces the same "pass back reasoning_content on subsequent
  // turns" contract as DeepSeek/Kimi-thinking. Without replay the upstream
  // 400s with "Param Incorrect: The reasoning_content in the thinking mode
  // must be passed back to the API."
  "xiaomi-mimo",
]);

const REASONING_REPLAY_MODEL_PATTERNS = [
  /deepseek-r1/i,
  /deepseek-reasoner/i,
  /deepseek-chat/i,
  /deepseek[-/]v4[-.](flash|pro)(-free)?/i,
  /zen\/deepseek-v4/i,
  /kimi-k2/i,
  /qwq/i,
  /qwen.*think/i,
  /glm.*think/i,
  // MiMo (Xiaomi) thinking models — defensive match if a wildcard route
  // assigns a non-`xiaomi-mimo` provider ID to a mimo-* model alias.
  /^mimo[-.]?v\d/i,
];

const DEEPSEEK_V4_MODEL_PATTERN = /deepseek[-/]v4[-.](flash|pro)/i;

export function isDeepSeekReasoningModel(params: {
  provider: string;
  model: string;
  thinkingEnabled?: boolean;
}): boolean {
  if (params.thinkingEnabled !== true) return false;
  return DEEPSEEK_V4_MODEL_PATTERN.test(params.model);
}

/**
 * Check if a provider/model combination requires reasoning replay.
 */
export function requiresReasoningReplay(params: {
  provider: string;
  model: string;
  thinkingEnabled?: boolean;
  supportsReasoning?: boolean;
  interleavedField?: string | null;
  allowLegacyFallback?: boolean;
}): boolean {
  const normalizedProvider = params.provider.trim().toLowerCase();
  const normalizedModel = params.model.trim();
  const normalizedInterleavedField =
    typeof params.interleavedField === "string" ? params.interleavedField.trim().toLowerCase() : "";

  // Explicit model signal from models.dev (preferred source of truth).
  if (normalizedInterleavedField === "reasoning_content") return true;
  if (normalizedInterleavedField === "reasoning_details") return false;

  // DeepSeek legacy reasoner family has an inverse contract: do not replay.
  if (/deepseek-reasoner/i.test(normalizedModel) || /deepseek-r1/i.test(normalizedModel)) {
    return false;
  }

  // Explicit known contract: DeepSeek V4 thinking with tool calls requires replay.
  if (isDeepSeekReasoningModel(params)) return true;

  const useLegacyFallback = params.allowLegacyFallback !== false;
  if (!useLegacyFallback) return false;

  if (REASONING_REPLAY_PROVIDERS.has(normalizedProvider)) return true;
  return REASONING_REPLAY_MODEL_PATTERNS.some((p) => p.test(normalizedModel));
}

// ──────────────── In-Memory Cache ────────────────

interface MemoryCacheEntry {
  reasoning: string;
  provider: string;
  model: string;
  expiresAt: number;
  createdAt: number;
}

type AssistantMessageLike = {
  role?: unknown;
  tool_calls?: unknown;
  reasoning_content?: unknown;
  reasoning?: unknown;
};

type AssistantMessageCacheContext = {
  requestId?: string;
  messageIndex?: number;
};

type ToolCallLike = {
  id?: unknown;
};

const memoryCache = new Map<string, MemoryCacheEntry>();
const MAX_MEMORY_ENTRIES = 200;
const MAX_ENTRY_BYTES = 10000;
const TTL_MS = 2 * 60 * 60 * 1000; // 2 hours

// ──────────────── Counters ────────────────

let hits = 0;
let misses = 0;
let replays = 0;

// ──────────────── Core Operations ────────────────

/**
 * Evict the oldest entry from the memory cache when full.
 */
function evictOldest(): void {
  let oldestKey: string | null = null;
  let oldestTime = Infinity;
  for (const [key, entry] of memoryCache) {
    if (entry.createdAt < oldestTime) {
      oldestTime = entry.createdAt;
      oldestKey = key;
    }
  }
  if (oldestKey) memoryCache.delete(oldestKey);
}

/**
 * Remove expired entries from memory cache.
 */
function purgeExpiredMemory(): void {
  const now = Date.now();
  for (const [key, entry] of memoryCache) {
    if (now >= entry.expiresAt) {
      memoryCache.delete(key);
    }
  }
}

/**
 * Cache a reasoning_content string for one tool_call ID.
 * Writes to memory and best-effort DB persistence.
 */
export function cacheReasoning(
  toolCallId: string,
  provider: string,
  model: string,
  reasoning: string
): void {
  cacheReasoningByKey(toolCallId, provider, model, reasoning);
}

export function cacheReasoningByKey(
  key: string,
  provider: string,
  model: string,
  reasoning: string
): void {
  if (!key || !reasoning) return;

  if (reasoning.length > MAX_ENTRY_BYTES) {
    reasoning = reasoning.slice(0, MAX_ENTRY_BYTES);
  }

  const now = Date.now();

  // Memory write
  if (memoryCache.size >= MAX_MEMORY_ENTRIES) {
    evictOldest();
  }
  memoryCache.set(key, {
    reasoning,
    provider,
    model,
    expiresAt: now + TTL_MS,
    createdAt: now,
  });

  try {
    setReasoningCache(key, provider, model, reasoning, TTL_MS);
  } catch {
    // DB persistence failure is non-fatal; memory cache still serves the hot path.
  }
}

function buildAssistantMessageCacheKey(requestId: string, messageIndex: number): string {
  return `request:${requestId}:message:${messageIndex}`;
}

/**
 * Cache reasoning for multiple tool_call IDs (same reasoning content).
 */
export function cacheReasoningBatch(
  toolCallIds: string[],
  provider: string,
  model: string,
  reasoning: string
): void {
  for (const id of toolCallIds) {
    if (id) cacheReasoning(id, provider, model, reasoning);
  }
}

/**
 * Capture reasoning_content from an assistant message.
 * Returns the number of cache keys written.
 */
export function cacheReasoningFromAssistantMessage(
  message: AssistantMessageLike | null | undefined,
  provider: string,
  model: string,
  context?: AssistantMessageCacheContext
): number {
  if (!message || message.role !== "assistant") {
    return 0;
  }

  const reasoning =
    typeof message.reasoning_content === "string" && message.reasoning_content.length > 0
      ? message.reasoning_content
      : typeof message.reasoning === "string" && message.reasoning.length > 0
        ? message.reasoning
        : "";
  if (!reasoning) return 0;

  const toolCallIds = Array.isArray(message.tool_calls)
    ? (message.tool_calls as ToolCallLike[])
        .map((toolCall) => (typeof toolCall.id === "string" ? toolCall.id : ""))
        .filter((id) => id.length > 0)
    : [];
  if (toolCallIds.length === 0) {
    const requestId = context?.requestId?.trim();
    const messageIndex = context?.messageIndex;
    if (!requestId || typeof messageIndex !== "number" || !Number.isInteger(messageIndex)) {
      return 0;
    }

    cacheReasoningByKey(
      buildAssistantMessageCacheKey(requestId, messageIndex),
      provider,
      model,
      reasoning
    );
    return 1;
  }

  cacheReasoningBatch(toolCallIds, provider, model, reasoning);
  return toolCallIds.length;
}

/**
 * Look up cached reasoning_content by tool_call_id.
 * Memory first → DB fallback → null (miss).
 */
export function lookupReasoning(toolCallId: string): string | null {
  if (!toolCallId) {
    misses++;
    return null;
  }

  // 1. Check memory
  const mem = memoryCache.get(toolCallId);
  if (mem) {
    if (Date.now() < mem.expiresAt) {
      hits++;
      return mem.reasoning;
    }
    // Expired in memory — remove
    memoryCache.delete(toolCallId);
  }

  // 2. Fallback to DB
  let dbResult: { reasoning: string; provider: string; model: string } | null = null;
  try {
    dbResult = getReasoningCache(toolCallId);
  } catch {
    // DB lookup failure is non-fatal; treat it as a cache miss.
  }
  if (dbResult) {
    hits++;
    let promotedReasoning = dbResult.reasoning;
    if (promotedReasoning.length > MAX_ENTRY_BYTES) {
      promotedReasoning = promotedReasoning.slice(0, MAX_ENTRY_BYTES);
    }
    // Promote back to memory for fast subsequent lookups
    memoryCache.set(toolCallId, {
      reasoning: promotedReasoning,
      provider: dbResult.provider,
      model: dbResult.model,
      expiresAt: Date.now() + TTL_MS,
      createdAt: Date.now(),
    });
    return promotedReasoning;
  }

  // 3. Miss
  misses++;
  return null;
}

/**
 * Record that a replay was performed (for dashboard metrics).
 */
export function recordReplay(): void {
  replays++;
}

// ──────────────── Stats & Dashboard ────────────────

/**
 * Get combined stats from memory + DB + counters.
 */
export function getReasoningCacheServiceStats(): {
  memoryEntries: number;
  dbEntries: number;
  totalEntries: number;
  totalChars: number;
  hits: number;
  misses: number;
  replays: number;
  replayRate: string;
  byProvider: Record<string, { entries: number; chars: number }>;
  byModel: Record<string, { entries: number; chars: number }>;
  oldestEntry: string | null;
  newestEntry: string | null;
} {
  // Purge expired memory entries before reporting
  purgeExpiredMemory();

  let dbStats = {
    totalEntries: 0,
    totalChars: 0,
    byProvider: {} as Record<string, { entries: number; chars: number }>,
    byModel: {} as Record<string, { entries: number; chars: number }>,
    oldestEntry: null as string | null,
    newestEntry: null as string | null,
  };
  try {
    dbStats = getReasoningCacheStats();
  } catch {
    // DB stats are unavailable; return memory counters with empty persisted stats.
  }

  const totalLookups = hits + misses;
  const replayRate = totalLookups > 0 ? ((replays / totalLookups) * 100).toFixed(1) : "0.0";

  return {
    memoryEntries: memoryCache.size,
    dbEntries: dbStats.totalEntries,
    totalEntries: dbStats.totalEntries,
    totalChars: dbStats.totalChars,
    hits,
    misses,
    replays,
    replayRate: `${replayRate}%`,
    byProvider: dbStats.byProvider,
    byModel: dbStats.byModel,
    oldestEntry: dbStats.oldestEntry,
    newestEntry: dbStats.newestEntry,
  };
}

/**
 * Get paginated entries (delegates to DB).
 */
export function getReasoningCacheServiceEntries(
  opts: {
    limit?: number;
    offset?: number;
    provider?: string;
    model?: string;
  } = {}
): unknown[] {
  try {
    return getReasoningCacheEntries(opts);
  } catch {
    return [];
  }
}

/**
 * Clear all reasoning cache entries (memory + DB).
 * Returns count of DB entries removed.
 */
export function clearReasoningCacheAll(provider?: string): number {
  // Clear memory
  if (provider) {
    for (const [key, entry] of memoryCache) {
      if (entry.provider === provider) memoryCache.delete(key);
    }
  } else {
    memoryCache.clear();
  }

  // Reset counters
  hits = 0;
  misses = 0;
  replays = 0;

  try {
    return clearAllReasoningCache(provider);
  } catch {
    return 0;
  }
}

/**
 * Delete one reasoning cache entry by tool_call_id from memory + DB.
 */
export function deleteReasoningCacheEntry(toolCallId: string): number {
  if (!toolCallId) return 0;
  const existedInMemory = memoryCache.delete(toolCallId);
  let deletedFromDb = 0;
  try {
    deletedFromDb = deleteReasoningCache(toolCallId);
  } catch {
    // Memory delete already happened; DB delete can be retried by a later cleanup.
  }
  return deletedFromDb + (existedInMemory && deletedFromDb === 0 ? 1 : 0);
}

/**
 * Cleanup expired entries from both memory and DB.
 * Called periodically (e.g., every 30 min from health-check).
 */
export function cleanupReasoningCache(): number {
  purgeExpiredMemory();
  try {
    return cleanupExpiredReasoning();
  } catch {
    return 0;
  }
}

// ──────────────── Auto-start periodic cleanup ────────────────
//
// server-init.ts was supposed to start the cleanup job, but that module is
// never imported anywhere (it is stranded/dead code).  As a result, the
// reasoning_cache SQLite table accumulates expired entries indefinitely.
//
// Fix: start the periodic cleanup directly from this module so it runs
// regardless of how the server boots.  On first import we run one
// immediate sweep, then schedule a 30-minute interval.
//
// See: src/lib/jobs/reasoningCacheCleanupJob.ts (the original job module,
// which also remains valid if server-init.ts ever gets wired in).

const DEFAULT_CLEANUP_INTERVAL_MS = 30 * 60 * 1000; // 30 min

function getCleanupIntervalMs(): number {
  const raw = process.env.OMNIROUTE_REASONING_CACHE_CLEANUP_INTERVAL_MS;
  const parsed = raw ? Number(raw) : Number.NaN;
  return Number.isFinite(parsed) && parsed >= 60_000 ? parsed : DEFAULT_CLEANUP_INTERVAL_MS;
}

function startAutoCleanup(): void {
  // Run once immediately on boot
  try {
    const deleted = cleanupReasoningCache();
    if (deleted > 0) {
      console.log(`[ReasoningCache] boot cleanup removed ${deleted} expired entries`);
    }
  } catch (error) {
    console.error("[ReasoningCache] boot cleanup failed:", error);
  }

  // Schedule periodic cleanup
  const timer = setInterval(() => {
    try {
      const deleted = cleanupReasoningCache();
      if (deleted > 0) {
        console.log(`[ReasoningCache] periodic cleanup removed ${deleted} expired entries`);
      }
    } catch (error) {
      console.error("[ReasoningCache] periodic cleanup failed:", error);
    }
  }, getCleanupIntervalMs());

  timer.unref?.();
}

// Start on module load — every consumer of reasoning cache benefits.
startAutoCleanup();