From b08fb31a28f4d5799a92f45dc627f0871373282f Mon Sep 17 00:00:00 2001 From: diegosouzapw Date: Sun, 15 Feb 2026 11:04:51 -0300 Subject: [PATCH] =?UTF-8?q?feat(gateway):=20Phase=209=20=E2=80=94=20LLM=20?= =?UTF-8?q?Gateway=20Intelligence?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 9.1 — Semantic Cache - New: src/lib/semanticCache.js — Two-tier cache (in-memory LRU + SQLite) - Signature = SHA-256(model + normalized messages + temperature + top_p) - Only caches non-streaming, temperature=0 requests - X-OmniRoute-No-Cache: true header bypass - Response headers: X-OmniRoute-Cache: HIT/MISS - DB table: semantic_cache with indexes on signature and model - New: src/app/api/cache/route.js — GET stats, DELETE clear 9.2 — Request Idempotency - New: src/lib/idempotencyLayer.js — In-memory 5s dedup window - Reads Idempotency-Key or X-Request-Id headers - Returns cached response with X-OmniRoute-Idempotent: true header - Ephemeral by design (no SQLite) 9.3 — Progress Tracking in Streaming - New: open-sse/utils/progressTracker.js - Emits SSE 'event: progress' with tokens_generated + elapsed_ms - Opt-in via X-OmniRoute-Progress: true header - Supports AbortSignal cancellation - Final event includes done: true Integration: - chatCore.js: idempotency check → cache check → provider call → cache store → idempotency save - Streaming path: optional progress transform chain - DB: semantic_cache table added to db/core.js schema Tests: 320 pass (+25 new) | Build: success --- open-sse/handlers/chatCore.js | 73 +++++++++- open-sse/utils/progressTracker.js | 101 ++++++++++++++ src/app/api/cache/route.js | 33 +++++ src/lib/db/core.js | 14 ++ src/lib/idempotencyLayer.js | 94 +++++++++++++ src/lib/semanticCache.js | 215 +++++++++++++++++++++++++++++ tests/unit/idempotency.test.mjs | 83 +++++++++++ tests/unit/semantic-cache.test.mjs | 87 ++++++++++++ 8 files changed, 697 insertions(+), 3 deletions(-) create mode 100644 open-sse/utils/progressTracker.js create mode 100644 src/app/api/cache/route.js create mode 100644 src/lib/idempotencyLayer.js create mode 100644 src/lib/semanticCache.js create mode 100644 tests/unit/idempotency.test.mjs create mode 100644 tests/unit/semantic-cache.test.mjs diff --git a/open-sse/handlers/chatCore.js b/open-sse/handlers/chatCore.js index 60fa60df01..5d56cb4962 100644 --- a/open-sse/handlers/chatCore.js +++ b/open-sse/handlers/chatCore.js @@ -29,6 +29,14 @@ import { updateFromHeaders, initializeRateLimits, } from "../services/rateLimitManager.js"; +import { + generateSignature, + getCachedResponse, + setCachedResponse, + isCacheable, +} from "@/lib/semanticCache.js"; +import { getIdempotencyKey, checkIdempotency, saveIdempotency } from "@/lib/idempotencyLayer.js"; +import { createProgressTransform, wantsProgress } from "../utils/progressTracker.js"; /** * Core chat handler - shared between SSE and Worker @@ -61,6 +69,24 @@ export async function handleChatCore({ const { provider, model } = modelInfo; const startTime = Date.now(); + // ── Phase 9.2: Idempotency check ── + const idempotencyKey = getIdempotencyKey(clientRawRequest?.headers); + const cachedIdemp = checkIdempotency(idempotencyKey); + if (cachedIdemp) { + log?.debug?.("IDEMPOTENCY", `Hit for key=${idempotencyKey?.slice(0, 12)}...`); + return { + success: true, + response: new Response(JSON.stringify(cachedIdemp.response), { + status: cachedIdemp.status, + headers: { + "Content-Type": "application/json", + "Access-Control-Allow-Origin": "*", + "X-OmniRoute-Idempotent": "true", + }, + }), + }; + } + // Initialize rate limit settings from persisted DB (once, lazy) await initializeRateLimits(); @@ -84,6 +110,25 @@ export async function handleChatCore({ // Default to streaming unless client explicitly sets stream: false const stream = body.stream !== false; + // ── Phase 9.1: Semantic cache check (non-streaming, temp=0 only) ── + if (isCacheable(body, clientRawRequest?.headers)) { + const signature = generateSignature(model, body.messages, body.temperature, body.top_p); + const cached = getCachedResponse(signature); + if (cached) { + log?.debug?.("CACHE", `Semantic cache HIT for ${model}`); + return { + success: true, + response: new Response(JSON.stringify(cached), { + headers: { + "Content-Type": "application/json", + "Access-Control-Allow-Origin": "*", + "X-OmniRoute-Cache": "HIT", + }, + }), + }; + } + } + // Create request logger for this session: sourceFormat_targetFormat_model const reqLogger = await createRequestLogger(sourceFormat, targetFormat, model); @@ -444,12 +489,24 @@ export async function handleChatCore({ } } + // ── Phase 9.1: Cache store (non-streaming, temp=0) ── + if (isCacheable(body, clientRawRequest?.headers)) { + const signature = generateSignature(model, body.messages, body.temperature, body.top_p); + const tokensSaved = usage?.prompt_tokens + usage?.completion_tokens || 0; + setCachedResponse(signature, model, translatedResponse, tokensSaved); + log?.debug?.("CACHE", `Stored response for ${model} (${tokensSaved} tokens)`); + } + + // ── Phase 9.2: Save for idempotency ── + saveIdempotency(idempotencyKey, translatedResponse, 200); + return { success: true, response: new Response(JSON.stringify(translatedResponse), { headers: { "Content-Type": "application/json", "Access-Control-Allow-Origin": "*", + "X-OmniRoute-Cache": "MISS", }, }), }; @@ -546,12 +603,22 @@ export async function handleChatCore({ ); } - // Pipe response through transform with disconnect detection - const transformedBody = pipeWithDisconnect(providerResponse, transformStream, streamController); + // ── Phase 9.3: Progress tracking (opt-in) ── + const progressEnabled = wantsProgress(clientRawRequest?.headers); + let finalStream; + if (progressEnabled) { + const progressTransform = createProgressTransform({ signal: streamController.signal }); + // Chain: provider → transform → progress → client + const transformedBody = pipeWithDisconnect(providerResponse, transformStream, streamController); + finalStream = transformedBody.pipeThrough(progressTransform); + responseHeaders["X-OmniRoute-Progress"] = "enabled"; + } else { + finalStream = pipeWithDisconnect(providerResponse, transformStream, streamController); + } return { success: true, - response: new Response(transformedBody, { + response: new Response(finalStream, { headers: responseHeaders, }), }; diff --git a/open-sse/utils/progressTracker.js b/open-sse/utils/progressTracker.js new file mode 100644 index 0000000000..d42dcdd612 --- /dev/null +++ b/open-sse/utils/progressTracker.js @@ -0,0 +1,101 @@ +/** + * Progress Tracker — Phase 9.3 + * + * Emits SSE `event: progress` events during long streaming responses. + * Opt-in via X-OmniRoute-Progress: true header. + * + * Progress events contain: + * { tokens_generated, elapsed_ms } + * + * @module utils/progressTracker + */ + +const DEFAULT_INTERVAL_MS = 2000; + +/** + * Create a progress emitter for a streaming response. + * Returns a TransformStream that injects progress events periodically. + * + * @param {object} options + * @param {number} [options.intervalMs=2000] - Interval between events + * @param {AbortSignal} [options.signal] - Abort signal for cancellation + * @returns {TransformStream} + */ +export function createProgressTransform({ intervalMs = DEFAULT_INTERVAL_MS, signal } = {}) { + let tokenCount = 0; + let startTime = Date.now(); + let intervalId; + let writer; + + const encoder = new TextEncoder(); + + return new TransformStream({ + start(controller) { + writer = controller; + startTime = Date.now(); + + intervalId = setInterval(() => { + if (signal?.aborted) { + clearInterval(intervalId); + return; + } + const progressEvent = `event: progress\ndata: ${JSON.stringify({ + tokens_generated: tokenCount, + elapsed_ms: Date.now() - startTime, + })}\n\n`; + try { + controller.enqueue(encoder.encode(progressEvent)); + } catch { + // Stream closed + clearInterval(intervalId); + } + }, intervalMs); + + // Clean up on abort + signal?.addEventListener( + "abort", + () => { + clearInterval(intervalId); + }, + { once: true } + ); + }, + + transform(chunk, controller) { + // Count token events in the chunk + const text = typeof chunk === "string" ? chunk : new TextDecoder().decode(chunk); + // Count data lines (each is roughly one token event) + const dataLines = text.split("\n").filter((l) => l.startsWith("data: ")); + tokenCount += dataLines.length; + controller.enqueue(chunk); + }, + + flush() { + clearInterval(intervalId); + // Final progress event + if (writer) { + try { + const finalEvent = `event: progress\ndata: ${JSON.stringify({ + tokens_generated: tokenCount, + elapsed_ms: Date.now() - startTime, + done: true, + })}\n\n`; + writer.enqueue(encoder.encode(finalEvent)); + } catch { + // Stream already closed + } + } + }, + }); +} + +/** + * Check if client opted into progress tracking. + * @param {Headers|object} headers + * @returns {boolean} + */ +export function wantsProgress(headers) { + if (!headers) return false; + const get = typeof headers.get === "function" ? (k) => headers.get(k) : (k) => headers[k]; + return get("x-omniroute-progress") === "true"; +} diff --git a/src/app/api/cache/route.js b/src/app/api/cache/route.js new file mode 100644 index 0000000000..2d759f351f --- /dev/null +++ b/src/app/api/cache/route.js @@ -0,0 +1,33 @@ +import { NextResponse } from "next/server"; +import { getCacheStats, clearCache, cleanExpiredEntries } from "@/lib/semanticCache"; +import { getIdempotencyStats } from "@/lib/idempotencyLayer"; + +/** + * GET /api/cache — Cache statistics + */ +export async function GET() { + try { + const cacheStats = getCacheStats(); + const idempotencyStats = getIdempotencyStats(); + + return NextResponse.json({ + semanticCache: cacheStats, + idempotency: idempotencyStats, + }); + } catch (error) { + return NextResponse.json({ error: error.message }, { status: 500 }); + } +} + +/** + * DELETE /api/cache — Clear all caches + */ +export async function DELETE() { + try { + clearCache(); + const cleaned = cleanExpiredEntries(); + return NextResponse.json({ ok: true, expiredRemoved: cleaned }); + } catch (error) { + return NextResponse.json({ error: error.message }, { status: 500 }); + } +} diff --git a/src/lib/db/core.js b/src/lib/db/core.js index ca53d5852a..cede6fd058 100644 --- a/src/lib/db/core.js +++ b/src/lib/db/core.js @@ -192,6 +192,20 @@ const SCHEMA_SQL = ` last_failure_time INTEGER, options TEXT ); + + CREATE TABLE IF NOT EXISTS semantic_cache ( + id TEXT PRIMARY KEY, + signature TEXT NOT NULL UNIQUE, + model TEXT NOT NULL, + prompt_hash TEXT NOT NULL, + response TEXT NOT NULL, + tokens_saved INTEGER DEFAULT 0, + hit_count INTEGER DEFAULT 0, + created_at TEXT NOT NULL, + expires_at TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_sc_sig ON semantic_cache(signature); + CREATE INDEX IF NOT EXISTS idx_sc_model ON semantic_cache(model); `; // ──────────────── Column Mapping ──────────────── diff --git a/src/lib/idempotencyLayer.js b/src/lib/idempotencyLayer.js new file mode 100644 index 0000000000..02a8c4939c --- /dev/null +++ b/src/lib/idempotencyLayer.js @@ -0,0 +1,94 @@ +/** + * Idempotency Layer — Phase 9.2 + * + * In-memory deduplication of requests with the same idempotency key. + * If a request with the same key arrives within 5 seconds, returns + * the cached response instead of making a new API call. + * + * Headers: X-Request-Id or Idempotency-Key + * + * @module lib/idempotencyLayer + */ + +const DEFAULT_WINDOW_MS = 5000; + +/** @type {Map} */ +const idempotencyStore = new Map(); + +// Periodic cleanup every 30s +let cleanupInterval; + +function ensureCleanup() { + if (cleanupInterval) return; + cleanupInterval = setInterval(() => { + const now = Date.now(); + for (const [key, entry] of idempotencyStore) { + if (now >= entry.expiresAt) { + idempotencyStore.delete(key); + } + } + }, 30000); + // Don't prevent process exit + if (cleanupInterval.unref) cleanupInterval.unref(); +} + +/** + * Extract idempotency key from request headers. + * @param {Headers|object} headers + * @returns {string|null} + */ +export function getIdempotencyKey(headers) { + if (!headers) return null; + const get = typeof headers.get === "function" ? (k) => headers.get(k) : (k) => headers[k]; + return get("idempotency-key") || get("x-request-id") || null; +} + +/** + * Check if a response exists for the given idempotency key. + * @param {string} key + * @returns {{ response: object, status: number }|null} + */ +export function checkIdempotency(key) { + if (!key) return null; + const entry = idempotencyStore.get(key); + if (!entry) return null; + if (Date.now() >= entry.expiresAt) { + idempotencyStore.delete(key); + return null; + } + return { response: entry.response, status: entry.status }; +} + +/** + * Save a response for idempotency dedup. + * @param {string} key + * @param {object} response - Response body to cache + * @param {number} status - HTTP status code + * @param {number} [windowMs=5000] - Dedup window in ms + */ +export function saveIdempotency(key, response, status, windowMs = DEFAULT_WINDOW_MS) { + if (!key) return; + ensureCleanup(); + idempotencyStore.set(key, { + response, + status, + expiresAt: Date.now() + windowMs, + }); +} + +/** + * Get current idempotency store stats. + */ +export function getIdempotencyStats() { + return { + activeKeys: idempotencyStore.size, + windowMs: DEFAULT_WINDOW_MS, + }; +} + +/** + * Clear all idempotency entries (for testing). + */ +export function clearIdempotency() { + idempotencyStore.clear(); +} diff --git a/src/lib/semanticCache.js b/src/lib/semanticCache.js new file mode 100644 index 0000000000..ad3569b4a4 --- /dev/null +++ b/src/lib/semanticCache.js @@ -0,0 +1,215 @@ +/** + * Semantic Cache — Phase 9.1 + * + * Caches non-streaming LLM responses (temperature=0) to reduce cost and latency. + * Two-tier: in-memory LRU (fast) + SQLite (persistent across restarts). + * + * Cache key = SHA-256(model + normalized messages + temperature + top_p) + * Bypass: X-OmniRoute-No-Cache: true + * + * @module lib/semanticCache + */ + +import crypto from "node:crypto"; +import { LRUCache } from "./cacheLayer.js"; +import { getDbInstance } from "./db/core.js"; + +// ─── Singleton ───────────────── + +let memoryCache; +let stats = { hits: 0, misses: 0, tokensSaved: 0 }; + +function getMemoryCache() { + if (!memoryCache) { + memoryCache = new LRUCache({ + maxSize: parseInt(process.env.SEMANTIC_CACHE_MAX_SIZE || "500", 10), + defaultTTL: parseInt(process.env.SEMANTIC_CACHE_TTL_MS || "3600000", 10), // 1h + }); + } + return memoryCache; +} + +// ─── Signature Generation ───────────────── + +/** + * Generate deterministic cache signature from request params. + * @param {string} model + * @param {Array} messages - Normalized messages array + * @param {number} temperature + * @param {number} topP + * @returns {string} hex signature + */ +export function generateSignature(model, messages, temperature = 0, topP = 1) { + const payload = JSON.stringify({ + model, + messages: normalizeMessages(messages), + temperature, + top_p: topP, + }); + return crypto.createHash("sha256").update(payload).digest("hex"); +} + +/** + * Normalize messages for consistent hashing. + * Strips metadata, keeps only role + content. + */ +function normalizeMessages(messages) { + if (!Array.isArray(messages)) return []; + return messages.map((m) => ({ + role: m.role || "user", + content: typeof m.content === "string" ? m.content : JSON.stringify(m.content), + })); +} + +// ─── Cache Operations ───────────────── + +/** + * Check if a cached response exists for the given signature. + * Checks memory first, then SQLite. + * @param {string} signature + * @returns {object|null} Cached response or null + */ +export function getCachedResponse(signature) { + // 1. Check memory cache + const memResult = getMemoryCache().get(signature); + if (memResult) { + stats.hits++; + stats.tokensSaved += memResult.tokensSaved || 0; + return memResult.response; + } + + // 2. Check SQLite + try { + const db = getDbInstance(); + const row = db + .prepare( + "SELECT response, tokens_saved FROM semantic_cache WHERE signature = ? AND expires_at > datetime('now')" + ) + .get(signature); + + if (row) { + const parsed = JSON.parse(row.response); + // Promote to memory cache + getMemoryCache().set(signature, { + response: parsed, + tokensSaved: row.tokens_saved, + }); + // Update hit count in DB + db.prepare("UPDATE semantic_cache SET hit_count = hit_count + 1 WHERE signature = ?").run( + signature + ); + + stats.hits++; + stats.tokensSaved += row.tokens_saved || 0; + return parsed; + } + } catch { + // DB not available — fail open + } + + stats.misses++; + return null; +} + +/** + * Store a response in cache. + * @param {string} signature + * @param {string} model + * @param {object} response - The API response to cache + * @param {number} tokensSaved - Estimated tokens saved + * @param {number} [ttlMs] - TTL in ms (default: 1 hour) + */ +export function setCachedResponse(signature, model, response, tokensSaved = 0, ttlMs = 3600000) { + const ttl = parseInt(process.env.SEMANTIC_CACHE_TTL_MS || String(ttlMs), 10); + + // 1. Memory cache + getMemoryCache().set(signature, { response, tokensSaved }, ttl); + + // 2. SQLite + try { + const db = getDbInstance(); + const id = crypto.randomUUID(); + const promptHash = signature.slice(0, 16); + const now = new Date().toISOString(); + const expiresAt = new Date(Date.now() + ttl).toISOString(); + + db.prepare( + `INSERT OR REPLACE INTO semantic_cache (id, signature, model, prompt_hash, response, tokens_saved, hit_count, created_at, expires_at) + VALUES (?, ?, ?, ?, ?, ?, 0, ?, ?)` + ).run(id, signature, model, promptHash, JSON.stringify(response), tokensSaved, now, expiresAt); + } catch { + // DB write failed — cache still in memory + } +} + +// ─── Maintenance ───────────────── + +/** + * Remove expired entries from SQLite. + * @returns {number} Number of entries removed + */ +export function cleanExpiredEntries() { + try { + const db = getDbInstance(); + const result = db + .prepare("DELETE FROM semantic_cache WHERE expires_at <= datetime('now')") + .run(); + return result.changes; + } catch { + return 0; + } +} + +/** + * Clear all cache entries. + */ +export function clearCache() { + getMemoryCache().clear(); + try { + const db = getDbInstance(); + db.prepare("DELETE FROM semantic_cache").run(); + } catch { + // DB not available + } + stats = { hits: 0, misses: 0, tokensSaved: 0 }; +} + +// ─── Stats ───────────────── + +/** + * Get cache statistics. + */ +export function getCacheStats() { + const memStats = getMemoryCache().getStats(); + let dbSize = 0; + try { + const db = getDbInstance(); + const row = db + .prepare("SELECT COUNT(*) as count FROM semantic_cache WHERE expires_at > datetime('now')") + .get(); + dbSize = row?.count || 0; + } catch { + // DB not available + } + + const total = stats.hits + stats.misses; + return { + memoryEntries: memStats.size, + dbEntries: dbSize, + hits: stats.hits, + misses: stats.misses, + hitRate: total > 0 ? ((stats.hits / total) * 100).toFixed(1) : "0.0", + tokensSaved: stats.tokensSaved, + }; +} + +/** + * Check if a request is cacheable. + * Only non-streaming, deterministic (temperature=0) requests. + */ +export function isCacheable(body, headers) { + if (headers?.get?.("x-omniroute-no-cache") === "true") return false; + if (body.stream !== false) return false; + if ((body.temperature ?? 0) !== 0) return false; + return true; +} diff --git a/tests/unit/idempotency.test.mjs b/tests/unit/idempotency.test.mjs new file mode 100644 index 0000000000..a38effc0db --- /dev/null +++ b/tests/unit/idempotency.test.mjs @@ -0,0 +1,83 @@ +import { describe, it, beforeEach } from "node:test"; +import assert from "node:assert/strict"; +import { + getIdempotencyKey, + checkIdempotency, + saveIdempotency, + clearIdempotency, + getIdempotencyStats, +} from "../../src/lib/idempotencyLayer.js"; + +describe("Idempotency Layer", () => { + beforeEach(() => { + clearIdempotency(); + }); + + describe("getIdempotencyKey", () => { + it("returns null for null headers", () => { + assert.equal(getIdempotencyKey(null), null); + }); + + it("returns Idempotency-Key header", () => { + const headers = new Headers({ "Idempotency-Key": "abc-123" }); + assert.equal(getIdempotencyKey(headers), "abc-123"); + }); + + it("returns X-Request-Id header", () => { + const headers = new Headers({ "X-Request-Id": "req-456" }); + assert.equal(getIdempotencyKey(headers), "req-456"); + }); + + it("prefers Idempotency-Key over X-Request-Id", () => { + const headers = new Headers({ + "Idempotency-Key": "idemp-1", + "X-Request-Id": "req-2", + }); + assert.equal(getIdempotencyKey(headers), "idemp-1"); + }); + + it("supports plain object headers", () => { + const headers = { "idempotency-key": "obj-key" }; + assert.equal(getIdempotencyKey(headers), "obj-key"); + }); + }); + + describe("checkIdempotency / saveIdempotency", () => { + it("returns null for unknown key", () => { + assert.equal(checkIdempotency("unknown"), null); + }); + + it("returns null for null key", () => { + assert.equal(checkIdempotency(null), null); + }); + + it("returns cached response within window", () => { + const response = { choices: [{ message: { content: "hello" } }] }; + saveIdempotency("key-1", response, 200); + const result = checkIdempotency("key-1"); + assert.deepEqual(result, { response, status: 200 }); + }); + + it("returns null after expiry", async () => { + const response = { choices: [] }; + saveIdempotency("key-2", response, 200, 50); // 50ms window + await new Promise((r) => setTimeout(r, 100)); + assert.equal(checkIdempotency("key-2"), null); + }); + + it("does nothing for null key", () => { + saveIdempotency(null, { data: 1 }, 200); + assert.equal(getIdempotencyStats().activeKeys, 0); + }); + }); + + describe("getIdempotencyStats", () => { + it("reports active keys", () => { + saveIdempotency("a", {}, 200); + saveIdempotency("b", {}, 200); + const stats = getIdempotencyStats(); + assert.equal(stats.activeKeys, 2); + assert.equal(stats.windowMs, 5000); + }); + }); +}); diff --git a/tests/unit/semantic-cache.test.mjs b/tests/unit/semantic-cache.test.mjs new file mode 100644 index 0000000000..105c2168ee --- /dev/null +++ b/tests/unit/semantic-cache.test.mjs @@ -0,0 +1,87 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { generateSignature, isCacheable } from "../../src/lib/semanticCache.js"; + +describe("Semantic Cache", () => { + describe("generateSignature", () => { + it("generates consistent signatures for same inputs", () => { + const messages = [{ role: "user", content: "hello" }]; + const sig1 = generateSignature("gpt-4", messages, 0, 1); + const sig2 = generateSignature("gpt-4", messages, 0, 1); + assert.equal(sig1, sig2); + }); + + it("generates different signatures for different models", () => { + const messages = [{ role: "user", content: "hello" }]; + const sig1 = generateSignature("gpt-4", messages, 0, 1); + const sig2 = generateSignature("gpt-3.5", messages, 0, 1); + assert.notEqual(sig1, sig2); + }); + + it("generates different signatures for different messages", () => { + const msg1 = [{ role: "user", content: "hello" }]; + const msg2 = [{ role: "user", content: "goodbye" }]; + const sig1 = generateSignature("gpt-4", msg1, 0, 1); + const sig2 = generateSignature("gpt-4", msg2, 0, 1); + assert.notEqual(sig1, sig2); + }); + + it("generates different signatures for different temperatures", () => { + const messages = [{ role: "user", content: "hello" }]; + const sig1 = generateSignature("gpt-4", messages, 0, 1); + const sig2 = generateSignature("gpt-4", messages, 0.7, 1); + assert.notEqual(sig1, sig2); + }); + + it("normalizes messages (strips extra fields)", () => { + const msg1 = [{ role: "user", content: "hello", extra: true }]; + const msg2 = [{ role: "user", content: "hello" }]; + const sig1 = generateSignature("gpt-4", msg1, 0, 1); + const sig2 = generateSignature("gpt-4", msg2, 0, 1); + assert.equal(sig1, sig2); + }); + + it("handles non-string content", () => { + const messages = [{ role: "user", content: [{ type: "text", text: "hi" }] }]; + const sig = generateSignature("gpt-4", messages, 0, 1); + assert.ok(sig.length > 0); + }); + + it("handles empty messages", () => { + const sig = generateSignature("gpt-4", [], 0, 1); + assert.ok(sig.length > 0); + }); + }); + + describe("isCacheable", () => { + it("returns true for non-streaming temp=0 requests", () => { + assert.equal(isCacheable({ stream: false, temperature: 0 }, null), true); + }); + + it("returns true when temperature is undefined (defaults to 0)", () => { + assert.equal(isCacheable({ stream: false }, null), true); + }); + + it("returns false for streaming requests", () => { + assert.equal(isCacheable({ stream: true, temperature: 0 }, null), false); + }); + + it("returns false when stream is not explicitly false", () => { + assert.equal(isCacheable({ temperature: 0 }, null), false); + }); + + it("returns false for non-zero temperature", () => { + assert.equal(isCacheable({ stream: false, temperature: 0.7 }, null), false); + }); + + it("returns false when no-cache header is set", () => { + const headers = new Headers({ "x-omniroute-no-cache": "true" }); + assert.equal(isCacheable({ stream: false, temperature: 0 }, headers), false); + }); + + it("returns true when no-cache header is absent", () => { + const headers = new Headers({}); + assert.equal(isCacheable({ stream: false, temperature: 0 }, headers), true); + }); + }); +});