From 07b190d07f9bd216c710822efa4d01900ea5da50 Mon Sep 17 00:00:00 2001 From: Paijo <14921983+oyi77@users.noreply.github.com> Date: Thu, 11 Jun 2026 07:40:49 +0700 Subject: [PATCH] Refactor: Extract chatCore phases into modular files (#3598) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Integrated into release/v3.8.21 — chatCore phase modularization. Adjusted: re-derive idempotencyKey for the save path after the check moved into the module (co-authored). Thanks @oyi77! --- open-sse/handlers/chatCore.ts | 350 +++--------------- open-sse/handlers/chatCore/idempotency.ts | 55 +++ .../chatCore/memorySkillsInjection.ts | 159 ++++++++ open-sse/handlers/chatCore/sanitization.ts | 63 ++++ open-sse/handlers/chatCore/semanticCache.ts | 91 +++++ 5 files changed, 416 insertions(+), 302 deletions(-) create mode 100644 open-sse/handlers/chatCore/idempotency.ts create mode 100644 open-sse/handlers/chatCore/memorySkillsInjection.ts create mode 100644 open-sse/handlers/chatCore/sanitization.ts create mode 100644 open-sse/handlers/chatCore/semanticCache.ts diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 2f911bfb6c..8a9239c05e 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -1,3 +1,7 @@ +import { injectMemoryAndSkills } from "./chatCore/memorySkillsInjection.ts"; +import { checkIdempotencyCache } from "./chatCore/idempotency.ts"; +import { checkSemanticCache } from "./chatCore/semanticCache.ts"; +import { sanitizeChatRequestBody } from "./chatCore/sanitization.ts"; import { CORS_HEADERS } from "../utils/cors.ts"; import { HEAP_PRESSURE_THRESHOLD_MB } from "../utils/heapPressure.ts"; import { normalizeHeaders } from "../utils/headers.ts"; @@ -176,7 +180,7 @@ import { isCacheableForRead, isCacheableForWrite, } from "@/lib/semanticCache"; -import { getIdempotencyKey, checkIdempotency, saveIdempotency } from "@/lib/idempotencyLayer"; +import { getIdempotencyKey, saveIdempotency } from "@/lib/idempotencyLayer"; import { createProgressTransform, wantsProgress } from "../utils/progressTracker.ts"; import { createPiiSseTransform } from "@/lib/streamingPiiTransform"; import { isFeatureFlagEnabled } from "@/shared/utils/featureFlags"; @@ -209,14 +213,6 @@ import { import { generateRequestId } from "@/shared/utils/requestId"; import { normalizePayloadForLog } from "@/lib/logPayloads"; import { extractFacts } from "@/lib/memory/extraction"; -import { injectMemory, shouldInjectMemory } from "@/lib/memory/injection"; -import { retrieveMemories } from "@/lib/memory/retrieval"; -import { - DEFAULT_MEMORY_SETTINGS, - getMemorySettings, - toMemoryRetrievalConfig, -} from "@/lib/memory/settings"; -import { injectSkills } from "@/lib/skills/injection"; import { handleToolCallExecution } from "@/lib/skills/interception"; import { OMNIROUTE_RESPONSE_HEADERS } from "@/shared/constants/headers"; import { @@ -1871,39 +1867,16 @@ export async function handleChatCore({ }; // ── Phase 9.2: Idempotency check ── - const idempotencyKey = getIdempotencyKey(clientRawRequest?.headers); - const cachedIdemp = checkIdempotency(idempotencyKey); - if (cachedIdemp) { - log?.debug?.("IDEMPOTENCY", `Hit for key=${idempotencyKey?.slice(0, 12)}...`); - const idempotentUsage = - cachedIdemp.response && typeof cachedIdemp.response === "object" - ? ((cachedIdemp.response as Record).usage as - | Record - | undefined) - : undefined; - const idempotentCost = idempotentUsage - ? await calculateCost(provider, model, idempotentUsage as Record, { - serviceTier: effectiveServiceTier, - }) - : 0; - return { - success: true, - response: new Response(JSON.stringify(cachedIdemp.response), { - status: cachedIdemp.status, - headers: { - "Content-Type": "application/json", - "X-OmniRoute-Idempotent": "true", - ...buildOmniRouteResponseMetaHeaders({ - provider, - model, - cacheHit: false, - latencyMs: Date.now() - startTime, - usage: idempotentUsage, - costUsd: idempotentCost, - }), - }, - }), - }; + const idempotencyHit = await checkIdempotencyCache({ + clientRawRequest, + provider, + model, + effectiveServiceTier, + startTime, + log, + }); + if (idempotencyHit) { + return idempotencyHit; } // T07: Inject connectionId into credentials so executors can rotate API keys @@ -2303,269 +2276,38 @@ export async function handleChatCore({ log?.debug?.("FORMAT", `${sourceFormat} → ${targetFormat} | stream=${stream}`); // ── Phase 9.1: Semantic cache check (temp=0, any streaming mode) ── - // Streaming responses are cached after assembly; cache hits return JSON regardless of stream flag. - if (semanticCacheEnabled && isCacheableForRead(body, clientRawRequest?.headers)) { - const signature = generateSignature( - model, - body.messages ?? body.input, - body.temperature, - body.top_p - ); - const cached = getCachedResponse(signature); - if (cached) { - log?.debug?.("CACHE", `Semantic cache HIT for ${model} (stream=${stream})`); - reqLogger.logConvertedResponse(cached as Record); - const cachedUsage = - extractUsageFromResponse(cached as Record, provider) || - ((cached as Record)?.usage as Record | undefined); - const cachedCost = cachedUsage - ? await calculateCost(provider, model, cachedUsage as Record, { - serviceTier: effectiveServiceTier, - }) - : 0; - persistAttemptLogs({ - status: 200, - tokens: (cached as Record)?.usage, - responseBody: cached, - providerRequest: null, - providerResponse: null, - clientResponse: cached, - cacheSource: "semantic", - }); - trackPendingRequest(model, provider, connectionId, false); - // #2952 — when the client requested a stream, serve the cached completion - // as an SSE stream (not a raw JSON body) so content + reasoning_content - // arrive in the streaming shape the client expects. Cache hits previously - // returned application/json regardless of the stream flag, which made - // OpenAI-compatible streaming clients lose reasoning_content. Non-OpenAI - // shapes (no `choices`) yield "" and fall back to the JSON body unchanged. - const cachedSse = stream ? synthesizeOpenAiSseFromJson(JSON.stringify(cached)) : ""; - const cacheHitMetaHeaders = buildOmniRouteResponseMetaHeaders({ - provider, - model, - cacheHit: true, - latencyMs: Date.now() - startTime, - usage: cachedUsage, - costUsd: cachedCost, - }); - return { - success: true, - response: new Response(cachedSse || JSON.stringify(cached), { - headers: { - "Content-Type": cachedSse ? "text/event-stream" : "application/json", - [OMNIROUTE_RESPONSE_HEADERS.cache]: "HIT", - ...cacheHitMetaHeaders, - }, - }), - }; - } - } - - // ── Common input sanitization (runs for ALL paths including passthrough) ── - // #994: Normalize between max_output_tokens and max_tokens for universal compatibility. - // For Responses API targets, max_output_tokens is the canonical field. For others, - // max_tokens is preferred. We handle normalization here to support passthrough - // paths where the translator is skipped. - const prefersResponsesTokenField = - sourceFormat === FORMATS.OPENAI_RESPONSES || targetFormat === FORMATS.OPENAI_RESPONSES; - - if (prefersResponsesTokenField) { - if (body.max_output_tokens === undefined) { - if (body.max_completion_tokens !== undefined) { - body.max_output_tokens = body.max_completion_tokens; - delete body.max_completion_tokens; - } else if (body.max_tokens !== undefined) { - body.max_output_tokens = body.max_tokens; - delete body.max_tokens; - } - } - } else if (body.max_output_tokens !== undefined) { - if (body.max_tokens === undefined) { - body.max_tokens = body.max_output_tokens; - } - delete body.max_output_tokens; - } - - // #291: Strip empty name fields from messages/input items - // Upstream providers (OpenAI, Codex) reject name:"" with 400 errors. - if (Array.isArray(body.messages)) { - body.messages = body.messages.map((msg: Record) => { - if (msg.name === "") { - const { name: _n, ...rest } = msg; - return rest; - } - return msg; - }); - } - if (Array.isArray(body.input)) { - body.input = body.input.map((item: Record) => { - if (item.name === "") { - const { name: _n, ...rest } = item; - return rest; - } - return item; - }); - } - // #346/#637: Strip tools with empty name - // Clients sometimes forward tool definitions with empty names, causing - // upstream providers to reject with 400 "Invalid 'tools[0].name': empty string." - if (Array.isArray(body.tools)) { - body.tools = body.tools.filter((tool: Record) => { - // Built-in Responses API tool types (web_search, file_search, computer, etc.) - // are identified solely by their `type` field and carry no name — preserve them. - const toolType = typeof tool.type === "string" ? tool.type : ""; - if (toolType && toolType !== "function" && !tool.function && tool.name === undefined) { - return true; - } - const fn = tool.function as Record | undefined; - const name = fn?.name ?? tool.name; - return name && String(name).trim().length > 0; - }); - - // Sanitize OpenAI-format function tool schemas before they reach strict - // upstream JSON Schema validators (e.g. Moonshot AI behind - // opencode-go/kimi-k2.6). See toolSchemaSanitizer.ts for the specific bug. - // sanitizeOpenAITool is safe to call on any input — it no-ops non-function - // tools (e.g. Responses API built-ins) and non-object values. - body.tools = body.tools.map((tool) => sanitizeOpenAITool(tool) as (typeof body.tools)[number]); + const cacheHit = await checkSemanticCache({ + semanticCacheEnabled, + body, + clientRawRequest, + model, + provider, + stream: !!stream, + reqLogger, + effectiveServiceTier, + connectionId, + startTime, + log, + persistAttemptLogs, + }); + if (cacheHit) { + return cacheHit; } + body = sanitizeChatRequestBody(body, sourceFormat, targetFormat); const memoryOwnerId = resolveMemoryOwnerId(apiKeyInfo as Record | null); - const memorySettings = memoryOwnerId - ? await getMemorySettings().catch(() => DEFAULT_MEMORY_SETTINGS) - : null; - - if ( - memoryOwnerId && - memorySettings && - shouldInjectMemory(body as Parameters[0], { - enabled: memorySettings.enabled && memorySettings.maxTokens > 0, - }) - ) { - try { - // Plan 21 FAIL #1 fix: extract the last user message and pass it as - // `query`. Without this, `config.query` is undefined in retrieveMemories - // and the semantic/hybrid branches (sqlite-vec + RRF, and Qdrant - // tier-2) never fire from the chat hot path — they only fire in the - // Playground (retrievePreview, which gets `query` as a positional arg). - const lastUserQuery = ((): string => { - // Responses API item types that are NOT user input — never accept - // their text as the retrieval query (e.g. function_call_output is the - // tool's reply, reasoning is the model's chain of thought). - const NON_USER_TYPES = new Set([ - "function_call", - "function_call_output", - "tool_call", - "tool_call_output", - "reasoning", - "computer_call", - "computer_call_output", - "web_search_call", - "file_search_call", - ]); - - function pickFrom(arr: unknown[]): string { - for (let i = arr.length - 1; i >= 0; i--) { - const item = arr[i] as Record | undefined; - if (!item) continue; - // Chat API: only role==="user" items. Responses API items often - // have type instead of role — skip non-user types like - // function_call_output so the tool's reply doesn't leak into the - // memory query. - if (item.role !== undefined && item.role !== "user") continue; - if (item.role === undefined && typeof item.type === "string") { - if (NON_USER_TYPES.has(item.type)) continue; - } - const content = item.content ?? item.text; - if (typeof content === "string" && content.trim().length > 0) { - return content; - } - if (Array.isArray(content)) { - const parts: string[] = []; - for (const p of content) { - if (typeof p === "string") { - parts.push(p); - } else if (p && typeof p === "object") { - const pp = p as Record; - // Skip non-text content parts (image_url, tool_use, etc.) - const ptype = typeof pp.type === "string" ? pp.type : ""; - if ( - ptype && - ptype !== "text" && - ptype !== "input_text" && - ptype !== "output_text" - ) { - continue; - } - const t = pp.text ?? pp.input_text; - if (typeof t === "string") parts.push(t); - } - } - if (parts.length > 0) return parts.join(" ").trim(); - } - } - return ""; - } - const b = body as Record; - if (Array.isArray(b.messages)) { - const r = pickFrom(b.messages); - if (r) return r; - } - if (Array.isArray(b.input)) { - const r = pickFrom(b.input); - if (r) return r; - } - return ""; - })(); - - const memories = await retrieveMemories( - memoryOwnerId, - toMemoryRetrievalConfig(memorySettings, { query: lastUserQuery }) - ); - if (memories.length > 0) { - const injected = injectMemory( - body as Parameters[0], - memories, - provider - ); - body = injected as typeof body; - log?.debug?.("MEMORY", `Injected ${memories.length} memories for key=${memoryOwnerId}`); - } - } catch (memErr) { - log?.debug?.( - "MEMORY", - `Memory injection skipped: ${memErr instanceof Error ? memErr.message : String(memErr)}` - ); - } - } - - if (memoryOwnerId && memorySettings?.skillsEnabled) { - const existingTools = Array.isArray(body.tools) ? body.tools : []; - const mergedTools = injectSkills({ - provider: getSkillsProviderForFormat(sourceFormat), - existingTools, - apiKeyId: memoryOwnerId, - model: typeof effectiveModel === "string" ? effectiveModel : undefined, - sourceFormat, - targetFormat, - backgroundReason, - messages: Array.isArray(body.messages) - ? body.messages - : Array.isArray(body.input) - ? body.input - : undefined, - }); - - if (mergedTools.length > existingTools.length) { - body = { - ...body, - tools: mergedTools, - }; - log?.debug?.("SKILLS", `Injected ${mergedTools.length - existingTools.length} skills`); - } - } - - trace("post_injection", { provider, model }); + const injectionResult = await injectMemoryAndSkills({ + body, + memoryOwnerId, + provider, + effectiveModel, + sourceFormat, + targetFormat, + backgroundReason, + log, + }); + body = injectionResult.body; + const memorySettings = injectionResult.memorySettings; // Translate request (pass reqLogger for intermediate logging) // ── Proactive Context Compression (Phase 4) ── @@ -5490,6 +5232,10 @@ export async function handleChatCore({ } // ── Phase 9.2: Save for idempotency ── + // The idempotency *check* moved into checkIdempotencyCache() during the + // chatCore modularization (#3598); re-derive the key here for the save path. + // getIdempotencyKey is pure (reads idempotency-key/x-request-id headers). + const idempotencyKey = getIdempotencyKey(clientRawRequest?.headers); saveIdempotency(idempotencyKey, translatedResponse, 200); reqLogger.logConvertedResponse(translatedResponse); persistAttemptLogs({ diff --git a/open-sse/handlers/chatCore/idempotency.ts b/open-sse/handlers/chatCore/idempotency.ts new file mode 100644 index 0000000000..fc4c347d6e --- /dev/null +++ b/open-sse/handlers/chatCore/idempotency.ts @@ -0,0 +1,55 @@ +import { getIdempotencyKey, checkIdempotency } from "@/lib/idempotencyLayer"; +import { calculateCost } from "@/lib/usage/costCalculator"; +import { buildOmniRouteResponseMetaHeaders } from "@/domain/omnirouteResponseMeta"; + +export async function checkIdempotencyCache({ + clientRawRequest, + provider, + model, + effectiveServiceTier, + startTime, + log, +}: { + clientRawRequest: unknown; + provider: string; + model: string; + effectiveServiceTier: unknown; + startTime: number; + log: unknown; +}) { + const idempotencyKey = getIdempotencyKey(clientRawRequest?.headers); + const cachedIdemp = checkIdempotency(idempotencyKey); + if (cachedIdemp) { + log?.debug?.("IDEMPOTENCY", `Hit for key=${idempotencyKey?.slice(0, 12)}...`); + const idempotentUsage = + cachedIdemp.response && typeof cachedIdemp.response === "object" + ? ((cachedIdemp.response as Record).usage as + | Record + | undefined) + : undefined; + const idempotentCost = idempotentUsage + ? await calculateCost(provider, model, idempotentUsage as Record, { + serviceTier: effectiveServiceTier, + }) + : 0; + return { + success: true, + response: new Response(JSON.stringify(cachedIdemp.response), { + status: cachedIdemp.status, + headers: { + "Content-Type": "application/json", + "X-OmniRoute-Idempotent": "true", + ...buildOmniRouteResponseMetaHeaders({ + provider, + model, + cacheHit: false, + latencyMs: Date.now() - startTime, + usage: idempotentUsage, + costUsd: idempotentCost, + }), + }, + }), + }; + } + return null; +} diff --git a/open-sse/handlers/chatCore/memorySkillsInjection.ts b/open-sse/handlers/chatCore/memorySkillsInjection.ts new file mode 100644 index 0000000000..28719d810d --- /dev/null +++ b/open-sse/handlers/chatCore/memorySkillsInjection.ts @@ -0,0 +1,159 @@ +import { retrieveMemories } from "@/lib/memory/retrieval"; +import { getMemorySettings, DEFAULT_MEMORY_SETTINGS, toMemoryRetrievalConfig } from "@/lib/memory/settings"; +import { injectMemory, shouldInjectMemory } from "@/lib/memory/injection"; +import { injectSkills } from "@/lib/skills/injection"; +import { FORMATS } from "../../translator/formats.ts"; + +export function getSkillsProviderForFormat(format: string): "openai" | "anthropic" | "google" | "other" { + switch (format) { + case FORMATS.CLAUDE: + return "anthropic"; + case FORMATS.GEMINI: + return "google"; + default: + return "openai"; + } +} + +export async function injectMemoryAndSkills({ + body, + memoryOwnerId, + provider, + effectiveModel, + sourceFormat, + targetFormat, + backgroundReason, + log, +}: { + body: Record; + memoryOwnerId: string | null; + provider: string; + effectiveModel: string; + sourceFormat: string; + targetFormat: string; + backgroundReason: string | null; + log: unknown; +}) { + const memorySettings = memoryOwnerId + ? await getMemorySettings().catch(() => DEFAULT_MEMORY_SETTINGS) + : null; + + if ( + memoryOwnerId && + memorySettings && + shouldInjectMemory(body as Parameters[0], { + enabled: memorySettings.enabled && memorySettings.maxTokens > 0, + }) + ) { + try { + const lastUserQuery = ((): string => { + const NON_USER_TYPES = new Set([ + "function_call", + "function_call_output", + "tool_call", + "tool_call_output", + "reasoning", + "computer_call", + "computer_call_output", + "web_search_call", + "file_search_call", + ]); + + function pickFrom(arr: unknown[]): string { + for (let i = arr.length - 1; i >= 0; i--) { + const item = arr[i] as Record | undefined; + if (!item) continue; + if (item.role !== undefined && item.role !== "user") continue; + if (item.role === undefined && typeof item.type === "string") { + if (NON_USER_TYPES.has(item.type)) continue; + } + const content = item.content ?? item.text; + if (typeof content === "string" && content.trim().length > 0) { + return content; + } + if (Array.isArray(content)) { + const parts: string[] = []; + for (const p of content) { + if (typeof p === "string") { + parts.push(p); + } else if (p && typeof p === "object") { + const pp = p as Record; + const ptype = typeof pp.type === "string" ? pp.type : ""; + if ( + ptype && + ptype !== "text" && + ptype !== "input_text" && + ptype !== "output_text" + ) { + continue; + } + const t = pp.text ?? pp.input_text; + if (typeof t === "string") parts.push(t); + } + } + if (parts.length > 0) return parts.join(" ").trim(); + } + } + return ""; + } + + if (Array.isArray(body.messages)) { + const r = pickFrom(body.messages); + if (r) return r; + } + if (Array.isArray(body.input)) { + const r = pickFrom(body.input); + if (r) return r; + } + return ""; + })(); + + const memories = await retrieveMemories( + memoryOwnerId, + toMemoryRetrievalConfig(memorySettings, { query: lastUserQuery }) + ); + if (memories.length > 0) { + const injected = injectMemory( + body as Parameters[0], + memories, + provider + ); + body = injected as typeof body; + log?.debug?.("MEMORY", `Injected ${memories.length} memories for key=${memoryOwnerId}`); + } + } catch (memErr) { + log?.debug?.( + "MEMORY", + `Memory injection skipped: ${memErr instanceof Error ? memErr.message : String(memErr)}` + ); + } + } + + if (memoryOwnerId && memorySettings?.skillsEnabled) { + const existingTools = Array.isArray(body.tools) ? body.tools : []; + const mergedTools = injectSkills({ + provider: getSkillsProviderForFormat(sourceFormat), + existingTools, + apiKeyId: memoryOwnerId, + model: typeof effectiveModel === "string" ? effectiveModel : undefined, + sourceFormat, + targetFormat, + backgroundReason, + messages: Array.isArray(body.messages) + ? body.messages + : Array.isArray(body.input) + ? body.input + : undefined, + }); + + if (mergedTools.length > existingTools.length) { + body = { + ...body, + tools: mergedTools, + }; + log?.debug?.("SKILLS", `Injected ${mergedTools.length - existingTools.length} skills`); + } + } + + return { body, memorySettings }; +} diff --git a/open-sse/handlers/chatCore/sanitization.ts b/open-sse/handlers/chatCore/sanitization.ts new file mode 100644 index 0000000000..62b43615ed --- /dev/null +++ b/open-sse/handlers/chatCore/sanitization.ts @@ -0,0 +1,63 @@ +import { FORMATS } from "../../translator/formats.ts"; +import { sanitizeOpenAITool } from "../../services/toolSchemaSanitizer.ts"; + +export function sanitizeChatRequestBody( + body: Record, + sourceFormat: string, + targetFormat: string +): Record { + const prefersResponsesTokenField = + sourceFormat === FORMATS.OPENAI_RESPONSES || targetFormat === FORMATS.OPENAI_RESPONSES; + + if (prefersResponsesTokenField) { + if (body.max_output_tokens === undefined) { + if (body.max_completion_tokens !== undefined) { + body.max_output_tokens = body.max_completion_tokens; + delete body.max_completion_tokens; + } else if (body.max_tokens !== undefined) { + body.max_output_tokens = body.max_tokens; + delete body.max_tokens; + } + } + } else if (body.max_output_tokens !== undefined) { + if (body.max_tokens === undefined) { + body.max_tokens = body.max_output_tokens; + } + delete body.max_output_tokens; + } + + if (Array.isArray(body.messages)) { + body.messages = body.messages.map((msg: Record) => { + if (msg.name === "") { + const { name: _n, ...rest } = msg; + return rest; + } + return msg; + }); + } + if (Array.isArray(body.input)) { + body.input = body.input.map((item: Record) => { + if (item.name === "") { + const { name: _n, ...rest } = item; + return rest; + } + return item; + }); + } + + if (Array.isArray(body.tools)) { + body.tools = body.tools.filter((tool: Record) => { + const toolType = typeof tool.type === "string" ? tool.type : ""; + if (toolType && toolType !== "function" && !tool.function && tool.name === undefined) { + return true; + } + const fn = tool.function as Record | undefined; + const name = fn?.name ?? tool.name; + return name && String(name).trim().length > 0; + }); + + body.tools = body.tools.map((tool) => sanitizeOpenAITool(tool) as (typeof body.tools)[number]); + } + + return body; +} diff --git a/open-sse/handlers/chatCore/semanticCache.ts b/open-sse/handlers/chatCore/semanticCache.ts new file mode 100644 index 0000000000..19d7b6021c --- /dev/null +++ b/open-sse/handlers/chatCore/semanticCache.ts @@ -0,0 +1,91 @@ +import { + generateSignature, + getCachedResponse, + isCacheableForRead, +} from "@/lib/semanticCache"; +import { calculateCost } from "@/lib/usage/costCalculator"; +import { trackPendingRequest } from "@/lib/usageDb"; +import { synthesizeOpenAiSseFromJson } from "../../utils/jsonToSse.ts"; +import { buildOmniRouteResponseMetaHeaders } from "@/domain/omnirouteResponseMeta"; +import { extractUsageFromResponse } from "../usageExtractor.ts"; +import { OMNIROUTE_RESPONSE_HEADERS } from "@/shared/constants/headers"; + +export async function checkSemanticCache({ + semanticCacheEnabled, + body, + clientRawRequest, + model, + provider, + stream, + reqLogger, + effectiveServiceTier, + connectionId, + startTime, + log, + persistAttemptLogs, +}: { + semanticCacheEnabled: boolean; + body: Record; + clientRawRequest: unknown; + model: string; + provider: string; + stream: boolean; + reqLogger: unknown; + effectiveServiceTier: unknown; + connectionId: string | null; + startTime: number; + log: unknown; + persistAttemptLogs: (args: unknown) => void; +}) { + if (semanticCacheEnabled && isCacheableForRead(body, clientRawRequest?.headers)) { + const signature = generateSignature( + model, + body.messages ?? body.input, + body.temperature, + body.top_p + ); + const cached = getCachedResponse(signature); + if (cached) { + log?.debug?.("CACHE", `Semantic cache HIT for ${model} (stream=${stream})`); + reqLogger.logConvertedResponse(cached as Record); + const cachedUsage = + extractUsageFromResponse(cached as Record, provider) || + ((cached as Record)?.usage as Record | undefined); + const cachedCost = cachedUsage + ? await calculateCost(provider, model, cachedUsage as Record, { + serviceTier: effectiveServiceTier, + }) + : 0; + persistAttemptLogs({ + status: 200, + tokens: (cached as Record)?.usage, + responseBody: cached, + providerRequest: null, + providerResponse: null, + clientResponse: cached, + cacheSource: "semantic", + }); + trackPendingRequest(model, provider, connectionId, false); + const cachedSse = stream ? synthesizeOpenAiSseFromJson(JSON.stringify(cached)) : ""; + const cacheHitMetaHeaders = buildOmniRouteResponseMetaHeaders({ + provider, + model, + cacheHit: true, + latencyMs: Date.now() - startTime, + usage: cachedUsage, + costUsd: cachedCost, + }); + return { + success: true, + response: new Response(cachedSse || JSON.stringify(cached), { + headers: { + "Content-Type": cachedSse ? "text/event-stream" : "application/json", + [OMNIROUTE_RESPONSE_HEADERS.cache]: "HIT", + ...cacheHitMetaHeaders, + }, + }), + }; + } + } + return null; +} \ No newline at end of file