diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index cf75785f0a..d762d8bd2d 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -147,7 +147,11 @@ import { getExplicitModelOutputCap, resolveInputTokenCapForGate, } from "@/lib/modelCapabilities.ts"; -import { checkRequestCapabilityFit, deriveRequestCapabilityRequirements, buildCapabilityMismatchMessage } from "@/shared/constants/capabilities/capabilityFilter.ts"; +import { + checkRequestCapabilityFit, + deriveRequestCapabilityRequirements, + buildCapabilityMismatchMessage, +} from "@/shared/constants/capabilities/capabilityFilter.ts"; import { isFeatureFlagEnabled } from "@/shared/utils/featureFlags.ts"; import { toPositiveInteger } from "../services/reasoningTokenBuffer.ts"; import { normalizeThinkingForModel } from "@/shared/constants/modelSpecs.ts"; @@ -421,6 +425,7 @@ export async function handleChatCore({ comboStrategy = null, isCombo = false, routingComboId = null, + sessionAffinityKey = null, comboStepId = null, comboExecutionKey = null, cachedSettings = null, @@ -884,6 +889,10 @@ export async function handleChatCore({ "x-omniroute-session-id" )) || null; const pipelineSessionId = explicitSessionIdHeader || skillRequestId; + const reasoningReplaySessionKey = sessionAffinityKey || explicitSessionIdHeader; + const reasoningCacheScope = reasoningReplaySessionKey + ? `api-key:${String(apiKeyInfo?.id ?? "local")}\x1f${String(reasoningReplaySessionKey)}` + : null; // persistAttemptLogs extracted to chatCore/attemptLogging.ts (#3501); bind the per-request context // once so the 16 call sites keep passing only the per-attempt args (byte-identical). const persistAttemptLogs = (args: PersistAttemptLogsArgs) => @@ -2042,6 +2051,7 @@ export async function handleChatCore({ preserveDeveloperRole, preserveCacheControl, copilotClient: copilotCompatibleReasoning, + reasoningCacheScope, } ); } @@ -2206,6 +2216,7 @@ export async function handleChatCore({ preserveCacheControl, signatureNamespace: connectionId, copilotClient: copilotCompatibleReasoning, + reasoningCacheScope, ...(preCompressionBody ? { preCompressionBody } : {}), } ); @@ -2635,8 +2646,11 @@ export async function handleChatCore({ } // === /Quota Share enforcement PRE-hook === if (isFeatureFlagEnabled("CAPABILITY_FILTER_ENABLED")) { - const fit = checkRequestCapabilityFit(getResolvedModelCapabilities({ provider, model: effectiveModel }), - deriveRequestCapabilityRequirements(body as Record), provider); + const fit = checkRequestCapabilityFit( + getResolvedModelCapabilities({ provider, model: effectiveModel }), + deriveRequestCapabilityRequirements(body as Record), + provider + ); if (!fit.compatible) { const msg = buildCapabilityMismatchMessage(fit.terminalReason!, provider, effectiveModel); log?.warn?.("CAPABILITY", msg); @@ -4378,16 +4392,23 @@ export async function handleChatCore({ // Reasoning Replay Cache (#1628): Capture reasoning_content from non-streaming responses // with tool_calls so it can be replayed on subsequent turns (DeepSeek V4, Kimi K2, etc.) try { - const firstChoice = translatedResponse?.choices?.[0]; + const cacheResponse = translatedResponse?.choices?.[0] + ? translatedResponse + : needsTranslation(responsePayloadFormat, FORMATS.OPENAI) + ? translateNonStreamingResponse( + responseBody, + responsePayloadFormat, + FORMATS.OPENAI, + responseToolNameMap + ) + : responseBody; + const firstChoice = cacheResponse?.choices?.[0]; const msg = firstChoice?.message; - // The response being cached now will be replayed as history on the *next* - // turn, where the read side (translator/index.ts) keys the lookup by the - // message's real position in that future `messages` array — i.e. right - // after everything the client sent this turn. - const bodyMessages = (body as { messages?: unknown[] } | null | undefined)?.messages; + const historyMessages = (translatedBody as { messages?: unknown[] } | null | undefined) + ?.messages; cacheReasoningFromAssistantMessage(msg, provider, model, { - requestId: skillRequestId, - messageIndex: Array.isArray(bodyMessages) ? bodyMessages.length : 0, + scope: reasoningCacheScope, + historyMessages: Array.isArray(historyMessages) ? historyMessages : [], }); } catch { // Cache capture is non-critical — never block the response @@ -4813,14 +4834,24 @@ export async function handleChatCore({ if (normalizedStreamStatus === 200 && streamResponseBody) { try { const streamBody = streamResponseBody as Record; - const choices = streamBody.choices as { message?: Record }[] | undefined; + const cacheStreamBody = Array.isArray(streamBody.choices) + ? streamBody + : needsTranslation(clientResponseFormat, FORMATS.OPENAI) + ? (translateNonStreamingResponse( + streamBody, + clientResponseFormat, + FORMATS.OPENAI, + responseToolNameMap + ) as Record) + : streamBody; + const choices = cacheStreamBody.choices as + { message?: Record }[] | undefined; const msg = choices?.[0]?.message; - // See the non-streaming capture above: messageIndex must match the - // position this message will occupy in the *next* turn's history. - const bodyMessages = (body as { messages?: unknown[] } | null | undefined)?.messages; + const historyMessages = (translatedBody as { messages?: unknown[] } | null | undefined) + ?.messages; cacheReasoningFromAssistantMessage(msg, provider, model, { - requestId: skillRequestId, - messageIndex: Array.isArray(bodyMessages) ? bodyMessages.length : 0, + scope: reasoningCacheScope, + historyMessages: Array.isArray(historyMessages) ? historyMessages : [], }); } catch { // Cache capture is non-critical — never block the stream diff --git a/open-sse/services/reasoningCache.ts b/open-sse/services/reasoningCache.ts index dc81b14e69..bdf870e3fb 100644 --- a/open-sse/services/reasoningCache.ts +++ b/open-sse/services/reasoningCache.ts @@ -13,6 +13,7 @@ * @see Issue #1628 */ +import { createHash } from "node:crypto"; import { clearAllReasoningCache, cleanupExpiredReasoning, @@ -137,8 +138,8 @@ type AssistantMessageLike = { }; type AssistantMessageCacheContext = { - requestId?: string; - messageIndex?: number; + scope?: string; + historyMessages?: AssistantMessageLike[]; }; type ToolCallLike = { @@ -234,8 +235,79 @@ export function cacheReasoningByKey( } } -function buildAssistantMessageCacheKey(requestId: string, messageIndex: number): string { - return `request:${requestId}:message:${messageIndex}`; +function stableCacheValue(value: unknown): unknown { + if (Array.isArray(value)) return value.map(stableCacheValue); + if (!value || typeof value !== "object") return value; + + const record = value as Record; + return Object.fromEntries( + Object.keys(record) + .filter((key) => key !== "reasoning" && key !== "reasoning_content") + .sort() + .map((key) => [key, stableCacheValue(record[key])]) + ); +} + +function canonicalizeMessageContent(content: unknown): unknown { + if (!Array.isArray(content)) return stableCacheValue(content ?? null); + + const textParts: string[] = []; + for (const part of content) { + if (typeof part === "string") { + textParts.push(part); + continue; + } + if (!part || typeof part !== "object") return stableCacheValue(content); + const record = part as Record; + if ( + (record.type === "text" || record.type === "input_text" || record.type === "output_text") && + typeof record.text === "string" + ) { + textParts.push(record.text); + continue; + } + return stableCacheValue(content); + } + return textParts.join(""); +} + +function canonicalizeHistoryMessage(message: AssistantMessageLike): unknown { + const record = message as Record; + const toolCalls = Array.isArray(record.tool_calls) + ? record.tool_calls.map((toolCall) => { + const call = toolCall as Record; + const fn = (call.function ?? {}) as Record; + return stableCacheValue({ + type: call.type, + function: { name: fn.name, arguments: fn.arguments }, + }); + }) + : undefined; + return stableCacheValue({ + role: record.role, + name: record.name, + content: canonicalizeMessageContent(record.content), + tool_calls: toolCalls, + }); +} + +export function buildAssistantMessageCacheKey( + scope: string | null | undefined, + messages: AssistantMessageLike[], + messageIndex: number +): string { + const normalizedScope = scope?.trim(); + if (!normalizedScope || !Number.isInteger(messageIndex) || messageIndex < 0) return ""; + const message = messages[messageIndex]; + if (!message || message.role !== "assistant") return ""; + + const transcript = messages.slice(0, messageIndex + 1).map(canonicalizeHistoryMessage); + const digest = createHash("sha256") + .update(normalizedScope) + .update("\x1f") + .update(JSON.stringify(transcript)) + .digest("hex"); + return `conversation:${digest}`; } /** @@ -282,18 +354,15 @@ export function cacheReasoningFromAssistantMessage( .filter((id) => id.length > 0) : []; if (toolCallIds.length === 0) { - const requestId = context?.requestId?.trim(); - const messageIndex = context?.messageIndex; - if (!requestId || typeof messageIndex !== "number" || !Number.isInteger(messageIndex)) { - return 0; - } + const scope = context?.scope?.trim(); + const historyMessages = context?.historyMessages; + if (!scope || !Array.isArray(historyMessages)) return 0; - cacheReasoningByKey( - buildAssistantMessageCacheKey(requestId, messageIndex), - provider, - model, - reasoning - ); + const messages = [...historyMessages, message]; + const cacheKey = buildAssistantMessageCacheKey(scope, messages, messages.length - 1); + if (!cacheKey) return 0; + + cacheReasoningByKey(cacheKey, provider, model, reasoning); return 1; } @@ -329,7 +398,8 @@ export function lookupReasoning(toolCallId: string): string | null { } // 2. Fallback to DB - let dbResult: { reasoning: string; provider: string; model: string } | null = null; + let dbResult: { reasoning: string; provider: string; model: string; expiresAt: string } | null = + null; try { dbResult = getReasoningCache(toolCallId); } catch { @@ -341,6 +411,11 @@ export function lookupReasoning(toolCallId: string): string | null { misses++; return null; } + const persistedExpiresAt = Date.parse(dbResult.expiresAt); + if (!Number.isFinite(persistedExpiresAt) || persistedExpiresAt <= Date.now()) { + misses++; + return null; + } hits++; let promotedReasoning = dbResult.reasoning; if (promotedReasoning.length > MAX_ENTRY_BYTES) { @@ -351,7 +426,7 @@ export function lookupReasoning(toolCallId: string): string | null { reasoning: promotedReasoning, provider: dbResult.provider, model: dbResult.model, - expiresAt: Date.now() + TTL_MS, + expiresAt: persistedExpiresAt, createdAt: Date.now(), }); return promotedReasoning; diff --git a/open-sse/translator/index.ts b/open-sse/translator/index.ts index 78c0bce43b..979fce6e5f 100644 --- a/open-sse/translator/index.ts +++ b/open-sse/translator/index.ts @@ -30,6 +30,7 @@ import { getResolvedModelCapabilities, supportsReasoning } from "../services/mod import { normalizeRoles } from "../services/roleNormalizer.ts"; import { hoistLeadingSystemMessage } from "./helpers/strictSystemHoist.ts"; import { + buildAssistantMessageCacheKey, lookupReasoning, recordReplay, requiresReasoningReplay, @@ -149,25 +150,6 @@ function normalizeOpenAIResponsesRequest(body) { return normalized; } -function getReasoningCacheRequestId(body: Record | null | undefined): string { - if (!body || typeof body !== "object") return ""; - - const requestId = - body._reasoningCacheRequestId ?? - body.reasoningCacheRequestId ?? - body.request_id ?? - body.requestId; - return typeof requestId === "string" ? requestId.trim() : ""; -} - -function getAssistantMessageCacheKey( - body: Record | null | undefined, - messageIndex: number -): string { - const requestId = getReasoningCacheRequestId(body); - return requestId ? `request:${requestId}:message:${messageIndex}` : ""; -} - function hasNonEmptyReasoningContent(message: Record): boolean { return typeof message.reasoning_content === "string" && message.reasoning_content.length > 0; } @@ -253,6 +235,7 @@ export function translateRequest( preserveCacheControl?: boolean; signatureNamespace?: string | null; preCompressionBody?: Record | null; + reasoningCacheScope?: string | null; /** UA-detected GitHub Copilot client. Forwarded to translators via the * transient `_copilotClient` credential flag (see openai-responses → openai). */ copilotClient?: boolean; @@ -268,13 +251,6 @@ export function translateRequest( const normalizedModel = String(model ?? ""); const isKimiCoding = normalizedProvider === "kimi-coding" || normalizedProvider === "kimi-coding-apikey"; - const requiresExplicitReasoningReplay = requiresReasoningReplay({ - provider: normalizedProvider, - model: normalizedModel, - allowLegacyFallback: false, - }); - const preserveResponsesReasoning = - sourceFormat === FORMATS.OPENAI_RESPONSES && requiresExplicitReasoningReplay; // Phase 2: Apply thinking budget control before normalization result = applyThinkingBudget(result); @@ -285,6 +261,29 @@ export function translateRequest( // Normalize thinking config: remove if lastMessage is not user normalizeThinkingConfig(result); + // Resolve the replay contract before Responses input is converted: conversion + // must know whether reasoning items are protocol history rather than display metadata. + const resolvedCapabilities = getResolvedModelCapabilities({ + provider: normalizedProvider, + model: normalizedModel, + }); + const replayRequirements = { + provider: normalizedProvider, + model: normalizedModel, + thinkingEnabled: hasThinkingConfig(result), + supportsReasoning: supportsReasoning({ + provider: normalizedProvider, + model: normalizedModel, + }), + interleavedField: resolvedCapabilities?.interleavedField ?? null, + }; + const isReasoner = requiresReasoningReplay(replayRequirements); + const requiresExplicitReasoningReplay = requiresReasoningReplay({ + ...replayRequirements, + allowLegacyFallback: false, + }); + const preserveResponsesReasoning = sourceFormat === FORMATS.OPENAI_RESPONSES && isReasoner; + // Ensure tool_calls have id; optionally normalize to 9-char for providers like Mistral ensureToolCallIds(result, { use9CharId }); @@ -424,24 +423,6 @@ export function translateRequest( } } - // Resolve reasoning-replay status up-front: it gates both the reasoning_content - // strip in filterToOpenAIFormat below (#4849 must NOT strip client reasoning for - // replay providers) and the cache re-injection further down. - const resolvedCapabilities = getResolvedModelCapabilities({ - provider: normalizedProvider, - model: normalizedModel, - }); - const isReasoner = requiresReasoningReplay({ - provider: normalizedProvider, - model: normalizedModel, - thinkingEnabled: hasThinkingConfig(result), - supportsReasoning: supportsReasoning({ - provider: normalizedProvider, - model: normalizedModel, - }), - interleavedField: resolvedCapabilities?.interleavedField ?? null, - }); - // Always normalize to clean OpenAI format when target is OpenAI // This handles hybrid requests (e.g., OpenAI messages + Claude tools) if (targetFormat === FORMATS.OPENAI) { @@ -653,7 +634,11 @@ export function translateRequest( const cacheKey = hasToolCalls ? msg.tool_calls[0]?.id - : getAssistantMessageCacheKey(result, messageIndex); + : buildAssistantMessageCacheKey( + options?.reasoningCacheScope, + result.messages, + messageIndex + ); if (cacheKey) { const cached = lookupReasoning(cacheKey); if (cached) { diff --git a/open-sse/translator/request/openai-responses.ts b/open-sse/translator/request/openai-responses.ts index 9c2822dc02..df65ab5f3c 100644 --- a/open-sse/translator/request/openai-responses.ts +++ b/open-sse/translator/request/openai-responses.ts @@ -235,21 +235,23 @@ export function openaiResponsesToOpenAIRequest( if (itemType === "message") { const role = toString(item.role); - // Flush pending assistant message with tool calls - if (currentAssistantMsg) { - messages.push(currentAssistantMsg); - currentAssistantMsg = null; - } - if (role !== "assistant" && pendingReasoningContent) { - messages.push({ - role: "assistant", - content: null, - reasoning_content: pendingReasoningContent, - }); - pendingReasoningContent = ""; + + if (role !== "assistant") { + if (currentAssistantMsg) { + messages.push(currentAssistantMsg); + currentAssistantMsg = null; + } + if (pendingReasoningContent) { + messages.push({ + role: "assistant", + content: null, + reasoning_content: pendingReasoningContent, + }); + pendingReasoningContent = ""; + } } - // Flush pending tool results + // Flush pending tool results before the next explicit message boundary. if (pendingToolResults.length > 0) { for (const toolResult of pendingToolResults) { messages.push(toolResult); @@ -292,12 +294,29 @@ export function openaiResponsesToOpenAIRequest( }) : item.content; - const message: JsonRecord = { role, content }; - if (role === "assistant" && pendingReasoningContent) { - message.reasoning_content = pendingReasoningContent; - pendingReasoningContent = ""; + if (role === "assistant") { + if (!currentAssistantMsg) { + currentAssistantMsg = { role, content }; + } else if (currentAssistantMsg.content == null && content != null) { + currentAssistantMsg.content = content; + } else if (content != null) { + const existingContent = currentAssistantMsg.content; + currentAssistantMsg.content = [ + ...(Array.isArray(existingContent) ? existingContent : [existingContent]), + ...(Array.isArray(content) ? content : [content]), + ]; + } + if (pendingReasoningContent) { + currentAssistantMsg.reasoning_content = appendReasoningContent( + currentAssistantMsg.reasoning_content, + pendingReasoningContent + ); + pendingReasoningContent = ""; + } + continue; } - messages.push(message); + + messages.push({ role, content }); continue; } diff --git a/src/lib/db/reasoningCache.ts b/src/lib/db/reasoningCache.ts index ae36c0bb9b..885bf5d141 100644 --- a/src/lib/db/reasoningCache.ts +++ b/src/lib/db/reasoningCache.ts @@ -103,18 +103,32 @@ export function setReasoningCache( */ export function getReasoningCache( toolCallId: string -): { reasoning: string; provider: string; model: string } | null { +): { reasoning: string; provider: string; model: string; expiresAt: string } | null { const db = getDbInstance(); const row = db .prepare( - `SELECT reasoning, provider, model FROM reasoning_cache + `SELECT reasoning, provider, model, expires_at FROM reasoning_cache WHERE tool_call_id = ? AND ${EXPIRES_AT_EPOCH_SQL} > unixepoch('now')` ) - .get(toolCallId) as { reasoning: string; provider: string; model: string } | undefined; + .get(toolCallId) as + | { + reasoning: string; + provider: string; + model: string; + expires_at: number | string; + } + | undefined; const PLACEHOLDER = "(prior reasoning summary unavailable)"; - if (row && row.reasoning && row.reasoning.trim() === PLACEHOLDER) return null; // ponytail: never replay the placeholder - return row ?? null; + if (row && row.reasoning && row.reasoning.trim() === PLACEHOLDER) return null; + return row + ? { + reasoning: row.reasoning, + provider: row.provider, + model: row.model, + expiresAt: epochSecondsToIso(row.expires_at), + } + : null; } /** diff --git a/src/sse/handlers/chat.ts b/src/sse/handlers/chat.ts index e1faaa115a..446dbb61ea 100644 --- a/src/sse/handlers/chat.ts +++ b/src/sse/handlers/chat.ts @@ -1562,6 +1562,7 @@ async function handleSingleModelChat( correlationId: runtimeOptions?.correlationId ?? null, modelPinned: runtimeOptions?.modelPinned ?? false, routingComboId: runtimeOptions?.routingComboId ?? null, + sessionAffinityKey: runtimeOptions.sessionAffinityKey ?? null, }); } catch (error) { releaseOAuthSession(); diff --git a/src/sse/handlers/chatHelpers.ts b/src/sse/handlers/chatHelpers.ts index 3bfebaa277..c554cfe856 100644 --- a/src/sse/handlers/chatHelpers.ts +++ b/src/sse/handlers/chatHelpers.ts @@ -421,6 +421,7 @@ export async function executeChatWithBreaker({ correlationId = null, modelPinned = false, routingComboId = null, + sessionAffinityKey = null, }: ExecuteChatWithBreakerOptions): Promise { let tlsFingerprintUsed = false; const normalizedTrafficType: TrafficType = @@ -476,6 +477,7 @@ export async function executeChatWithBreaker({ correlationId, modelPinned, routingComboId, + sessionAffinityKey, skipResourcePressureGuard: true, onCredentialsRefreshed: async (newCreds: any) => { await updateProviderCredentials(credentials.connectionId, { diff --git a/tests/unit/chatcore-translation-paths.test.ts b/tests/unit/chatcore-translation-paths.test.ts index 0ea88229b4..ff13bba00f 100644 --- a/tests/unit/chatcore-translation-paths.test.ts +++ b/tests/unit/chatcore-translation-paths.test.ts @@ -29,6 +29,8 @@ const { clearModelLock, isModelLocked } = await import("../../open-sse/services/accountFallback.ts"); const { saveModelsDevCapabilities, clearModelsDevCapabilities } = await import("../../src/lib/modelsDevSync.ts"); +// Dynamic import is required after TEST_DATA_DIR is initialized above. +const { clearReasoningCacheAll } = await import("../../open-sse/services/reasoningCache.ts"); const { getBackgroundDegradationConfig, setBackgroundDegradationConfig, @@ -256,6 +258,7 @@ async function resetStorage() { clearIdempotency(); clearInflight(); clearModelsDevCapabilities(); + clearReasoningCacheAll(); setBackgroundDegradationConfig(originalBackgroundConfig); resetBackgroundStats(); globalThis.setTimeout = originalSetTimeout; @@ -305,6 +308,7 @@ async function invokeChatCore({ connectionId = null, onCredentialsRefreshed = null, onRequestSuccess = null, + sessionAffinityKey = null, }: any = {}) { const calls: any[] = []; @@ -348,6 +352,7 @@ async function invokeChatCore({ connectionId, apiKeyInfo, userAgent, + sessionAffinityKey, isCombo, comboStrategy, onCredentialsRefreshed, @@ -563,6 +568,178 @@ test("chatCore applies Responses input policy to openai-compatible targets", asy assert.equal(input.find((item) => item.type === "function_call")?.id, undefined); } }); + +test("chatCore replays no-tool reasoning across public Responses turns", async () => { + saveModelsDevCapabilities({ + deepseek: { + "deepseek-v4-pro": { + ...capabilityEntry(128_000), + reasoning: true, + interleaved_field: null, + }, + }, + }); + const sessionAffinityKey = "header:reasoning-replay-session"; + const apiKeyInfo = { id: "reasoning-replay-key" }; + const responseFactory = () => + new Response( + JSON.stringify({ + id: "chatcmpl-reasoning", + object: "chat.completion", + choices: [ + { + index: 0, + message: { + role: "assistant", + content: "Hello! How can I help?", + reasoning_content: "Authentic upstream reasoning", + }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 4, completion_tokens: 2, total_tokens: 6 }, + }), + { status: 200, headers: { "Content-Type": "application/json" } } + ); + + const first = await invokeChatCore({ + provider: "deepseek", + model: "deepseek-v4-pro", + endpoint: "/v1/responses", + body: { + model: "deepseek-v4-pro", + stream: false, + reasoning: { effort: "high" }, + input: [{ type: "message", role: "user", content: [{ type: "input_text", text: "hi" }] }], + }, + apiKeyInfo, + sessionAffinityKey, + responseFactory, + }); + assert.equal(first.result.success, true); + + const second = await invokeChatCore({ + provider: "deepseek", + model: "deepseek-v4-pro", + endpoint: "/v1/responses", + body: { + model: "deepseek-v4-pro", + stream: false, + reasoning: { effort: "high" }, + input: [ + { type: "message", role: "user", content: [{ type: "input_text", text: "hi" }] }, + { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "Hello! How can I help?" }], + }, + { + type: "message", + role: "user", + content: [{ type: "input_text", text: "tell me more" }], + }, + ], + }, + apiKeyInfo, + sessionAffinityKey, + responseFactory, + }); + + assert.equal(second.result.success, true); + assert.equal(second.call.body.messages[1].reasoning_content, "Authentic upstream reasoning"); +}); +test("chatCore captures streaming no-tool reasoning for Responses replay", async () => { + saveModelsDevCapabilities({ + deepseek: { + "deepseek-v4-pro": { + ...capabilityEntry(128_000), + reasoning: true, + interleaved_field: null, + }, + }, + }); + const sessionAffinityKey = "header:streaming-reasoning-replay-session"; + const apiKeyInfo = { id: "streaming-reasoning-replay-key" }; + const streamResponseFactory = () => + new Response( + [ + `data: ${JSON.stringify({ + id: "chatcmpl-stream-reasoning", + object: "chat.completion.chunk", + choices: [ + { + index: 0, + delta: { + role: "assistant", + reasoning_content: "Authentic streaming reasoning", + }, + }, + ], + })}`, + `data: ${JSON.stringify({ + id: "chatcmpl-stream-reasoning", + object: "chat.completion.chunk", + choices: [{ index: 0, delta: { content: "Streamed answer" } }], + })}`, + `data: ${JSON.stringify({ + id: "chatcmpl-stream-reasoning", + object: "chat.completion.chunk", + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + })}`, + "data: [DONE]", + "", + ].join("\n\n"), + { status: 200, headers: { "Content-Type": "text/event-stream" } } + ); + + const first = await invokeChatCore({ + provider: "deepseek", + model: "deepseek-v4-pro", + endpoint: "/v1/responses", + body: { + model: "deepseek-v4-pro", + stream: true, + reasoning: { effort: "high" }, + input: [{ type: "message", role: "user", content: [{ type: "input_text", text: "hi" }] }], + }, + apiKeyInfo, + sessionAffinityKey, + responseFactory: streamResponseFactory, + }); + assert.equal(first.result.success, true); + await first.result.response.text(); + await flushAsyncSideEffects(); + + const second = await invokeChatCore({ + provider: "deepseek", + model: "deepseek-v4-pro", + endpoint: "/v1/responses", + body: { + model: "deepseek-v4-pro", + stream: false, + reasoning: { effort: "high" }, + input: [ + { type: "message", role: "user", content: [{ type: "input_text", text: "hi" }] }, + { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "Streamed answer" }], + }, + { + type: "message", + role: "user", + content: [{ type: "input_text", text: "tell me more" }], + }, + ], + }, + apiKeyInfo, + sessionAffinityKey, + responseFactory: () => buildOpenAIResponse(false), + }); + + assert.equal(second.result.success, true); + assert.equal(second.call.body.messages[1].reasoning_content, "Authentic streaming reasoning"); +}); test("chatCore preserves opted-in encrypted reasoning for Codex", async () => { const { call, result } = await invokeChatCore({ provider: "codex", diff --git a/tests/unit/moonshot-k3.test.ts b/tests/unit/moonshot-k3.test.ts index 55ed7f0275..d85b67855b 100644 --- a/tests/unit/moonshot-k3.test.ts +++ b/tests/unit/moonshot-k3.test.ts @@ -18,6 +18,7 @@ import { normalizeMoonshotRequest } from "../../open-sse/executors/moonshot.ts"; import { cacheReasoning, cacheReasoningByKey, + buildAssistantMessageCacheKey, deleteReasoningCacheEntry, requiresReasoningReplay, } from "../../open-sse/services/reasoningCache.ts"; @@ -263,8 +264,16 @@ test("video_url is preserved only for Moonshot's OpenAI-compatible extension", ( }); test("Moonshot keeps empty partial assistant prefixes without replaying reasoning", () => { - const requestId = "moonshot-partial-prefix"; - const cacheKey = `request:${requestId}:message:0`; + const scope = "api-key:test\x1fpartial-prefix"; + const messages = [ + { + role: "assistant", + content: "", + name: "Kal'tsit", + partial: true, + }, + ]; + const cacheKey = buildAssistantMessageCacheKey(scope, messages, 0); cacheReasoningByKey(cacheKey, "moonshot", "kimi-k3", "unrelated prior reasoning"); try { @@ -272,20 +281,12 @@ test("Moonshot keeps empty partial assistant prefixes without replaying reasonin "openai", "openai", "kimi-k3", - { - _reasoningCacheRequestId: requestId, - messages: [ - { - role: "assistant", - content: "", - name: "Kal'tsit", - partial: true, - }, - ], - }, + { messages }, false, null, - "moonshot" + "moonshot", + null, + { reasoningCacheScope: scope } ) as { messages: Array> }; assert.equal(output.messages.length, 1); diff --git a/tests/unit/reasoning-cache.test.ts b/tests/unit/reasoning-cache.test.ts index 1f5b1ce55d..9c144596b4 100644 --- a/tests/unit/reasoning-cache.test.ts +++ b/tests/unit/reasoning-cache.test.ts @@ -18,6 +18,7 @@ process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "reasoning-cache-test // ──────────── Direct service import ──────────── import { + buildAssistantMessageCacheKey, cacheReasoningFromAssistantMessage, cacheReasoning, cacheReasoningByKey, @@ -110,6 +111,30 @@ describe("Reasoning Replay Cache — Service Layer", () => { assert.equal(stats.dbEntries, 1); }); + it("should preserve SQLite expiry when promoting an entry to memory", () => { + clearReasoningCacheAll(); + const realDateNow = Date.now; + const startedAt = realDateNow(); + setReasoningCache( + "call_db_short_ttl", + "deepseek", + "deepseek-v4-pro", + "Short-lived DB reasoning", + 5_000 + ); + + try { + assert.equal(lookupReasoning("call_db_short_ttl"), "Short-lived DB reasoning"); + getDbInstance() + .prepare("DELETE FROM reasoning_cache WHERE tool_call_id = ?") + .run("call_db_short_ttl"); + Date.now = () => startedAt + 6_000; + assert.equal(lookupReasoning("call_db_short_ttl"), null); + } finally { + Date.now = realDateNow; + } + }); + it("should return null for unknown tool_call_id", () => { const result = lookupReasoning("call_nonexistent"); assert.equal(result, null); @@ -204,21 +229,30 @@ describe("Reasoning Replay Cache — Service Layer", () => { assert.equal(lookupReasoning("call_capture_alias"), "Alias reasoning"); }); - it("should cache assistant reasoning without tool calls by request and message index", () => { + it("should cache assistant reasoning without tool calls by scoped transcript", () => { clearReasoningCacheAll(); + const scope = "api-key:test:session:test"; + const historyMessages = [{ role: "user", content: "hi" }]; + const assistantMessage = { + role: "assistant", + content: "Hello!", + reasoning_content: "No tool call reasoning", + }; const cached = cacheReasoningFromAssistantMessage( - { - role: "assistant", - reasoning_content: "No tool call reasoning", - }, + assistantMessage, "deepseek", - "deepseek-reasoner", - { requestId: "req_no_tools", messageIndex: 3 } + "deepseek-v4-pro", + { scope, historyMessages } + ); + const cacheKey = buildAssistantMessageCacheKey( + scope, + [...historyMessages, assistantMessage], + historyMessages.length ); assert.equal(cached, 1); - assert.equal(lookupReasoning("request:req_no_tools:message:3"), "No tool call reasoning"); + assert.equal(lookupReasoning(cacheKey), "No tool call reasoning"); }); it("should skip assistant reasoning without tool calls when stable key context is absent", () => { @@ -591,6 +625,62 @@ describe("Reasoning Replay Cache — Translator Replay", () => { assert.equal(getReasoningCacheServiceStats().replays, 1); }); + it("should preserve DeepSeek Responses reasoning before Chat conversion", () => { + clearReasoningCacheAll(); + clearModelsDevCapabilities(); + cacheReasoning( + "call_ds_responses", + "deepseek", + "deepseek-v4-flash", + "Conflicting cached reasoning" + ); + const statsBeforeTranslation = getReasoningCacheServiceStats(); + + const translated = translateRequest( + FORMATS.OPENAI_RESPONSES, + FORMATS.OPENAI, + "deepseek-v4-flash", + { + reasoning: { effort: "high" }, + input: [ + { + type: "reasoning", + summary: [{ type: "summary_text", text: "Client DeepSeek reasoning" }], + }, + { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "I will inspect" }], + }, + { + type: "function_call", + call_id: "call_ds_responses", + name: "read_file", + arguments: "{}", + }, + { type: "function_call_output", call_id: "call_ds_responses", output: "contents" }, + { + type: "message", + role: "user", + content: [{ type: "input_text", text: "Continue" }], + }, + ], + }, + false, + null, + "deepseek" + ); + + const assistant = translated.messages.find((message) => message.role === "assistant"); + assert.equal(assistant.reasoning_content, "Client DeepSeek reasoning"); + assert.equal(assistant.content[0].text, "I will inspect"); + assert.equal(assistant.tool_calls[0].id, "call_ds_responses"); + const statsAfterTranslation = getReasoningCacheServiceStats(); + assert.equal(statsAfterTranslation.hits, statsBeforeTranslation.hits); + assert.equal(statsAfterTranslation.misses, statsBeforeTranslation.misses); + assert.equal(statsAfterTranslation.replays, statsBeforeTranslation.replays); + }); + it("should preserve client-provided reasoning content", () => { clearReasoningCacheAll(); clearModelsDevCapabilities(); @@ -604,6 +694,7 @@ describe("Reasoning Replay Cache — Translator Replay", () => { }, }); cacheReasoning("call_preserve", "deepseek", "deepseek-reasoner", "Cached reasoning"); + const statsBeforeTranslation = getReasoningCacheServiceStats(); const translated = translateRequest( FORMATS.OPENAI, @@ -633,6 +724,9 @@ describe("Reasoning Replay Cache — Translator Replay", () => { assert.equal(translated.messages[1].reasoning_content, "Client reasoning"); assert.equal(getReasoningCacheServiceStats().replays, 0); + const statsAfterTranslation = getReasoningCacheServiceStats(); + assert.equal(statsAfterTranslation.hits, statsBeforeTranslation.hits); + assert.equal(statsAfterTranslation.misses, statsBeforeTranslation.misses); }); it("should inject cached reasoning for Qwen and GLM thinking models", () => { @@ -851,9 +945,7 @@ describe("Reasoning Replay Cache — Translator Replay", () => { ); }); - it("should replay cached reasoning for a plain (non-tool-call) DeepSeek turn when available (#1682)", () => { - // When a request_id-keyed cache entry exists for the plain turn, the real - // reasoning is replayed instead of the placeholder. + it("should replay cached reasoning for a plain DeepSeek turn when available", () => { clearReasoningCacheAll(); clearModelsDevCapabilities(); saveModelsDevCapabilities({ @@ -865,49 +957,32 @@ describe("Reasoning Replay Cache — Translator Replay", () => { }), }, }); - // The non-tool-call cache key is built as `getAssistantMessageCacheKey(result, messageIndex)` - // where messageIndex is the assistant message's real position in the `messages` - // array (index 1 here: user, assistant, user) — matching what the write side - // (chatCore.ts) now caches under once the response is generated. - cacheReasoning( - "request:req-plain-1:message:1", - "deepseek", - "deepseek-v4-pro", - "Real cached plain-turn reasoning" - ); + const scope = "api-key:test:session:plain"; + const messages = [ + { role: "user", content: "hi" }, + { role: "assistant", content: "Hello! How can I help?" }, + { role: "user", content: "tell me more" }, + ]; + const cacheKey = buildAssistantMessageCacheKey(scope, messages, 1); + cacheReasoning(cacheKey, "deepseek", "deepseek-v4-pro", "Real cached plain-turn reasoning"); const translated = translateRequest( FORMATS.OPENAI, FORMATS.OPENAI, "deepseek-v4-pro", - { - request_id: "req-plain-1", - messages: [ - { role: "user", content: "hi" }, - { role: "assistant", content: "Hello! How can I help?" }, - { role: "user", content: "tell me more" }, - ], - }, + { messages }, false, null, - "deepseek" + "deepseek", + null, + { reasoningCacheScope: scope } ); - assert.equal( - translated.messages[1].reasoning_content, - "Real cached plain-turn reasoning", - "plain DeepSeek assistant turn should replay the real cached reasoning when present" - ); + assert.equal(translated.messages[1].reasoning_content, "Real cached plain-turn reasoning"); assert.equal(getReasoningCacheServiceStats().replays, 1); }); - it("write side (chatCore's messageIndex) and read side (translateRequest) agree on the same key end-to-end", () => { - // Regression for a mismatch where chatCore.ts always cached under - // `messageIndex: 0` (the position of the response within *its own* choices - // array) while translateRequest's read side looked up the message's real - // position in the *next* turn's full history — the two never agreed once a - // conversation went past its first assistant turn, so replay silently - // fell back to the placeholder in real multi-turn usage. + it("writes and reads the same no-tool transcript key for Chat history", () => { clearReasoningCacheAll(); clearModelsDevCapabilities(); saveModelsDevCapabilities({ @@ -919,36 +994,83 @@ describe("Reasoning Replay Cache — Translator Replay", () => { }), }, }); - - // Turn 1: the incoming request has a single user message (length 1), so - // the assistant response chatCore is about to cache will occupy index 1 - // once it's appended to history for turn 2 — mirroring - // `messageIndex: bodyMessages.length` in chatCore.ts. - const turn1RequestBody = { messages: [{ role: "user", content: "hi" }] }; + const scope = "api-key:test:session:chat"; + const historyMessages = [{ role: "user", content: "hi" }]; cacheReasoningFromAssistantMessage( { role: "assistant", content: "Hello! How can I help?", reasoning_content: "real reasoning" }, "deepseek", "deepseek-v4-pro", - { requestId: "req-e2e-1", messageIndex: turn1RequestBody.messages.length } + { scope, historyMessages } ); - // Turn 2: client replays the full history including the cached assistant - // turn, now genuinely at index 1. const translated = translateRequest( FORMATS.OPENAI, FORMATS.OPENAI, "deepseek-v4-pro", { - request_id: "req-e2e-1", messages: [ - { role: "user", content: "hi" }, + ...historyMessages, { role: "assistant", content: "Hello! How can I help?" }, { role: "user", content: "tell me more" }, ], }, false, null, - "deepseek" + "deepseek", + null, + { reasoningCacheScope: scope } + ); + + assert.equal(translated.messages[1].reasoning_content, "real reasoning"); + assert.equal(getReasoningCacheServiceStats().replays, 1); + }); + + it("writes a Chat response and replays it from Responses history", () => { + clearReasoningCacheAll(); + clearModelsDevCapabilities(); + saveModelsDevCapabilities({ + deepseek: { + "deepseek-v4-pro": buildCapability({ + interleaved_field: "reasoning_content", + reasoning: true, + tool_call: true, + }), + }, + }); + const scope = "api-key:test:session:responses"; + const historyMessages = [{ role: "user", content: [{ type: "text", text: "hi" }] }]; + cacheReasoningFromAssistantMessage( + { role: "assistant", content: "Hello! How can I help?", reasoning_content: "real reasoning" }, + "deepseek", + "deepseek-v4-pro", + { scope, historyMessages } + ); + + const translated = translateRequest( + FORMATS.OPENAI_RESPONSES, + FORMATS.OPENAI, + "deepseek-v4-pro", + { + reasoning: { effort: "high" }, + input: [ + { type: "message", role: "user", content: [{ type: "input_text", text: "hi" }] }, + { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "Hello! How can I help?" }], + }, + { + type: "message", + role: "user", + content: [{ type: "input_text", text: "tell me more" }], + }, + ], + }, + false, + null, + "deepseek", + null, + { reasoningCacheScope: scope } ); assert.equal(translated.messages[1].reasoning_content, "real reasoning"); diff --git a/tests/unit/translator-helper-branches.test.ts b/tests/unit/translator-helper-branches.test.ts index 33871a3766..bf3bfea581 100644 --- a/tests/unit/translator-helper-branches.test.ts +++ b/tests/unit/translator-helper-branches.test.ts @@ -8,8 +8,12 @@ const geminiHelper = await import("../../open-sse/translator/helpers/geminiHelpe const toolCallHelper = await import("../../open-sse/translator/helpers/toolCallHelper.ts"); const { FORMATS } = await import("../../open-sse/translator/formats.ts"); const { translateRequest } = await import("../../open-sse/translator/index.ts"); -const { cacheReasoningByKey, clearReasoningCacheAll, getReasoningCacheServiceStats } = - await import("../../open-sse/services/reasoningCache.ts"); +const { + buildAssistantMessageCacheKey, + cacheReasoningByKey, + clearReasoningCacheAll, + getReasoningCacheServiceStats, +} = await import("../../open-sse/services/reasoningCache.ts"); const { clearModelsDevCapabilities, saveModelsDevCapabilities } = await import("../../src/lib/modelsDevSync.ts"); @@ -639,8 +643,13 @@ test("translateRequest replays cached reasoning-only messages when interleaved f }), }, }); + const scope = "api-key:test:session:helper-deepseek"; + const messages = [ + { role: "user", content: "solve this" }, + { role: "assistant", content: "answer", reasoning_content: "" }, + ]; cacheReasoningByKey( - "request:req_reasoning_only:message:1", + buildAssistantMessageCacheKey(scope, messages, 1), "deepseek", "deepseek-v4-flash", "cached reasoning only" @@ -650,16 +659,12 @@ test("translateRequest replays cached reasoning-only messages when interleaved f FORMATS.OPENAI, FORMATS.OPENAI, "deepseek-v4-flash", - { - _reasoningCacheRequestId: "req_reasoning_only", - messages: [ - { role: "user", content: "solve this" }, - { role: "assistant", content: "answer", reasoning_content: "" }, - ], - }, + { messages }, false, null, - "deepseek" + "deepseek", + null, + { reasoningCacheScope: scope } ); assert.equal(result.messages[1].reasoning_content, "cached reasoning only"); @@ -670,8 +675,13 @@ test("translateRequest replays cached reasoning-only messages when interleaved f test("translateRequest does not replay reasoning-only messages for non-DeepSeek models", () => { clearReasoningCacheAll(); + const scope = "api-key:test:session:helper-kimi"; + const messages = [ + { role: "user", content: "solve this" }, + { role: "assistant", content: "answer", reasoning_content: "" }, + ]; cacheReasoningByKey( - "request:req_kimi_reasoning_only:message:0", + buildAssistantMessageCacheKey(scope, messages, 1), "kimi", "kimi-k2.6", "cached kimi reasoning" @@ -681,16 +691,12 @@ test("translateRequest does not replay reasoning-only messages for non-DeepSeek FORMATS.OPENAI, FORMATS.OPENAI, "kimi-k2.6", - { - _reasoningCacheRequestId: "req_kimi_reasoning_only", - messages: [ - { role: "user", content: "solve this" }, - { role: "assistant", content: "answer", reasoning_content: "" }, - ], - }, + { messages }, false, null, - "kimi" + "kimi", + null, + { reasoningCacheScope: scope } ); assert.equal(result.messages[1].reasoning_content, undefined); diff --git a/tests/unit/translator-openai-responses-req.test.ts b/tests/unit/translator-openai-responses-req.test.ts index b12038606b..3aa5178a06 100644 --- a/tests/unit/translator-openai-responses-req.test.ts +++ b/tests/unit/translator-openai-responses-req.test.ts @@ -85,6 +85,94 @@ test("Responses -> Chat converts instructions, inputs, function calls, outputs, }); }); +test("Responses -> Chat keeps assistant text, reasoning, and function calls in one turn", () => { + const result = openaiResponsesToOpenAIRequest( + "gpt-4o", + { + input: [ + { + type: "reasoning", + summary: [{ type: "summary_text", text: "Inspect first" }], + }, + { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "I will inspect" }], + }, + { type: "function_call", call_id: "call_1", name: "read_file", arguments: "{}" }, + { type: "function_call", call_id: "call_2", name: "search", arguments: "{}" }, + { type: "function_call_output", call_id: "call_1", output: "contents" }, + ], + }, + false, + { _preserveReasoningContent: true } + ) as { messages: Array> }; + + assert.equal(result.messages.length, 2); + assert.deepEqual(result.messages[0], { + role: "assistant", + content: [{ type: "text", text: "I will inspect" }], + reasoning_content: "Inspect first", + tool_calls: [ + { + id: "call_1", + type: "function", + function: { name: "read_file", arguments: "{}" }, + }, + { + id: "call_2", + type: "function", + function: { name: "search", arguments: "{}" }, + }, + ], + }); + assert.deepEqual(result.messages[1], { + role: "tool", + tool_call_id: "call_1", + content: "contents", + }); +}); + +test("Responses -> Chat merges assistant text that follows a function call", () => { + const result = openaiResponsesToOpenAIRequest( + "gpt-4o", + { + input: [ + { type: "function_call", call_id: "call_1", name: "read_file", arguments: "{}" }, + { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "I inspected it" }], + }, + { + type: "reasoning", + summary: [{ type: "summary_text", text: "Inspection complete" }], + }, + { type: "message", role: "user", content: [{ type: "input_text", text: "Continue" }] }, + ], + }, + false, + { _preserveReasoningContent: true } + ) as { messages: Array> }; + + assert.deepEqual(result.messages[0], { + role: "assistant", + content: [{ type: "text", text: "I inspected it" }], + tool_calls: [ + { + id: "call_1", + type: "function", + function: { name: "read_file", arguments: "{}" }, + }, + ], + reasoning_content: "Inspection complete", + }); + assert.deepEqual(result.messages[1], { + role: "user", + content: [{ type: "text", text: "Continue" }], + }); +}); + test("Responses -> Chat filters orphan tool outputs and supports role-based message items", () => { const result = openaiResponsesToOpenAIRequest( "gpt-4o", @@ -925,7 +1013,12 @@ test("Responses -> Chat: tool_search is mapped to a Chat function tool, not drop input: [{ role: "user", content: [{ type: "input_text", text: "hello" }] }], tools: [ { type: "tool_search", name: "search" }, - { type: "function", name: "foo", description: "A function", parameters: { type: "object" } }, + { + type: "function", + name: "foo", + description: "A function", + parameters: { type: "object" }, + }, ], }, false, @@ -934,7 +1027,11 @@ test("Responses -> Chat: tool_search is mapped to a Chat function tool, not drop const tools = result.tools as any[]; assert.ok(Array.isArray(tools), "tools array must be present"); - assert.equal(tools.some((t) => t.type === "tool_search"), false, "raw tool_search type must not survive"); + assert.equal( + tools.some((t) => t.type === "tool_search"), + false, + "raw tool_search type must not survive" + ); assert.equal(tools.length, 2, "mapped tool_search function + the function tool must remain"); const toolSearch = tools.find((t) => t.function?.name === "search"); assert.ok(toolSearch, "tool_search must be mapped to a Chat function tool named after it");