fix(reasoning): preserve and replay assistant turns (#10045)

This commit is contained in:
Ke Jin
2026-08-13 18:52:12 +08:00
committed by GitHub
parent 7366bb6c3a
commit 1a8d38655d
12 changed files with 722 additions and 192 deletions

View File

@@ -147,7 +147,11 @@ import {
getExplicitModelOutputCap,
resolveInputTokenCapForGate,
} from "@/lib/modelCapabilities.ts";
import { checkRequestCapabilityFit, deriveRequestCapabilityRequirements, buildCapabilityMismatchMessage } from "@/shared/constants/capabilities/capabilityFilter.ts";
import {
checkRequestCapabilityFit,
deriveRequestCapabilityRequirements,
buildCapabilityMismatchMessage,
} from "@/shared/constants/capabilities/capabilityFilter.ts";
import { isFeatureFlagEnabled } from "@/shared/utils/featureFlags.ts";
import { toPositiveInteger } from "../services/reasoningTokenBuffer.ts";
import { normalizeThinkingForModel } from "@/shared/constants/modelSpecs.ts";
@@ -421,6 +425,7 @@ export async function handleChatCore({
comboStrategy = null,
isCombo = false,
routingComboId = null,
sessionAffinityKey = null,
comboStepId = null,
comboExecutionKey = null,
cachedSettings = null,
@@ -884,6 +889,10 @@ export async function handleChatCore({
"x-omniroute-session-id"
)) || null;
const pipelineSessionId = explicitSessionIdHeader || skillRequestId;
const reasoningReplaySessionKey = sessionAffinityKey || explicitSessionIdHeader;
const reasoningCacheScope = reasoningReplaySessionKey
? `api-key:${String(apiKeyInfo?.id ?? "local")}\x1f${String(reasoningReplaySessionKey)}`
: null;
// persistAttemptLogs extracted to chatCore/attemptLogging.ts (#3501); bind the per-request context
// once so the 16 call sites keep passing only the per-attempt args (byte-identical).
const persistAttemptLogs = (args: PersistAttemptLogsArgs) =>
@@ -2042,6 +2051,7 @@ export async function handleChatCore({
preserveDeveloperRole,
preserveCacheControl,
copilotClient: copilotCompatibleReasoning,
reasoningCacheScope,
}
);
}
@@ -2206,6 +2216,7 @@ export async function handleChatCore({
preserveCacheControl,
signatureNamespace: connectionId,
copilotClient: copilotCompatibleReasoning,
reasoningCacheScope,
...(preCompressionBody ? { preCompressionBody } : {}),
}
);
@@ -2635,8 +2646,11 @@ export async function handleChatCore({
}
// === /Quota Share enforcement PRE-hook ===
if (isFeatureFlagEnabled("CAPABILITY_FILTER_ENABLED")) {
const fit = checkRequestCapabilityFit(getResolvedModelCapabilities({ provider, model: effectiveModel }),
deriveRequestCapabilityRequirements(body as Record<string, unknown>), provider);
const fit = checkRequestCapabilityFit(
getResolvedModelCapabilities({ provider, model: effectiveModel }),
deriveRequestCapabilityRequirements(body as Record<string, unknown>),
provider
);
if (!fit.compatible) {
const msg = buildCapabilityMismatchMessage(fit.terminalReason!, provider, effectiveModel);
log?.warn?.("CAPABILITY", msg);
@@ -4378,16 +4392,23 @@ export async function handleChatCore({
// Reasoning Replay Cache (#1628): Capture reasoning_content from non-streaming responses
// with tool_calls so it can be replayed on subsequent turns (DeepSeek V4, Kimi K2, etc.)
try {
const firstChoice = translatedResponse?.choices?.[0];
const cacheResponse = translatedResponse?.choices?.[0]
? translatedResponse
: needsTranslation(responsePayloadFormat, FORMATS.OPENAI)
? translateNonStreamingResponse(
responseBody,
responsePayloadFormat,
FORMATS.OPENAI,
responseToolNameMap
)
: responseBody;
const firstChoice = cacheResponse?.choices?.[0];
const msg = firstChoice?.message;
// The response being cached now will be replayed as history on the *next*
// turn, where the read side (translator/index.ts) keys the lookup by the
// message's real position in that future `messages` array — i.e. right
// after everything the client sent this turn.
const bodyMessages = (body as { messages?: unknown[] } | null | undefined)?.messages;
const historyMessages = (translatedBody as { messages?: unknown[] } | null | undefined)
?.messages;
cacheReasoningFromAssistantMessage(msg, provider, model, {
requestId: skillRequestId,
messageIndex: Array.isArray(bodyMessages) ? bodyMessages.length : 0,
scope: reasoningCacheScope,
historyMessages: Array.isArray(historyMessages) ? historyMessages : [],
});
} catch {
// Cache capture is non-critical — never block the response
@@ -4813,14 +4834,24 @@ export async function handleChatCore({
if (normalizedStreamStatus === 200 && streamResponseBody) {
try {
const streamBody = streamResponseBody as Record<string, unknown>;
const choices = streamBody.choices as { message?: Record<string, unknown> }[] | undefined;
const cacheStreamBody = Array.isArray(streamBody.choices)
? streamBody
: needsTranslation(clientResponseFormat, FORMATS.OPENAI)
? (translateNonStreamingResponse(
streamBody,
clientResponseFormat,
FORMATS.OPENAI,
responseToolNameMap
) as Record<string, unknown>)
: streamBody;
const choices = cacheStreamBody.choices as
{ message?: Record<string, unknown> }[] | undefined;
const msg = choices?.[0]?.message;
// See the non-streaming capture above: messageIndex must match the
// position this message will occupy in the *next* turn's history.
const bodyMessages = (body as { messages?: unknown[] } | null | undefined)?.messages;
const historyMessages = (translatedBody as { messages?: unknown[] } | null | undefined)
?.messages;
cacheReasoningFromAssistantMessage(msg, provider, model, {
requestId: skillRequestId,
messageIndex: Array.isArray(bodyMessages) ? bodyMessages.length : 0,
scope: reasoningCacheScope,
historyMessages: Array.isArray(historyMessages) ? historyMessages : [],
});
} catch {
// Cache capture is non-critical — never block the stream

View File

@@ -13,6 +13,7 @@
* @see Issue #1628
*/
import { createHash } from "node:crypto";
import {
clearAllReasoningCache,
cleanupExpiredReasoning,
@@ -137,8 +138,8 @@ type AssistantMessageLike = {
};
type AssistantMessageCacheContext = {
requestId?: string;
messageIndex?: number;
scope?: string;
historyMessages?: AssistantMessageLike[];
};
type ToolCallLike = {
@@ -234,8 +235,79 @@ export function cacheReasoningByKey(
}
}
function buildAssistantMessageCacheKey(requestId: string, messageIndex: number): string {
return `request:${requestId}:message:${messageIndex}`;
function stableCacheValue(value: unknown): unknown {
if (Array.isArray(value)) return value.map(stableCacheValue);
if (!value || typeof value !== "object") return value;
const record = value as Record<string, unknown>;
return Object.fromEntries(
Object.keys(record)
.filter((key) => key !== "reasoning" && key !== "reasoning_content")
.sort()
.map((key) => [key, stableCacheValue(record[key])])
);
}
function canonicalizeMessageContent(content: unknown): unknown {
if (!Array.isArray(content)) return stableCacheValue(content ?? null);
const textParts: string[] = [];
for (const part of content) {
if (typeof part === "string") {
textParts.push(part);
continue;
}
if (!part || typeof part !== "object") return stableCacheValue(content);
const record = part as Record<string, unknown>;
if (
(record.type === "text" || record.type === "input_text" || record.type === "output_text") &&
typeof record.text === "string"
) {
textParts.push(record.text);
continue;
}
return stableCacheValue(content);
}
return textParts.join("");
}
function canonicalizeHistoryMessage(message: AssistantMessageLike): unknown {
const record = message as Record<string, unknown>;
const toolCalls = Array.isArray(record.tool_calls)
? record.tool_calls.map((toolCall) => {
const call = toolCall as Record<string, unknown>;
const fn = (call.function ?? {}) as Record<string, unknown>;
return stableCacheValue({
type: call.type,
function: { name: fn.name, arguments: fn.arguments },
});
})
: undefined;
return stableCacheValue({
role: record.role,
name: record.name,
content: canonicalizeMessageContent(record.content),
tool_calls: toolCalls,
});
}
export function buildAssistantMessageCacheKey(
scope: string | null | undefined,
messages: AssistantMessageLike[],
messageIndex: number
): string {
const normalizedScope = scope?.trim();
if (!normalizedScope || !Number.isInteger(messageIndex) || messageIndex < 0) return "";
const message = messages[messageIndex];
if (!message || message.role !== "assistant") return "";
const transcript = messages.slice(0, messageIndex + 1).map(canonicalizeHistoryMessage);
const digest = createHash("sha256")
.update(normalizedScope)
.update("\x1f")
.update(JSON.stringify(transcript))
.digest("hex");
return `conversation:${digest}`;
}
/**
@@ -282,18 +354,15 @@ export function cacheReasoningFromAssistantMessage(
.filter((id) => id.length > 0)
: [];
if (toolCallIds.length === 0) {
const requestId = context?.requestId?.trim();
const messageIndex = context?.messageIndex;
if (!requestId || typeof messageIndex !== "number" || !Number.isInteger(messageIndex)) {
return 0;
}
const scope = context?.scope?.trim();
const historyMessages = context?.historyMessages;
if (!scope || !Array.isArray(historyMessages)) return 0;
cacheReasoningByKey(
buildAssistantMessageCacheKey(requestId, messageIndex),
provider,
model,
reasoning
);
const messages = [...historyMessages, message];
const cacheKey = buildAssistantMessageCacheKey(scope, messages, messages.length - 1);
if (!cacheKey) return 0;
cacheReasoningByKey(cacheKey, provider, model, reasoning);
return 1;
}
@@ -329,7 +398,8 @@ export function lookupReasoning(toolCallId: string): string | null {
}
// 2. Fallback to DB
let dbResult: { reasoning: string; provider: string; model: string } | null = null;
let dbResult: { reasoning: string; provider: string; model: string; expiresAt: string } | null =
null;
try {
dbResult = getReasoningCache(toolCallId);
} catch {
@@ -341,6 +411,11 @@ export function lookupReasoning(toolCallId: string): string | null {
misses++;
return null;
}
const persistedExpiresAt = Date.parse(dbResult.expiresAt);
if (!Number.isFinite(persistedExpiresAt) || persistedExpiresAt <= Date.now()) {
misses++;
return null;
}
hits++;
let promotedReasoning = dbResult.reasoning;
if (promotedReasoning.length > MAX_ENTRY_BYTES) {
@@ -351,7 +426,7 @@ export function lookupReasoning(toolCallId: string): string | null {
reasoning: promotedReasoning,
provider: dbResult.provider,
model: dbResult.model,
expiresAt: Date.now() + TTL_MS,
expiresAt: persistedExpiresAt,
createdAt: Date.now(),
});
return promotedReasoning;

View File

@@ -30,6 +30,7 @@ import { getResolvedModelCapabilities, supportsReasoning } from "../services/mod
import { normalizeRoles } from "../services/roleNormalizer.ts";
import { hoistLeadingSystemMessage } from "./helpers/strictSystemHoist.ts";
import {
buildAssistantMessageCacheKey,
lookupReasoning,
recordReplay,
requiresReasoningReplay,
@@ -149,25 +150,6 @@ function normalizeOpenAIResponsesRequest(body) {
return normalized;
}
function getReasoningCacheRequestId(body: Record<string, unknown> | null | undefined): string {
if (!body || typeof body !== "object") return "";
const requestId =
body._reasoningCacheRequestId ??
body.reasoningCacheRequestId ??
body.request_id ??
body.requestId;
return typeof requestId === "string" ? requestId.trim() : "";
}
function getAssistantMessageCacheKey(
body: Record<string, unknown> | null | undefined,
messageIndex: number
): string {
const requestId = getReasoningCacheRequestId(body);
return requestId ? `request:${requestId}:message:${messageIndex}` : "";
}
function hasNonEmptyReasoningContent(message: Record<string, unknown>): boolean {
return typeof message.reasoning_content === "string" && message.reasoning_content.length > 0;
}
@@ -253,6 +235,7 @@ export function translateRequest(
preserveCacheControl?: boolean;
signatureNamespace?: string | null;
preCompressionBody?: Record<string, unknown> | null;
reasoningCacheScope?: string | null;
/** UA-detected GitHub Copilot client. Forwarded to translators via the
* transient `_copilotClient` credential flag (see openai-responses → openai). */
copilotClient?: boolean;
@@ -268,13 +251,6 @@ export function translateRequest(
const normalizedModel = String(model ?? "");
const isKimiCoding =
normalizedProvider === "kimi-coding" || normalizedProvider === "kimi-coding-apikey";
const requiresExplicitReasoningReplay = requiresReasoningReplay({
provider: normalizedProvider,
model: normalizedModel,
allowLegacyFallback: false,
});
const preserveResponsesReasoning =
sourceFormat === FORMATS.OPENAI_RESPONSES && requiresExplicitReasoningReplay;
// Phase 2: Apply thinking budget control before normalization
result = applyThinkingBudget(result);
@@ -285,6 +261,29 @@ export function translateRequest(
// Normalize thinking config: remove if lastMessage is not user
normalizeThinkingConfig(result);
// Resolve the replay contract before Responses input is converted: conversion
// must know whether reasoning items are protocol history rather than display metadata.
const resolvedCapabilities = getResolvedModelCapabilities({
provider: normalizedProvider,
model: normalizedModel,
});
const replayRequirements = {
provider: normalizedProvider,
model: normalizedModel,
thinkingEnabled: hasThinkingConfig(result),
supportsReasoning: supportsReasoning({
provider: normalizedProvider,
model: normalizedModel,
}),
interleavedField: resolvedCapabilities?.interleavedField ?? null,
};
const isReasoner = requiresReasoningReplay(replayRequirements);
const requiresExplicitReasoningReplay = requiresReasoningReplay({
...replayRequirements,
allowLegacyFallback: false,
});
const preserveResponsesReasoning = sourceFormat === FORMATS.OPENAI_RESPONSES && isReasoner;
// Ensure tool_calls have id; optionally normalize to 9-char for providers like Mistral
ensureToolCallIds(result, { use9CharId });
@@ -424,24 +423,6 @@ export function translateRequest(
}
}
// Resolve reasoning-replay status up-front: it gates both the reasoning_content
// strip in filterToOpenAIFormat below (#4849 must NOT strip client reasoning for
// replay providers) and the cache re-injection further down.
const resolvedCapabilities = getResolvedModelCapabilities({
provider: normalizedProvider,
model: normalizedModel,
});
const isReasoner = requiresReasoningReplay({
provider: normalizedProvider,
model: normalizedModel,
thinkingEnabled: hasThinkingConfig(result),
supportsReasoning: supportsReasoning({
provider: normalizedProvider,
model: normalizedModel,
}),
interleavedField: resolvedCapabilities?.interleavedField ?? null,
});
// Always normalize to clean OpenAI format when target is OpenAI
// This handles hybrid requests (e.g., OpenAI messages + Claude tools)
if (targetFormat === FORMATS.OPENAI) {
@@ -653,7 +634,11 @@ export function translateRequest(
const cacheKey = hasToolCalls
? msg.tool_calls[0]?.id
: getAssistantMessageCacheKey(result, messageIndex);
: buildAssistantMessageCacheKey(
options?.reasoningCacheScope,
result.messages,
messageIndex
);
if (cacheKey) {
const cached = lookupReasoning(cacheKey);
if (cached) {

View File

@@ -235,21 +235,23 @@ export function openaiResponsesToOpenAIRequest(
if (itemType === "message") {
const role = toString(item.role);
// Flush pending assistant message with tool calls
if (currentAssistantMsg) {
messages.push(currentAssistantMsg);
currentAssistantMsg = null;
}
if (role !== "assistant" && pendingReasoningContent) {
messages.push({
role: "assistant",
content: null,
reasoning_content: pendingReasoningContent,
});
pendingReasoningContent = "";
if (role !== "assistant") {
if (currentAssistantMsg) {
messages.push(currentAssistantMsg);
currentAssistantMsg = null;
}
if (pendingReasoningContent) {
messages.push({
role: "assistant",
content: null,
reasoning_content: pendingReasoningContent,
});
pendingReasoningContent = "";
}
}
// Flush pending tool results
// Flush pending tool results before the next explicit message boundary.
if (pendingToolResults.length > 0) {
for (const toolResult of pendingToolResults) {
messages.push(toolResult);
@@ -292,12 +294,29 @@ export function openaiResponsesToOpenAIRequest(
})
: item.content;
const message: JsonRecord = { role, content };
if (role === "assistant" && pendingReasoningContent) {
message.reasoning_content = pendingReasoningContent;
pendingReasoningContent = "";
if (role === "assistant") {
if (!currentAssistantMsg) {
currentAssistantMsg = { role, content };
} else if (currentAssistantMsg.content == null && content != null) {
currentAssistantMsg.content = content;
} else if (content != null) {
const existingContent = currentAssistantMsg.content;
currentAssistantMsg.content = [
...(Array.isArray(existingContent) ? existingContent : [existingContent]),
...(Array.isArray(content) ? content : [content]),
];
}
if (pendingReasoningContent) {
currentAssistantMsg.reasoning_content = appendReasoningContent(
currentAssistantMsg.reasoning_content,
pendingReasoningContent
);
pendingReasoningContent = "";
}
continue;
}
messages.push(message);
messages.push({ role, content });
continue;
}

View File

@@ -103,18 +103,32 @@ export function setReasoningCache(
*/
export function getReasoningCache(
toolCallId: string
): { reasoning: string; provider: string; model: string } | null {
): { reasoning: string; provider: string; model: string; expiresAt: string } | null {
const db = getDbInstance();
const row = db
.prepare(
`SELECT reasoning, provider, model FROM reasoning_cache
`SELECT reasoning, provider, model, expires_at FROM reasoning_cache
WHERE tool_call_id = ? AND ${EXPIRES_AT_EPOCH_SQL} > unixepoch('now')`
)
.get(toolCallId) as { reasoning: string; provider: string; model: string } | undefined;
.get(toolCallId) as
| {
reasoning: string;
provider: string;
model: string;
expires_at: number | string;
}
| undefined;
const PLACEHOLDER = "(prior reasoning summary unavailable)";
if (row && row.reasoning && row.reasoning.trim() === PLACEHOLDER) return null; // ponytail: never replay the placeholder
return row ?? null;
if (row && row.reasoning && row.reasoning.trim() === PLACEHOLDER) return null;
return row
? {
reasoning: row.reasoning,
provider: row.provider,
model: row.model,
expiresAt: epochSecondsToIso(row.expires_at),
}
: null;
}
/**

View File

@@ -1562,6 +1562,7 @@ async function handleSingleModelChat(
correlationId: runtimeOptions?.correlationId ?? null,
modelPinned: runtimeOptions?.modelPinned ?? false,
routingComboId: runtimeOptions?.routingComboId ?? null,
sessionAffinityKey: runtimeOptions.sessionAffinityKey ?? null,
});
} catch (error) {
releaseOAuthSession();

View File

@@ -421,6 +421,7 @@ export async function executeChatWithBreaker({
correlationId = null,
modelPinned = false,
routingComboId = null,
sessionAffinityKey = null,
}: ExecuteChatWithBreakerOptions): Promise<ExecuteChatWithBreakerResult> {
let tlsFingerprintUsed = false;
const normalizedTrafficType: TrafficType =
@@ -476,6 +477,7 @@ export async function executeChatWithBreaker({
correlationId,
modelPinned,
routingComboId,
sessionAffinityKey,
skipResourcePressureGuard: true,
onCredentialsRefreshed: async (newCreds: any) => {
await updateProviderCredentials(credentials.connectionId, {

View File

@@ -29,6 +29,8 @@ const { clearModelLock, isModelLocked } =
await import("../../open-sse/services/accountFallback.ts");
const { saveModelsDevCapabilities, clearModelsDevCapabilities } =
await import("../../src/lib/modelsDevSync.ts");
// Dynamic import is required after TEST_DATA_DIR is initialized above.
const { clearReasoningCacheAll } = await import("../../open-sse/services/reasoningCache.ts");
const {
getBackgroundDegradationConfig,
setBackgroundDegradationConfig,
@@ -256,6 +258,7 @@ async function resetStorage() {
clearIdempotency();
clearInflight();
clearModelsDevCapabilities();
clearReasoningCacheAll();
setBackgroundDegradationConfig(originalBackgroundConfig);
resetBackgroundStats();
globalThis.setTimeout = originalSetTimeout;
@@ -305,6 +308,7 @@ async function invokeChatCore({
connectionId = null,
onCredentialsRefreshed = null,
onRequestSuccess = null,
sessionAffinityKey = null,
}: any = {}) {
const calls: any[] = [];
@@ -348,6 +352,7 @@ async function invokeChatCore({
connectionId,
apiKeyInfo,
userAgent,
sessionAffinityKey,
isCombo,
comboStrategy,
onCredentialsRefreshed,
@@ -563,6 +568,178 @@ test("chatCore applies Responses input policy to openai-compatible targets", asy
assert.equal(input.find((item) => item.type === "function_call")?.id, undefined);
}
});
test("chatCore replays no-tool reasoning across public Responses turns", async () => {
saveModelsDevCapabilities({
deepseek: {
"deepseek-v4-pro": {
...capabilityEntry(128_000),
reasoning: true,
interleaved_field: null,
},
},
});
const sessionAffinityKey = "header:reasoning-replay-session";
const apiKeyInfo = { id: "reasoning-replay-key" };
const responseFactory = () =>
new Response(
JSON.stringify({
id: "chatcmpl-reasoning",
object: "chat.completion",
choices: [
{
index: 0,
message: {
role: "assistant",
content: "Hello! How can I help?",
reasoning_content: "Authentic upstream reasoning",
},
finish_reason: "stop",
},
],
usage: { prompt_tokens: 4, completion_tokens: 2, total_tokens: 6 },
}),
{ status: 200, headers: { "Content-Type": "application/json" } }
);
const first = await invokeChatCore({
provider: "deepseek",
model: "deepseek-v4-pro",
endpoint: "/v1/responses",
body: {
model: "deepseek-v4-pro",
stream: false,
reasoning: { effort: "high" },
input: [{ type: "message", role: "user", content: [{ type: "input_text", text: "hi" }] }],
},
apiKeyInfo,
sessionAffinityKey,
responseFactory,
});
assert.equal(first.result.success, true);
const second = await invokeChatCore({
provider: "deepseek",
model: "deepseek-v4-pro",
endpoint: "/v1/responses",
body: {
model: "deepseek-v4-pro",
stream: false,
reasoning: { effort: "high" },
input: [
{ type: "message", role: "user", content: [{ type: "input_text", text: "hi" }] },
{
type: "message",
role: "assistant",
content: [{ type: "output_text", text: "Hello! How can I help?" }],
},
{
type: "message",
role: "user",
content: [{ type: "input_text", text: "tell me more" }],
},
],
},
apiKeyInfo,
sessionAffinityKey,
responseFactory,
});
assert.equal(second.result.success, true);
assert.equal(second.call.body.messages[1].reasoning_content, "Authentic upstream reasoning");
});
test("chatCore captures streaming no-tool reasoning for Responses replay", async () => {
saveModelsDevCapabilities({
deepseek: {
"deepseek-v4-pro": {
...capabilityEntry(128_000),
reasoning: true,
interleaved_field: null,
},
},
});
const sessionAffinityKey = "header:streaming-reasoning-replay-session";
const apiKeyInfo = { id: "streaming-reasoning-replay-key" };
const streamResponseFactory = () =>
new Response(
[
`data: ${JSON.stringify({
id: "chatcmpl-stream-reasoning",
object: "chat.completion.chunk",
choices: [
{
index: 0,
delta: {
role: "assistant",
reasoning_content: "Authentic streaming reasoning",
},
},
],
})}`,
`data: ${JSON.stringify({
id: "chatcmpl-stream-reasoning",
object: "chat.completion.chunk",
choices: [{ index: 0, delta: { content: "Streamed answer" } }],
})}`,
`data: ${JSON.stringify({
id: "chatcmpl-stream-reasoning",
object: "chat.completion.chunk",
choices: [{ index: 0, delta: {}, finish_reason: "stop" }],
})}`,
"data: [DONE]",
"",
].join("\n\n"),
{ status: 200, headers: { "Content-Type": "text/event-stream" } }
);
const first = await invokeChatCore({
provider: "deepseek",
model: "deepseek-v4-pro",
endpoint: "/v1/responses",
body: {
model: "deepseek-v4-pro",
stream: true,
reasoning: { effort: "high" },
input: [{ type: "message", role: "user", content: [{ type: "input_text", text: "hi" }] }],
},
apiKeyInfo,
sessionAffinityKey,
responseFactory: streamResponseFactory,
});
assert.equal(first.result.success, true);
await first.result.response.text();
await flushAsyncSideEffects();
const second = await invokeChatCore({
provider: "deepseek",
model: "deepseek-v4-pro",
endpoint: "/v1/responses",
body: {
model: "deepseek-v4-pro",
stream: false,
reasoning: { effort: "high" },
input: [
{ type: "message", role: "user", content: [{ type: "input_text", text: "hi" }] },
{
type: "message",
role: "assistant",
content: [{ type: "output_text", text: "Streamed answer" }],
},
{
type: "message",
role: "user",
content: [{ type: "input_text", text: "tell me more" }],
},
],
},
apiKeyInfo,
sessionAffinityKey,
responseFactory: () => buildOpenAIResponse(false),
});
assert.equal(second.result.success, true);
assert.equal(second.call.body.messages[1].reasoning_content, "Authentic streaming reasoning");
});
test("chatCore preserves opted-in encrypted reasoning for Codex", async () => {
const { call, result } = await invokeChatCore({
provider: "codex",

View File

@@ -18,6 +18,7 @@ import { normalizeMoonshotRequest } from "../../open-sse/executors/moonshot.ts";
import {
cacheReasoning,
cacheReasoningByKey,
buildAssistantMessageCacheKey,
deleteReasoningCacheEntry,
requiresReasoningReplay,
} from "../../open-sse/services/reasoningCache.ts";
@@ -263,8 +264,16 @@ test("video_url is preserved only for Moonshot's OpenAI-compatible extension", (
});
test("Moonshot keeps empty partial assistant prefixes without replaying reasoning", () => {
const requestId = "moonshot-partial-prefix";
const cacheKey = `request:${requestId}:message:0`;
const scope = "api-key:test\x1fpartial-prefix";
const messages = [
{
role: "assistant",
content: "",
name: "Kal'tsit",
partial: true,
},
];
const cacheKey = buildAssistantMessageCacheKey(scope, messages, 0);
cacheReasoningByKey(cacheKey, "moonshot", "kimi-k3", "unrelated prior reasoning");
try {
@@ -272,20 +281,12 @@ test("Moonshot keeps empty partial assistant prefixes without replaying reasonin
"openai",
"openai",
"kimi-k3",
{
_reasoningCacheRequestId: requestId,
messages: [
{
role: "assistant",
content: "",
name: "Kal'tsit",
partial: true,
},
],
},
{ messages },
false,
null,
"moonshot"
"moonshot",
null,
{ reasoningCacheScope: scope }
) as { messages: Array<Record<string, unknown>> };
assert.equal(output.messages.length, 1);

View File

@@ -18,6 +18,7 @@ process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "reasoning-cache-test
// ──────────── Direct service import ────────────
import {
buildAssistantMessageCacheKey,
cacheReasoningFromAssistantMessage,
cacheReasoning,
cacheReasoningByKey,
@@ -110,6 +111,30 @@ describe("Reasoning Replay Cache — Service Layer", () => {
assert.equal(stats.dbEntries, 1);
});
it("should preserve SQLite expiry when promoting an entry to memory", () => {
clearReasoningCacheAll();
const realDateNow = Date.now;
const startedAt = realDateNow();
setReasoningCache(
"call_db_short_ttl",
"deepseek",
"deepseek-v4-pro",
"Short-lived DB reasoning",
5_000
);
try {
assert.equal(lookupReasoning("call_db_short_ttl"), "Short-lived DB reasoning");
getDbInstance()
.prepare("DELETE FROM reasoning_cache WHERE tool_call_id = ?")
.run("call_db_short_ttl");
Date.now = () => startedAt + 6_000;
assert.equal(lookupReasoning("call_db_short_ttl"), null);
} finally {
Date.now = realDateNow;
}
});
it("should return null for unknown tool_call_id", () => {
const result = lookupReasoning("call_nonexistent");
assert.equal(result, null);
@@ -204,21 +229,30 @@ describe("Reasoning Replay Cache — Service Layer", () => {
assert.equal(lookupReasoning("call_capture_alias"), "Alias reasoning");
});
it("should cache assistant reasoning without tool calls by request and message index", () => {
it("should cache assistant reasoning without tool calls by scoped transcript", () => {
clearReasoningCacheAll();
const scope = "api-key:test:session:test";
const historyMessages = [{ role: "user", content: "hi" }];
const assistantMessage = {
role: "assistant",
content: "Hello!",
reasoning_content: "No tool call reasoning",
};
const cached = cacheReasoningFromAssistantMessage(
{
role: "assistant",
reasoning_content: "No tool call reasoning",
},
assistantMessage,
"deepseek",
"deepseek-reasoner",
{ requestId: "req_no_tools", messageIndex: 3 }
"deepseek-v4-pro",
{ scope, historyMessages }
);
const cacheKey = buildAssistantMessageCacheKey(
scope,
[...historyMessages, assistantMessage],
historyMessages.length
);
assert.equal(cached, 1);
assert.equal(lookupReasoning("request:req_no_tools:message:3"), "No tool call reasoning");
assert.equal(lookupReasoning(cacheKey), "No tool call reasoning");
});
it("should skip assistant reasoning without tool calls when stable key context is absent", () => {
@@ -591,6 +625,62 @@ describe("Reasoning Replay Cache — Translator Replay", () => {
assert.equal(getReasoningCacheServiceStats().replays, 1);
});
it("should preserve DeepSeek Responses reasoning before Chat conversion", () => {
clearReasoningCacheAll();
clearModelsDevCapabilities();
cacheReasoning(
"call_ds_responses",
"deepseek",
"deepseek-v4-flash",
"Conflicting cached reasoning"
);
const statsBeforeTranslation = getReasoningCacheServiceStats();
const translated = translateRequest(
FORMATS.OPENAI_RESPONSES,
FORMATS.OPENAI,
"deepseek-v4-flash",
{
reasoning: { effort: "high" },
input: [
{
type: "reasoning",
summary: [{ type: "summary_text", text: "Client DeepSeek reasoning" }],
},
{
type: "message",
role: "assistant",
content: [{ type: "output_text", text: "I will inspect" }],
},
{
type: "function_call",
call_id: "call_ds_responses",
name: "read_file",
arguments: "{}",
},
{ type: "function_call_output", call_id: "call_ds_responses", output: "contents" },
{
type: "message",
role: "user",
content: [{ type: "input_text", text: "Continue" }],
},
],
},
false,
null,
"deepseek"
);
const assistant = translated.messages.find((message) => message.role === "assistant");
assert.equal(assistant.reasoning_content, "Client DeepSeek reasoning");
assert.equal(assistant.content[0].text, "I will inspect");
assert.equal(assistant.tool_calls[0].id, "call_ds_responses");
const statsAfterTranslation = getReasoningCacheServiceStats();
assert.equal(statsAfterTranslation.hits, statsBeforeTranslation.hits);
assert.equal(statsAfterTranslation.misses, statsBeforeTranslation.misses);
assert.equal(statsAfterTranslation.replays, statsBeforeTranslation.replays);
});
it("should preserve client-provided reasoning content", () => {
clearReasoningCacheAll();
clearModelsDevCapabilities();
@@ -604,6 +694,7 @@ describe("Reasoning Replay Cache — Translator Replay", () => {
},
});
cacheReasoning("call_preserve", "deepseek", "deepseek-reasoner", "Cached reasoning");
const statsBeforeTranslation = getReasoningCacheServiceStats();
const translated = translateRequest(
FORMATS.OPENAI,
@@ -633,6 +724,9 @@ describe("Reasoning Replay Cache — Translator Replay", () => {
assert.equal(translated.messages[1].reasoning_content, "Client reasoning");
assert.equal(getReasoningCacheServiceStats().replays, 0);
const statsAfterTranslation = getReasoningCacheServiceStats();
assert.equal(statsAfterTranslation.hits, statsBeforeTranslation.hits);
assert.equal(statsAfterTranslation.misses, statsBeforeTranslation.misses);
});
it("should inject cached reasoning for Qwen and GLM thinking models", () => {
@@ -851,9 +945,7 @@ describe("Reasoning Replay Cache — Translator Replay", () => {
);
});
it("should replay cached reasoning for a plain (non-tool-call) DeepSeek turn when available (#1682)", () => {
// When a request_id-keyed cache entry exists for the plain turn, the real
// reasoning is replayed instead of the placeholder.
it("should replay cached reasoning for a plain DeepSeek turn when available", () => {
clearReasoningCacheAll();
clearModelsDevCapabilities();
saveModelsDevCapabilities({
@@ -865,49 +957,32 @@ describe("Reasoning Replay Cache — Translator Replay", () => {
}),
},
});
// The non-tool-call cache key is built as `getAssistantMessageCacheKey(result, messageIndex)`
// where messageIndex is the assistant message's real position in the `messages`
// array (index 1 here: user, assistant, user) — matching what the write side
// (chatCore.ts) now caches under once the response is generated.
cacheReasoning(
"request:req-plain-1:message:1",
"deepseek",
"deepseek-v4-pro",
"Real cached plain-turn reasoning"
);
const scope = "api-key:test:session:plain";
const messages = [
{ role: "user", content: "hi" },
{ role: "assistant", content: "Hello! How can I help?" },
{ role: "user", content: "tell me more" },
];
const cacheKey = buildAssistantMessageCacheKey(scope, messages, 1);
cacheReasoning(cacheKey, "deepseek", "deepseek-v4-pro", "Real cached plain-turn reasoning");
const translated = translateRequest(
FORMATS.OPENAI,
FORMATS.OPENAI,
"deepseek-v4-pro",
{
request_id: "req-plain-1",
messages: [
{ role: "user", content: "hi" },
{ role: "assistant", content: "Hello! How can I help?" },
{ role: "user", content: "tell me more" },
],
},
{ messages },
false,
null,
"deepseek"
"deepseek",
null,
{ reasoningCacheScope: scope }
);
assert.equal(
translated.messages[1].reasoning_content,
"Real cached plain-turn reasoning",
"plain DeepSeek assistant turn should replay the real cached reasoning when present"
);
assert.equal(translated.messages[1].reasoning_content, "Real cached plain-turn reasoning");
assert.equal(getReasoningCacheServiceStats().replays, 1);
});
it("write side (chatCore's messageIndex) and read side (translateRequest) agree on the same key end-to-end", () => {
// Regression for a mismatch where chatCore.ts always cached under
// `messageIndex: 0` (the position of the response within *its own* choices
// array) while translateRequest's read side looked up the message's real
// position in the *next* turn's full history — the two never agreed once a
// conversation went past its first assistant turn, so replay silently
// fell back to the placeholder in real multi-turn usage.
it("writes and reads the same no-tool transcript key for Chat history", () => {
clearReasoningCacheAll();
clearModelsDevCapabilities();
saveModelsDevCapabilities({
@@ -919,36 +994,83 @@ describe("Reasoning Replay Cache — Translator Replay", () => {
}),
},
});
// Turn 1: the incoming request has a single user message (length 1), so
// the assistant response chatCore is about to cache will occupy index 1
// once it's appended to history for turn 2 — mirroring
// `messageIndex: bodyMessages.length` in chatCore.ts.
const turn1RequestBody = { messages: [{ role: "user", content: "hi" }] };
const scope = "api-key:test:session:chat";
const historyMessages = [{ role: "user", content: "hi" }];
cacheReasoningFromAssistantMessage(
{ role: "assistant", content: "Hello! How can I help?", reasoning_content: "real reasoning" },
"deepseek",
"deepseek-v4-pro",
{ requestId: "req-e2e-1", messageIndex: turn1RequestBody.messages.length }
{ scope, historyMessages }
);
// Turn 2: client replays the full history including the cached assistant
// turn, now genuinely at index 1.
const translated = translateRequest(
FORMATS.OPENAI,
FORMATS.OPENAI,
"deepseek-v4-pro",
{
request_id: "req-e2e-1",
messages: [
{ role: "user", content: "hi" },
...historyMessages,
{ role: "assistant", content: "Hello! How can I help?" },
{ role: "user", content: "tell me more" },
],
},
false,
null,
"deepseek"
"deepseek",
null,
{ reasoningCacheScope: scope }
);
assert.equal(translated.messages[1].reasoning_content, "real reasoning");
assert.equal(getReasoningCacheServiceStats().replays, 1);
});
it("writes a Chat response and replays it from Responses history", () => {
clearReasoningCacheAll();
clearModelsDevCapabilities();
saveModelsDevCapabilities({
deepseek: {
"deepseek-v4-pro": buildCapability({
interleaved_field: "reasoning_content",
reasoning: true,
tool_call: true,
}),
},
});
const scope = "api-key:test:session:responses";
const historyMessages = [{ role: "user", content: [{ type: "text", text: "hi" }] }];
cacheReasoningFromAssistantMessage(
{ role: "assistant", content: "Hello! How can I help?", reasoning_content: "real reasoning" },
"deepseek",
"deepseek-v4-pro",
{ scope, historyMessages }
);
const translated = translateRequest(
FORMATS.OPENAI_RESPONSES,
FORMATS.OPENAI,
"deepseek-v4-pro",
{
reasoning: { effort: "high" },
input: [
{ type: "message", role: "user", content: [{ type: "input_text", text: "hi" }] },
{
type: "message",
role: "assistant",
content: [{ type: "output_text", text: "Hello! How can I help?" }],
},
{
type: "message",
role: "user",
content: [{ type: "input_text", text: "tell me more" }],
},
],
},
false,
null,
"deepseek",
null,
{ reasoningCacheScope: scope }
);
assert.equal(translated.messages[1].reasoning_content, "real reasoning");

View File

@@ -8,8 +8,12 @@ const geminiHelper = await import("../../open-sse/translator/helpers/geminiHelpe
const toolCallHelper = await import("../../open-sse/translator/helpers/toolCallHelper.ts");
const { FORMATS } = await import("../../open-sse/translator/formats.ts");
const { translateRequest } = await import("../../open-sse/translator/index.ts");
const { cacheReasoningByKey, clearReasoningCacheAll, getReasoningCacheServiceStats } =
await import("../../open-sse/services/reasoningCache.ts");
const {
buildAssistantMessageCacheKey,
cacheReasoningByKey,
clearReasoningCacheAll,
getReasoningCacheServiceStats,
} = await import("../../open-sse/services/reasoningCache.ts");
const { clearModelsDevCapabilities, saveModelsDevCapabilities } =
await import("../../src/lib/modelsDevSync.ts");
@@ -639,8 +643,13 @@ test("translateRequest replays cached reasoning-only messages when interleaved f
}),
},
});
const scope = "api-key:test:session:helper-deepseek";
const messages = [
{ role: "user", content: "solve this" },
{ role: "assistant", content: "answer", reasoning_content: "" },
];
cacheReasoningByKey(
"request:req_reasoning_only:message:1",
buildAssistantMessageCacheKey(scope, messages, 1),
"deepseek",
"deepseek-v4-flash",
"cached reasoning only"
@@ -650,16 +659,12 @@ test("translateRequest replays cached reasoning-only messages when interleaved f
FORMATS.OPENAI,
FORMATS.OPENAI,
"deepseek-v4-flash",
{
_reasoningCacheRequestId: "req_reasoning_only",
messages: [
{ role: "user", content: "solve this" },
{ role: "assistant", content: "answer", reasoning_content: "" },
],
},
{ messages },
false,
null,
"deepseek"
"deepseek",
null,
{ reasoningCacheScope: scope }
);
assert.equal(result.messages[1].reasoning_content, "cached reasoning only");
@@ -670,8 +675,13 @@ test("translateRequest replays cached reasoning-only messages when interleaved f
test("translateRequest does not replay reasoning-only messages for non-DeepSeek models", () => {
clearReasoningCacheAll();
const scope = "api-key:test:session:helper-kimi";
const messages = [
{ role: "user", content: "solve this" },
{ role: "assistant", content: "answer", reasoning_content: "" },
];
cacheReasoningByKey(
"request:req_kimi_reasoning_only:message:0",
buildAssistantMessageCacheKey(scope, messages, 1),
"kimi",
"kimi-k2.6",
"cached kimi reasoning"
@@ -681,16 +691,12 @@ test("translateRequest does not replay reasoning-only messages for non-DeepSeek
FORMATS.OPENAI,
FORMATS.OPENAI,
"kimi-k2.6",
{
_reasoningCacheRequestId: "req_kimi_reasoning_only",
messages: [
{ role: "user", content: "solve this" },
{ role: "assistant", content: "answer", reasoning_content: "" },
],
},
{ messages },
false,
null,
"kimi"
"kimi",
null,
{ reasoningCacheScope: scope }
);
assert.equal(result.messages[1].reasoning_content, undefined);

View File

@@ -85,6 +85,94 @@ test("Responses -> Chat converts instructions, inputs, function calls, outputs,
});
});
test("Responses -> Chat keeps assistant text, reasoning, and function calls in one turn", () => {
const result = openaiResponsesToOpenAIRequest(
"gpt-4o",
{
input: [
{
type: "reasoning",
summary: [{ type: "summary_text", text: "Inspect first" }],
},
{
type: "message",
role: "assistant",
content: [{ type: "output_text", text: "I will inspect" }],
},
{ type: "function_call", call_id: "call_1", name: "read_file", arguments: "{}" },
{ type: "function_call", call_id: "call_2", name: "search", arguments: "{}" },
{ type: "function_call_output", call_id: "call_1", output: "contents" },
],
},
false,
{ _preserveReasoningContent: true }
) as { messages: Array<Record<string, unknown>> };
assert.equal(result.messages.length, 2);
assert.deepEqual(result.messages[0], {
role: "assistant",
content: [{ type: "text", text: "I will inspect" }],
reasoning_content: "Inspect first",
tool_calls: [
{
id: "call_1",
type: "function",
function: { name: "read_file", arguments: "{}" },
},
{
id: "call_2",
type: "function",
function: { name: "search", arguments: "{}" },
},
],
});
assert.deepEqual(result.messages[1], {
role: "tool",
tool_call_id: "call_1",
content: "contents",
});
});
test("Responses -> Chat merges assistant text that follows a function call", () => {
const result = openaiResponsesToOpenAIRequest(
"gpt-4o",
{
input: [
{ type: "function_call", call_id: "call_1", name: "read_file", arguments: "{}" },
{
type: "message",
role: "assistant",
content: [{ type: "output_text", text: "I inspected it" }],
},
{
type: "reasoning",
summary: [{ type: "summary_text", text: "Inspection complete" }],
},
{ type: "message", role: "user", content: [{ type: "input_text", text: "Continue" }] },
],
},
false,
{ _preserveReasoningContent: true }
) as { messages: Array<Record<string, unknown>> };
assert.deepEqual(result.messages[0], {
role: "assistant",
content: [{ type: "text", text: "I inspected it" }],
tool_calls: [
{
id: "call_1",
type: "function",
function: { name: "read_file", arguments: "{}" },
},
],
reasoning_content: "Inspection complete",
});
assert.deepEqual(result.messages[1], {
role: "user",
content: [{ type: "text", text: "Continue" }],
});
});
test("Responses -> Chat filters orphan tool outputs and supports role-based message items", () => {
const result = openaiResponsesToOpenAIRequest(
"gpt-4o",
@@ -925,7 +1013,12 @@ test("Responses -> Chat: tool_search is mapped to a Chat function tool, not drop
input: [{ role: "user", content: [{ type: "input_text", text: "hello" }] }],
tools: [
{ type: "tool_search", name: "search" },
{ type: "function", name: "foo", description: "A function", parameters: { type: "object" } },
{
type: "function",
name: "foo",
description: "A function",
parameters: { type: "object" },
},
],
},
false,
@@ -934,7 +1027,11 @@ test("Responses -> Chat: tool_search is mapped to a Chat function tool, not drop
const tools = result.tools as any[];
assert.ok(Array.isArray(tools), "tools array must be present");
assert.equal(tools.some((t) => t.type === "tool_search"), false, "raw tool_search type must not survive");
assert.equal(
tools.some((t) => t.type === "tool_search"),
false,
"raw tool_search type must not survive"
);
assert.equal(tools.length, 2, "mapped tool_search function + the function tool must remain");
const toolSearch = tools.find((t) => t.function?.name === "search");
assert.ok(toolSearch, "tool_search must be mapped to a Chat function tool named after it");