diff --git a/changelog.d/fixes/12307-semantic-cache-output-contract.md b/changelog.d/fixes/12307-semantic-cache-output-contract.md new file mode 100644 index 0000000000..2a9fe60d7b --- /dev/null +++ b/changelog.d/fixes/12307-semantic-cache-output-contract.md @@ -0,0 +1 @@ +- fix(cache): fold `response_format`/Responses-API `text.format` into the semantic cache signature so a `temp=0` request can no longer be served a stored response body with a different output schema (#12307) diff --git a/open-sse/handlers/chatCore/semanticCache.ts b/open-sse/handlers/chatCore/semanticCache.ts index 7913741117..36afc9c77a 100644 --- a/open-sse/handlers/chatCore/semanticCache.ts +++ b/open-sse/handlers/chatCore/semanticCache.ts @@ -1,4 +1,9 @@ -import { generateSignature, getCachedResponse, isCacheableForRead } from "@/lib/semanticCache"; +import { + generateSignature, + getCachedResponse, + isCacheableForRead, + outputContractOf, +} from "@/lib/semanticCache"; import { calculateCost } from "@/lib/usage/costCalculator"; import { finalizePendingScope, type PendingRequestScope } from "@/lib/usage/pendingRequestScope"; import { synthesizeOpenAiSseFromJson } from "../../utils/jsonToSse.ts"; @@ -25,13 +30,7 @@ export async function checkSemanticCache({ semanticCacheEnabled: boolean; // Only the fields this read path actually touches are named; everything else // on the request body stays `unknown` via the index signature. - body: Record & { - temperature?: number; - top_p?: number; - tool_choice?: unknown; - tools?: unknown; - response_format?: unknown; - }; + body: Record & { temperature?: number; top_p?: number }; clientRawRequest: { headers?: unknown } | null; model: string; provider: string; @@ -54,7 +53,7 @@ export async function checkSemanticCache({ body.temperature, body.top_p, apiKeyId ?? undefined, - { toolChoice: body.tool_choice, tools: body.tools, responseFormat: body.response_format } + outputContractOf(body) ); const cached = getCachedResponse(signature); if (cached) { diff --git a/open-sse/handlers/chatCore/semanticCacheStore.ts b/open-sse/handlers/chatCore/semanticCacheStore.ts index ceb97adc72..e5b7e2cc6b 100644 --- a/open-sse/handlers/chatCore/semanticCacheStore.ts +++ b/open-sse/handlers/chatCore/semanticCacheStore.ts @@ -10,6 +10,7 @@ */ import { generateSignature as defaultGenerateSignature, + outputContractOf, setCachedResponse as defaultSetCachedResponse, isCacheableForWrite as defaultIsCacheableForWrite, isTruncatedCompletion as defaultIsTruncatedCompletion, @@ -23,9 +24,6 @@ type CacheBody = { input?: unknown; temperature?: number; top_p?: number; - tool_choice?: unknown; - tools?: unknown; - response_format?: unknown; }; type UsageLike = { prompt_tokens?: number; completion_tokens?: number } | null | undefined; @@ -74,11 +72,7 @@ export function storeSemanticCacheResponse( args.body.temperature, args.body.top_p, args.apiKeyId ?? undefined, - { - toolChoice: args.body.tool_choice, - tools: args.body.tools, - responseFormat: args.body.response_format, - } + outputContractOf(args.body) ); const tokensSaved = args.usage?.prompt_tokens + args.usage?.completion_tokens || 0; deps.setCachedResponse(signature, args.model, args.translatedResponse, tokensSaved); diff --git a/open-sse/handlers/chatCore/streamingSemanticCacheStore.ts b/open-sse/handlers/chatCore/streamingSemanticCacheStore.ts index 0447b2543b..3c3bd3473b 100644 --- a/open-sse/handlers/chatCore/streamingSemanticCacheStore.ts +++ b/open-sse/handlers/chatCore/streamingSemanticCacheStore.ts @@ -11,6 +11,7 @@ */ import { generateSignature as defaultGenerateSignature, + outputContractOf, setCachedResponse as defaultSetCachedResponse, isCacheableForWrite as defaultIsCacheableForWrite, isTruncatedStreamBody as defaultIsTruncatedStreamBody, @@ -24,9 +25,6 @@ type CacheBody = { input?: unknown; temperature?: number; top_p?: number; - tool_choice?: unknown; - tools?: unknown; - response_format?: unknown; }; export interface StreamingSemanticCacheStoreDeps { @@ -77,11 +75,7 @@ function writeStreamingCacheEntry( args.body.temperature, args.body.top_p, args.apiKeyId ?? undefined, - { - toolChoice: args.body.tool_choice, - tools: args.body.tools, - responseFormat: args.body.response_format, - } + outputContractOf(args.body) ); const tokensSaved = streamTokensSaved(args.streamUsage); deps.setCachedResponse(sig, args.model, cleanBody, tokensSaved); diff --git a/src/lib/semanticCache.ts b/src/lib/semanticCache.ts index 236226ddbe..d0bdaaa822 100644 --- a/src/lib/semanticCache.ts +++ b/src/lib/semanticCache.ts @@ -6,7 +6,8 @@ * are cached after assembly; cache hits always return JSON. * Two-tier: in-memory LRU (fast) + SQLite (persistent across restarts). * - * Cache key = SHA-256(model + normalized messages + temperature + top_p) + * Cache key = SHA-256(model + normalized messages + temperature + top_p + * + output contract, when present — see outputContractOf, #12307) * Bypass: X-OmniRoute-No-Cache: true * * @module lib/semanticCache @@ -142,11 +143,48 @@ export function clearMemoryCache(): void { * (#12734). Without these, a cached response produced under one `tool_choice`/`tools`/ * `response_format` could be replayed for a later request that forbids or changes that * behavior (e.g. a cached `tool_calls` response served to a `tool_choice: "none"` request). + * + * The snake_case fields mirror the raw request body shape and are what `outputContractOf` + * (#12307) fills in; the camelCase fields are the pre-existing (#12734) call-site shape. + * `generateSignature` folds both spellings in so neither call style silently drops a field. */ export interface SignatureConstraints { toolChoice?: unknown; tools?: unknown; responseFormat?: unknown; + tool_choice?: unknown; + response_format?: unknown; + text_format?: unknown; +} + +/** + * The parts of a request that decide what a *valid response* looks like. + * Two calls that agree on the conversation but disagree here are not + * interchangeable and must not share a cache entry (#12307): a request for + * {color, wheels} must not be served a stored {value: "..."} body, and a + * tool-calling request must not be served the body of one without tools. + * + * Returns null when the request carries none of these, so plain-chat + * signatures — and every cache entry already written for them — are unchanged. + */ +export function outputContractOf(body: unknown): SignatureConstraints | null { + const record = asRecord(body); + const text = asRecord(record.text); + const contract: SignatureConstraints = {}; + // Both spellings are set for each field so callers built against either the + // pre-existing (#12734) camelCase constraints shape or this snake_case one + // (matching the raw request body) can read the field they expect. + if (record.response_format != null) { + contract.response_format = record.response_format; + contract.responseFormat = record.response_format; + } + if (text.format != null) contract.text_format = text.format; + if (record.tools != null) contract.tools = record.tools; + if (record.tool_choice != null) { + contract.tool_choice = record.tool_choice; + contract.toolChoice = record.tool_choice; + } + return Object.keys(contract).length > 0 ? contract : null; } /** Normalize a single tool definition, keeping only the fields that define its policy. */ @@ -181,8 +219,9 @@ function normalizeTools(tools: unknown): unknown { * @param {number} temperature * @param {number} topP * @param {string} [apiKeyId] - API key ID for per-key isolation (prevents cross-user cache hits) - * @param {SignatureConstraints} [constraints] - tool_choice/tools/response_format (#12734): - * these change model behavior and must not collide with a signature computed without them. + * @param {SignatureConstraints} [constraints] - tool_choice/tools/response_format (#12734) + * plus the Responses-API `text.format` spelling (#12307): these change model behavior + * and must not collide with a signature computed without them. * @returns {string} hex signature */ export function generateSignature( @@ -191,16 +230,17 @@ export function generateSignature( temperature = 0, topP = 1, apiKeyId?: string, - constraints?: SignatureConstraints + constraints?: SignatureConstraints | null ) { const payload = JSON.stringify({ model, messages: normalizeConversation(conversation), temperature, top_p: topP, - tool_choice: constraints?.toolChoice, + tool_choice: constraints?.toolChoice ?? constraints?.tool_choice, tools: normalizeTools(constraints?.tools), - response_format: constraints?.responseFormat, + response_format: constraints?.responseFormat ?? constraints?.response_format, + text_format: constraints?.text_format, }); const digest = crypto.createHash("sha256").update(payload).digest("hex"); // Per-key cache isolation (#3740) namespaces the signature with the apiKeyId as a diff --git a/tests/unit/12307-semantic-cache-output-contract.test.ts b/tests/unit/12307-semantic-cache-output-contract.test.ts new file mode 100644 index 0000000000..849ca8b7ee --- /dev/null +++ b/tests/unit/12307-semantic-cache-output-contract.test.ts @@ -0,0 +1,113 @@ +// Regression for #12307: the semantic-cache signature ignored response_format / +// tools, so two temp=0 requests with identical messages but different response +// schemas shared a cache key — the second was served the first's stored body +// under a 200, violating the schema it asked for. The signature now folds an +// "output contract" (response_format, text.format, tools, tool_choice) into the +// digest when present, and stays byte-identical to the legacy key when absent +// so existing plain-chat cache entries survive the upgrade. +import test from "node:test"; +import assert from "node:assert/strict"; + +const { generateSignature, outputContractOf } = await import("../../src/lib/semanticCache.ts"); +const { storeSemanticCacheResponse } = await import( + "../../open-sse/handlers/chatCore/semanticCacheStore.ts" +); +const { storeStreamingSemanticCacheResponse } = await import( + "../../open-sse/handlers/chatCore/streamingSemanticCacheStore.ts" +); + +const MESSAGES = [{ role: "user", content: "Describe a red bicycle." }]; + +const schemaOf = (properties: Record) => ({ + type: "json_schema", + json_schema: { name: "d", strict: true, schema: { type: "object", properties } }, +}); + +test("different response schemas over identical messages produce different signatures", () => { + const a = generateSignature("m", MESSAGES, 0, 1, "key", + outputContractOf({ response_format: schemaOf({ value: { type: "string" } }) })); + const b = generateSignature("m", MESSAGES, 0, 1, "key", + outputContractOf({ response_format: schemaOf({ color: { type: "string" }, wheels: { type: "integer" } }) })); + assert.notEqual(a, b); +}); + +test("identical response schemas still share a signature", () => { + const contract = () => outputContractOf({ response_format: schemaOf({ value: { type: "string" } }) }); + assert.equal( + generateSignature("m", MESSAGES, 0, 1, "key", contract()), + generateSignature("m", MESSAGES, 0, 1, "key", contract()) + ); +}); + +test("tools presence splits the signature from a tool-less request", () => { + const withTools = generateSignature("m", MESSAGES, 0, 1, "key", + outputContractOf({ tools: [{ type: "function", function: { name: "f", parameters: {} } }] })); + const without = generateSignature("m", MESSAGES, 0, 1, "key", outputContractOf({})); + assert.notEqual(withTools, without); +}); + +test("plain chat keeps the legacy signature — existing cache entries stay valid", () => { + const legacy = generateSignature("m", MESSAGES, 0, 1, "key"); + const withNullContract = generateSignature("m", MESSAGES, 0, 1, "key", outputContractOf({ stream: true })); + assert.equal(legacy, withNullContract); +}); + +test("outputContractOf returns null when no shape-determining field is present", () => { + assert.equal(outputContractOf({ model: "m", messages: MESSAGES, temperature: 0 }), null); + assert.equal(outputContractOf(null), null); + assert.equal(outputContractOf("nonsense"), null); +}); + +test("outputContractOf collects the Responses-API text.format spelling too", () => { + const contract = outputContractOf({ text: { format: { type: "json_schema", schema: {} } } }); + assert.ok(contract && typeof contract === "object"); + assert.ok("text_format" in (contract as Record)); +}); + +function recordingDeps() { + const calls: unknown[][] = []; + return { + calls, + deps: { + isCacheableForWrite: () => true, + isSmallEnoughForSemanticCache: () => true, + generateSignature: (...args: unknown[]) => { + calls.push(args); + return "sig"; + }, + setCachedResponse: () => {}, + }, + }; +} + +test("non-streaming store path forwards the output contract to the signature", () => { + const { calls, deps } = recordingDeps(); + const body = { messages: MESSAGES, temperature: 0, top_p: 1, response_format: schemaOf({ v: { type: "string" } }) }; + storeSemanticCacheResponse( + { enabled: true, body, headers: {}, translatedResponse: { ok: true }, model: "m", apiKeyId: "key" }, + deps as never + ); + assert.equal(calls.length, 1); + assert.deepEqual(calls[0][5], outputContractOf(body)); +}); + +test("streaming store path forwards the output contract to the signature", () => { + const { calls, deps } = recordingDeps(); + const body = { messages: MESSAGES, temperature: 0, top_p: 1, response_format: schemaOf({ v: { type: "string" } }) }; + storeStreamingSemanticCacheResponse( + { + enabled: true, + streamStatus: 200, + streamResponseBody: { ok: true }, + body, + headers: {}, + model: "m", + apiKeyId: "key", + streamUsage: null, + log: null, + } as never, + deps as never + ); + assert.equal(calls.length, 1); + assert.deepEqual(calls[0][5], outputContractOf(body)); +});