fix(cache): fold the response output contract into the semantic cache signature (#12309)

* fix(cache): fold the response output contract into the semantic cache signature

The signature hashed only {model, messages, temperature, top_p}, so two temp=0
requests with identical messages but different response_format shared a cache
key: the second was served the first's stored body under a 200, violating the
schema it asked for. tools/tool_choice had the same exposure.

generateSignature now takes an optional output contract — response_format,
text.format, tools, tool_choice, collected by outputContractOf() — and folds it
into the digest only when present, so plain-chat signatures (and every cache
entry already written for them) are unchanged. All three call sites pass it;
read/write symmetry is preserved because bodyForCacheWrite snapshots the same
body object the read path hashed (#cache-signature-asymmetry).

Closes #12307

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

* docs(changelog): add fragment for #12307 semantic-cache output-contract fix

Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com>

* fix(cache): populate both constraint spellings in outputContractOf

The merge with #12734 left generateSignature reading the camelCase
constraints (toolChoice/responseFormat) with a snake_case fallback, but
outputContractOf only filled the snake_case keys, so the #12734
"signature is called with tool_choice/tools/response_format from body"
store tests failed on the merged branch. Set both spellings so either
caller shape reads the value it expects.

Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com>

---------

Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com>
Co-authored-by: amirrezakm <amirrezakm@users.noreply.github.com>

changelog.d/fixes/12307-semantic-cache-output-contract.md

@@ -0,0 +1 @@
+- fix(cache): fold `response_format`/Responses-API `text.format` into the semantic cache signature so a `temp=0` request can no longer be served a stored response body with a different output schema (#12307)

open-sse/handlers/chatCore/semanticCache.ts

@@ -1,4 +1,9 @@
-import { generateSignature, getCachedResponse, isCacheableForRead } from "@/lib/semanticCache";
+import {
+  generateSignature,
+  getCachedResponse,
+  isCacheableForRead,
+  outputContractOf,
+} from "@/lib/semanticCache";
 import { calculateCost } from "@/lib/usage/costCalculator";
 import { finalizePendingScope, type PendingRequestScope } from "@/lib/usage/pendingRequestScope";
 import { synthesizeOpenAiSseFromJson } from "../../utils/jsonToSse.ts";
@@ -25,13 +30,7 @@ export async function checkSemanticCache({
   semanticCacheEnabled: boolean;
   // Only the fields this read path actually touches are named; everything else
   // on the request body stays `unknown` via the index signature.
-  body: Record<string, unknown> & {
-    temperature?: number;
-    top_p?: number;
-    tool_choice?: unknown;
-    tools?: unknown;
-    response_format?: unknown;
-  };
+  body: Record<string, unknown> & { temperature?: number; top_p?: number };
   clientRawRequest: { headers?: unknown } | null;
   model: string;
   provider: string;
@@ -54,7 +53,7 @@ export async function checkSemanticCache({
       body.temperature,
       body.top_p,
       apiKeyId ?? undefined,
-      { toolChoice: body.tool_choice, tools: body.tools, responseFormat: body.response_format }
+      outputContractOf(body)
     );
     const cached = getCachedResponse(signature);
     if (cached) {

open-sse/handlers/chatCore/semanticCacheStore.ts

@@ -10,6 +10,7 @@
  */
 import {
   generateSignature as defaultGenerateSignature,
+  outputContractOf,
   setCachedResponse as defaultSetCachedResponse,
   isCacheableForWrite as defaultIsCacheableForWrite,
   isTruncatedCompletion as defaultIsTruncatedCompletion,
@@ -23,9 +24,6 @@ type CacheBody = {
   input?: unknown;
   temperature?: number;
   top_p?: number;
-  tool_choice?: unknown;
-  tools?: unknown;
-  response_format?: unknown;
 };
 
 type UsageLike = { prompt_tokens?: number; completion_tokens?: number } | null | undefined;
@@ -74,11 +72,7 @@ export function storeSemanticCacheResponse(
     args.body.temperature,
     args.body.top_p,
     args.apiKeyId ?? undefined,
-    {
-      toolChoice: args.body.tool_choice,
-      tools: args.body.tools,
-      responseFormat: args.body.response_format,
-    }
+    outputContractOf(args.body)
   );
   const tokensSaved = args.usage?.prompt_tokens + args.usage?.completion_tokens || 0;
   deps.setCachedResponse(signature, args.model, args.translatedResponse, tokensSaved);

open-sse/handlers/chatCore/streamingSemanticCacheStore.ts

@@ -11,6 +11,7 @@
  */
 import {
   generateSignature as defaultGenerateSignature,
+  outputContractOf,
   setCachedResponse as defaultSetCachedResponse,
   isCacheableForWrite as defaultIsCacheableForWrite,
   isTruncatedStreamBody as defaultIsTruncatedStreamBody,
@@ -24,9 +25,6 @@ type CacheBody = {
   input?: unknown;
   temperature?: number;
   top_p?: number;
-  tool_choice?: unknown;
-  tools?: unknown;
-  response_format?: unknown;
 };
 
 export interface StreamingSemanticCacheStoreDeps {
@@ -77,11 +75,7 @@ function writeStreamingCacheEntry(
       args.body.temperature,
       args.body.top_p,
       args.apiKeyId ?? undefined,
-      {
-        toolChoice: args.body.tool_choice,
-        tools: args.body.tools,
-        responseFormat: args.body.response_format,
-      }
+      outputContractOf(args.body)
     );
     const tokensSaved = streamTokensSaved(args.streamUsage);
     deps.setCachedResponse(sig, args.model, cleanBody, tokensSaved);

src/lib/semanticCache.ts

@@ -6,7 +6,8 @@
  * are cached after assembly; cache hits always return JSON.
  * Two-tier: in-memory LRU (fast) + SQLite (persistent across restarts).
  *
- * Cache key = SHA-256(model + normalized messages + temperature + top_p)
+ * Cache key = SHA-256(model + normalized messages + temperature + top_p
+ *             + output contract, when present — see outputContractOf, #12307)
  * Bypass: X-OmniRoute-No-Cache: true
  *
  * @module lib/semanticCache
@@ -142,11 +143,48 @@ export function clearMemoryCache(): void {
  * (#12734). Without these, a cached response produced under one `tool_choice`/`tools`/
  * `response_format` could be replayed for a later request that forbids or changes that
  * behavior (e.g. a cached `tool_calls` response served to a `tool_choice: "none"` request).
+ *
+ * The snake_case fields mirror the raw request body shape and are what `outputContractOf`
+ * (#12307) fills in; the camelCase fields are the pre-existing (#12734) call-site shape.
+ * `generateSignature` folds both spellings in so neither call style silently drops a field.
  */
 export interface SignatureConstraints {
   toolChoice?: unknown;
   tools?: unknown;
   responseFormat?: unknown;
+  tool_choice?: unknown;
+  response_format?: unknown;
+  text_format?: unknown;
+}
+
+/**
+ * The parts of a request that decide what a *valid response* looks like.
+ * Two calls that agree on the conversation but disagree here are not
+ * interchangeable and must not share a cache entry (#12307): a request for
+ * {color, wheels} must not be served a stored {value: "..."} body, and a
+ * tool-calling request must not be served the body of one without tools.
+ *
+ * Returns null when the request carries none of these, so plain-chat
+ * signatures — and every cache entry already written for them — are unchanged.
+ */
+export function outputContractOf(body: unknown): SignatureConstraints | null {
+  const record = asRecord(body);
+  const text = asRecord(record.text);
+  const contract: SignatureConstraints = {};
+  // Both spellings are set for each field so callers built against either the
+  // pre-existing (#12734) camelCase constraints shape or this snake_case one
+  // (matching the raw request body) can read the field they expect.
+  if (record.response_format != null) {
+    contract.response_format = record.response_format;
+    contract.responseFormat = record.response_format;
+  }
+  if (text.format != null) contract.text_format = text.format;
+  if (record.tools != null) contract.tools = record.tools;
+  if (record.tool_choice != null) {
+    contract.tool_choice = record.tool_choice;
+    contract.toolChoice = record.tool_choice;
+  }
+  return Object.keys(contract).length > 0 ? contract : null;
 }
 
 /** Normalize a single tool definition, keeping only the fields that define its policy. */
@@ -181,8 +219,9 @@ function normalizeTools(tools: unknown): unknown {
  * @param {number} temperature
  * @param {number} topP
  * @param {string} [apiKeyId] - API key ID for per-key isolation (prevents cross-user cache hits)
- * @param {SignatureConstraints} [constraints] - tool_choice/tools/response_format (#12734):
- *   these change model behavior and must not collide with a signature computed without them.
+ * @param {SignatureConstraints} [constraints] - tool_choice/tools/response_format (#12734)
+ *   plus the Responses-API `text.format` spelling (#12307): these change model behavior
+ *   and must not collide with a signature computed without them.
  * @returns {string} hex signature
  */
 export function generateSignature(
@@ -191,16 +230,17 @@ export function generateSignature(
   temperature = 0,
   topP = 1,
   apiKeyId?: string,
-  constraints?: SignatureConstraints
+  constraints?: SignatureConstraints | null
 ) {
   const payload = JSON.stringify({
     model,
     messages: normalizeConversation(conversation),
     temperature,
     top_p: topP,
-    tool_choice: constraints?.toolChoice,
+    tool_choice: constraints?.toolChoice ?? constraints?.tool_choice,
     tools: normalizeTools(constraints?.tools),
-    response_format: constraints?.responseFormat,
+    response_format: constraints?.responseFormat ?? constraints?.response_format,
+    text_format: constraints?.text_format,
   });
   const digest = crypto.createHash("sha256").update(payload).digest("hex");
   // Per-key cache isolation (#3740) namespaces the signature with the apiKeyId as a

tests/unit/12307-semantic-cache-output-contract.test.ts

@@ -0,0 +1,113 @@
+// Regression for #12307: the semantic-cache signature ignored response_format /
+// tools, so two temp=0 requests with identical messages but different response
+// schemas shared a cache key — the second was served the first's stored body
+// under a 200, violating the schema it asked for. The signature now folds an
+// "output contract" (response_format, text.format, tools, tool_choice) into the
+// digest when present, and stays byte-identical to the legacy key when absent
+// so existing plain-chat cache entries survive the upgrade.
+import test from "node:test";
+import assert from "node:assert/strict";
+
+const { generateSignature, outputContractOf } = await import("../../src/lib/semanticCache.ts");
+const { storeSemanticCacheResponse } = await import(
+  "../../open-sse/handlers/chatCore/semanticCacheStore.ts"
+);
+const { storeStreamingSemanticCacheResponse } = await import(
+  "../../open-sse/handlers/chatCore/streamingSemanticCacheStore.ts"
+);
+
+const MESSAGES = [{ role: "user", content: "Describe a red bicycle." }];
+
+const schemaOf = (properties: Record<string, unknown>) => ({
+  type: "json_schema",
+  json_schema: { name: "d", strict: true, schema: { type: "object", properties } },
+});
+
+test("different response schemas over identical messages produce different signatures", () => {
+  const a = generateSignature("m", MESSAGES, 0, 1, "key",
+    outputContractOf({ response_format: schemaOf({ value: { type: "string" } }) }));
+  const b = generateSignature("m", MESSAGES, 0, 1, "key",
+    outputContractOf({ response_format: schemaOf({ color: { type: "string" }, wheels: { type: "integer" } }) }));
+  assert.notEqual(a, b);
+});
+
+test("identical response schemas still share a signature", () => {
+  const contract = () => outputContractOf({ response_format: schemaOf({ value: { type: "string" } }) });
+  assert.equal(
+    generateSignature("m", MESSAGES, 0, 1, "key", contract()),
+    generateSignature("m", MESSAGES, 0, 1, "key", contract())
+  );
+});
+
+test("tools presence splits the signature from a tool-less request", () => {
+  const withTools = generateSignature("m", MESSAGES, 0, 1, "key",
+    outputContractOf({ tools: [{ type: "function", function: { name: "f", parameters: {} } }] }));
+  const without = generateSignature("m", MESSAGES, 0, 1, "key", outputContractOf({}));
+  assert.notEqual(withTools, without);
+});
+
+test("plain chat keeps the legacy signature — existing cache entries stay valid", () => {
+  const legacy = generateSignature("m", MESSAGES, 0, 1, "key");
+  const withNullContract = generateSignature("m", MESSAGES, 0, 1, "key", outputContractOf({ stream: true }));
+  assert.equal(legacy, withNullContract);
+});
+
+test("outputContractOf returns null when no shape-determining field is present", () => {
+  assert.equal(outputContractOf({ model: "m", messages: MESSAGES, temperature: 0 }), null);
+  assert.equal(outputContractOf(null), null);
+  assert.equal(outputContractOf("nonsense"), null);
+});
+
+test("outputContractOf collects the Responses-API text.format spelling too", () => {
+  const contract = outputContractOf({ text: { format: { type: "json_schema", schema: {} } } });
+  assert.ok(contract && typeof contract === "object");
+  assert.ok("text_format" in (contract as Record<string, unknown>));
+});
+
+function recordingDeps() {
+  const calls: unknown[][] = [];
+  return {
+    calls,
+    deps: {
+      isCacheableForWrite: () => true,
+      isSmallEnoughForSemanticCache: () => true,
+      generateSignature: (...args: unknown[]) => {
+        calls.push(args);
+        return "sig";
+      },
+      setCachedResponse: () => {},
+    },
+  };
+}
+
+test("non-streaming store path forwards the output contract to the signature", () => {
+  const { calls, deps } = recordingDeps();
+  const body = { messages: MESSAGES, temperature: 0, top_p: 1, response_format: schemaOf({ v: { type: "string" } }) };
+  storeSemanticCacheResponse(
+    { enabled: true, body, headers: {}, translatedResponse: { ok: true }, model: "m", apiKeyId: "key" },
+    deps as never
+  );
+  assert.equal(calls.length, 1);
+  assert.deepEqual(calls[0][5], outputContractOf(body));
+});
+
+test("streaming store path forwards the output contract to the signature", () => {
+  const { calls, deps } = recordingDeps();
+  const body = { messages: MESSAGES, temperature: 0, top_p: 1, response_format: schemaOf({ v: { type: "string" } }) };
+  storeStreamingSemanticCacheResponse(
+    {
+      enabled: true,
+      streamStatus: 200,
+      streamResponseBody: { ok: true },
+      body,
+      headers: {},
+      model: "m",
+      apiKeyId: "key",
+      streamUsage: null,
+      log: null,
+    } as never,
+    deps as never
+  );
+  assert.equal(calls.length, 1);
+  assert.deepEqual(calls[0][5], outputContractOf(body));
+});