fix(usage): port #12910 exact-id finalization into the dual-layer cache

Ports the single-layer cache's #12910 fix into the new dual-layer
checkSemanticCache: finalize the exact pending request by id
(finalizePendingScope) instead of an ambiguous (model, provider,
connectionId) tuple, which could finalize the wrong in-flight
request when connectionId is null or multiple requests share a
connection.

Co-authored-by: Jihyun Son <initguru@gmail.com>
Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com>

changelog.d/fixes/12910-semantic-cache-exact-id-finalize.md

@@ -0,0 +1 @@
+- fix(usage): finalize semantic cache hits by exact request id — `checkSemanticCache` now uses `finalizePendingScope(pendingScope, ...)` instead of an ambiguous (model, provider, connectionId) tuple, fixing wrong-request finalization when connectionId is null or multiple requests are in flight on the same connection (#12910)

open-sse/handlers/chatCore.ts

@@ -1230,7 +1230,7 @@ export async function handleChatCore({
     stream: !!stream,
     reqLogger,
     effectiveServiceTier,
-    connectionId,
+    pendingScope,
     startTime,
     log,
     persistAttemptLogs,

open-sse/handlers/chatCore/semanticCache.ts

@@ -5,7 +5,7 @@ import {
   recordSemanticCacheHit,
 } from "@/lib/semanticCache";
 import { calculateCost } from "@/lib/usage/costCalculator";
-import { trackPendingRequest } from "@/lib/usageDb";
+import { finalizePendingScope, type PendingRequestScope } from "@/lib/usage/pendingRequestScope";
 import { synthesizeOpenAiSseFromJson } from "../../utils/jsonToSse.ts";
 import { attachOmniRouteMetaHeaders } from "@/domain/omnirouteResponseMeta";
 import { extractUsageFromResponse } from "../usageExtractor.ts";
@@ -21,7 +21,7 @@ export async function checkSemanticCache({
   stream,
   reqLogger,
   effectiveServiceTier,
-  connectionId,
+  pendingScope,
   startTime,
   log,
   persistAttemptLogs,
@@ -44,7 +44,7 @@ export async function checkSemanticCache({
   stream: boolean;
   reqLogger: { logConvertedResponse: (response: Record<string, unknown>) => void };
   effectiveServiceTier: string | null | undefined;
-  connectionId: string | null;
+  pendingScope: PendingRequestScope;
   startTime: number;
   log: { debug?: (...args: unknown[]) => void } | null;
   persistAttemptLogs: (args: unknown) => void;
@@ -112,7 +112,14 @@ export async function checkSemanticCache({
         clientResponse: cached,
         cacheSource: hitType === "semantic" ? "semantic_similarity" : "semantic",
       });
-      trackPendingRequest(model, provider, connectionId, false);
+      // Finalize by exact request id (#12910): a (model, provider, connectionId)
+      // tuple can match the wrong in-flight request when connectionId is null or
+      // multiple requests share the same connection.
+      finalizePendingScope(pendingScope, {
+        status: 200,
+        providerResponse: cached,
+        clientResponse: cached,
+      });
 
       const cachedSse = stream
         ? managerResult.entry

tests/unit/chatcore-semantic-cache.test.ts

@@ -42,7 +42,12 @@ function makeBaseArgs(overrides: Record<string, unknown> = {}) {
       },
     },
     effectiveServiceTier: undefined,
-    connectionId: null as string | null,
+    pendingScope: {
+      id: null,
+      model: "gpt-4o",
+      provider: "openai",
+      connectionId: null,
+    },
     startTime: Date.now(),
     log: {
       debug: () => {
@@ -162,7 +167,12 @@ function makeHitArgs(overrides: Record<string, unknown> = {}) {
       },
     },
     effectiveServiceTier: undefined,
-    connectionId: null as string | null,
+    pendingScope: {
+      id: null,
+      model: "gpt-4o",
+      provider: "openai",
+      connectionId: null,
+    },
     startTime: Date.now() - 5,
     log: {
       debug: (...a: unknown[]) => {
@@ -499,6 +509,56 @@ test("checkSemanticCache HIT includes X-OmniRoute-Cache-Latency: synthetic heade
   );
 });
 
+test("checkSemanticCache HIT finalizes the exact pending request by id (#12910)", async () => {
+  clearCache();
+  const {
+    clearPendingRequests,
+    getPendingById,
+    trackPendingRequest: trackPending,
+  } = await import("../../src/lib/usage/usageHistory.ts");
+  const { getCompletedDetails } = await import("../../src/lib/usage/completedRequestDetails.ts");
+  clearPendingRequests();
+  try {
+    const cached = {
+      id: "chatcmpl-cached-finalize",
+      choices: [
+        {
+          index: 0,
+          message: { role: "assistant", content: "finalize answer" },
+          finish_reason: "stop",
+        },
+      ],
+      usage: { prompt_tokens: 5, completion_tokens: 5, total_tokens: 10 },
+    };
+    const pendingId = trackPending("gpt-4o", "openai", "account-a", true);
+    assert.ok(pendingId);
+    const { args } = makeHitArgs({
+      body: {
+        model: "gpt-4o",
+        messages: [{ role: "user", content: "hit query finalize" }],
+        temperature: 0,
+      },
+      pendingScope: {
+        id: pendingId,
+        model: "gpt-4o",
+        provider: "openai",
+        connectionId: "account-a",
+      },
+    });
+    seedHit(args, cached);
+
+    const result = await checkSemanticCache(args as Parameters<typeof checkSemanticCache>[0]);
+    assert.ok(result);
+    assert.equal(getPendingById().has(pendingId as string), false);
+    const completed = getCompletedDetails().get(pendingId as string);
+    assert.ok(completed);
+    assert.equal(completed.status, 200);
+    assert.deepEqual(completed.clientResponse, cached);
+  } finally {
+    clearPendingRequests();
+  }
+});
+
 // ─── tool_choice / tools / response_format must be part of the signature (#12734) ────────────
 
 test("#12734: cached tool_calls response must NOT be replayed for tool_choice: 'none'", async () => {