diff --git a/changelog.d/fixes/12910-semantic-cache-exact-id-finalize.md b/changelog.d/fixes/12910-semantic-cache-exact-id-finalize.md new file mode 100644 index 0000000000..63fa5174d9 --- /dev/null +++ b/changelog.d/fixes/12910-semantic-cache-exact-id-finalize.md @@ -0,0 +1 @@ +- fix(usage): finalize semantic cache hits by exact request id — `checkSemanticCache` now uses `finalizePendingScope(pendingScope, ...)` instead of an ambiguous (model, provider, connectionId) tuple, fixing wrong-request finalization when connectionId is null or multiple requests are in flight on the same connection (#12910) diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 7ef9ef5541..f10b6f05dc 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -1230,7 +1230,7 @@ export async function handleChatCore({ stream: !!stream, reqLogger, effectiveServiceTier, - connectionId, + pendingScope, startTime, log, persistAttemptLogs, diff --git a/open-sse/handlers/chatCore/semanticCache.ts b/open-sse/handlers/chatCore/semanticCache.ts index 7cb4e9c6f5..f47fa85831 100644 --- a/open-sse/handlers/chatCore/semanticCache.ts +++ b/open-sse/handlers/chatCore/semanticCache.ts @@ -5,7 +5,7 @@ import { recordSemanticCacheHit, } from "@/lib/semanticCache"; import { calculateCost } from "@/lib/usage/costCalculator"; -import { trackPendingRequest } from "@/lib/usageDb"; +import { finalizePendingScope, type PendingRequestScope } from "@/lib/usage/pendingRequestScope"; import { synthesizeOpenAiSseFromJson } from "../../utils/jsonToSse.ts"; import { attachOmniRouteMetaHeaders } from "@/domain/omnirouteResponseMeta"; import { extractUsageFromResponse } from "../usageExtractor.ts"; @@ -21,7 +21,7 @@ export async function checkSemanticCache({ stream, reqLogger, effectiveServiceTier, - connectionId, + pendingScope, startTime, log, persistAttemptLogs, @@ -44,7 +44,7 @@ export async function checkSemanticCache({ stream: boolean; reqLogger: { logConvertedResponse: (response: Record) => void }; effectiveServiceTier: string | null | undefined; - connectionId: string | null; + pendingScope: PendingRequestScope; startTime: number; log: { debug?: (...args: unknown[]) => void } | null; persistAttemptLogs: (args: unknown) => void; @@ -112,7 +112,14 @@ export async function checkSemanticCache({ clientResponse: cached, cacheSource: hitType === "semantic" ? "semantic_similarity" : "semantic", }); - trackPendingRequest(model, provider, connectionId, false); + // Finalize by exact request id (#12910): a (model, provider, connectionId) + // tuple can match the wrong in-flight request when connectionId is null or + // multiple requests share the same connection. + finalizePendingScope(pendingScope, { + status: 200, + providerResponse: cached, + clientResponse: cached, + }); const cachedSse = stream ? managerResult.entry diff --git a/tests/unit/chatcore-semantic-cache.test.ts b/tests/unit/chatcore-semantic-cache.test.ts index a4f67fef77..e8e2221d57 100644 --- a/tests/unit/chatcore-semantic-cache.test.ts +++ b/tests/unit/chatcore-semantic-cache.test.ts @@ -42,7 +42,12 @@ function makeBaseArgs(overrides: Record = {}) { }, }, effectiveServiceTier: undefined, - connectionId: null as string | null, + pendingScope: { + id: null, + model: "gpt-4o", + provider: "openai", + connectionId: null, + }, startTime: Date.now(), log: { debug: () => { @@ -162,7 +167,12 @@ function makeHitArgs(overrides: Record = {}) { }, }, effectiveServiceTier: undefined, - connectionId: null as string | null, + pendingScope: { + id: null, + model: "gpt-4o", + provider: "openai", + connectionId: null, + }, startTime: Date.now() - 5, log: { debug: (...a: unknown[]) => { @@ -499,6 +509,56 @@ test("checkSemanticCache HIT includes X-OmniRoute-Cache-Latency: synthetic heade ); }); +test("checkSemanticCache HIT finalizes the exact pending request by id (#12910)", async () => { + clearCache(); + const { + clearPendingRequests, + getPendingById, + trackPendingRequest: trackPending, + } = await import("../../src/lib/usage/usageHistory.ts"); + const { getCompletedDetails } = await import("../../src/lib/usage/completedRequestDetails.ts"); + clearPendingRequests(); + try { + const cached = { + id: "chatcmpl-cached-finalize", + choices: [ + { + index: 0, + message: { role: "assistant", content: "finalize answer" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 5, completion_tokens: 5, total_tokens: 10 }, + }; + const pendingId = trackPending("gpt-4o", "openai", "account-a", true); + assert.ok(pendingId); + const { args } = makeHitArgs({ + body: { + model: "gpt-4o", + messages: [{ role: "user", content: "hit query finalize" }], + temperature: 0, + }, + pendingScope: { + id: pendingId, + model: "gpt-4o", + provider: "openai", + connectionId: "account-a", + }, + }); + seedHit(args, cached); + + const result = await checkSemanticCache(args as Parameters[0]); + assert.ok(result); + assert.equal(getPendingById().has(pendingId as string), false); + const completed = getCompletedDetails().get(pendingId as string); + assert.ok(completed); + assert.equal(completed.status, 200); + assert.deepEqual(completed.clientResponse, cached); + } finally { + clearPendingRequests(); + } +}); + // ─── tool_choice / tools / response_format must be part of the signature (#12734) ──────────── test("#12734: cached tool_calls response must NOT be replayed for tool_choice: 'none'", async () => {