From 2b2d947faf551ff1662c43361163d3181e8e71ee Mon Sep 17 00:00:00 2001 From: "Bob.Hou" Date: Tue, 11 Aug 2026 08:55:07 -0400 Subject: [PATCH] fix(cache): add latency marker + per-key bypass for semantic cache (#8984) * fix(cache): add latency marker + per-key bypass for semantic cache Semantic cache silently corrupts latency measurements: a 10s upstream call served from cache looks like 19ms. Three fixes: A. Latency marker: cache HIT responses now carry X-OmniRoute-Cache-Latency: synthetic so measurement tools can distinguish real vs cached latency. B. Per-key bypass: new apiKeys.cacheDefaultMode ('legacy' | 'bypass') lets latency-sensitive clients opt out of cache reads entirely. - DB column + migration (134) - rowParser parseCacheDefaultMode - API create default + PATCH update - checkSemanticCache returns null on bypass C. Type safety: ApiKeyRow/ApiKeyView/params updated, superRefine guard includes cacheDefaultMode. Cache write path intentionally unchanged: apiKeyId is already in the cache signature (semanticCache.ts:140), so per-key isolation prevents cross-key pollution. Changed test files: - tests/unit/chatcore-semantic-cache.test.ts (3 new tests) Signed-off-by: Minxi Hou * docs: document semantic cache latency impact + bypass configuration --------- Signed-off-by: Minxi Hou --- config/quality/file-size-baseline.json | 1 + docs/reference/API_REFERENCE.md | 36 ++++++ open-sse/handlers/chatCore.ts | 2 + open-sse/handlers/chatCore/semanticCache.ts | 7 ++ src/app/api/keys/[id]/route.ts | 3 + src/app/api/keys/route.ts | 1 + src/lib/db/apiKeyColumnFallbacks.ts | 1 + src/lib/db/apiKeys.ts | 24 +++- src/lib/db/apiKeys/rowParsers.ts | 4 + .../143_api_key_cache_default_mode.sql | 5 + src/shared/constants/headers.ts | 1 + src/shared/validation/schemas/keys.ts | 2 + tests/unit/chatcore-semantic-cache.test.ts | 114 ++++++++++++++++-- 13 files changed, 193 insertions(+), 8 deletions(-) create mode 100644 src/lib/db/migrations/143_api_key_cache_default_mode.sql diff --git a/config/quality/file-size-baseline.json b/config/quality/file-size-baseline.json index 42443bb7a0..49f68b2bdb 100644 --- a/config/quality/file-size-baseline.json +++ b/config/quality/file-size-baseline.json @@ -1,4 +1,5 @@ { + "_rebaseline_2026_08_09_8984_api_key_cache_mode": "PR #8984 own growth during the 2026-08-09 rebase: src/lib/db/apiKeys.ts 1529->1545 (+16 = the per-key apiKeys.cacheDefaultMode column + its row parsers and cascade wiring; additive at the existing connection write/read chokepoints). Covered by tests/unit/chatcore-semantic-cache.test.ts. (chatCore.ts stays at the pre-existing base-red ceiling — upstream tip already exceeds the frozen 5042, this PR only adds +2 on top; not re-bumped per the no-inherit-ratchet rule.)", "_rebaseline_2026_08_09_9207_breaker_halfopen_recovery": "PR #9207 own growth during the 2026-08-09 rebase: open-sse/services/accountFallback.ts 1978->2020 (+42 = recordProviderSuccess now also transitions the provider circuit breaker from HALF_OPEN to CLOSED when a request succeeds, so the breaker is not stuck half-open after repeated failures; the transition and its reset wiring grow the existing provider-success path, not extractable). Covered by tests/unit/provider-breaker-halfopen-recovery.test.ts.", "_rebaseline_2026_08_09_9296_adobe_media_capabilities": "PR #9296 (artickc, fix/adobe-firefly-model-capabilities) own growth: src/app/api/v1/models/catalog.ts 1590->1597 (+7). The image and video catalog serializers now expose the already-normalized Adobe Firefly discovery capability data (media_capabilities, plus the existing video modality/size fields) at their only response-emission chokepoints. The discovery parser and capability normalization remain in open-sse/services/adobeFireflyModels.ts; extracting these seven serialization fields would obscure the catalog contract. Covered by tests/unit/adobe-firefly.test.ts and tests/unit/image-upscale.test.ts.", "_rebaseline_2026_08_08_v3850_base_drift_batch_9757": "Base drift on release/v3.8.50, not own growth: the 08-06..08-08 merge batches grew 12 already-frozen (or newly-landed) files without carrying their rebaselines \u2014 the dedicated rebaseline PR #9616 was closed as 'superseded' but its file-size entries never actually reached the base, and later merges (#8894 combos page, #9539 EditConnectionModal, #8895 models route, #9294/#9293 catalog, #9541 db/core, #8970 tokenHealthCheck, #8925 mcp schemas+server, #8890 accountFallback, #9467 chat.ts, #8931 openai-to-kiro, ProxyRegistryManager) kept growing them. All 12 values re-measured on THIS branch's tree (= pure tip + this PR's 1-line chat.ts fix, which adds zero lines). This PR's own source changes (chat.ts identifier restore, stream.ts format carve-out) do not grow any frozen file past these values.", diff --git a/docs/reference/API_REFERENCE.md b/docs/reference/API_REFERENCE.md index 921819ee46..3eceb2ef74 100644 --- a/docs/reference/API_REFERENCE.md +++ b/docs/reference/API_REFERENCE.md @@ -481,6 +481,42 @@ Response example: } ``` +### Latency impact + +A semantic cache HIT serves the response from cache **without an upstream +call**, so the reported `X-OmniRoute-Response-Latency` is near-zero +(regardless of the original upstream latency). Latency-sensitive clients +(benchmarking, p50/p99 monitoring) should check the +`X-OmniRoute-Cache-Latency` response header: + +| Value | Meaning | +|-------|---------| +| `synthetic` | Response served from cache; latency is not real upstream time | +| *(absent)* | Response from real upstream call | + +### Per-key cache bypass + +API keys can opt out of semantic cache reads via `cacheDefaultMode`: + +| Value | Behavior | +|-------|----------| +| `legacy` | Normal cache behavior (default) | +| `bypass` | Skip cache lookup entirely; always hit upstream | + +Set at key creation (`POST /api/keys`) or update (`PATCH /api/keys/[id]`): + +```json +{ "cacheDefaultMode": "bypass" } +``` + +### Per-request bypass + +Any request can bypass the cache regardless of key settings: + +``` +X-OmniRoute-No-Cache: true +``` + --- ## Dashboard & Management diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 5f3a0b76e3..ec583d1873 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -1063,6 +1063,8 @@ export async function handleChatCore({ log, persistAttemptLogs, apiKeyId: apiKeyInfo?.id ?? undefined, + cacheDefaultMode: (apiKeyInfo as { cacheDefaultMode?: "legacy" | "bypass" } | null) + ?.cacheDefaultMode, }); if (cacheHit) { return cacheHit; diff --git a/open-sse/handlers/chatCore/semanticCache.ts b/open-sse/handlers/chatCore/semanticCache.ts index 5a3de78e25..fbcf53fedb 100644 --- a/open-sse/handlers/chatCore/semanticCache.ts +++ b/open-sse/handlers/chatCore/semanticCache.ts @@ -24,6 +24,7 @@ export async function checkSemanticCache({ log, persistAttemptLogs, apiKeyId, + cacheDefaultMode, }: { semanticCacheEnabled: boolean; // Only the fields this read path actually touches are named; everything else @@ -40,7 +41,10 @@ export async function checkSemanticCache({ log: { debug?: (...args: unknown[]) => void } | null; persistAttemptLogs: (args: unknown) => void; apiKeyId?: string | null; + cacheDefaultMode?: "legacy" | "bypass" | null; }) { + // Per-key bypass: skip cache lookup entirely when the API key opts out. + if (cacheDefaultMode === "bypass") return null; if (semanticCacheEnabled && isCacheableForRead(body, clientRawRequest?.headers)) { const signature = generateSignature( model, @@ -75,6 +79,9 @@ export async function checkSemanticCache({ const headers: Record = { "Content-Type": cachedSse ? "text/event-stream" : "application/json", [OMNIROUTE_RESPONSE_HEADERS.cache]: "HIT", + // Marker for latency measurement tools: this response served from cache + // has synthetic (near-zero) latency, not real upstream latency. + [OMNIROUTE_RESPONSE_HEADERS.cacheLatency]: "synthetic", }; // A cache HIT serves WITHOUT an upstream call, so the incremental cost billed to // the client is 0 (consumers that sum X-OmniRoute-Response-Cost must not charge for diff --git a/src/app/api/keys/[id]/route.ts b/src/app/api/keys/[id]/route.ts index 501e820fa9..5b8283553d 100644 --- a/src/app/api/keys/[id]/route.ts +++ b/src/app/api/keys/[id]/route.ts @@ -82,6 +82,7 @@ export async function PATCH(request, { params }) { allowedEndpoints, streamDefaultMode, compressionEnabled, + cacheDefaultMode, disableNonPublicModels, allowUsageCommand, usageLimitEnabled, @@ -109,6 +110,7 @@ export async function PATCH(request, { params }) { if (allowedEndpoints !== undefined) payload.allowedEndpoints = allowedEndpoints; if (streamDefaultMode !== undefined) payload.streamDefaultMode = streamDefaultMode; if (compressionEnabled !== undefined) payload.compressionEnabled = compressionEnabled; + if (cacheDefaultMode !== undefined) payload.cacheDefaultMode = cacheDefaultMode; if (disableNonPublicModels !== undefined) payload.disableNonPublicModels = disableNonPublicModels; if (allowUsageCommand !== undefined) payload.allowUsageCommand = allowUsageCommand; @@ -145,6 +147,7 @@ export async function PATCH(request, { params }) { ...(allowedEndpoints !== undefined && { allowedEndpoints }), ...(streamDefaultMode !== undefined && { streamDefaultMode }), ...(compressionEnabled !== undefined && { compressionEnabled }), + ...(cacheDefaultMode !== undefined && { cacheDefaultMode }), ...(disableNonPublicModels !== undefined && { disableNonPublicModels }), ...(allowUsageCommand !== undefined && { allowUsageCommand }), ...(usageLimitEnabled !== undefined && { usageLimitEnabled }), diff --git a/src/app/api/keys/route.ts b/src/app/api/keys/route.ts index 1fb71de220..6e0f34ea11 100644 --- a/src/app/api/keys/route.ts +++ b/src/app/api/keys/route.ts @@ -126,6 +126,7 @@ export async function POST(request) { chaosModeEnabled: chaosModeEnabled === true, streamDefaultMode: "legacy", compressionEnabled: true, + cacheDefaultMode: "legacy", }, { status: 201 } ); diff --git a/src/lib/db/apiKeyColumnFallbacks.ts b/src/lib/db/apiKeyColumnFallbacks.ts index 21a36d31f4..af23f89908 100644 --- a/src/lib/db/apiKeyColumnFallbacks.ts +++ b/src/lib/db/apiKeyColumnFallbacks.ts @@ -24,6 +24,7 @@ export const API_KEY_COLUMN_FALLBACKS = [ { name: "allowed_endpoints", definition: "allowed_endpoints TEXT" }, { name: "allowed_quotas", definition: "allowed_quotas TEXT NOT NULL DEFAULT '[]'" }, { name: "stream_default_mode", definition: "stream_default_mode TEXT NOT NULL DEFAULT 'legacy'" }, + { name: "cache_default_mode", definition: "cache_default_mode TEXT NOT NULL DEFAULT 'legacy'" }, { name: "disable_non_public_models", definition: "disable_non_public_models INTEGER NOT NULL DEFAULT 0", diff --git a/src/lib/db/apiKeys.ts b/src/lib/db/apiKeys.ts index d6e30c56d5..5869216933 100644 --- a/src/lib/db/apiKeys.ts +++ b/src/lib/db/apiKeys.ts @@ -46,6 +46,7 @@ import { parseNullableTimestamp, parseIsBanned, parseStreamDefaultMode, + parseCacheDefaultMode, parseChaosModeEnabled, parseCompressionEnabled, } from "./apiKeys/rowParsers"; @@ -99,6 +100,7 @@ interface ApiKeyMetadata { proxyId: string | null; allowedEndpoints: string[]; streamDefaultMode: "legacy" | "json"; + cacheDefaultMode: "legacy" | "bypass"; disableNonPublicModels: boolean; allowUsageCommand: boolean; usageLimitEnabled: boolean; @@ -137,6 +139,8 @@ interface ApiKeyRow extends JsonRecord { proxy_id?: unknown; stream_default_mode?: unknown; streamDefaultMode?: unknown; + cache_default_mode?: unknown; + cacheDefaultMode?: unknown; allow_usage_command?: unknown; allowUsageCommand?: unknown; usage_limit_enabled?: unknown; @@ -190,6 +194,7 @@ interface ApiKeyView extends JsonRecord { expiresAt?: string | null; allowedEndpoints: string[]; streamDefaultMode: "legacy" | "json"; + cacheDefaultMode: "legacy" | "bypass"; disableNonPublicModels?: boolean; allowUsageCommand?: boolean; usageLimitEnabled?: boolean; @@ -404,7 +409,7 @@ function getPreparedStatements(db: ApiKeysDbLike): ApiKeysStatements { "SELECT id, expires_at, revoked_at, is_active, is_banned FROM api_keys WHERE key = ? OR key_hash = ?", ); _stmtGetKeyMetadata = db.prepare( - "SELECT id, name, machine_id, allowed_models, blocked_models, allowed_combos, allowed_connections, allowed_quotas, no_log, auto_resolve, is_active, access_schedule, max_requests_per_day, max_requests_per_minute, throttle_delay_ms, max_sessions, revoked_at, expires_at, ip_allowlist, scopes, rate_limits, is_banned, key_hash, allowed_endpoints, stream_default_mode, disable_non_public_models, allow_usage_command, usage_limit_enabled, daily_usage_limit_usd, weekly_usage_limit_usd, chaos_mode_enabled, compression_enabled, proxy_id FROM api_keys WHERE key = ? OR key_hash = ?", + "SELECT id, name, machine_id, allowed_models, blocked_models, allowed_combos, allowed_connections, allowed_quotas, no_log, auto_resolve, is_active, access_schedule, max_requests_per_day, max_requests_per_minute, throttle_delay_ms, max_sessions, revoked_at, expires_at, ip_allowlist, scopes, rate_limits, is_banned, key_hash, allowed_endpoints, stream_default_mode, cache_default_mode, disable_non_public_models, allow_usage_command, usage_limit_enabled, daily_usage_limit_usd, weekly_usage_limit_usd, chaos_mode_enabled, compression_enabled, proxy_id FROM api_keys WHERE key = ? OR key_hash = ?", ); _stmtInsertKey = db.prepare( "INSERT INTO api_keys (id, name, key, machine_id, allowed_models, no_log, created_at, key_prefix, key_hash, scopes) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", @@ -459,6 +464,7 @@ export async function getApiKeys(limit?: number, offset?: number) { camelRow.scopes = parseStringList((camelRow as JsonRecord).scopes); camelRow.allowedEndpoints = parseStringList((camelRow as JsonRecord).allowedEndpoints); camelRow.streamDefaultMode = parseStreamDefaultMode((camelRow as JsonRecord).streamDefaultMode); + camelRow.cacheDefaultMode = parseCacheDefaultMode((camelRow as JsonRecord).cacheDefaultMode); camelRow.disableNonPublicModels = parseDisableNonPublicModels( (camelRow as JsonRecord).disableNonPublicModels, ); @@ -572,6 +578,7 @@ export async function getApiKeyById(id: string) { camelRow.scopes = parseStringList((camelRow as JsonRecord).scopes); camelRow.allowedEndpoints = parseStringList((camelRow as JsonRecord).allowedEndpoints); camelRow.streamDefaultMode = parseStreamDefaultMode((camelRow as JsonRecord).streamDefaultMode); + camelRow.cacheDefaultMode = parseCacheDefaultMode((camelRow as JsonRecord).cacheDefaultMode); camelRow.disableNonPublicModels = parseDisableNonPublicModels( (camelRow as JsonRecord).disableNonPublicModels, ); @@ -701,6 +708,7 @@ export async function updateApiKeyPermissions( proxyId?: string | null; allowedEndpoints?: string[] | null; streamDefaultMode?: "legacy" | "json" | null; + cacheDefaultMode?: "legacy" | "bypass" | null; disableNonPublicModels?: boolean; allowUsageCommand?: boolean; usageLimitEnabled?: boolean; @@ -739,6 +747,8 @@ export async function updateApiKeyPermissions( allowedEndpoints: (update as { allowedEndpoints?: string[] | null }).allowedEndpoints, streamDefaultMode: (update as { streamDefaultMode?: "legacy" | "json" | null }) .streamDefaultMode, + cacheDefaultMode: (update as { cacheDefaultMode?: "legacy" | "bypass" | null }) + .cacheDefaultMode, disableNonPublicModels: (update as { disableNonPublicModels?: boolean }) .disableNonPublicModels, allowUsageCommand: (update as { allowUsageCommand?: boolean }).allowUsageCommand, @@ -772,6 +782,7 @@ export async function updateApiKeyPermissions( (normalized as Record).proxyId === undefined && (normalized as Record).allowedEndpoints === undefined && (normalized as Record).streamDefaultMode === undefined && + (normalized as Record).cacheDefaultMode === undefined && normalized.disableNonPublicModels === undefined && normalized.allowUsageCommand === undefined && normalized.chaosModeEnabled === undefined && @@ -804,6 +815,7 @@ export async function updateApiKeyPermissions( scopes?: string; proxyId?: string | null; streamDefaultMode?: "legacy" | "json"; + cacheDefaultMode?: "legacy" | "bypass"; disableNonPublicModels?: number; allowUsageCommand?: number; usageLimitEnabled?: number; @@ -954,6 +966,12 @@ export async function updateApiKeyPermissions( params.streamDefaultMode = parseStreamDefaultMode(streamDefaultModeUpdate); } + const cacheDefaultModeUpdate = (normalized as Record).cacheDefaultMode; + if (cacheDefaultModeUpdate !== undefined) { + updates.push("cache_default_mode = @cacheDefaultMode"); + params.cacheDefaultMode = parseCacheDefaultMode(cacheDefaultModeUpdate); + } + const scopesUpdate = (normalized as Record).scopes; const nextScopes: string[] = Array.isArray(scopesUpdate) ? (scopesUpdate as unknown[]).filter((s): s is string => typeof s === "string") @@ -1316,6 +1334,7 @@ export async function getApiKeyMetadata( proxyId: null, allowedEndpoints: [], streamDefaultMode: "legacy", + cacheDefaultMode: "legacy", disableNonPublicModels: false, allowUsageCommand: false, usageLimitEnabled: false, @@ -1389,6 +1408,9 @@ export async function getApiKeyMetadata( streamDefaultMode: parseStreamDefaultMode( (record as JsonRecord).stream_default_mode ?? (record as JsonRecord).streamDefaultMode, ), + cacheDefaultMode: parseCacheDefaultMode( + (record as JsonRecord).cache_default_mode ?? (record as JsonRecord).cacheDefaultMode + ), disableNonPublicModels: parseDisableNonPublicModels( (record as JsonRecord).disable_non_public_models ?? (record as JsonRecord).disableNonPublicModels, diff --git a/src/lib/db/apiKeys/rowParsers.ts b/src/lib/db/apiKeys/rowParsers.ts index 9551714bc2..9f43787421 100644 --- a/src/lib/db/apiKeys/rowParsers.ts +++ b/src/lib/db/apiKeys/rowParsers.ts @@ -169,3 +169,7 @@ export function parseIsBanned(value: unknown): boolean { export function parseStreamDefaultMode(value: unknown): "legacy" | "json" { return value === "json" ? "json" : "legacy"; } + +export function parseCacheDefaultMode(value: unknown): "legacy" | "bypass" { + return value === "bypass" ? "bypass" : "legacy"; +} diff --git a/src/lib/db/migrations/143_api_key_cache_default_mode.sql b/src/lib/db/migrations/143_api_key_cache_default_mode.sql new file mode 100644 index 0000000000..b239a330bc --- /dev/null +++ b/src/lib/db/migrations/143_api_key_cache_default_mode.sql @@ -0,0 +1,5 @@ +-- 143: Per-API-key cache bypass for semantic cache (Phase 9.1). +-- "legacy" = cache behaves as normal (default). "bypass" = skip cache lookup +-- entirely, used by latency-sensitive clients that must measure real upstream time. + +ALTER TABLE api_keys ADD COLUMN cache_default_mode TEXT NOT NULL DEFAULT 'legacy'; diff --git a/src/shared/constants/headers.ts b/src/shared/constants/headers.ts index 4d25068eb7..a4b6b50ff2 100644 --- a/src/shared/constants/headers.ts +++ b/src/shared/constants/headers.ts @@ -1,6 +1,7 @@ export const OMNIROUTE_RESPONSE_HEADERS = { cache: "X-OmniRoute-Cache", cacheHit: "X-OmniRoute-Cache-Hit", + cacheLatency: "X-OmniRoute-Cache-Latency", compression: "X-OmniRoute-Compression", costSaved: "X-OmniRoute-Cost-Saved", decision: "X-OmniRoute-Decision", diff --git a/src/shared/validation/schemas/keys.ts b/src/shared/validation/schemas/keys.ts index f2147f155a..793f3b8b96 100644 --- a/src/shared/validation/schemas/keys.ts +++ b/src/shared/validation/schemas/keys.ts @@ -107,6 +107,7 @@ export const updateKeyPermissionsSchema = z allowedEndpoints: z.array(z.string().trim().min(1).max(64)).max(20).optional(), streamDefaultMode: z.enum(["legacy", "json"]).optional(), compressionEnabled: z.boolean().optional(), + cacheDefaultMode: z.enum(["legacy", "bypass"]).optional(), disableNonPublicModels: z.boolean().optional(), allowUsageCommand: z.boolean().optional(), usageLimitEnabled: z.boolean().optional(), @@ -133,6 +134,7 @@ export const updateKeyPermissionsSchema = z value.allowedEndpoints === undefined && value.streamDefaultMode === undefined && value.compressionEnabled === undefined && + value.cacheDefaultMode === undefined && value.disableNonPublicModels === undefined && value.allowUsageCommand === undefined && value.usageLimitEnabled === undefined && diff --git a/tests/unit/chatcore-semantic-cache.test.ts b/tests/unit/chatcore-semantic-cache.test.ts index 1ca36cc072..032bfcf9f9 100644 --- a/tests/unit/chatcore-semantic-cache.test.ts +++ b/tests/unit/chatcore-semantic-cache.test.ts @@ -14,9 +14,8 @@ const core = await import("../../src/lib/db/core.ts"); // Seeding the real cache (no mock.module under the Stryker tap-runner) lets us drive the // HIT branch deterministically: setCachedResponse populates the in-memory cache that // getCachedResponse checks first, so the signature checkSemanticCache rebuilds resolves. -const { generateSignature, setCachedResponse, clearCache } = await import( - "../../src/lib/semanticCache.ts" -); +const { generateSignature, setCachedResponse, clearCache } = + await import("../../src/lib/semanticCache.ts"); const { OMNIROUTE_RESPONSE_HEADERS } = await import("../../src/shared/constants/headers.ts"); const { calculateCost } = await import("../../src/lib/usage/costCalculator.ts"); const { formatOmniRouteCost } = await import("../../src/domain/omnirouteResponseMeta.ts"); @@ -148,7 +147,11 @@ function makeHitArgs(overrides: Record = {}) { const debugCalls: unknown[][] = []; const args = { semanticCacheEnabled: true, - body: { model: "gpt-4o", messages: [{ role: "user", content: "cached query" }], temperature: 0 }, + body: { + model: "gpt-4o", + messages: [{ role: "user", content: "cached query" }], + temperature: 0, + }, clientRawRequest: { headers: {} }, model: "gpt-4o", provider: "openai", @@ -198,7 +201,11 @@ test("checkSemanticCache returns a non-streaming JSON HIT with cache headers + l usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }, }; const { args, persistCalls, convertedCalls, debugCalls } = makeHitArgs({ - body: { model: "gpt-4o", messages: [{ role: "user", content: "hit query one" }], temperature: 0 }, + body: { + model: "gpt-4o", + messages: [{ role: "user", content: "hit query one" }], + temperature: 0, + }, stream: false, }); seedHit(args, cached); @@ -251,7 +258,11 @@ test("checkSemanticCache returns a streaming SSE HIT (text/event-stream) when st usage: { prompt_tokens: 3, completion_tokens: 4, total_tokens: 7 }, }; const { args, persistCalls } = makeHitArgs({ - body: { model: "gpt-4o", messages: [{ role: "user", content: "hit query stream" }], temperature: 0 }, + body: { + model: "gpt-4o", + messages: [{ role: "user", content: "hit query stream" }], + temperature: 0, + }, stream: true, }); seedHit(args, cached); @@ -279,7 +290,11 @@ test("checkSemanticCache HITs even when the cached body has no usage (cost falls const cached = { id: "chatcmpl-cached-no-usage", choices: [ - { index: 0, message: { role: "assistant", content: "no-usage answer" }, finish_reason: "stop" }, + { + index: 0, + message: { role: "assistant", content: "no-usage answer" }, + finish_reason: "stop", + }, ], }; const { args, persistCalls } = makeHitArgs({ @@ -392,3 +407,88 @@ test("checkSemanticCache isolates HITs per apiKeyId (no cross-key cache sharing) const hitA = await checkSemanticCache(argsA2 as Parameters[0]); assert.ok(hitA, "keyA must resolve its own cached entry"); }); + +// ─── cacheDefaultMode: per-key bypass (W3 fix) ────────────────────────────── + +test("checkSemanticCache returns null when cacheDefaultMode is 'bypass' (even with cacheable body)", async () => { + clearCache(); + const cached = { + id: "chatcmpl-bypass", + choices: [ + { index: 0, message: { role: "assistant", content: "bypassed" }, finish_reason: "stop" }, + ], + usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }, + }; + const { args } = makeHitArgs({ + semanticCacheEnabled: true, + body: { + model: "gpt-4o", + messages: [{ role: "user", content: "bypass query" }], + temperature: 0, + }, + cacheDefaultMode: "bypass", + }); + seedHit(args, cached); + + const result = await checkSemanticCache(args as Parameters[0]); + + assert.equal(result, null, "cacheDefaultMode=bypass -> cache lookup skipped -> null"); +}); + +test("checkSemanticCache returns a HIT when cacheDefaultMode is 'legacy' (default behavior preserved)", async () => { + clearCache(); + const cached = { + id: "chatcmpl-legacy", + choices: [ + { index: 0, message: { role: "assistant", content: "legacy hit" }, finish_reason: "stop" }, + ], + usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }, + }; + const { args } = makeHitArgs({ + semanticCacheEnabled: true, + body: { + model: "gpt-4o", + messages: [{ role: "user", content: "legacy query" }], + temperature: 0, + }, + cacheDefaultMode: "legacy", + }); + seedHit(args, cached); + + const result = await checkSemanticCache(args as Parameters[0]); + + assert.ok(result, "cacheDefaultMode=legacy -> normal cache lookup -> HIT"); + assert.equal(result.success, true); +}); + +// ─── cacheLatency marker header (W3 fix) ───────────────────────────────────── + +test("checkSemanticCache HIT includes X-OmniRoute-Cache-Latency: synthetic header", async () => { + clearCache(); + const cached = { + id: "chatcmpl-latency", + choices: [ + { index: 0, message: { role: "assistant", content: "latency test" }, finish_reason: "stop" }, + ], + usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }, + }; + const { args } = makeHitArgs({ + semanticCacheEnabled: true, + body: { + model: "gpt-4o", + messages: [{ role: "user", content: "latency query" }], + temperature: 0, + }, + }); + seedHit(args, cached); + + const result = await checkSemanticCache(args as Parameters[0]); + + assert.ok(result, "HIT -> non-null result"); + const res = result.response as Response; + assert.equal( + res.headers.get(OMNIROUTE_RESPONSE_HEADERS.cacheLatency), + "synthetic", + "HIT response carries X-OmniRoute-Cache-Latency: synthetic marker" + ); +});