mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-09-21 22:32:22 +03:00
04cc8aab67
* fix(cache): fold the response output contract into the semantic cache signature
The signature hashed only {model, messages, temperature, top_p}, so two temp=0
requests with identical messages but different response_format shared a cache
key: the second was served the first's stored body under a 200, violating the
schema it asked for. tools/tool_choice had the same exposure.
generateSignature now takes an optional output contract — response_format,
text.format, tools, tool_choice, collected by outputContractOf() — and folds it
into the digest only when present, so plain-chat signatures (and every cache
entry already written for them) are unchanged. All three call sites pass it;
read/write symmetry is preserved because bodyForCacheWrite snapshots the same
body object the read path hashed (#cache-signature-asymmetry).
Closes #12307
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
* docs(changelog): add fragment for #12307 semantic-cache output-contract fix
Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com>
* fix(cache): populate both constraint spellings in outputContractOf
The merge with #12734 left generateSignature reading the camelCase
constraints (toolChoice/responseFormat) with a snake_case fallback, but
outputContractOf only filled the snake_case keys, so the #12734
"signature is called with tool_choice/tools/response_format from body"
store tests failed on the merged branch. Set both spellings so either
caller shape reads the value it expects.
Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com>
---------
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com>
Co-authored-by: amirrezakm <amirrezakm@users.noreply.github.com>
81 lines
2.9 KiB
TypeScript
81 lines
2.9 KiB
TypeScript
/**
|
|
* chatCore semantic-cache store (Quality Gate v2 / Fase 9 — chatCore god-file decomposition,
|
|
* #3501).
|
|
*
|
|
* Extracted from handleChatCore's non-streaming success path (Phase 9.1): when semantic caching is
|
|
* enabled and the request/response are cacheable, store the translated response under its signature
|
|
* so a later temp=0 request can be served from cache. Side-effect only (cache write + debug log);
|
|
* no early-return, no outer-variable reassignment. Behaviour is byte-identical to the previous
|
|
* inline block, including the `prompt + completion || 0` token-saved precedence.
|
|
*/
|
|
import {
|
|
generateSignature as defaultGenerateSignature,
|
|
outputContractOf,
|
|
setCachedResponse as defaultSetCachedResponse,
|
|
isCacheableForWrite as defaultIsCacheableForWrite,
|
|
isTruncatedCompletion as defaultIsTruncatedCompletion,
|
|
} from "@/lib/semanticCache";
|
|
import { isSmallEnoughForSemanticCache as defaultIsSmallEnough } from "../../utils/estimateSize.ts";
|
|
|
|
type LoggerLike = { debug?: (...args: unknown[]) => void } | null | undefined;
|
|
|
|
type CacheBody = {
|
|
messages?: unknown;
|
|
input?: unknown;
|
|
temperature?: number;
|
|
top_p?: number;
|
|
};
|
|
|
|
type UsageLike = { prompt_tokens?: number; completion_tokens?: number } | null | undefined;
|
|
|
|
export interface SemanticCacheStoreDeps {
|
|
isCacheableForWrite: typeof defaultIsCacheableForWrite;
|
|
/** Optional so pre-existing callers/tests with partial deps keep working. */
|
|
isTruncatedCompletion?: typeof defaultIsTruncatedCompletion;
|
|
isSmallEnoughForSemanticCache: typeof defaultIsSmallEnough;
|
|
generateSignature: typeof defaultGenerateSignature;
|
|
setCachedResponse: typeof defaultSetCachedResponse;
|
|
}
|
|
|
|
const DEFAULT_DEPS: SemanticCacheStoreDeps = {
|
|
isCacheableForWrite: defaultIsCacheableForWrite,
|
|
isTruncatedCompletion: defaultIsTruncatedCompletion,
|
|
isSmallEnoughForSemanticCache: defaultIsSmallEnough,
|
|
generateSignature: defaultGenerateSignature,
|
|
setCachedResponse: defaultSetCachedResponse,
|
|
};
|
|
|
|
export function storeSemanticCacheResponse(
|
|
args: {
|
|
enabled: boolean;
|
|
body: CacheBody;
|
|
headers: unknown;
|
|
translatedResponse: unknown;
|
|
model: string;
|
|
apiKeyId?: string;
|
|
usage?: UsageLike;
|
|
log?: LoggerLike;
|
|
},
|
|
deps: SemanticCacheStoreDeps = DEFAULT_DEPS
|
|
): void {
|
|
if (
|
|
!args.enabled ||
|
|
!deps.isCacheableForWrite(args.body, args.headers) ||
|
|
(deps.isTruncatedCompletion ?? defaultIsTruncatedCompletion)(args.translatedResponse) ||
|
|
!deps.isSmallEnoughForSemanticCache(args.translatedResponse)
|
|
) {
|
|
return;
|
|
}
|
|
const signature = deps.generateSignature(
|
|
args.model,
|
|
args.body.messages ?? args.body.input,
|
|
args.body.temperature,
|
|
args.body.top_p,
|
|
args.apiKeyId ?? undefined,
|
|
outputContractOf(args.body)
|
|
);
|
|
const tokensSaved = args.usage?.prompt_tokens + args.usage?.completion_tokens || 0;
|
|
deps.setCachedResponse(signature, args.model, args.translatedResponse, tokensSaved);
|
|
args.log?.debug?.("CACHE", `Stored response for ${args.model} (${tokensSaved} tokens)`);
|
|
}
|