mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-19 21:52:21 +03:00
7db3ba615b
* fix(sse): stop thrashing the provider prompt cache for caching-aware clients - shouldPreserveCacheControl: preserve client cache_control markers for every combo strategy. The deterministic-strategy gate forced marker rewrites whose per-request breakpoint positions are not stable turn-over-turn, thrashing the upstream prompt cache (observed in production as ~200k cache_write tokens per turn on quota-share combos). Preserving is never worse: on a stable target the client's breakpoints advance deterministically; on a target switch both approaches miss equally. - prepareClaudeRequest: translator-path opt-in fallback — when preserve-mode has nothing to preserve (client sent no cache_control anywhere), apply the standard heuristic so requests never ship with zero cache breakpoints. The claude-code-compatible relay path keeps its no-supplement contract. - anthropic-beta: forward the client-negotiated context-1m-2025-08-07 through the allowlist merge so a /model <id>[1m] client keeps its long-context negotiation behind the proxy (never forced when the client did not send it). * fix(sse): surface cache tokens in non-streaming OpenAI usage + finalize pending on quota-share block - translateNonStreamingResponse (claude→openai): fold cache_read into prompt_tokens and expose prompt_tokens_details.cached_tokens / cache_creation_tokens, mirroring the streaming contract (#1426/#2215). Non-streaming OpenAI clients behind a cached Claude upstream previously saw prompt_tokens=<uncached remainder> (e.g. 23 for a ~9k request) with no cache visibility. - chatCore quota-share block: finalize the pending-request slot before returning the policy 429 — the path never reaches upstream and the orphaned pending lingered as a status-0 call-log row until the reaper swept it. Validated live on the staging box (repro: blocked key → orphan row; after: clean). --------- Co-authored-by: diegosouzapw <diegosouzapw24@gmail.com>
69 lines
2.8 KiB
TypeScript
69 lines
2.8 KiB
TypeScript
/**
|
|
* Non-streaming claude→openai usage must surface prompt-cache tokens.
|
|
*
|
|
* The streaming translator already folds cache_read into prompt_tokens and
|
|
* exposes prompt_tokens_details (#1426/#2215). The non-streaming path built the
|
|
* usage object from input_tokens/output_tokens only, so an OpenAI-format client
|
|
* behind a cached Claude upstream saw prompt_tokens=<uncached remainder> (e.g.
|
|
* 23 for a ~9k-token request) and no cache visibility at all — billing/telemetry
|
|
* built on the OpenAI usage contract under-reported by orders of magnitude.
|
|
*
|
|
* Contract mirrored from the streaming path:
|
|
* prompt_tokens = input_tokens + cache_read_input_tokens
|
|
* prompt_tokens_details.cached_tokens = cache_read_input_tokens
|
|
* prompt_tokens_details.cache_creation_tokens = cache_creation_input_tokens
|
|
* (cache_creation stays OUT of prompt_tokens — #2215: providers price it
|
|
* differently and folding it in broke min-token accounting.)
|
|
*/
|
|
import { describe, test } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { translateNonStreamingResponse } from "../../open-sse/handlers/responseTranslator.ts";
|
|
import { FORMATS } from "../../open-sse/translator/formats.ts";
|
|
|
|
function claudeResponse(usage: Record<string, number>) {
|
|
return {
|
|
id: "msg_test",
|
|
model: "claude-sonnet-5",
|
|
stop_reason: "end_turn",
|
|
content: [{ type: "text", text: "ok" }],
|
|
usage,
|
|
};
|
|
}
|
|
|
|
describe("non-streaming claude→openai usage cache tokens", () => {
|
|
test("folds cache_read into prompt_tokens and exposes details", () => {
|
|
const out = translateNonStreamingResponse(
|
|
claudeResponse({
|
|
input_tokens: 2,
|
|
output_tokens: 6,
|
|
cache_read_input_tokens: 8941,
|
|
cache_creation_input_tokens: 27,
|
|
}),
|
|
FORMATS.CLAUDE,
|
|
FORMATS.OPENAI
|
|
) as Record<string, unknown>;
|
|
|
|
const usage = out.usage as Record<string, unknown>;
|
|
assert.equal(usage.prompt_tokens, 8943, "prompt_tokens = input + cache_read");
|
|
assert.equal(usage.completion_tokens, 6);
|
|
assert.equal(usage.total_tokens, 8949);
|
|
const details = usage.prompt_tokens_details as Record<string, unknown>;
|
|
assert.ok(details, "prompt_tokens_details must be present when cache fields exist");
|
|
assert.equal(details.cached_tokens, 8941);
|
|
assert.equal(details.cache_creation_tokens, 27);
|
|
});
|
|
|
|
test("no cache fields → plain usage without details (unchanged shape)", () => {
|
|
const out = translateNonStreamingResponse(
|
|
claudeResponse({ input_tokens: 10, output_tokens: 5 }),
|
|
FORMATS.CLAUDE,
|
|
FORMATS.OPENAI
|
|
) as Record<string, unknown>;
|
|
|
|
const usage = out.usage as Record<string, unknown>;
|
|
assert.equal(usage.prompt_tokens, 10);
|
|
assert.equal(usage.total_tokens, 15);
|
|
assert.equal(usage.prompt_tokens_details, undefined);
|
|
});
|
|
});
|