mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-07-31 20:32:20 +03:00
Add a per-connection cache capability override (supportsPromptCaching, cacheControlPassthrough) stored in provider_specific_data.cache, consulted first by providerSupportsCaching() / providerHonorsOpenAIFormatCacheControl() before falling back to the hardcoded CACHING_PROVIDERS name sets. Unblocks prompt_cache_key injection, the compression cache-aware guard, and cache_control passthrough for openai-compatible-chat-<uuid>-style custom connections that can never match the hardcoded provider-name sets. Default (no override) is byte-identical to current behavior.
This commit is contained in:
committed by
GitHub
parent
98966fdac9
commit
6c8392fa45
1
changelog.d/features/6880-connection-cache-override.md
Normal file
1
changelog.d/features/6880-connection-cache-override.md
Normal file
@@ -0,0 +1 @@
|
||||
- **feat(providers):** let a custom/openai-compatible connection opt into prompt-cache behavior via a per-connection `cache` capability override, unblocking `prompt_cache_key` injection, the compression cache-aware guard, and `cache_control` passthrough for `openai-compatible-chat-<uuid>`-style connections. (thanks @andrea-kingautomation)
|
||||
@@ -232,7 +232,10 @@ import { getProviderCredentials, extractSessionAffinityKey } from "@/sse/service
|
||||
import { deleteSessionAccountAffinity } from "@/lib/db/sessionAccountAffinity";
|
||||
import { getCacheControlSettings } from "@/lib/cacheControlSettings";
|
||||
import { guardrailRegistry } from "@/lib/guardrails";
|
||||
import { shouldPreserveCacheControl } from "../utils/cacheControlPolicy.ts";
|
||||
import {
|
||||
shouldPreserveCacheControl,
|
||||
resolveConnectionCacheOverride,
|
||||
} from "../utils/cacheControlPolicy.ts";
|
||||
import { getCachedSettings } from "@/lib/db/readCache";
|
||||
import { applyCodexGlobalFastServiceTier } from "@/lib/providers/codexFastTier";
|
||||
import { buildUpstreamHeadersForExecute as buildUpstreamHeadersForExecuteFor } from "./chatCore/upstreamExecuteHeaders.ts";
|
||||
@@ -1179,12 +1182,15 @@ export async function handleChatCore({
|
||||
if (compressionHeader) {
|
||||
log?.debug?.("COMPRESSION", `x-omniroute-compression header: ${compressionHeader}`);
|
||||
}
|
||||
const connectionCacheOverride = resolveConnectionCacheOverride(
|
||||
credentials?.providerSpecificData
|
||||
);
|
||||
const modeBeforeOutputTransform = selectCompressionStrategy(
|
||||
config,
|
||||
compressionComboKey,
|
||||
estimatedTokens,
|
||||
body as Record<string, unknown>,
|
||||
{ provider, targetFormat, model: effectiveModel },
|
||||
{ provider, targetFormat, model: effectiveModel, connectionCacheOverride },
|
||||
namedCombos,
|
||||
compressionHeader
|
||||
);
|
||||
@@ -1283,7 +1289,7 @@ export async function handleChatCore({
|
||||
compressionComboKey,
|
||||
estimatedTokens,
|
||||
compressionInputBody,
|
||||
{ provider, targetFormat, model: effectiveModel },
|
||||
{ provider, targetFormat, model: effectiveModel, connectionCacheOverride },
|
||||
namedCombos,
|
||||
compressionHeader,
|
||||
{
|
||||
@@ -1322,7 +1328,7 @@ export async function handleChatCore({
|
||||
// #3890: in a caching context, never compress the system prompt (cacheable prefix)
|
||||
// even if the operator disabled preserveSystemPrompt — honors the cache-aware flag
|
||||
// that selectCompressionStrategy can only partially apply via the mode string.
|
||||
const cacheCtx = { provider, targetFormat, model: effectiveModel };
|
||||
const cacheCtx = { provider, targetFormat, model: effectiveModel, connectionCacheOverride };
|
||||
const compressionConfig = resolveCacheAwareConfig(config, compressionInputBody, cacheCtx);
|
||||
const result = await applyCompressionAsync(compressionInputBody, mode, {
|
||||
model: effectiveModel,
|
||||
@@ -1463,6 +1469,7 @@ export async function handleChatCore({
|
||||
effectiveModel,
|
||||
mode,
|
||||
stats: result.stats,
|
||||
connectionCacheOverride,
|
||||
log,
|
||||
});
|
||||
log?.info?.(
|
||||
@@ -1658,6 +1665,7 @@ export async function handleChatCore({
|
||||
// Determine if we should preserve client-side cache_control headers
|
||||
// Fetch settings from DB to get user preference
|
||||
const cacheControlMode = await getCacheControlSettings().catch(() => "auto" as const);
|
||||
const connectionCacheOverride = resolveConnectionCacheOverride(credentials?.providerSpecificData);
|
||||
const preserveCacheControl = shouldPreserveCacheControl({
|
||||
userAgent,
|
||||
isCombo,
|
||||
@@ -1665,6 +1673,7 @@ export async function handleChatCore({
|
||||
targetProvider: provider,
|
||||
targetFormat,
|
||||
settings: { alwaysPreserveClientCache: cacheControlMode },
|
||||
connectionCacheOverride,
|
||||
});
|
||||
|
||||
if (preserveCacheControl) {
|
||||
|
||||
@@ -8,6 +8,8 @@
|
||||
* affects the request. Behaviour is byte-identical to the previous inline block.
|
||||
*/
|
||||
|
||||
import type { ConnectionCacheOverride } from "../../utils/cacheControlPolicy.ts";
|
||||
|
||||
type LoggerLike = { debug?: (...args: unknown[]) => void } | null | undefined;
|
||||
|
||||
export function recordCompressionCacheStats(args: {
|
||||
@@ -17,6 +19,7 @@ export function recordCompressionCacheStats(args: {
|
||||
effectiveModel: string | null | undefined;
|
||||
mode: string;
|
||||
stats: { originalTokens: number; compressedTokens: number };
|
||||
connectionCacheOverride?: ConnectionCacheOverride | null;
|
||||
log?: LoggerLike;
|
||||
}): void {
|
||||
void (async () => {
|
||||
@@ -27,6 +30,7 @@ export function recordCompressionCacheStats(args: {
|
||||
provider: args.provider,
|
||||
targetFormat: args.targetFormat,
|
||||
model: args.effectiveModel,
|
||||
connectionCacheOverride: args.connectionCacheOverride ?? null,
|
||||
});
|
||||
const tokensSavedCompression = Math.max(
|
||||
0,
|
||||
|
||||
@@ -15,12 +15,19 @@ import {
|
||||
resolvePayloadRuleProtocols,
|
||||
} from "../../services/payloadRules.ts";
|
||||
import { getEffectiveToolLimit, getKnownToolLimit } from "../../services/toolLimitDetector.ts";
|
||||
import { providerSupportsCaching } from "../../utils/cacheControlPolicy.ts";
|
||||
import {
|
||||
providerSupportsCaching,
|
||||
resolveConnectionCacheOverride,
|
||||
type ConnectionCacheOverride,
|
||||
} from "../../utils/cacheControlPolicy.ts";
|
||||
import { FORMATS } from "../../translator/formats.ts";
|
||||
|
||||
type LoggerLike = { debug?: (...args: unknown[]) => void } | null | undefined;
|
||||
type Body = Record<string, unknown>;
|
||||
type CredentialsLike = { apiKey?: unknown; accessToken?: unknown } | null | undefined;
|
||||
type CredentialsLike =
|
||||
| { apiKey?: unknown; accessToken?: unknown; providerSpecificData?: Record<string, unknown> | null }
|
||||
| null
|
||||
| undefined;
|
||||
|
||||
function buildAppliedRulesSummary(
|
||||
applied: Array<{ type: string; path: string; value?: unknown }>
|
||||
@@ -100,11 +107,12 @@ function backfillQwenOAuthUser(
|
||||
async function injectPromptCacheKey(
|
||||
bodyToSend: Body,
|
||||
provider: string | null | undefined,
|
||||
targetFormat: string
|
||||
targetFormat: string,
|
||||
connectionCacheOverride: ConnectionCacheOverride | null
|
||||
): Promise<Body> {
|
||||
if (
|
||||
targetFormat === FORMATS.OPENAI &&
|
||||
providerSupportsCaching(provider) &&
|
||||
providerSupportsCaching(provider, undefined, connectionCacheOverride) &&
|
||||
!bodyToSend.prompt_cache_key &&
|
||||
Array.isArray(bodyToSend.messages) &&
|
||||
!["nvidia", "codex", "xai"].includes(provider)
|
||||
@@ -162,7 +170,8 @@ export async function prepareUpstreamBody(opts: {
|
||||
|
||||
bodyToSend = truncateToolList(bodyToSend, provider, bypassDefaultToolLimit ?? false, log);
|
||||
bodyToSend = backfillQwenOAuthUser(bodyToSend, provider, credentials, log);
|
||||
bodyToSend = await injectPromptCacheKey(bodyToSend, provider, targetFormat);
|
||||
const connectionCacheOverride = resolveConnectionCacheOverride(credentials?.providerSpecificData);
|
||||
bodyToSend = await injectPromptCacheKey(bodyToSend, provider, targetFormat, connectionCacheOverride);
|
||||
|
||||
return bodyToSend;
|
||||
}
|
||||
|
||||
@@ -6,7 +6,10 @@
|
||||
* @exports CachingContext, CacheAwareStrategy, detectCachingContext, getCacheAwareStrategy
|
||||
*/
|
||||
|
||||
import { providerSupportsCaching } from "../../utils/cacheControlPolicy.ts";
|
||||
import {
|
||||
providerSupportsCaching,
|
||||
type ConnectionCacheOverride,
|
||||
} from "../../utils/cacheControlPolicy.ts";
|
||||
|
||||
type JsonRecord = Record<string, unknown>;
|
||||
|
||||
@@ -14,6 +17,7 @@ export interface CachingDetectionContext {
|
||||
provider?: string | null;
|
||||
targetFormat?: string | null;
|
||||
model?: string | null;
|
||||
connectionCacheOverride?: ConnectionCacheOverride | null;
|
||||
}
|
||||
|
||||
export interface CachingContext {
|
||||
@@ -94,7 +98,7 @@ export function detectCachingContext(
|
||||
hasCacheControl: hasCacheControl(body),
|
||||
provider,
|
||||
targetFormat,
|
||||
isCachingProvider: providerSupportsCaching(provider, targetFormat),
|
||||
isCachingProvider: providerSupportsCaching(provider, targetFormat, context.connectionCacheOverride),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -9,7 +9,10 @@ import {
|
||||
prepareClaudeRequest,
|
||||
} from "./helpers/claudeHelper.ts";
|
||||
import { filterToOpenAIFormat } from "./helpers/openaiHelper.ts";
|
||||
import { providerHonorsOpenAIFormatCacheControl } from "../utils/cacheControlPolicy.ts";
|
||||
import {
|
||||
providerHonorsOpenAIFormatCacheControl,
|
||||
resolveConnectionCacheOverride,
|
||||
} from "../utils/cacheControlPolicy.ts";
|
||||
import {
|
||||
coerceToolSchemas,
|
||||
injectEmptyReasoningContentForToolCalls,
|
||||
@@ -171,6 +174,9 @@ export function translateRequest(
|
||||
let result = body;
|
||||
const use9CharId = options?.normalizeToolCallId === true;
|
||||
const preserveDeveloperRole = options?.preserveDeveloperRole;
|
||||
const connectionCacheOverride = resolveConnectionCacheOverride(
|
||||
(credentials as { providerSpecificData?: unknown } | null)?.providerSpecificData
|
||||
);
|
||||
|
||||
// Phase 2: Apply thinking budget control before normalization
|
||||
result = applyThinkingBudget(result);
|
||||
@@ -246,7 +252,7 @@ export function translateRequest(
|
||||
// stripped.
|
||||
const preserveCacheControl =
|
||||
options?.preserveCacheControl === true &&
|
||||
providerHonorsOpenAIFormatCacheControl(provider);
|
||||
providerHonorsOpenAIFormatCacheControl(provider, connectionCacheOverride);
|
||||
const step1Credentials =
|
||||
options?.copilotClient || hasTargetHint || preserveCacheControl
|
||||
? {
|
||||
@@ -313,7 +319,8 @@ export function translateRequest(
|
||||
// requested upstream; generic/implicit-cache OpenAI providers stay stripped.
|
||||
result = filterToOpenAIFormat(result, {
|
||||
preserveCacheControl:
|
||||
options?.preserveCacheControl === true && providerHonorsOpenAIFormatCacheControl(provider),
|
||||
options?.preserveCacheControl === true &&
|
||||
providerHonorsOpenAIFormatCacheControl(provider, connectionCacheOverride),
|
||||
// #4849 regression guard: keep client reasoning_content for replay providers.
|
||||
preserveReasoningContent: isReasoner,
|
||||
});
|
||||
|
||||
@@ -123,14 +123,54 @@ const OPENAI_FORMAT_CACHE_CONTROL_PROVIDERS = new Set([
|
||||
"xiaomi-mimo",
|
||||
]);
|
||||
|
||||
/**
|
||||
* Per-connection override for cache behavior, resolved from the connection's
|
||||
* `provider_specific_data.cache` JSON sub-object (see `resolveConnectionCacheOverride`).
|
||||
* Lets an operator opt a custom/openai-compatible connection into prompt-cache
|
||||
* behavior that the hardcoded provider-name sets above can never match (#6880).
|
||||
*/
|
||||
export interface ConnectionCacheOverride {
|
||||
supportsPromptCaching?: boolean;
|
||||
cacheControlPassthrough?: "strip" | "openai-format" | "claude-format";
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract and validate a `ConnectionCacheOverride` from a connection's
|
||||
* `providerSpecificData` bag. Returns `null` when absent/malformed so every
|
||||
* call site can safely pass the result straight through.
|
||||
*/
|
||||
export function resolveConnectionCacheOverride(
|
||||
providerSpecificData: unknown
|
||||
): ConnectionCacheOverride | null {
|
||||
if (!providerSpecificData || typeof providerSpecificData !== "object") return null;
|
||||
const cache = (providerSpecificData as Record<string, unknown>).cache;
|
||||
if (!cache || typeof cache !== "object" || Array.isArray(cache)) return null;
|
||||
const record = cache as Record<string, unknown>;
|
||||
const result: ConnectionCacheOverride = {};
|
||||
if (typeof record.supportsPromptCaching === "boolean") {
|
||||
result.supportsPromptCaching = record.supportsPromptCaching;
|
||||
}
|
||||
if (
|
||||
record.cacheControlPassthrough === "strip" ||
|
||||
record.cacheControlPassthrough === "openai-format" ||
|
||||
record.cacheControlPassthrough === "claude-format"
|
||||
) {
|
||||
result.cacheControlPassthrough = record.cacheControlPassthrough;
|
||||
}
|
||||
return Object.keys(result).length > 0 ? result : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether `cache_control` markers should be PASSED THROUGH the OpenAI-format
|
||||
* translation for this provider (vs. stripped). Used to gate the request-side
|
||||
* passthrough so generic / implicit-cache OpenAI providers keep getting cleaned.
|
||||
*/
|
||||
export function providerHonorsOpenAIFormatCacheControl(
|
||||
provider: string | null | undefined
|
||||
provider: string | null | undefined,
|
||||
connectionCacheOverride?: ConnectionCacheOverride | null
|
||||
): boolean {
|
||||
if (connectionCacheOverride?.cacheControlPassthrough === "openai-format") return true;
|
||||
if (connectionCacheOverride?.cacheControlPassthrough === "strip") return false;
|
||||
if (!provider) return false;
|
||||
return OPENAI_FORMAT_CACHE_CONTROL_PROVIDERS.has(provider.toLowerCase());
|
||||
}
|
||||
@@ -159,8 +199,12 @@ export function isClaudeCodeClient(userAgent: string | null | undefined): boolea
|
||||
*/
|
||||
export function providerSupportsCaching(
|
||||
provider: string | null | undefined,
|
||||
targetFormat?: string | null
|
||||
targetFormat?: string | null,
|
||||
connectionCacheOverride?: ConnectionCacheOverride | null
|
||||
): boolean {
|
||||
if (typeof connectionCacheOverride?.supportsPromptCaching === "boolean") {
|
||||
return connectionCacheOverride.supportsPromptCaching;
|
||||
}
|
||||
if (!provider) return false;
|
||||
if (CACHING_PROVIDERS.has(provider.toLowerCase())) return true;
|
||||
// All Claude-protocol providers support prompt caching
|
||||
@@ -195,6 +239,7 @@ export function shouldPreserveCacheControl({
|
||||
targetProvider,
|
||||
targetFormat,
|
||||
settings,
|
||||
connectionCacheOverride,
|
||||
}: {
|
||||
userAgent: string | null | undefined;
|
||||
isCombo: boolean;
|
||||
@@ -202,6 +247,7 @@ export function shouldPreserveCacheControl({
|
||||
targetProvider: string | null | undefined;
|
||||
targetFormat?: string | null;
|
||||
settings?: CacheControlSettings;
|
||||
connectionCacheOverride?: ConnectionCacheOverride | null;
|
||||
}): boolean {
|
||||
// User override takes precedence
|
||||
if (settings?.alwaysPreserveClientCache === "always") {
|
||||
@@ -218,7 +264,7 @@ export function shouldPreserveCacheControl({
|
||||
}
|
||||
|
||||
// Target provider must support caching
|
||||
if (!providerSupportsCaching(targetProvider, targetFormat)) {
|
||||
if (!providerSupportsCaching(targetProvider, targetFormat, connectionCacheOverride)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
@@ -129,6 +129,54 @@ export function normalizeRequestDefaults(
|
||||
return Object.keys(normalized).length > 0 ? normalized : undefined;
|
||||
}
|
||||
|
||||
const CACHE_PASSTHROUGH_VALUES = new Set(["strip", "openai-format", "claude-format"]);
|
||||
|
||||
// #6880 — per-connection prompt-cache capability override: strip unknown keys / invalid
|
||||
// types, drop the sub-object entirely when nothing valid survives.
|
||||
export function normalizeCacheOverride(value: unknown): JsonRecord | undefined {
|
||||
const record = asRecord(value);
|
||||
if (Object.keys(record).length === 0) return undefined;
|
||||
|
||||
const normalized: JsonRecord = {};
|
||||
if (typeof record.supportsPromptCaching === "boolean") {
|
||||
normalized.supportsPromptCaching = record.supportsPromptCaching;
|
||||
}
|
||||
if (
|
||||
typeof record.cacheControlPassthrough === "string" &&
|
||||
CACHE_PASSTHROUGH_VALUES.has(record.cacheControlPassthrough)
|
||||
) {
|
||||
normalized.cacheControlPassthrough = record.cacheControlPassthrough;
|
||||
}
|
||||
|
||||
return Object.keys(normalized).length > 0 ? normalized : undefined;
|
||||
}
|
||||
|
||||
// #6880 — extracted so normalizeProviderSpecificData() stays under the
|
||||
// max-lines-per-function gate: normalizes the two nested-object sub-fields
|
||||
// (requestDefaults, cache) in one pass.
|
||||
function normalizeNestedSubObjects(
|
||||
provider: string | null | undefined,
|
||||
normalized: JsonRecord
|
||||
): void {
|
||||
if ("requestDefaults" in normalized) {
|
||||
const requestDefaults = normalizeRequestDefaults(provider, normalized.requestDefaults);
|
||||
if (requestDefaults) {
|
||||
normalized.requestDefaults = requestDefaults;
|
||||
} else {
|
||||
delete normalized.requestDefaults;
|
||||
}
|
||||
}
|
||||
|
||||
if ("cache" in normalized) {
|
||||
const cache = normalizeCacheOverride(normalized.cache);
|
||||
if (cache) {
|
||||
normalized.cache = cache;
|
||||
} else {
|
||||
delete normalized.cache;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function normalizeProviderSpecificData(
|
||||
provider: string | null | undefined,
|
||||
value: unknown
|
||||
@@ -138,14 +186,7 @@ export function normalizeProviderSpecificData(
|
||||
|
||||
const normalized: JsonRecord = { ...record };
|
||||
|
||||
if ("requestDefaults" in normalized) {
|
||||
const requestDefaults = normalizeRequestDefaults(provider, normalized.requestDefaults);
|
||||
if (requestDefaults) {
|
||||
normalized.requestDefaults = requestDefaults;
|
||||
} else {
|
||||
delete normalized.requestDefaults;
|
||||
}
|
||||
}
|
||||
normalizeNestedSubObjects(provider, normalized);
|
||||
|
||||
if ("openaiStoreEnabled" in normalized && typeof normalized.openaiStoreEnabled !== "boolean") {
|
||||
delete normalized.openaiStoreEnabled;
|
||||
|
||||
@@ -15,6 +15,44 @@ function isHttpUrl(value: string): boolean {
|
||||
|
||||
const CODEX_REASONING_EFFORT_VALUES = new Set(["none", "low", "medium", "high", "xhigh", "max"]);
|
||||
const REQUEST_DEFAULT_SERVICE_TIER_VALUES = new Set(["default", "priority", "fast", "flex"]);
|
||||
const CACHE_PASSTHROUGH_VALUES = new Set(["strip", "openai-format", "claude-format"]);
|
||||
|
||||
// #6880 — per-connection prompt-cache capability override, extracted so
|
||||
// validateProviderSpecificData() stays under the complexity gate.
|
||||
function validateCacheBlock(data: Record<string, unknown>, ctx: z.RefinementCtx): void {
|
||||
const cache = data.cache;
|
||||
if (cache === undefined) return;
|
||||
if (!cache || typeof cache !== "object" || Array.isArray(cache)) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: "providerSpecificData.cache must be an object",
|
||||
path: ["cache"],
|
||||
});
|
||||
return;
|
||||
}
|
||||
const cacheRecord = cache as Record<string, unknown>;
|
||||
const supportsPromptCaching = cacheRecord.supportsPromptCaching;
|
||||
if (supportsPromptCaching !== undefined && typeof supportsPromptCaching !== "boolean") {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: "providerSpecificData.cache.supportsPromptCaching must be a boolean",
|
||||
path: ["cache", "supportsPromptCaching"],
|
||||
});
|
||||
}
|
||||
const cacheControlPassthrough = cacheRecord.cacheControlPassthrough;
|
||||
if (
|
||||
cacheControlPassthrough !== undefined &&
|
||||
(typeof cacheControlPassthrough !== "string" ||
|
||||
!CACHE_PASSTHROUGH_VALUES.has(cacheControlPassthrough))
|
||||
) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message:
|
||||
'providerSpecificData.cache.cacheControlPassthrough must be one of "strip", "openai-format", "claude-format"',
|
||||
path: ["cache", "cacheControlPassthrough"],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export function validateProviderSpecificData(
|
||||
data: Record<string, unknown> | undefined,
|
||||
@@ -163,6 +201,8 @@ export function validateProviderSpecificData(
|
||||
}
|
||||
}
|
||||
|
||||
validateCacheBlock(data, ctx);
|
||||
|
||||
const consoleApiKey = data.consoleApiKey;
|
||||
if (consoleApiKey !== undefined && consoleApiKey !== null && typeof consoleApiKey !== "string") {
|
||||
ctx.addIssue({
|
||||
|
||||
207
tests/unit/connection-cache-override-6880.test.ts
Normal file
207
tests/unit/connection-cache-override-6880.test.ts
Normal file
@@ -0,0 +1,207 @@
|
||||
import { describe, test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import type { z } from "zod";
|
||||
import {
|
||||
providerSupportsCaching,
|
||||
providerHonorsOpenAIFormatCacheControl,
|
||||
resolveConnectionCacheOverride,
|
||||
shouldPreserveCacheControl,
|
||||
} from "../../open-sse/utils/cacheControlPolicy.ts";
|
||||
import {
|
||||
detectCachingContext,
|
||||
getCacheAwareStrategy,
|
||||
} from "../../open-sse/services/compression/cachingAware.ts";
|
||||
import { validateProviderSpecificData } from "../../src/shared/validation/providerSpecificData.ts";
|
||||
import { normalizeProviderSpecificData } from "../../src/lib/providers/requestDefaults.ts";
|
||||
|
||||
// Regression for #6880: a custom/openai-compatible connection (provider id like
|
||||
// `openai-compatible-chat-<uuid>`) can never match the hardcoded CACHING_PROVIDERS /
|
||||
// OPENAI_FORMAT_CACHE_CONTROL_PROVIDERS name sets in cacheControlPolicy.ts, so cache
|
||||
// behaviors (prompt_cache_key injection, the compression cache-aware guard, and
|
||||
// cache_control passthrough) are permanently disabled for that class of connections with
|
||||
// no way to opt in. This adds a per-connection `cache` capability override consulted
|
||||
// first by the policy functions, defaulting to today's hardcoded-set behavior.
|
||||
|
||||
function collectIssues(): { ctx: z.RefinementCtx; issues: Array<{ path: (string | number)[]; message: string }> } {
|
||||
const issues: Array<{ path: (string | number)[]; message: string }> = [];
|
||||
const ctx = {
|
||||
addIssue: (issue: { path?: (string | number)[]; message: string }) => {
|
||||
issues.push({ path: issue.path ?? [], message: issue.message });
|
||||
},
|
||||
} as unknown as z.RefinementCtx;
|
||||
return { ctx, issues };
|
||||
}
|
||||
|
||||
describe("#6880 resolveConnectionCacheOverride", () => {
|
||||
test("returns null for undefined/non-object/empty cache", () => {
|
||||
assert.equal(resolveConnectionCacheOverride(undefined), null);
|
||||
assert.equal(resolveConnectionCacheOverride(null), null);
|
||||
assert.equal(resolveConnectionCacheOverride("nope"), null);
|
||||
assert.equal(resolveConnectionCacheOverride({}), null);
|
||||
assert.equal(resolveConnectionCacheOverride({ cache: null }), null);
|
||||
assert.equal(resolveConnectionCacheOverride({ cache: [] }), null);
|
||||
assert.equal(resolveConnectionCacheOverride({ cache: {} }), null);
|
||||
});
|
||||
|
||||
test("extracts valid fields and drops invalid/unknown values", () => {
|
||||
const result = resolveConnectionCacheOverride({
|
||||
cache: {
|
||||
supportsPromptCaching: true,
|
||||
cacheControlPassthrough: "openai-format",
|
||||
unknownField: "ignored",
|
||||
},
|
||||
});
|
||||
assert.deepEqual(result, {
|
||||
supportsPromptCaching: true,
|
||||
cacheControlPassthrough: "openai-format",
|
||||
});
|
||||
|
||||
const invalid = resolveConnectionCacheOverride({
|
||||
cache: { supportsPromptCaching: "yes", cacheControlPassthrough: "bogus" },
|
||||
});
|
||||
assert.equal(invalid, null);
|
||||
});
|
||||
});
|
||||
|
||||
describe("#6880 providerSupportsCaching override", () => {
|
||||
test("unblocks a custom openai-compatible connection when the override opts in", () => {
|
||||
assert.equal(
|
||||
providerSupportsCaching("openai-compatible-chat-abc123", undefined, {
|
||||
supportsPromptCaching: true,
|
||||
}),
|
||||
true
|
||||
);
|
||||
});
|
||||
|
||||
test("no override -> default hardcoded-set behavior is unchanged", () => {
|
||||
assert.equal(providerSupportsCaching("openai-compatible-chat-abc123"), false);
|
||||
});
|
||||
|
||||
test("explicit opt-out overrides the hardcoded set", () => {
|
||||
assert.equal(providerSupportsCaching("claude", undefined, { supportsPromptCaching: false }), false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("#6880 providerHonorsOpenAIFormatCacheControl override", () => {
|
||||
test("openai-format override enables passthrough for a non-hardcoded provider", () => {
|
||||
assert.equal(
|
||||
providerHonorsOpenAIFormatCacheControl("grok-custom", { cacheControlPassthrough: "openai-format" }),
|
||||
true
|
||||
);
|
||||
});
|
||||
|
||||
test("strip override disables passthrough", () => {
|
||||
assert.equal(
|
||||
providerHonorsOpenAIFormatCacheControl("grok-custom", { cacheControlPassthrough: "strip" }),
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
test("no override -> default hardcoded-set behavior is unchanged", () => {
|
||||
assert.equal(providerHonorsOpenAIFormatCacheControl("grok-custom"), false);
|
||||
assert.equal(providerHonorsOpenAIFormatCacheControl("alibaba"), true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("#6880 shouldPreserveCacheControl override", () => {
|
||||
test("preserves cache_control for a non-hardcoded provider when override opts in", () => {
|
||||
const result = shouldPreserveCacheControl({
|
||||
userAgent: "claude-code/1.0",
|
||||
isCombo: false,
|
||||
targetProvider: "openai-compatible-chat-abc123",
|
||||
targetFormat: "openai",
|
||||
connectionCacheOverride: { supportsPromptCaching: true },
|
||||
});
|
||||
assert.equal(result, true);
|
||||
});
|
||||
|
||||
test("no override -> non-hardcoded provider still not preserved", () => {
|
||||
const result = shouldPreserveCacheControl({
|
||||
userAgent: "claude-code/1.0",
|
||||
isCombo: false,
|
||||
targetProvider: "openai-compatible-chat-abc123",
|
||||
targetFormat: "openai",
|
||||
});
|
||||
assert.equal(result, false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("#6880 compression cache-aware guard", () => {
|
||||
test("detectCachingContext reports isCachingProvider=true when the override opts in", () => {
|
||||
const ctx = detectCachingContext(
|
||||
{ messages: [{ role: "user", content: "hi" }] },
|
||||
{
|
||||
provider: "openai-compatible-chat-xyz",
|
||||
targetFormat: "openai",
|
||||
connectionCacheOverride: { supportsPromptCaching: true },
|
||||
}
|
||||
);
|
||||
assert.equal(ctx.isCachingProvider, true);
|
||||
});
|
||||
|
||||
test("detectCachingContext without override keeps default (non-caching) behavior", () => {
|
||||
const ctx = detectCachingContext(
|
||||
{ messages: [{ role: "user", content: "hi" }] },
|
||||
{ provider: "openai-compatible-chat-xyz", targetFormat: "openai" }
|
||||
);
|
||||
assert.equal(ctx.isCachingProvider, false);
|
||||
});
|
||||
|
||||
test("getCacheAwareStrategy protects the cacheable prefix for an overridden context", () => {
|
||||
const ctx = detectCachingContext(
|
||||
{ messages: [{ role: "user", content: "hi" }] },
|
||||
{
|
||||
provider: "openai-compatible-chat-xyz",
|
||||
targetFormat: "openai",
|
||||
connectionCacheOverride: { supportsPromptCaching: true },
|
||||
}
|
||||
);
|
||||
const strategy = getCacheAwareStrategy("aggressive", ctx);
|
||||
assert.equal(strategy.skipSystemPrompt, true);
|
||||
assert.equal(strategy.deterministicOnly, true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("#6880 validateProviderSpecificData cache block", () => {
|
||||
test("accepts a well-formed cache block", () => {
|
||||
const { ctx, issues } = collectIssues();
|
||||
validateProviderSpecificData(
|
||||
{ cache: { supportsPromptCaching: true, cacheControlPassthrough: "openai-format" } },
|
||||
ctx
|
||||
);
|
||||
assert.deepEqual(issues, []);
|
||||
});
|
||||
|
||||
test("rejects a non-object cache", () => {
|
||||
const { ctx, issues } = collectIssues();
|
||||
validateProviderSpecificData({ cache: "nope" }, ctx);
|
||||
assert.equal(issues.length, 1);
|
||||
assert.deepEqual(issues[0]?.path, ["cache"]);
|
||||
});
|
||||
|
||||
test("rejects an invalid cacheControlPassthrough value", () => {
|
||||
const { ctx, issues } = collectIssues();
|
||||
validateProviderSpecificData({ cache: { cacheControlPassthrough: "bogus" } }, ctx);
|
||||
assert.equal(issues.length, 1);
|
||||
assert.deepEqual(issues[0]?.path, ["cache", "cacheControlPassthrough"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("#6880 normalizeProviderSpecificData cache block", () => {
|
||||
test("strips an invalid cache sub-object down to nothing (key deleted)", () => {
|
||||
const normalized = normalizeProviderSpecificData("openai-compatible-chat-xyz", {
|
||||
cache: { supportsPromptCaching: "yes", cacheControlPassthrough: "bogus" },
|
||||
});
|
||||
assert.equal(normalized?.cache, undefined);
|
||||
});
|
||||
|
||||
test("preserves a valid cache sub-object", () => {
|
||||
const normalized = normalizeProviderSpecificData("openai-compatible-chat-xyz", {
|
||||
cache: { supportsPromptCaching: true, cacheControlPassthrough: "openai-format", junk: 1 },
|
||||
});
|
||||
assert.deepEqual(normalized?.cache, {
|
||||
supportsPromptCaching: true,
|
||||
cacheControlPassthrough: "openai-format",
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user