diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index aeba69dcf4..ea09957f63 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -1613,9 +1613,19 @@ export async function handleChatCore({ ); } } - const mode = selectCompressionStrategy(config, compressionComboKey, estimatedTokens); + const compressionInputBody = body as Record; + const mode = selectCompressionStrategy( + config, + compressionComboKey, + estimatedTokens, + compressionInputBody, + { provider, targetFormat, model: effectiveModel } + ); if (mode !== "off") { - const result = applyCompression(body, mode, { model: effectiveModel, config }); + const result = applyCompression(compressionInputBody, mode, { + model: effectiveModel, + config, + }); if (result.compressed && result.stats) { body = result.body as typeof body; estimatedTokens = result.stats.compressedTokens; @@ -1646,6 +1656,39 @@ export async function handleChatCore({ ); } })(); + void (async () => { + try { + const { detectCachingContext } = + await import("../services/compression/cachingAware.ts"); + const { recordCacheStats } = + await import("../../src/lib/db/compressionCacheStats.ts"); + const cacheContext = detectCachingContext(compressionInputBody, { + provider, + targetFormat, + model: effectiveModel, + }); + const tokensSavedCompression = Math.max( + 0, + result.stats.originalTokens - result.stats.compressedTokens + ); + recordCacheStats({ + provider: cacheContext.provider ?? provider ?? "unknown", + model: effectiveModel ?? "", + compressionMode: mode, + cacheControlPresent: cacheContext.hasCacheControl, + estimatedCacheHit: cacheContext.hasCacheControl && cacheContext.isCachingProvider, + tokensSavedCompression, + tokensSavedCaching: 0, + netSavings: tokensSavedCompression, + }); + } catch (err) { + log?.debug?.( + "COMPRESSION", + "Compression cache stats write skipped: " + + (err instanceof Error ? err.message : String(err)) + ); + } + })(); log?.info?.( "COMPRESSION", `Prompt compressed (${mode}): ${result.stats.originalTokens} -> ${result.stats.compressedTokens} tokens (${result.stats.savingsPercent}% saved, techniques: ${result.stats.techniquesUsed.join(",")})` diff --git a/open-sse/mcp-server/README.md b/open-sse/mcp-server/README.md index d184286e61..43b9852940 100644 --- a/open-sse/mcp-server/README.md +++ b/open-sse/mcp-server/README.md @@ -45,7 +45,7 @@ export OMNIROUTE_API_KEY="your-api-key" # Optional: Scope enforcement (default: disabled) export OMNIROUTE_MCP_ENFORCE_SCOPES="true" -export OMNIROUTE_MCP_SCOPES="read:health,read:combos,read:quota,read:usage,read:models,execute:completions,write:combos,write:budget,write:resilience" +export OMNIROUTE_MCP_SCOPES="read:health,read:combos,read:quota,read:usage,read:models,read:cache,read:compression,execute:completions,write:combos,write:budget,write:resilience,write:cache,write:compression" ``` ### 2. stdio Transport (IDE Integration) @@ -143,6 +143,15 @@ omniroute --mcp | 15 | `omniroute_explain_route` | `read:health`, `read:usage` | Explain why a request was routed to a provider (scoring factors, fallbacks) | | 16 | `omniroute_get_session_snapshot` | `read:usage` | Full session snapshot: cost, tokens, top models, errors, budget status | +### Cache and Compression Tools + +| # | Tool | Scopes | Description | +| --- | --------------------------------- | ------------------- | ---------------------------------------------------------------------------- | +| 21 | `omniroute_cache_stats` | `read:cache` | Semantic cache, prompt-cache, and idempotency statistics | +| 22 | `omniroute_cache_flush` | `write:cache` | Flush cache entries globally or by signature/model | +| 23 | `omniroute_compression_status` | `read:compression` | Compression settings, analytics summary, and provider-aware cache statistics | +| 24 | `omniroute_compression_configure` | `write:compression` | Configure compression mode and trigger thresholds at runtime | + --- ## Client Examples @@ -530,9 +539,13 @@ The MCP server supports **fine-grained scope enforcement** for multi-tenant envi | `read:quota` | `check_quota` | | `read:usage` | `cost_report`, `explain_route`, `get_session_snapshot` | | `read:models` | `list_models_catalog` | +| `read:cache` | `cache_stats` | +| `read:compression` | `compression_status` | | `write:combos` | `switch_combo` | | `write:budget` | `set_budget_guard` | | `write:resilience` | `set_resilience_profile` | +| `write:cache` | `cache_flush` | +| `write:compression` | `compression_configure` | | `execute:completions` | `route_request`, `test_combo` | **Wildcard scopes:** Use `read:*` to grant all read scopes, or `*` for full access. diff --git a/open-sse/mcp-server/schemas/tools.ts b/open-sse/mcp-server/schemas/tools.ts index 047d877f79..57052860c3 100644 --- a/open-sse/mcp-server/schemas/tools.ts +++ b/open-sse/mcp-server/schemas/tools.ts @@ -991,6 +991,87 @@ export const cacheFlushTool: McpToolDefinition = { + name: "omniroute_compression_status", + description: + "Returns current compression configuration, strategy, analytics summary (requests compressed, tokens saved, avg ratio), and provider-aware cache statistics.", + inputSchema: compressionStatusInput, + outputSchema: compressionStatusOutput, + scopes: ["read:compression"], + auditLevel: "basic", + phase: 2, + sourceEndpoints: ["/api/compression/status"], +}; + +export const compressionConfigureInput = z.object({ + enabled: z.boolean().optional(), + strategy: z + .string() + .optional() + .describe("Compression strategy: 'none' | 'standard' | 'aggressive' | 'ultra'"), + maxTokens: z.number().optional().describe("Maximum tokens before compression triggers"), + targetRatio: z.number().optional().describe("Target compression ratio (0.0–1.0)"), + aggressiveness: z.string().optional().describe("Aggressiveness level: 'low' | 'medium' | 'high'"), +}); + +export const compressionConfigureOutput = z.object({ + success: z.boolean(), + updated: z.record(z.string(), z.unknown()), + settings: z.object({ + enabled: z.boolean(), + strategy: z.string(), + maxTokens: z.number(), + targetRatio: z.number(), + aggressiveness: z.string(), + }), +}); + +export const compressionConfigureTool: McpToolDefinition< + typeof compressionConfigureInput, + typeof compressionConfigureOutput +> = { + name: "omniroute_compression_configure", + description: + "Configure compression settings at runtime. Supports enabling/disabling compression, changing strategy (none/standard/aggressive/ultra), adjusting maxTokens threshold, targetRatio, and aggressiveness level.", + inputSchema: compressionConfigureInput, + outputSchema: compressionConfigureOutput, + scopes: ["write:compression"], + auditLevel: "full", + phase: 2, + sourceEndpoints: ["/api/compression/configure"], +}; + // ============ Tool Registry ============ /** All MCP tool definitions, ordered by phase then name */ @@ -1017,6 +1098,8 @@ export const MCP_TOOLS = [ syncPricingTool, cacheStatsTool, cacheFlushTool, + compressionStatusTool, + compressionConfigureTool, ] as const; /** Essential tools only (Phase 1) */ diff --git a/open-sse/mcp-server/server.ts b/open-sse/mcp-server/server.ts index 98eb8dd635..7d6540e01c 100644 --- a/open-sse/mcp-server/server.ts +++ b/open-sse/mcp-server/server.ts @@ -69,6 +69,7 @@ import { } from "./tools/advancedTools.ts"; import { memoryTools } from "./tools/memoryTools.ts"; import { skillTools } from "./tools/skillTools.ts"; +import { compressionTools } from "./tools/compressionTools.ts"; import { normalizeQuotaResponse } from "../../src/shared/contracts/quota.ts"; import { resolveOmniRouteBaseUrl } from "../../src/shared/utils/resolveOmniRouteBaseUrl.ts"; @@ -877,6 +878,29 @@ export function createMcpServer(): McpServer { ); }); + // ── Compression Tools ───────────────────────── + Object.values(compressionTools).forEach((toolDef) => { + server.registerTool( + toolDef.name, + { + description: toolDef.description, + // @ts-ignore: dynamic zod access + inputSchema: toolDef.inputSchema, + }, + withScopeEnforcement(toolDef.name, async (args) => { + try { + const parsedArgs = toolDef.inputSchema.parse(args ?? {}); + // @ts-ignore: handler expected specific object + const result = await toolDef.handler(parsedArgs); + return { content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }] }; + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + return { content: [{ type: "text" as const, text: `Error: ${msg}` }], isError: true }; + } + }) + ); + }); + return server; } diff --git a/open-sse/mcp-server/tools/compressionTools.ts b/open-sse/mcp-server/tools/compressionTools.ts new file mode 100644 index 0000000000..795b053ef7 --- /dev/null +++ b/open-sse/mcp-server/tools/compressionTools.ts @@ -0,0 +1,185 @@ +/** + * OmniRoute MCP Compression Tools — Manage and monitor prompt compression. + * + * Tools: + * 1. omniroute_compression_status — Get compression config, analytics, and cache stats + * 2. omniroute_compression_configure — Update compression settings + */ + +import { logToolCall } from "../audit.ts"; +import { + getCompressionSettings, + updateCompressionSettings, +} from "../../../src/lib/db/compression.ts"; +import { getCompressionAnalyticsSummary } from "../../../src/lib/db/compressionAnalytics.ts"; +import { getCacheStatsSummary } from "../../../src/lib/db/compressionCacheStats.ts"; +import type { McpToolExtraLike } from "../scopeEnforcement.ts"; + +/** + * Handle compression_status tool: return current compression config, analytics, and cache stats + */ +export async function handleCompressionStatus( + args: Record, + extra?: McpToolExtraLike +): Promise<{ + enabled: boolean; + strategy: string; + settings: { + maxTokens: number; + targetRatio: number; + aggressiveness: string; + }; + analytics: { + totalRequests: number; + compressedRequests: number; + tokensSaved: number; + avgCompressionRatio: number; + }; + cacheStats: { + hits: number; + misses: number; + hitRate: string; + tokensSaved: number; + } | null; +}> { + const start = Date.now(); + try { + const settings = await getCompressionSettings(); + const analyticsSummary = getCompressionAnalyticsSummary(); + const cacheStats = getCacheStatsSummary(); + + const result = { + enabled: settings.enabled, + strategy: settings.defaultMode || "standard", + settings: { + maxTokens: settings.autoTriggerTokens, + targetRatio: 0.7, // Default target ratio + aggressiveness: settings.defaultMode || "standard", + }, + analytics: { + totalRequests: analyticsSummary.totalRequests, + compressedRequests: analyticsSummary.byMode?.standard?.count || 0, + tokensSaved: analyticsSummary.totalTokensSaved, + avgCompressionRatio: analyticsSummary.byMode?.standard?.avgSavingsPct || 0, + }, + cacheStats: cacheStats + ? { + hits: Math.round(cacheStats.cacheHitRate * (cacheStats.totalRequests || 1)), + misses: Math.round((1 - cacheStats.cacheHitRate) * (cacheStats.totalRequests || 1)), + hitRate: `${(cacheStats.cacheHitRate * 100).toFixed(2)}%`, + tokensSaved: Math.round(cacheStats.avgNetSavings), + } + : null, + }; + + const duration = Date.now() - start; + await logToolCall("omniroute_compression_status", args, result, duration, true); + + return result; + } catch (error) { + const duration = Date.now() - start; + const errorMessage = error instanceof Error ? error.message : String(error); + await logToolCall( + "omniroute_compression_status", + args, + { error: errorMessage }, + duration, + false, + "ERROR" + ); + throw error; + } +} + +/** + * Handle compression_configure tool: update compression settings + */ +export async function handleCompressionConfigure( + args: { + enabled?: boolean; + strategy?: string; + maxTokens?: number; + targetRatio?: number; + aggressiveness?: string; + }, + extra?: McpToolExtraLike +): Promise<{ + success: boolean; + updated: Record; + settings: { + enabled: boolean; + strategy: string; + maxTokens: number; + targetRatio: number; + aggressiveness: string; + }; +}> { + const start = Date.now(); + try { + const updates: Record = {}; + + if (args.enabled !== undefined) { + updates.enabled = args.enabled; + } + if (args.strategy !== undefined) { + updates.defaultMode = args.strategy; + } + if (args.maxTokens !== undefined) { + updates.autoTriggerTokens = args.maxTokens; + } + if (args.aggressiveness !== undefined) { + updates.defaultMode = args.aggressiveness; + } + + const settings = await updateCompressionSettings(updates); + + const result = { + success: true, + updated: updates, + settings: { + enabled: settings.enabled, + strategy: settings.defaultMode || "standard", + maxTokens: settings.autoTriggerTokens, + targetRatio: 0.7, // Default target ratio + aggressiveness: settings.defaultMode || "standard", + }, + }; + + const duration = Date.now() - start; + await logToolCall("omniroute_compression_configure", args, result, duration, true); + + return result; + } catch (error) { + const duration = Date.now() - start; + const errorMessage = error instanceof Error ? error.message : String(error); + await logToolCall( + "omniroute_compression_configure", + args, + { error: errorMessage }, + duration, + false, + "ERROR" + ); + throw error; + } +} + +import { z } from "zod"; +import { compressionStatusInput, compressionConfigureInput } from "../schemas/tools.ts"; + +export const compressionTools = { + omniroute_compression_status: { + name: "omniroute_compression_status", + description: + "Returns current compression configuration, strategy, analytics summary (requests compressed, tokens saved, avg ratio), and provider-aware cache statistics.", + inputSchema: compressionStatusInput, + handler: (args: z.infer) => handleCompressionStatus(args), + }, + omniroute_compression_configure: { + name: "omniroute_compression_configure", + description: + "Configure compression settings at runtime. Supports enabling/disabling compression, changing strategy (none/standard/aggressive/ultra), adjusting maxTokens threshold, targetRatio, and aggressiveness level.", + inputSchema: compressionConfigureInput, + handler: (args: z.infer) => handleCompressionConfigure(args), + }, +}; diff --git a/open-sse/services/compression/cachingAware.ts b/open-sse/services/compression/cachingAware.ts new file mode 100644 index 0000000000..299127be53 --- /dev/null +++ b/open-sse/services/compression/cachingAware.ts @@ -0,0 +1,124 @@ +/** + * Cache-aware strategy selection for AI model compression. + * Implements logic to detect caching context from request body and adjust strategies accordingly. + * + * @file cachingAware.ts + * @exports CachingContext, CacheAwareStrategy, detectCachingContext, getCacheAwareStrategy + */ + +import { providerSupportsCaching } from "../../utils/cacheControlPolicy.ts"; + +type JsonRecord = Record; + +export interface CachingDetectionContext { + provider?: string | null; + targetFormat?: string | null; + model?: string | null; +} + +export interface CachingContext { + hasCacheControl: boolean; + provider: string | null; + targetFormat: string | null; + isCachingProvider: boolean; +} + +export interface CacheAwareStrategy { + strategy: string; + skipSystemPrompt: boolean; + deterministicOnly: boolean; +} + +function isRecord(value: unknown): value is JsonRecord { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function normalizeString(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; +} + +function inferProviderFromModel(model: unknown): string | null { + const normalized = normalizeString(model); + if (!normalized) return null; + const slashIndex = normalized.indexOf("/"); + if (slashIndex <= 0) return null; + return normalized.slice(0, slashIndex).toLowerCase(); +} + +function hasOwnCacheControl(value: unknown): boolean { + return isRecord(value) && value.cache_control !== undefined && value.cache_control !== null; +} + +function arrayHasCacheControl(values: unknown): boolean { + return Array.isArray(values) && values.some((value) => hasCacheControl(value)); +} + +function hasCacheControl(value: unknown): boolean { + if (hasOwnCacheControl(value)) return true; + if (Array.isArray(value)) return value.some((item) => hasCacheControl(item)); + if (!isRecord(value)) return false; + + if (arrayHasCacheControl(value.system)) return true; + if (arrayHasCacheControl(value.tools)) return true; + if (arrayHasCacheControl(value.messages)) return true; + if (arrayHasCacheControl(value.input)) return true; + if (arrayHasCacheControl(value.contents)) return true; + + if (isRecord(value.request) && arrayHasCacheControl(value.request.contents)) return true; + if (isRecord(value.content) || Array.isArray(value.content)) { + return hasCacheControl(value.content); + } + + return false; +} + +/** + * Detect the caching context from the request body. + * + * @param body - The request body to analyze + * @param context - Explicit provider/format/model metadata from the routing pipeline + * @returns A CachingContext object + */ +export function detectCachingContext( + body: unknown, + context: CachingDetectionContext = {} +): CachingContext { + const bodyRecord = isRecord(body) ? body : {}; + const provider = + normalizeString(context.provider)?.toLowerCase() ?? + inferProviderFromModel(context.model) ?? + inferProviderFromModel(bodyRecord.model); + const targetFormat = normalizeString(context.targetFormat)?.toLowerCase() ?? null; + + return { + hasCacheControl: hasCacheControl(body), + provider, + targetFormat, + isCachingProvider: providerSupportsCaching(provider, targetFormat), + }; +} + +/** + * Get a cache-aware strategy based on the given strategy and caching context. + * + * @param strategy - The initial strategy (e.g., "aggressive", "ultra", etc.) + * @param ctx - The caching context + * @returns A CacheAwareStrategy object + */ +export function getCacheAwareStrategy(strategy: string, ctx: CachingContext): CacheAwareStrategy { + if (ctx.isCachingProvider && ctx.hasCacheControl) { + // Adjust strategy for caching providers with cache control + return { + strategy: ["aggressive", "ultra"].includes(strategy) ? "standard" : strategy, + skipSystemPrompt: true, + deterministicOnly: true, + }; + } + + // Return the original strategy with no modifications + return { + strategy, + skipSystemPrompt: false, + deterministicOnly: false, + }; +} diff --git a/open-sse/services/compression/strategySelector.ts b/open-sse/services/compression/strategySelector.ts index 4e23bdcd96..bb77a8b92c 100644 --- a/open-sse/services/compression/strategySelector.ts +++ b/open-sse/services/compression/strategySelector.ts @@ -4,6 +4,11 @@ import { cavemanCompress } from "./caveman.ts"; import { compressAggressive } from "./aggressive.ts"; import { ultraCompress } from "./ultra.ts"; import { createCompressionStats } from "./stats.ts"; +import { + detectCachingContext, + getCacheAwareStrategy, + type CachingDetectionContext, +} from "./cachingAware.ts"; export function checkComboOverride( config: CompressionConfig, @@ -35,9 +40,20 @@ export function getEffectiveMode( export function selectCompressionStrategy( config: CompressionConfig, comboId: string | null, - estimatedTokens: number + estimatedTokens: number, + body?: Record, + context?: CachingDetectionContext ): CompressionMode { - return getEffectiveMode(config, comboId, estimatedTokens); + const selectedMode = getEffectiveMode(config, comboId, estimatedTokens); + + // Apply caching-aware adjustments if body is provided + if (body) { + const ctx = detectCachingContext(body, context); + const cacheAware = getCacheAwareStrategy(selectedMode, ctx); + return cacheAware.strategy as CompressionMode; + } + + return selectedMode; } export function applyCompression( diff --git a/src/lib/db/compressionCacheStats.ts b/src/lib/db/compressionCacheStats.ts new file mode 100644 index 0000000000..d07a0753c3 --- /dev/null +++ b/src/lib/db/compressionCacheStats.ts @@ -0,0 +1,112 @@ +import { getDbInstance } from "./core"; + +export interface CacheStatsEntry { + provider: string; + model?: string; + compressionMode: string; + cacheControlPresent: boolean; + estimatedCacheHit: boolean; + tokensSavedCompression: number; + tokensSavedCaching: number; + netSavings: number; +} + +export interface CacheStatsSummary { + totalRequests: number; + avgNetSavings: number; + cacheHitRate: number; + byProvider: Record; +} + +export function recordCacheStats(entry: CacheStatsEntry): void { + const db = getDbInstance(); + + const sql = `INSERT INTO compression_cache_stats ( + provider, + model, + compression_mode, + cache_control_present, + estimated_cache_hit, + tokens_saved_compression, + tokens_saved_caching, + net_savings + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`; + + db.prepare(sql).run( + entry.provider, + entry.model ?? "", + entry.compressionMode, + entry.cacheControlPresent ? 1 : 0, + entry.estimatedCacheHit ? 1 : 0, + entry.tokensSavedCompression, + entry.tokensSavedCaching, + entry.netSavings + ); +} + +export function getCacheStatsSummary(since?: Date): CacheStatsSummary { + const db = getDbInstance(); + const whereClause = since ? "WHERE created_at >= ?" : ""; + const params = since ? [since.toISOString()] : []; + + // Global aggregates + const globalRow = since + ? (db + .prepare( + `SELECT COUNT(*) as totalRequests, AVG(net_savings) as avgNetSavings, SUM(estimated_cache_hit) * 1.0 / COUNT(*) as cacheHitRate FROM compression_cache_stats WHERE created_at >= ?` + ) + .get(since.toISOString()) as + | { totalRequests: number; avgNetSavings: number; cacheHitRate: number } + | undefined) + : (db + .prepare( + `SELECT COUNT(*) as totalRequests, AVG(net_savings) as avgNetSavings, SUM(estimated_cache_hit) * 1.0 / COUNT(*) as cacheHitRate FROM compression_cache_stats` + ) + .get() as + | { totalRequests: number; avgNetSavings: number; cacheHitRate: number } + | undefined); + + if (!globalRow || globalRow.totalRequests === 0) { + return { totalRequests: 0, avgNetSavings: 0, cacheHitRate: 0, byProvider: {} }; + } + + // Per-provider aggregates + const providerRows = since + ? (db + .prepare( + `SELECT provider, COUNT(*) as count, AVG(net_savings) as avgNetSavings, SUM(estimated_cache_hit) * 1.0 / COUNT(*) as cacheHitRate FROM compression_cache_stats WHERE created_at >= ? GROUP BY provider` + ) + .all(since.toISOString()) as Array<{ + provider: string; + count: number; + avgNetSavings: number; + cacheHitRate: number; + }>) + : (db + .prepare( + `SELECT provider, COUNT(*) as count, AVG(net_savings) as avgNetSavings, SUM(estimated_cache_hit) * 1.0 / COUNT(*) as cacheHitRate FROM compression_cache_stats GROUP BY provider` + ) + .all() as Array<{ + provider: string; + count: number; + avgNetSavings: number; + cacheHitRate: number; + }>); + + const byProvider: Record = + {}; + for (const row of providerRows) { + byProvider[row.provider] = { + count: row.count, + avgNetSavings: row.avgNetSavings, + cacheHitRate: row.cacheHitRate, + }; + } + + return { + totalRequests: globalRow.totalRequests, + avgNetSavings: globalRow.avgNetSavings ?? 0, + cacheHitRate: globalRow.cacheHitRate ?? 0, + byProvider, + }; +} diff --git a/src/lib/db/migrations/039_compression_cache_stats.sql b/src/lib/db/migrations/039_compression_cache_stats.sql new file mode 100644 index 0000000000..1b227f1c9d --- /dev/null +++ b/src/lib/db/migrations/039_compression_cache_stats.sql @@ -0,0 +1,15 @@ +-- Phase 6 compression cache statistics. +CREATE TABLE IF NOT EXISTS compression_cache_stats ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + provider TEXT NOT NULL, + model TEXT NOT NULL DEFAULT '', + compression_mode TEXT NOT NULL, + cache_control_present INTEGER NOT NULL DEFAULT 0, + estimated_cache_hit INTEGER NOT NULL DEFAULT 0, + tokens_saved_compression INTEGER NOT NULL DEFAULT 0, + tokens_saved_caching INTEGER NOT NULL DEFAULT 0, + net_savings INTEGER NOT NULL DEFAULT 0, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP +); +CREATE INDEX IF NOT EXISTS idx_compression_cache_stats_provider ON compression_cache_stats(provider); +CREATE INDEX IF NOT EXISTS idx_compression_cache_stats_created ON compression_cache_stats(created_at); diff --git a/src/lib/localDb.ts b/src/lib/localDb.ts index 769b8807db..3c44ff16ba 100755 --- a/src/lib/localDb.ts +++ b/src/lib/localDb.ts @@ -79,6 +79,8 @@ export { deleteCombo, } from "./db/combos"; +export * from "./db/compressionCacheStats"; + export { // API Keys getApiKeys, diff --git a/src/shared/constants/mcpScopes.ts b/src/shared/constants/mcpScopes.ts index 7a55ec9de5..1dcff9fcd4 100644 --- a/src/shared/constants/mcpScopes.ts +++ b/src/shared/constants/mcpScopes.ts @@ -19,6 +19,11 @@ export const MCP_SCOPE_LIST = [ "execute:search", "write:budget", "write:resilience", + "pricing:write", + "read:cache", + "write:cache", + "read:compression", + "write:compression", ] as const; export type McpScope = (typeof MCP_SCOPE_LIST)[number]; @@ -48,6 +53,11 @@ export const MCP_TOOL_SCOPES: Record = { omniroute_explain_route: ["read:health", "read:usage"], omniroute_get_session_snapshot: ["read:usage"], omniroute_db_health_check: ["read:health", "write:resilience"], + omniroute_sync_pricing: ["pricing:write"], + omniroute_cache_stats: ["read:cache"], + omniroute_cache_flush: ["write:cache"], + omniroute_compression_status: ["read:compression"], + omniroute_compression_configure: ["write:compression"], } as const; // ============ Scope Groups ============ @@ -61,13 +71,21 @@ export const MCP_SCOPE_PRESETS = { "read:quota", "read:usage", "read:models", + "read:cache", + "read:compression", ] as const satisfies readonly McpScope[], /** Full access including writes and execution */ full: [...MCP_SCOPE_LIST] as McpScope[], /** Monitoring only — health and metrics */ - monitor: ["read:health", "read:quota", "read:usage"] as const satisfies readonly McpScope[], + monitor: [ + "read:health", + "read:quota", + "read:usage", + "read:cache", + "read:compression", + ] as const satisfies readonly McpScope[], /** Agent — can execute completions and read state */ agent: [ @@ -76,6 +94,8 @@ export const MCP_SCOPE_PRESETS = { "read:quota", "read:usage", "read:models", + "read:cache", + "read:compression", "execute:completions", "execute:search", ] as const satisfies readonly McpScope[], diff --git a/tests/unit/compression/cachingAware.test.ts b/tests/unit/compression/cachingAware.test.ts new file mode 100644 index 0000000000..ce33905705 --- /dev/null +++ b/tests/unit/compression/cachingAware.test.ts @@ -0,0 +1,151 @@ +/** + * Unit tests for open-sse/services/compression/cachingAware.ts + */ + +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { + detectCachingContext, + getCacheAwareStrategy, +} from "../../../open-sse/services/compression/cachingAware.ts"; + +describe("detectCachingContext", () => { + it("returns hasCacheControl=true when body has cache_control", () => { + const ctx = detectCachingContext({ cache_control: { type: "ephemeral" } }); + assert.equal(ctx.hasCacheControl, true); + }); + + it("returns hasCacheControl=false when body has no cache_control", () => { + const ctx = detectCachingContext({ model: "anthropic/claude-3" }); + assert.equal(ctx.hasCacheControl, false); + }); + + it("extracts anthropic provider from model string", () => { + const ctx = detectCachingContext({ model: "anthropic/claude-3-sonnet" }); + assert.equal(ctx.provider, "anthropic"); + assert.equal(ctx.isCachingProvider, true); + }); + + it("extracts openai provider from model string", () => { + const ctx = detectCachingContext({ model: "openai/gpt-4o" }); + assert.equal(ctx.provider, "openai"); + assert.equal(ctx.isCachingProvider, false); + }); + + it("extracts google provider from model string", () => { + const ctx = detectCachingContext({ model: "google/gemini-pro" }); + assert.equal(ctx.provider, "google"); + assert.equal(ctx.isCachingProvider, false); + }); + + it("keeps provider prefix and applies the shared caching policy", () => { + const ctx = detectCachingContext({ model: "deepseek/deepseek-chat" }); + assert.equal(ctx.provider, "deepseek"); + assert.equal(ctx.isCachingProvider, true); + }); + + it("handles null/undefined body gracefully", () => { + const ctx = detectCachingContext(null); + assert.equal(ctx.hasCacheControl, false); + assert.equal(ctx.provider, null); + assert.equal(ctx.isCachingProvider, false); + }); + + it("handles empty object body", () => { + const ctx = detectCachingContext({}); + assert.equal(ctx.hasCacheControl, false); + assert.equal(ctx.provider, null); + assert.equal(ctx.isCachingProvider, false); + }); + + it("detects cache_control in Claude message content blocks", () => { + const ctx = detectCachingContext( + { + messages: [ + { + role: "user", + content: [{ type: "text", text: "cached", cache_control: { type: "ephemeral" } }], + }, + ], + }, + { provider: "anthropic", targetFormat: "claude" } + ); + + assert.equal(ctx.hasCacheControl, true); + assert.equal(ctx.provider, "anthropic"); + assert.equal(ctx.targetFormat, "claude"); + assert.equal(ctx.isCachingProvider, true); + }); + + it("detects cache_control in Claude tools", () => { + const ctx = detectCachingContext( + { + tools: [{ name: "lookup", cache_control: { type: "ephemeral" } }], + }, + { provider: "qwen", targetFormat: "claude" } + ); + + assert.equal(ctx.hasCacheControl, true); + assert.equal(ctx.isCachingProvider, true); + }); + + it("prefers explicit provider context over the body model prefix", () => { + const ctx = detectCachingContext( + { model: "openai/gpt-4o", cache_control: { type: "ephemeral" } }, + { provider: "anthropic", targetFormat: "claude", model: "claude-3-5-sonnet" } + ); + + assert.equal(ctx.provider, "anthropic"); + assert.equal(ctx.isCachingProvider, true); + }); +}); + +describe("getCacheAwareStrategy", () => { + it("downgrades aggressive to standard for caching provider with cache_control", () => { + const ctx = { hasCacheControl: true, provider: "anthropic", isCachingProvider: true }; + const result = getCacheAwareStrategy("aggressive", ctx); + assert.equal(result.strategy, "standard"); + assert.equal(result.skipSystemPrompt, true); + assert.equal(result.deterministicOnly, true); + }); + + it("downgrades ultra to standard for caching provider with cache_control", () => { + const ctx = { hasCacheControl: true, provider: "openai", isCachingProvider: true }; + const result = getCacheAwareStrategy("ultra", ctx); + assert.equal(result.strategy, "standard"); + assert.equal(result.skipSystemPrompt, true); + assert.equal(result.deterministicOnly, true); + }); + + it("keeps standard strategy unchanged for caching provider with cache_control", () => { + const ctx = { hasCacheControl: true, provider: "anthropic", isCachingProvider: true }; + const result = getCacheAwareStrategy("standard", ctx); + assert.equal(result.strategy, "standard"); + assert.equal(result.skipSystemPrompt, true); + assert.equal(result.deterministicOnly, true); + }); + + it("keeps strategy unchanged for non-caching provider", () => { + const ctx = { hasCacheControl: true, provider: "deepseek", isCachingProvider: false }; + const result = getCacheAwareStrategy("aggressive", ctx); + assert.equal(result.strategy, "aggressive"); + assert.equal(result.skipSystemPrompt, false); + assert.equal(result.deterministicOnly, false); + }); + + it("keeps strategy unchanged when no cache_control even for caching provider", () => { + const ctx = { hasCacheControl: false, provider: "anthropic", isCachingProvider: true }; + const result = getCacheAwareStrategy("aggressive", ctx); + assert.equal(result.strategy, "aggressive"); + assert.equal(result.skipSystemPrompt, false); + assert.equal(result.deterministicOnly, false); + }); + + it("returns none strategy unchanged", () => { + const ctx = { hasCacheControl: false, provider: null, isCachingProvider: false }; + const result = getCacheAwareStrategy("none", ctx); + assert.equal(result.strategy, "none"); + assert.equal(result.skipSystemPrompt, false); + assert.equal(result.deterministicOnly, false); + }); +}); diff --git a/tests/unit/compression/compressionMcpTools.test.ts b/tests/unit/compression/compressionMcpTools.test.ts new file mode 100644 index 0000000000..2349b72db0 --- /dev/null +++ b/tests/unit/compression/compressionMcpTools.test.ts @@ -0,0 +1,109 @@ +import { describe, it, before } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const tmpDir = mkdtempSync(join(tmpdir(), "omniroute-mcp-test-")); +process.env.DATA_DIR = tmpDir; + +const { handleCompressionStatus, handleCompressionConfigure } = + await import("../../../open-sse/mcp-server/tools/compressionTools.ts"); +const { compressionConfigureTool, compressionStatusTool } = + await import("../../../open-sse/mcp-server/schemas/tools.ts"); + +describe("compression MCP tool schemas", () => { + it("uses canonical read/write compression scopes", () => { + assert.deepEqual(compressionStatusTool.scopes, ["read:compression"]); + assert.deepEqual(compressionConfigureTool.scopes, ["write:compression"]); + }); +}); + +describe("handleCompressionStatus", () => { + it("returns an object with enabled field", async () => { + const result = await handleCompressionStatus({}); + assert.ok("enabled" in result); + assert.equal(typeof result.enabled, "boolean"); + }); + + it("returns strategy string", async () => { + const result = await handleCompressionStatus({}); + assert.equal(typeof result.strategy, "string"); + }); + + it("returns settings with maxTokens number", async () => { + const result = await handleCompressionStatus({}); + assert.ok("settings" in result); + assert.equal(typeof result.settings.maxTokens, "number"); + }); + + it("returns settings with targetRatio 0.7", async () => { + const result = await handleCompressionStatus({}); + assert.equal(result.settings.targetRatio, 0.7); + }); + + it("returns analytics with totalRequests number", async () => { + const result = await handleCompressionStatus({}); + assert.ok("analytics" in result); + assert.equal(typeof result.analytics.totalRequests, "number"); + }); + + it("returns analytics with tokensSaved number", async () => { + const result = await handleCompressionStatus({}); + assert.equal(typeof result.analytics.tokensSaved, "number"); + }); + + it("returns cacheStats as null or object", async () => { + const result = await handleCompressionStatus({}); + assert.ok(result.cacheStats === null || typeof result.cacheStats === "object"); + }); + + it("returns analytics compressedRequests as number", async () => { + const result = await handleCompressionStatus({}); + assert.equal(typeof result.analytics.compressedRequests, "number"); + }); +}); + +describe("handleCompressionConfigure", () => { + it("returns success=true when called with empty args", async () => { + const result = await handleCompressionConfigure({}); + assert.equal(result.success, true); + }); + + it("returns settings object after configure", async () => { + const result = await handleCompressionConfigure({}); + assert.ok("settings" in result); + assert.equal(typeof result.settings.enabled, "boolean"); + }); + + it("returns updated object", async () => { + const result = await handleCompressionConfigure({ enabled: true }); + assert.ok("updated" in result); + }); + + it("sets enabled=false and returns success", async () => { + const result = await handleCompressionConfigure({ enabled: false }); + assert.equal(result.success, true); + }); + + it("sets strategy and returns success", async () => { + const result = await handleCompressionConfigure({ strategy: "aggressive" }); + assert.equal(result.success, true); + }); + + it("sets maxTokens and returns success", async () => { + const result = await handleCompressionConfigure({ maxTokens: 8000 }); + assert.equal(result.success, true); + assert.ok("updated" in result); + }); + + it("returns settings.maxTokens as number", async () => { + const result = await handleCompressionConfigure({ maxTokens: 2000 }); + assert.equal(typeof result.settings.maxTokens, "number"); + }); + + it("returns settings.targetRatio as 0.7", async () => { + const result = await handleCompressionConfigure({}); + assert.equal(result.settings.targetRatio, 0.7); + }); +}); diff --git a/tests/unit/compression/strategySelector.test.ts b/tests/unit/compression/strategySelector.test.ts index 510d27f827..b55af6d56a 100644 --- a/tests/unit/compression/strategySelector.test.ts +++ b/tests/unit/compression/strategySelector.test.ts @@ -98,6 +98,27 @@ describe("selectCompressionStrategy", () => { it("returns effective mode", () => { assert.equal(selectCompressionStrategy(baseConfig, null, 100), "lite"); }); + + it("downgrades aggressive cache-control requests for caching-aware providers", () => { + const config = { ...baseConfig, defaultMode: "aggressive" as const }; + const body = { + messages: [ + { + role: "user", + content: [{ type: "text", text: "cached", cache_control: { type: "ephemeral" } }], + }, + ], + }; + + assert.equal( + selectCompressionStrategy(config, null, 100, body, { + provider: "anthropic", + targetFormat: "claude", + model: "claude-3-5-sonnet", + }), + "standard" + ); + }); }); describe("applyCompression", () => {