diff --git a/Dockerfile b/Dockerfile index 3e3331a5df..135c6abceb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -57,7 +57,8 @@ LABEL org.opencontainers.image.title="omniroute" \ ENV NODE_ENV=production ENV PORT=20128 ENV HOSTNAME=0.0.0.0 -ENV NODE_OPTIONS="--max-old-space-size=256" +ENV OMNIROUTE_MEMORY_MB=1024 +ENV NODE_OPTIONS="--max-old-space-size=${OMNIROUTE_MEMORY_MB}" # Data directory inside Docker — must match the volume mount in docker-compose.yml ENV DATA_DIR=/app/data diff --git a/open-sse/config/providerRegistry.ts b/open-sse/config/providerRegistry.ts index f857287ff4..e121fc077a 100644 --- a/open-sse/config/providerRegistry.ts +++ b/open-sse/config/providerRegistry.ts @@ -580,7 +580,7 @@ function mapStainlessArch() { // ── Registry ────────────────────────────────────────────────────────────── -export const REGISTRY: Record = { +const _REGISTRY_EAGER: Record = { // ─── OAuth Providers ─────────────────────────────────────────────────── kie: { id: "kie", @@ -4248,12 +4248,14 @@ export const REGISTRY: Record = { }, }; +export const REGISTRY: Record = _REGISTRY_EAGER; + // ── Generator Functions ─────────────────────────────────────────────────── /** Generate legacy PROVIDERS object shape for constants.js backward compatibility */ export function generateLegacyProviders(): Record { const providers: Record = {}; - for (const [id, entry] of Object.entries(REGISTRY)) { + for (const [id, entry] of Object.entries(_REGISTRY_EAGER)) { const p: LegacyProvider = { format: entry.format }; // URL(s) @@ -4307,7 +4309,7 @@ export function generateLegacyProviders(): Record { /** Generate PROVIDER_MODELS map (alias → model list) */ export function generateModels(): Record { const models: Record = {}; - for (const entry of Object.values(REGISTRY)) { + for (const entry of Object.values(_REGISTRY_EAGER)) { if (entry.models && entry.models.length > 0) { const key = entry.alias || entry.id; // If alias already exists, don't overwrite (first wins) @@ -4322,7 +4324,7 @@ export function generateModels(): Record { /** Generate PROVIDER_ID_TO_ALIAS map */ export function generateAliasMap(): Record { const map: Record = {}; - for (const entry of Object.values(REGISTRY)) { + for (const entry of Object.values(_REGISTRY_EAGER)) { map[entry.id] = entry.alias || entry.id; } return map; @@ -4364,33 +4366,40 @@ export function isLocalProvider(baseUrl?: string | null): boolean { } /** Set of provider IDs with passthroughModels enabled — 404s are model-specific, not account-level. */ -const _passthroughProviderIds: Set | null = (() => { +let _passthroughProviderIds: Set | null = null; +function ensurePassthroughProviderIds(): Set { + if (_passthroughProviderIds) return _passthroughProviderIds; try { const ids = new Set(); - for (const entry of Object.values(REGISTRY)) { + for (const entry of Object.values(_REGISTRY_EAGER)) { if (entry.passthroughModels) ids.add(entry.id); } - return ids; + _passthroughProviderIds = ids; } catch { - return null; + _passthroughProviderIds = new Set(); } -})(); - + return _passthroughProviderIds; +} export function getPassthroughProviders(): Set { - return _passthroughProviderIds ?? new Set(); + return ensurePassthroughProviderIds(); } // ── Registry Lookup Helpers ─────────────────────────────────────────────── const _byAlias = new Map(); -for (const entry of Object.values(REGISTRY)) { - if (entry.alias && entry.alias !== entry.id) { - _byAlias.set(entry.alias, entry); +let _byAliasPopulated = false; +function ensureByAliasPopulated(): void { + if (_byAliasPopulated) return; + _byAliasPopulated = true; + for (const entry of Object.values(_REGISTRY_EAGER)) { + if (entry.alias && entry.alias !== entry.id) { + _byAlias.set(entry.alias, entry); + } } } - /** Get registry entry by provider ID or alias */ export function getRegistryEntry(provider: string): RegistryEntry | null { + ensureByAliasPopulated(); return REGISTRY[provider] || _byAlias.get(provider) || null; } @@ -4402,10 +4411,15 @@ export function getRegisteredProviders(): string[] { // Precomputed map: modelId → unsupportedParams (O(1) lookup instead of O(N×M) scan). // Built once at module load from all registry entries. const _unsupportedParamsMap = new Map(); -for (const entry of Object.values(REGISTRY)) { - for (const model of entry.models) { - if (model.unsupportedParams && !_unsupportedParamsMap.has(model.id)) { - _unsupportedParamsMap.set(model.id, model.unsupportedParams); +let _unsupportedParamsPopulated = false; +function ensureUnsupportedParamsPopulated(): void { + if (_unsupportedParamsPopulated) return; + _unsupportedParamsPopulated = true; + for (const entry of Object.values(_REGISTRY_EAGER)) { + for (const model of entry.models) { + if (model.unsupportedParams && !_unsupportedParamsMap.has(model.id)) { + _unsupportedParamsMap.set(model.id, model.unsupportedParams); + } } } } @@ -4417,6 +4431,7 @@ for (const entry of Object.values(REGISTRY)) { * Returns empty array if no restrictions are defined. */ export function getUnsupportedParams(provider: string, modelId: string): readonly string[] { + ensureUnsupportedParamsPopulated(); // 1. Check current provider's registry (exact match) const entry = getRegistryEntry(provider); const modelEntry = entry?.models.find((m) => m.id === modelId); diff --git a/open-sse/services/comboMetrics.ts b/open-sse/services/comboMetrics.ts index 6e33aafdb2..6c6e8e71da 100644 --- a/open-sse/services/comboMetrics.ts +++ b/open-sse/services/comboMetrics.ts @@ -188,6 +188,40 @@ function toMetricView( // In-memory store const metrics = new Map(); const shadowMetrics = new Map(); +const MAX_METRICS_ENTRIES = 500; +const METRICS_TTL_MS = 60 * 60 * 1000; // 1 hour + +function evictOldestMetric(targetMap: Map): void { + let oldest: string | null = null; + let oldestTime = Infinity; + for (const [name, entry] of targetMap) { + const t = entry.lastUsedAt ? new Date(entry.lastUsedAt).getTime() : Date.now(); + if (t < oldestTime) { oldestTime = t; oldest = name; } + } + if (oldest) { + metrics.delete(oldest); + shadowMetrics.delete(oldest); + } +} + +const _metricsCleanupTimer = setInterval(() => { + const now = Date.now(); + for (const [name, entry] of metrics) { + const lastUsed = entry.lastUsedAt ? new Date(entry.lastUsedAt).getTime() : now; + if (now - lastUsed > METRICS_TTL_MS) { + metrics.delete(name); + shadowMetrics.delete(name); + } + } + for (const [name, entry] of shadowMetrics) { + const lastUsed = entry.lastUsedAt ? new Date(entry.lastUsedAt).getTime() : now; + if (now - lastUsed > METRICS_TTL_MS) { + metrics.delete(name); + shadowMetrics.delete(name); + } + } +}, 5 * 60 * 1000); // every 5 minutes +_metricsCleanupTimer.unref?.(); // Don't prevent process exit /** * Record a combo request result. @@ -217,6 +251,9 @@ export function recordComboRequest( target?: ComboRequestTargetMeta | null; } ): void { + if (!metrics.has(comboName) && metrics.size >= MAX_METRICS_ENTRIES) { + evictOldestMetric(metrics); + } if (!metrics.has(comboName)) { metrics.set(comboName, createComboEntry(strategy)); } @@ -283,6 +320,9 @@ export function recordComboShadowRequest( target?: ComboRequestTargetMeta | null; } ): void { + if (!shadowMetrics.has(comboName) && shadowMetrics.size >= MAX_METRICS_ENTRIES) { + evictOldestMetric(shadowMetrics); + } if (!shadowMetrics.has(comboName)) { shadowMetrics.set(comboName, createShadowEntry()); } @@ -396,6 +436,9 @@ export function getAllComboMetrics(): Record { * Record detected prompt intent for a combo (used by multilingual routing analytics). */ export function recordComboIntent(comboName: string, intent: string): void { + if (!metrics.has(comboName) && metrics.size >= MAX_METRICS_ENTRIES) { + evictOldestMetric(metrics); + } if (!metrics.has(comboName)) { metrics.set(comboName, createComboEntry("priority")); } @@ -418,6 +461,7 @@ export function resetComboMetrics(comboName: string): void { * Reset all combo metrics. */ export function resetAllComboMetrics(): void { + clearInterval(_metricsCleanupTimer); metrics.clear(); shadowMetrics.clear(); } diff --git a/open-sse/services/usage.ts b/open-sse/services/usage.ts index 1f3cade0f6..1e8401c7a4 100644 --- a/open-sse/services/usage.ts +++ b/open-sse/services/usage.ts @@ -1821,6 +1821,28 @@ const _antigravityAvailableModelsInflight = new Map>(); const _antigravityCreditProbeCache = new Map(); const _antigravityCreditProbeInflight = new Map>(); +// ── Proactive TTL purging for module-level caches ────────────────────────── +// All 4 data caches only evict on read (passive TTL). This interval proactively +// purges stale entries so keys accessed once and never again don't leak memory. +// The 2 inflight Maps (availableModelsInflight, creditProbeInflight) self-clean +// when the Promise resolves/rejects, so they are NOT touched here. +const _usageCacheCleanupTimer = setInterval(() => { + const now = Date.now(); + for (const [key, entry] of _geminiCliSubCache) { + if (now - entry.fetchedAt > GEMINI_CLI_CACHE_TTL_MS) _geminiCliSubCache.delete(key); + } + for (const [key, entry] of _antigravitySubCache) { + if (now - entry.fetchedAt > ANTIGRAVITY_CACHE_TTL_MS) _antigravitySubCache.delete(key); + } + for (const [key, entry] of _antigravityAvailableModelsCache) { + if (now - entry.fetchedAt > ANTIGRAVITY_MODELS_CACHE_TTL_MS) _antigravityAvailableModelsCache.delete(key); + } + for (const [key, entry] of _antigravityCreditProbeCache) { + if (now - entry.fetchedAt > ANTIGRAVITY_CREDIT_PROBE_TTL_MS) _antigravityCreditProbeCache.delete(key); + } +}, 5 * 60 * 1000); // every 5 minutes +_usageCacheCleanupTimer.unref?.(); // Don't prevent process exit + interface AntigravityUsageOptions { forceRefresh?: boolean; } diff --git a/scripts/check-permissions.sh b/scripts/check-permissions.sh index 13eedf45b0..88e2dba4bc 100755 --- a/scripts/check-permissions.sh +++ b/scripts/check-permissions.sh @@ -1,6 +1,13 @@ #!/bin/sh set -e +# ── Memory limit override ────────────────────────────────────────────── +# If OMNIROUTE_MEMORY_MB is set, build NODE_OPTIONS dynamically so the +# user can tune heap size via environment without editing the Dockerfile. +if [ -n "$OMNIROUTE_MEMORY_MB" ]; then + export NODE_OPTIONS="${NODE_OPTIONS:-} --max-old-space-size=${OMNIROUTE_MEMORY_MB}" +fi + if [ -d "/app/data" ] && [ ! -w "/app/data" ]; then echo "WARNING: /app/data is not writable by the current user (UID $(id -u))." echo "Run this on the Docker host to fix:" diff --git a/tests/unit/services/combo-metrics-memory.test.ts b/tests/unit/services/combo-metrics-memory.test.ts new file mode 100644 index 0000000000..bc6b2390f9 --- /dev/null +++ b/tests/unit/services/combo-metrics-memory.test.ts @@ -0,0 +1,276 @@ +/** + * Tests for comboMetrics memory management — eviction and TTL behavior. + * + * Verifies that: + * - Recording and retrieval work correctly + * - MAX_METRICS_ENTRIES (500) cap is enforced via eviction of oldest entries + * - Shadow metrics are isolated from production metrics + * - resetComboMetrics and resetAllComboMetrics clear state correctly + * - getAllComboMetrics returns all entries + * - recordComboIntent tracks intent counts + */ +import test from "node:test"; +import assert from "node:assert/strict"; + +const mod = await import("../../../open-sse/services/comboMetrics.ts"); +const { + recordComboRequest, + recordComboShadowRequest, + getComboMetrics, + getAllComboMetrics, + resetComboMetrics, + resetAllComboMetrics, + recordComboIntent, +} = mod; + +// Reset all state before each test group to avoid cross-test pollution. +resetAllComboMetrics(); + +// ─── Basic record + retrieve ──────────────────────────────────────────────── + +test("recordComboRequest: basic recording and retrieval", () => { + resetAllComboMetrics(); + recordComboRequest("test-combo", "gpt-4", { + success: true, + latencyMs: 100, + }); + const metrics = getComboMetrics("test-combo"); + assert.ok(metrics, "metrics should exist for recorded combo"); + assert.equal(metrics.totalRequests, 1, "should have 1 total request"); + assert.equal(metrics.totalSuccesses, 1, "should have 1 success"); + assert.equal(metrics.totalFailures, 0, "should have 0 failures"); + assert.equal(metrics.successRate, 100, "success rate should be 100%"); + assert.equal(metrics.avgLatencyMs, 100, "avg latency should be 100ms"); +}); + +test("recordComboRequest: tracks per-model metrics", () => { + resetAllComboMetrics(); + recordComboRequest("combo-1", "gpt-4", { success: true, latencyMs: 100 }); + recordComboRequest("combo-1", "claude-3", { success: false, latencyMs: 200 }); + const metrics = getComboMetrics("combo-1"); + assert.ok(metrics); + assert.equal(metrics.totalRequests, 2, "should have 2 total requests"); + assert.equal(metrics.successRate, 50, "success rate should be 50%"); + assert.ok(metrics.byModel["gpt-4"], "should have gpt-4 model metrics"); + assert.ok(metrics.byModel["claude-3"], "should have claude-3 model metrics"); + assert.equal(metrics.byModel["gpt-4"].successRate, 100); + assert.equal(metrics.byModel["claude-3"].successRate, 0); +}); + +test("recordComboRequest: handles null model string gracefully", () => { + resetAllComboMetrics(); + recordComboRequest("combo-null", null, { success: true, latencyMs: 50 }); + const metrics = getComboMetrics("combo-null"); + assert.ok(metrics, "metrics should exist even with null model"); + assert.equal(metrics.totalRequests, 1); + // null model should not create per-model entries + assert.equal(Object.keys(metrics.byModel).length, 0, "should have no model entries"); +}); + +test("recordComboRequest: tracks fallback count", () => { + resetAllComboMetrics(); + recordComboRequest("combo-fb", "gpt-4", { + success: true, + latencyMs: 200, + fallbackCount: 2, + }); + const metrics = getComboMetrics("combo-fb"); + assert.ok(metrics); + assert.equal(metrics.totalFallbacks, 2, "should track 2 fallbacks"); + // fallbackRate = (totalFallbacks / totalRequests) * 100 = (2/1) * 100 = 200 + assert.equal(metrics.fallbackRate, 200, "fallback rate should be 200% (2 fallbacks on 1 request)"); +}); + +// ─── getComboMetrics returns null for unknown combos ───────────────────────── + +test("getComboMetrics: returns null for unknown combo", () => { + const metrics = getComboMetrics("nonexistent-combo-xyz"); + assert.equal(metrics, null, "unknown combo should return null"); +}); + +// ─── resetComboMetrics ─────────────────────────────────────────────────────── + +test("resetComboMetrics: clears a specific combo", () => { + resetAllComboMetrics(); + recordComboRequest("to-keep", "gpt-4", { success: true, latencyMs: 100 }); + recordComboRequest("to-reset", "gpt-4", { success: true, latencyMs: 100 }); + resetComboMetrics("to-reset"); + assert.ok(getComboMetrics("to-keep"), "to-keep should still exist"); + assert.equal(getComboMetrics("to-reset"), null, "to-reset should be cleared"); +}); + +// ─── resetAllComboMetrics ──────────────────────────────────────────────────── + +test("resetAllComboMetrics: clears all production and shadow metrics", () => { + recordComboRequest("combo-a", "gpt-4", { success: true, latencyMs: 100 }); + recordComboShadowRequest("shadow-a", "gpt-4", { success: true, latencyMs: 100 }); + assert.ok(getComboMetrics("combo-a"), "combo-a should exist before reset"); + assert.ok(getComboMetrics("shadow-a"), "shadow-a should exist before reset"); + resetAllComboMetrics(); + assert.equal(getComboMetrics("combo-a"), null, "combo-a should be cleared"); + assert.equal(getComboMetrics("shadow-a"), null, "shadow-a should be cleared"); +}); + +// ─── getAllComboMetrics ─────────────────────────────────────────────────────── + +test("getAllComboMetrics: returns all production and shadow entries", () => { + resetAllComboMetrics(); + recordComboRequest("combo-x", "gpt-4", { success: true, latencyMs: 100 }); + recordComboRequest("combo-y", "claude-3", { success: false, latencyMs: 200 }); + recordComboShadowRequest("shadow-x", "gpt-4", { success: true, latencyMs: 150 }); + const all = getAllComboMetrics(); + assert.ok(all["combo-x"], "should include combo-x"); + assert.ok(all["combo-y"], "should include combo-y"); + assert.ok(all["shadow-x"], "should include shadow-x from shadow metrics"); +}); + +// ─── Shadow metrics ────────────────────────────────────────────────────────── + +test("recordComboShadowRequest: tracks shadow metrics separately from production", () => { + resetAllComboMetrics(); + recordComboRequest("prod-combo", "gpt-4", { success: true, latencyMs: 100 }); + recordComboShadowRequest("prod-combo", "claude-3", { success: false, latencyMs: 200 }); + const metrics = getComboMetrics("prod-combo"); + assert.ok(metrics); + // Production sees only gpt-4 + assert.equal(metrics.totalRequests, 1, "production should have 1 request"); + assert.equal(metrics.successRate, 100, "production success rate should be 100%"); + // Shadow is separate + assert.ok(metrics.shadow, "shadow metrics should be present"); + assert.equal(metrics.shadow.totalRequests, 1, "shadow should have 1 request"); + assert.equal(metrics.shadow.successRate, 0, "shadow success rate should be 0%"); +}); + +test("recordComboShadowRequest: shadow-only combo returns in getAllComboMetrics", () => { + resetAllComboMetrics(); + recordComboShadowRequest("shadow-only", "gpt-4", { success: true, latencyMs: 50 }); + const all = getAllComboMetrics(); + assert.ok(all["shadow-only"], "shadow-only combo should appear in getAllComboMetrics"); +}); + +// ─── recordComboIntent ─────────────────────────────────────────────────────── + +test("recordComboIntent: tracks intent counts on existing combo", () => { + resetAllComboMetrics(); + recordComboRequest("intent-combo", "gpt-4", { success: true, latencyMs: 100 }); + recordComboIntent("intent-combo", "chat"); + recordComboIntent("intent-combo", "chat"); + recordComboIntent("intent-combo", "code"); + const metrics = getComboMetrics("intent-combo"); + assert.ok(metrics); + assert.equal(metrics.intentCounts["chat"], 2, "should count 2 chat intents"); + assert.equal(metrics.intentCounts["code"], 1, "should count 1 code intent"); +}); + +test("recordComboIntent: creates combo entry if not yet recorded", () => { + resetAllComboMetrics(); + recordComboIntent("new-intent-combo", "search"); + const metrics = getComboMetrics("new-intent-combo"); + assert.ok(metrics, "combo should be created by recordComboIntent"); + assert.equal(metrics.intentCounts["search"], 1); +}); + +// ─── Eviction: MAX_METRICS_ENTRIES cap ─────────────────────────────────────── + +test("eviction: inserting a new combo at capacity evicts the oldest entry", () => { + resetAllComboMetrics(); + const MAX = 500; + // Fill to capacity with unique combos + for (let i = 0; i < MAX; i++) { + recordComboRequest(`fill-${i}`, "gpt-4", { success: true, latencyMs: 10 }); + } + // All 500 should be present + assert.ok(getComboMetrics("fill-0"), "first inserted combo should exist at capacity"); + assert.ok(getComboMetrics(`fill-${MAX - 1}`), "last inserted combo should exist at capacity"); + + // Insert one more — should trigger eviction of the oldest entry + recordComboRequest("new-after-capacity", "gpt-4", { success: true, latencyMs: 50 }); + const all = getAllComboMetrics(); + const totalProduction = Object.keys(all).filter((k) => { + const m = all[k]; + return m && m.totalRequests > 0; + }).length; + // Map size should not exceed MAX (the new one replaced the oldest) + assert.ok(totalProduction <= MAX, `production combos (${totalProduction}) should not exceed cap (${MAX})`); + assert.ok( + getComboMetrics("new-after-capacity"), + "newly inserted combo should exist after eviction" + ); +}); + +test("eviction: shadow metrics respect their own MAX_METRICS_ENTRIES cap", () => { + resetAllComboMetrics(); + const MAX = 500; + // Fill shadow metrics to capacity + for (let i = 0; i < MAX; i++) { + recordComboShadowRequest(`shadow-fill-${i}`, "gpt-4", { + success: true, + latencyMs: 10, + }); + } + assert.ok( + getComboMetrics("shadow-fill-0"), + "first shadow combo should exist at capacity" + ); + + // Insert one more shadow — should trigger eviction + recordComboShadowRequest("shadow-after-capacity", "gpt-4", { + success: true, + latencyMs: 50, + }); + assert.ok( + getComboMetrics("shadow-after-capacity"), + "newly inserted shadow combo should exist after eviction" + ); +}); + +test("eviction: recordComboIntent respects MAX_METRICS_ENTRIES cap", () => { + resetAllComboMetrics(); + const MAX = 500; + // Fill production metrics to capacity + for (let i = 0; i < MAX; i++) { + recordComboRequest(`intent-fill-${i}`, "gpt-4", { success: true, latencyMs: 10 }); + } + // Adding intent for a NEW combo should trigger eviction + recordComboIntent("intent-after-capacity", "chat"); + assert.ok( + getComboMetrics("intent-after-capacity"), + "newly created intent combo should exist after eviction" + ); +}); + +test("eviction: updating an existing combo at capacity does NOT evict", () => { + resetAllComboMetrics(); + const MAX = 500; + // Fill to capacity + for (let i = 0; i < MAX; i++) { + recordComboRequest(`update-${i}`, "gpt-4", { success: true, latencyMs: 10 }); + } + // Update an existing combo — should NOT trigger eviction since it's not a new key + recordComboRequest("update-0", "gpt-4", { success: false, latencyMs: 50 }); + const metrics = getComboMetrics("update-0"); + assert.ok(metrics, "updated combo should still exist"); + assert.equal(metrics.totalRequests, 2, "updated combo should have 2 requests"); +}); + +// ─── Strategy tracking ─────────────────────────────────────────────────────── + +test("recordComboRequest: stores the routing strategy", () => { + resetAllComboMetrics(); + recordComboRequest("strat-combo", "gpt-4", { + success: true, + latencyMs: 100, + strategy: "weighted", + }); + const metrics = getComboMetrics("strat-combo"); + assert.ok(metrics); + assert.equal(metrics.strategy, "weighted", "should store the strategy"); +}); + +test("recordComboRequest: defaults strategy to 'priority'", () => { + resetAllComboMetrics(); + recordComboRequest("default-strat", "gpt-4", { success: true, latencyMs: 100 }); + const metrics = getComboMetrics("default-strat"); + assert.ok(metrics); + assert.equal(metrics.strategy, "priority", "default strategy should be 'priority'"); +});