Merge pull request #2965 from soyelmismo/fix/oom-memory-leak-caches

fix(oom): prevent unbounded memory growth causing heap OOM crashes
This commit is contained in:
Diego Rodrigues de Sa e Souza
2026-05-31 10:32:29 -03:00
committed by diegosouzapw
parent fa038e069d
commit a6de2a58c5
6 changed files with 385 additions and 20 deletions

View File

@@ -57,7 +57,8 @@ LABEL org.opencontainers.image.title="omniroute" \
ENV NODE_ENV=production
ENV PORT=20128
ENV HOSTNAME=0.0.0.0
ENV NODE_OPTIONS="--max-old-space-size=256"
ENV OMNIROUTE_MEMORY_MB=1024
ENV NODE_OPTIONS="--max-old-space-size=${OMNIROUTE_MEMORY_MB}"
# Data directory inside Docker — must match the volume mount in docker-compose.yml
ENV DATA_DIR=/app/data

View File

@@ -589,7 +589,7 @@ function mapStainlessArch() {
// ── Registry ──────────────────────────────────────────────────────────────
export const REGISTRY: Record<string, RegistryEntry> = {
const _REGISTRY_EAGER: Record<string, RegistryEntry> = {
// ─── OAuth Providers ───────────────────────────────────────────────────
kie: {
id: "kie",
@@ -4292,12 +4292,14 @@ export const REGISTRY: Record<string, RegistryEntry> = {
},
};
export const REGISTRY: Record<string, RegistryEntry> = _REGISTRY_EAGER;
// ── Generator Functions ───────────────────────────────────────────────────
/** Generate legacy PROVIDERS object shape for constants.js backward compatibility */
export function generateLegacyProviders(): Record<string, LegacyProvider> {
const providers: Record<string, LegacyProvider> = {};
for (const [id, entry] of Object.entries(REGISTRY)) {
for (const [id, entry] of Object.entries(_REGISTRY_EAGER)) {
const p: LegacyProvider = { format: entry.format };
// URL(s)
@@ -4351,7 +4353,7 @@ export function generateLegacyProviders(): Record<string, LegacyProvider> {
/** Generate PROVIDER_MODELS map (alias → model list) */
export function generateModels(): Record<string, RegistryModel[]> {
const models: Record<string, RegistryModel[]> = {};
for (const entry of Object.values(REGISTRY)) {
for (const entry of Object.values(_REGISTRY_EAGER)) {
if (entry.models && entry.models.length > 0) {
const key = entry.alias || entry.id;
// If alias already exists, don't overwrite (first wins)
@@ -4366,7 +4368,7 @@ export function generateModels(): Record<string, RegistryModel[]> {
/** Generate PROVIDER_ID_TO_ALIAS map */
export function generateAliasMap(): Record<string, string> {
const map: Record<string, string> = {};
for (const entry of Object.values(REGISTRY)) {
for (const entry of Object.values(_REGISTRY_EAGER)) {
map[entry.id] = entry.alias || entry.id;
}
return map;
@@ -4408,33 +4410,40 @@ export function isLocalProvider(baseUrl?: string | null): boolean {
}
/** Set of provider IDs with passthroughModels enabled — 404s are model-specific, not account-level. */
const _passthroughProviderIds: Set<string> | null = (() => {
let _passthroughProviderIds: Set<string> | null = null;
function ensurePassthroughProviderIds(): Set<string> {
if (_passthroughProviderIds) return _passthroughProviderIds;
try {
const ids = new Set<string>();
for (const entry of Object.values(REGISTRY)) {
for (const entry of Object.values(_REGISTRY_EAGER)) {
if (entry.passthroughModels) ids.add(entry.id);
}
return ids;
_passthroughProviderIds = ids;
} catch {
return null;
_passthroughProviderIds = new Set<string>();
}
})();
return _passthroughProviderIds;
}
export function getPassthroughProviders(): Set<string> {
return _passthroughProviderIds ?? new Set<string>();
return ensurePassthroughProviderIds();
}
// ── Registry Lookup Helpers ───────────────────────────────────────────────
const _byAlias = new Map<string, RegistryEntry>();
for (const entry of Object.values(REGISTRY)) {
if (entry.alias && entry.alias !== entry.id) {
_byAlias.set(entry.alias, entry);
let _byAliasPopulated = false;
function ensureByAliasPopulated(): void {
if (_byAliasPopulated) return;
_byAliasPopulated = true;
for (const entry of Object.values(_REGISTRY_EAGER)) {
if (entry.alias && entry.alias !== entry.id) {
_byAlias.set(entry.alias, entry);
}
}
}
/** Get registry entry by provider ID or alias */
export function getRegistryEntry(provider: string): RegistryEntry | null {
ensureByAliasPopulated();
return REGISTRY[provider] || _byAlias.get(provider) || null;
}
@@ -4446,10 +4455,15 @@ export function getRegisteredProviders(): string[] {
// Precomputed map: modelId → unsupportedParams (O(1) lookup instead of O(N×M) scan).
// Built once at module load from all registry entries.
const _unsupportedParamsMap = new Map<string, readonly string[]>();
for (const entry of Object.values(REGISTRY)) {
for (const model of entry.models) {
if (model.unsupportedParams && !_unsupportedParamsMap.has(model.id)) {
_unsupportedParamsMap.set(model.id, model.unsupportedParams);
let _unsupportedParamsPopulated = false;
function ensureUnsupportedParamsPopulated(): void {
if (_unsupportedParamsPopulated) return;
_unsupportedParamsPopulated = true;
for (const entry of Object.values(_REGISTRY_EAGER)) {
for (const model of entry.models) {
if (model.unsupportedParams && !_unsupportedParamsMap.has(model.id)) {
_unsupportedParamsMap.set(model.id, model.unsupportedParams);
}
}
}
}
@@ -4461,6 +4475,7 @@ for (const entry of Object.values(REGISTRY)) {
* Returns empty array if no restrictions are defined.
*/
export function getUnsupportedParams(provider: string, modelId: string): readonly string[] {
ensureUnsupportedParamsPopulated();
// 1. Check current provider's registry (exact match)
const entry = getRegistryEntry(provider);
const modelEntry = entry?.models.find((m) => m.id === modelId);

View File

@@ -188,6 +188,40 @@ function toMetricView<T extends ModelMetrics>(
// In-memory store
const metrics = new Map<string, ComboMetricsEntry>();
const shadowMetrics = new Map<string, ComboShadowMetricsEntry>();
const MAX_METRICS_ENTRIES = 500;
const METRICS_TTL_MS = 60 * 60 * 1000; // 1 hour
function evictOldestMetric(targetMap: Map<string, { lastUsedAt: string | null }>): void {
let oldest: string | null = null;
let oldestTime = Infinity;
for (const [name, entry] of targetMap) {
const t = entry.lastUsedAt ? new Date(entry.lastUsedAt).getTime() : Date.now();
if (t < oldestTime) { oldestTime = t; oldest = name; }
}
if (oldest) {
metrics.delete(oldest);
shadowMetrics.delete(oldest);
}
}
const _metricsCleanupTimer = setInterval(() => {
const now = Date.now();
for (const [name, entry] of metrics) {
const lastUsed = entry.lastUsedAt ? new Date(entry.lastUsedAt).getTime() : now;
if (now - lastUsed > METRICS_TTL_MS) {
metrics.delete(name);
shadowMetrics.delete(name);
}
}
for (const [name, entry] of shadowMetrics) {
const lastUsed = entry.lastUsedAt ? new Date(entry.lastUsedAt).getTime() : now;
if (now - lastUsed > METRICS_TTL_MS) {
metrics.delete(name);
shadowMetrics.delete(name);
}
}
}, 5 * 60 * 1000); // every 5 minutes
_metricsCleanupTimer.unref?.(); // Don't prevent process exit
/**
* Record a combo request result.
@@ -217,6 +251,9 @@ export function recordComboRequest(
target?: ComboRequestTargetMeta | null;
}
): void {
if (!metrics.has(comboName) && metrics.size >= MAX_METRICS_ENTRIES) {
evictOldestMetric(metrics);
}
if (!metrics.has(comboName)) {
metrics.set(comboName, createComboEntry(strategy));
}
@@ -283,6 +320,9 @@ export function recordComboShadowRequest(
target?: ComboRequestTargetMeta | null;
}
): void {
if (!shadowMetrics.has(comboName) && shadowMetrics.size >= MAX_METRICS_ENTRIES) {
evictOldestMetric(shadowMetrics);
}
if (!shadowMetrics.has(comboName)) {
shadowMetrics.set(comboName, createShadowEntry());
}
@@ -396,6 +436,9 @@ export function getAllComboMetrics(): Record<string, ComboMetricsView | null> {
* Record detected prompt intent for a combo (used by multilingual routing analytics).
*/
export function recordComboIntent(comboName: string, intent: string): void {
if (!metrics.has(comboName) && metrics.size >= MAX_METRICS_ENTRIES) {
evictOldestMetric(metrics);
}
if (!metrics.has(comboName)) {
metrics.set(comboName, createComboEntry("priority"));
}
@@ -418,6 +461,7 @@ export function resetComboMetrics(comboName: string): void {
* Reset all combo metrics.
*/
export function resetAllComboMetrics(): void {
clearInterval(_metricsCleanupTimer);
metrics.clear();
shadowMetrics.clear();
}

View File

@@ -1821,6 +1821,28 @@ const _antigravityAvailableModelsInflight = new Map<string, Promise<unknown>>();
const _antigravityCreditProbeCache = new Map<string, { data: number | null; fetchedAt: number }>();
const _antigravityCreditProbeInflight = new Map<string, Promise<number | null>>();
// ── Proactive TTL purging for module-level caches ──────────────────────────
// All 4 data caches only evict on read (passive TTL). This interval proactively
// purges stale entries so keys accessed once and never again don't leak memory.
// The 2 inflight Maps (availableModelsInflight, creditProbeInflight) self-clean
// when the Promise resolves/rejects, so they are NOT touched here.
const _usageCacheCleanupTimer = setInterval(() => {
const now = Date.now();
for (const [key, entry] of _geminiCliSubCache) {
if (now - entry.fetchedAt > GEMINI_CLI_CACHE_TTL_MS) _geminiCliSubCache.delete(key);
}
for (const [key, entry] of _antigravitySubCache) {
if (now - entry.fetchedAt > ANTIGRAVITY_CACHE_TTL_MS) _antigravitySubCache.delete(key);
}
for (const [key, entry] of _antigravityAvailableModelsCache) {
if (now - entry.fetchedAt > ANTIGRAVITY_MODELS_CACHE_TTL_MS) _antigravityAvailableModelsCache.delete(key);
}
for (const [key, entry] of _antigravityCreditProbeCache) {
if (now - entry.fetchedAt > ANTIGRAVITY_CREDIT_PROBE_TTL_MS) _antigravityCreditProbeCache.delete(key);
}
}, 5 * 60 * 1000); // every 5 minutes
_usageCacheCleanupTimer.unref?.(); // Don't prevent process exit
interface AntigravityUsageOptions {
forceRefresh?: boolean;
}

View File

@@ -1,6 +1,13 @@
#!/bin/sh
set -e
# ── Memory limit override ──────────────────────────────────────────────
# If OMNIROUTE_MEMORY_MB is set, build NODE_OPTIONS dynamically so the
# user can tune heap size via environment without editing the Dockerfile.
if [ -n "$OMNIROUTE_MEMORY_MB" ]; then
export NODE_OPTIONS="${NODE_OPTIONS:-} --max-old-space-size=${OMNIROUTE_MEMORY_MB}"
fi
if [ -d "/app/data" ] && [ ! -w "/app/data" ]; then
echo "WARNING: /app/data is not writable by the current user (UID $(id -u))."
echo "Run this on the Docker host to fix:"

View File

@@ -0,0 +1,276 @@
/**
* Tests for comboMetrics memory management — eviction and TTL behavior.
*
* Verifies that:
* - Recording and retrieval work correctly
* - MAX_METRICS_ENTRIES (500) cap is enforced via eviction of oldest entries
* - Shadow metrics are isolated from production metrics
* - resetComboMetrics and resetAllComboMetrics clear state correctly
* - getAllComboMetrics returns all entries
* - recordComboIntent tracks intent counts
*/
import test from "node:test";
import assert from "node:assert/strict";
const mod = await import("../../../open-sse/services/comboMetrics.ts");
const {
recordComboRequest,
recordComboShadowRequest,
getComboMetrics,
getAllComboMetrics,
resetComboMetrics,
resetAllComboMetrics,
recordComboIntent,
} = mod;
// Reset all state before each test group to avoid cross-test pollution.
resetAllComboMetrics();
// ─── Basic record + retrieve ────────────────────────────────────────────────
test("recordComboRequest: basic recording and retrieval", () => {
resetAllComboMetrics();
recordComboRequest("test-combo", "gpt-4", {
success: true,
latencyMs: 100,
});
const metrics = getComboMetrics("test-combo");
assert.ok(metrics, "metrics should exist for recorded combo");
assert.equal(metrics.totalRequests, 1, "should have 1 total request");
assert.equal(metrics.totalSuccesses, 1, "should have 1 success");
assert.equal(metrics.totalFailures, 0, "should have 0 failures");
assert.equal(metrics.successRate, 100, "success rate should be 100%");
assert.equal(metrics.avgLatencyMs, 100, "avg latency should be 100ms");
});
test("recordComboRequest: tracks per-model metrics", () => {
resetAllComboMetrics();
recordComboRequest("combo-1", "gpt-4", { success: true, latencyMs: 100 });
recordComboRequest("combo-1", "claude-3", { success: false, latencyMs: 200 });
const metrics = getComboMetrics("combo-1");
assert.ok(metrics);
assert.equal(metrics.totalRequests, 2, "should have 2 total requests");
assert.equal(metrics.successRate, 50, "success rate should be 50%");
assert.ok(metrics.byModel["gpt-4"], "should have gpt-4 model metrics");
assert.ok(metrics.byModel["claude-3"], "should have claude-3 model metrics");
assert.equal(metrics.byModel["gpt-4"].successRate, 100);
assert.equal(metrics.byModel["claude-3"].successRate, 0);
});
test("recordComboRequest: handles null model string gracefully", () => {
resetAllComboMetrics();
recordComboRequest("combo-null", null, { success: true, latencyMs: 50 });
const metrics = getComboMetrics("combo-null");
assert.ok(metrics, "metrics should exist even with null model");
assert.equal(metrics.totalRequests, 1);
// null model should not create per-model entries
assert.equal(Object.keys(metrics.byModel).length, 0, "should have no model entries");
});
test("recordComboRequest: tracks fallback count", () => {
resetAllComboMetrics();
recordComboRequest("combo-fb", "gpt-4", {
success: true,
latencyMs: 200,
fallbackCount: 2,
});
const metrics = getComboMetrics("combo-fb");
assert.ok(metrics);
assert.equal(metrics.totalFallbacks, 2, "should track 2 fallbacks");
// fallbackRate = (totalFallbacks / totalRequests) * 100 = (2/1) * 100 = 200
assert.equal(metrics.fallbackRate, 200, "fallback rate should be 200% (2 fallbacks on 1 request)");
});
// ─── getComboMetrics returns null for unknown combos ─────────────────────────
test("getComboMetrics: returns null for unknown combo", () => {
const metrics = getComboMetrics("nonexistent-combo-xyz");
assert.equal(metrics, null, "unknown combo should return null");
});
// ─── resetComboMetrics ───────────────────────────────────────────────────────
test("resetComboMetrics: clears a specific combo", () => {
resetAllComboMetrics();
recordComboRequest("to-keep", "gpt-4", { success: true, latencyMs: 100 });
recordComboRequest("to-reset", "gpt-4", { success: true, latencyMs: 100 });
resetComboMetrics("to-reset");
assert.ok(getComboMetrics("to-keep"), "to-keep should still exist");
assert.equal(getComboMetrics("to-reset"), null, "to-reset should be cleared");
});
// ─── resetAllComboMetrics ────────────────────────────────────────────────────
test("resetAllComboMetrics: clears all production and shadow metrics", () => {
recordComboRequest("combo-a", "gpt-4", { success: true, latencyMs: 100 });
recordComboShadowRequest("shadow-a", "gpt-4", { success: true, latencyMs: 100 });
assert.ok(getComboMetrics("combo-a"), "combo-a should exist before reset");
assert.ok(getComboMetrics("shadow-a"), "shadow-a should exist before reset");
resetAllComboMetrics();
assert.equal(getComboMetrics("combo-a"), null, "combo-a should be cleared");
assert.equal(getComboMetrics("shadow-a"), null, "shadow-a should be cleared");
});
// ─── getAllComboMetrics ───────────────────────────────────────────────────────
test("getAllComboMetrics: returns all production and shadow entries", () => {
resetAllComboMetrics();
recordComboRequest("combo-x", "gpt-4", { success: true, latencyMs: 100 });
recordComboRequest("combo-y", "claude-3", { success: false, latencyMs: 200 });
recordComboShadowRequest("shadow-x", "gpt-4", { success: true, latencyMs: 150 });
const all = getAllComboMetrics();
assert.ok(all["combo-x"], "should include combo-x");
assert.ok(all["combo-y"], "should include combo-y");
assert.ok(all["shadow-x"], "should include shadow-x from shadow metrics");
});
// ─── Shadow metrics ──────────────────────────────────────────────────────────
test("recordComboShadowRequest: tracks shadow metrics separately from production", () => {
resetAllComboMetrics();
recordComboRequest("prod-combo", "gpt-4", { success: true, latencyMs: 100 });
recordComboShadowRequest("prod-combo", "claude-3", { success: false, latencyMs: 200 });
const metrics = getComboMetrics("prod-combo");
assert.ok(metrics);
// Production sees only gpt-4
assert.equal(metrics.totalRequests, 1, "production should have 1 request");
assert.equal(metrics.successRate, 100, "production success rate should be 100%");
// Shadow is separate
assert.ok(metrics.shadow, "shadow metrics should be present");
assert.equal(metrics.shadow.totalRequests, 1, "shadow should have 1 request");
assert.equal(metrics.shadow.successRate, 0, "shadow success rate should be 0%");
});
test("recordComboShadowRequest: shadow-only combo returns in getAllComboMetrics", () => {
resetAllComboMetrics();
recordComboShadowRequest("shadow-only", "gpt-4", { success: true, latencyMs: 50 });
const all = getAllComboMetrics();
assert.ok(all["shadow-only"], "shadow-only combo should appear in getAllComboMetrics");
});
// ─── recordComboIntent ───────────────────────────────────────────────────────
test("recordComboIntent: tracks intent counts on existing combo", () => {
resetAllComboMetrics();
recordComboRequest("intent-combo", "gpt-4", { success: true, latencyMs: 100 });
recordComboIntent("intent-combo", "chat");
recordComboIntent("intent-combo", "chat");
recordComboIntent("intent-combo", "code");
const metrics = getComboMetrics("intent-combo");
assert.ok(metrics);
assert.equal(metrics.intentCounts["chat"], 2, "should count 2 chat intents");
assert.equal(metrics.intentCounts["code"], 1, "should count 1 code intent");
});
test("recordComboIntent: creates combo entry if not yet recorded", () => {
resetAllComboMetrics();
recordComboIntent("new-intent-combo", "search");
const metrics = getComboMetrics("new-intent-combo");
assert.ok(metrics, "combo should be created by recordComboIntent");
assert.equal(metrics.intentCounts["search"], 1);
});
// ─── Eviction: MAX_METRICS_ENTRIES cap ───────────────────────────────────────
test("eviction: inserting a new combo at capacity evicts the oldest entry", () => {
resetAllComboMetrics();
const MAX = 500;
// Fill to capacity with unique combos
for (let i = 0; i < MAX; i++) {
recordComboRequest(`fill-${i}`, "gpt-4", { success: true, latencyMs: 10 });
}
// All 500 should be present
assert.ok(getComboMetrics("fill-0"), "first inserted combo should exist at capacity");
assert.ok(getComboMetrics(`fill-${MAX - 1}`), "last inserted combo should exist at capacity");
// Insert one more — should trigger eviction of the oldest entry
recordComboRequest("new-after-capacity", "gpt-4", { success: true, latencyMs: 50 });
const all = getAllComboMetrics();
const totalProduction = Object.keys(all).filter((k) => {
const m = all[k];
return m && m.totalRequests > 0;
}).length;
// Map size should not exceed MAX (the new one replaced the oldest)
assert.ok(totalProduction <= MAX, `production combos (${totalProduction}) should not exceed cap (${MAX})`);
assert.ok(
getComboMetrics("new-after-capacity"),
"newly inserted combo should exist after eviction"
);
});
test("eviction: shadow metrics respect their own MAX_METRICS_ENTRIES cap", () => {
resetAllComboMetrics();
const MAX = 500;
// Fill shadow metrics to capacity
for (let i = 0; i < MAX; i++) {
recordComboShadowRequest(`shadow-fill-${i}`, "gpt-4", {
success: true,
latencyMs: 10,
});
}
assert.ok(
getComboMetrics("shadow-fill-0"),
"first shadow combo should exist at capacity"
);
// Insert one more shadow — should trigger eviction
recordComboShadowRequest("shadow-after-capacity", "gpt-4", {
success: true,
latencyMs: 50,
});
assert.ok(
getComboMetrics("shadow-after-capacity"),
"newly inserted shadow combo should exist after eviction"
);
});
test("eviction: recordComboIntent respects MAX_METRICS_ENTRIES cap", () => {
resetAllComboMetrics();
const MAX = 500;
// Fill production metrics to capacity
for (let i = 0; i < MAX; i++) {
recordComboRequest(`intent-fill-${i}`, "gpt-4", { success: true, latencyMs: 10 });
}
// Adding intent for a NEW combo should trigger eviction
recordComboIntent("intent-after-capacity", "chat");
assert.ok(
getComboMetrics("intent-after-capacity"),
"newly created intent combo should exist after eviction"
);
});
test("eviction: updating an existing combo at capacity does NOT evict", () => {
resetAllComboMetrics();
const MAX = 500;
// Fill to capacity
for (let i = 0; i < MAX; i++) {
recordComboRequest(`update-${i}`, "gpt-4", { success: true, latencyMs: 10 });
}
// Update an existing combo — should NOT trigger eviction since it's not a new key
recordComboRequest("update-0", "gpt-4", { success: false, latencyMs: 50 });
const metrics = getComboMetrics("update-0");
assert.ok(metrics, "updated combo should still exist");
assert.equal(metrics.totalRequests, 2, "updated combo should have 2 requests");
});
// ─── Strategy tracking ───────────────────────────────────────────────────────
test("recordComboRequest: stores the routing strategy", () => {
resetAllComboMetrics();
recordComboRequest("strat-combo", "gpt-4", {
success: true,
latencyMs: 100,
strategy: "weighted",
});
const metrics = getComboMetrics("strat-combo");
assert.ok(metrics);
assert.equal(metrics.strategy, "weighted", "should store the strategy");
});
test("recordComboRequest: defaults strategy to 'priority'", () => {
resetAllComboMetrics();
recordComboRequest("default-strat", "gpt-4", { success: true, latencyMs: 100 });
const metrics = getComboMetrics("default-strat");
assert.ok(metrics);
assert.equal(metrics.strategy, "priority", "default strategy should be 'priority'");
});