mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-07-26 09:52:11 +03:00
Merge pull request #2965 from soyelmismo/fix/oom-memory-leak-caches
fix(oom): prevent unbounded memory growth causing heap OOM crashes
This commit is contained in:
committed by
diegosouzapw
parent
fa038e069d
commit
a6de2a58c5
@@ -57,7 +57,8 @@ LABEL org.opencontainers.image.title="omniroute" \
|
||||
ENV NODE_ENV=production
|
||||
ENV PORT=20128
|
||||
ENV HOSTNAME=0.0.0.0
|
||||
ENV NODE_OPTIONS="--max-old-space-size=256"
|
||||
ENV OMNIROUTE_MEMORY_MB=1024
|
||||
ENV NODE_OPTIONS="--max-old-space-size=${OMNIROUTE_MEMORY_MB}"
|
||||
|
||||
# Data directory inside Docker — must match the volume mount in docker-compose.yml
|
||||
ENV DATA_DIR=/app/data
|
||||
|
||||
@@ -589,7 +589,7 @@ function mapStainlessArch() {
|
||||
|
||||
// ── Registry ──────────────────────────────────────────────────────────────
|
||||
|
||||
export const REGISTRY: Record<string, RegistryEntry> = {
|
||||
const _REGISTRY_EAGER: Record<string, RegistryEntry> = {
|
||||
// ─── OAuth Providers ───────────────────────────────────────────────────
|
||||
kie: {
|
||||
id: "kie",
|
||||
@@ -4292,12 +4292,14 @@ export const REGISTRY: Record<string, RegistryEntry> = {
|
||||
},
|
||||
};
|
||||
|
||||
export const REGISTRY: Record<string, RegistryEntry> = _REGISTRY_EAGER;
|
||||
|
||||
// ── Generator Functions ───────────────────────────────────────────────────
|
||||
|
||||
/** Generate legacy PROVIDERS object shape for constants.js backward compatibility */
|
||||
export function generateLegacyProviders(): Record<string, LegacyProvider> {
|
||||
const providers: Record<string, LegacyProvider> = {};
|
||||
for (const [id, entry] of Object.entries(REGISTRY)) {
|
||||
for (const [id, entry] of Object.entries(_REGISTRY_EAGER)) {
|
||||
const p: LegacyProvider = { format: entry.format };
|
||||
|
||||
// URL(s)
|
||||
@@ -4351,7 +4353,7 @@ export function generateLegacyProviders(): Record<string, LegacyProvider> {
|
||||
/** Generate PROVIDER_MODELS map (alias → model list) */
|
||||
export function generateModels(): Record<string, RegistryModel[]> {
|
||||
const models: Record<string, RegistryModel[]> = {};
|
||||
for (const entry of Object.values(REGISTRY)) {
|
||||
for (const entry of Object.values(_REGISTRY_EAGER)) {
|
||||
if (entry.models && entry.models.length > 0) {
|
||||
const key = entry.alias || entry.id;
|
||||
// If alias already exists, don't overwrite (first wins)
|
||||
@@ -4366,7 +4368,7 @@ export function generateModels(): Record<string, RegistryModel[]> {
|
||||
/** Generate PROVIDER_ID_TO_ALIAS map */
|
||||
export function generateAliasMap(): Record<string, string> {
|
||||
const map: Record<string, string> = {};
|
||||
for (const entry of Object.values(REGISTRY)) {
|
||||
for (const entry of Object.values(_REGISTRY_EAGER)) {
|
||||
map[entry.id] = entry.alias || entry.id;
|
||||
}
|
||||
return map;
|
||||
@@ -4408,33 +4410,40 @@ export function isLocalProvider(baseUrl?: string | null): boolean {
|
||||
}
|
||||
|
||||
/** Set of provider IDs with passthroughModels enabled — 404s are model-specific, not account-level. */
|
||||
const _passthroughProviderIds: Set<string> | null = (() => {
|
||||
let _passthroughProviderIds: Set<string> | null = null;
|
||||
function ensurePassthroughProviderIds(): Set<string> {
|
||||
if (_passthroughProviderIds) return _passthroughProviderIds;
|
||||
try {
|
||||
const ids = new Set<string>();
|
||||
for (const entry of Object.values(REGISTRY)) {
|
||||
for (const entry of Object.values(_REGISTRY_EAGER)) {
|
||||
if (entry.passthroughModels) ids.add(entry.id);
|
||||
}
|
||||
return ids;
|
||||
_passthroughProviderIds = ids;
|
||||
} catch {
|
||||
return null;
|
||||
_passthroughProviderIds = new Set<string>();
|
||||
}
|
||||
})();
|
||||
|
||||
return _passthroughProviderIds;
|
||||
}
|
||||
export function getPassthroughProviders(): Set<string> {
|
||||
return _passthroughProviderIds ?? new Set<string>();
|
||||
return ensurePassthroughProviderIds();
|
||||
}
|
||||
|
||||
// ── Registry Lookup Helpers ───────────────────────────────────────────────
|
||||
|
||||
const _byAlias = new Map<string, RegistryEntry>();
|
||||
for (const entry of Object.values(REGISTRY)) {
|
||||
if (entry.alias && entry.alias !== entry.id) {
|
||||
_byAlias.set(entry.alias, entry);
|
||||
let _byAliasPopulated = false;
|
||||
function ensureByAliasPopulated(): void {
|
||||
if (_byAliasPopulated) return;
|
||||
_byAliasPopulated = true;
|
||||
for (const entry of Object.values(_REGISTRY_EAGER)) {
|
||||
if (entry.alias && entry.alias !== entry.id) {
|
||||
_byAlias.set(entry.alias, entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Get registry entry by provider ID or alias */
|
||||
export function getRegistryEntry(provider: string): RegistryEntry | null {
|
||||
ensureByAliasPopulated();
|
||||
return REGISTRY[provider] || _byAlias.get(provider) || null;
|
||||
}
|
||||
|
||||
@@ -4446,10 +4455,15 @@ export function getRegisteredProviders(): string[] {
|
||||
// Precomputed map: modelId → unsupportedParams (O(1) lookup instead of O(N×M) scan).
|
||||
// Built once at module load from all registry entries.
|
||||
const _unsupportedParamsMap = new Map<string, readonly string[]>();
|
||||
for (const entry of Object.values(REGISTRY)) {
|
||||
for (const model of entry.models) {
|
||||
if (model.unsupportedParams && !_unsupportedParamsMap.has(model.id)) {
|
||||
_unsupportedParamsMap.set(model.id, model.unsupportedParams);
|
||||
let _unsupportedParamsPopulated = false;
|
||||
function ensureUnsupportedParamsPopulated(): void {
|
||||
if (_unsupportedParamsPopulated) return;
|
||||
_unsupportedParamsPopulated = true;
|
||||
for (const entry of Object.values(_REGISTRY_EAGER)) {
|
||||
for (const model of entry.models) {
|
||||
if (model.unsupportedParams && !_unsupportedParamsMap.has(model.id)) {
|
||||
_unsupportedParamsMap.set(model.id, model.unsupportedParams);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4461,6 +4475,7 @@ for (const entry of Object.values(REGISTRY)) {
|
||||
* Returns empty array if no restrictions are defined.
|
||||
*/
|
||||
export function getUnsupportedParams(provider: string, modelId: string): readonly string[] {
|
||||
ensureUnsupportedParamsPopulated();
|
||||
// 1. Check current provider's registry (exact match)
|
||||
const entry = getRegistryEntry(provider);
|
||||
const modelEntry = entry?.models.find((m) => m.id === modelId);
|
||||
|
||||
@@ -188,6 +188,40 @@ function toMetricView<T extends ModelMetrics>(
|
||||
// In-memory store
|
||||
const metrics = new Map<string, ComboMetricsEntry>();
|
||||
const shadowMetrics = new Map<string, ComboShadowMetricsEntry>();
|
||||
const MAX_METRICS_ENTRIES = 500;
|
||||
const METRICS_TTL_MS = 60 * 60 * 1000; // 1 hour
|
||||
|
||||
function evictOldestMetric(targetMap: Map<string, { lastUsedAt: string | null }>): void {
|
||||
let oldest: string | null = null;
|
||||
let oldestTime = Infinity;
|
||||
for (const [name, entry] of targetMap) {
|
||||
const t = entry.lastUsedAt ? new Date(entry.lastUsedAt).getTime() : Date.now();
|
||||
if (t < oldestTime) { oldestTime = t; oldest = name; }
|
||||
}
|
||||
if (oldest) {
|
||||
metrics.delete(oldest);
|
||||
shadowMetrics.delete(oldest);
|
||||
}
|
||||
}
|
||||
|
||||
const _metricsCleanupTimer = setInterval(() => {
|
||||
const now = Date.now();
|
||||
for (const [name, entry] of metrics) {
|
||||
const lastUsed = entry.lastUsedAt ? new Date(entry.lastUsedAt).getTime() : now;
|
||||
if (now - lastUsed > METRICS_TTL_MS) {
|
||||
metrics.delete(name);
|
||||
shadowMetrics.delete(name);
|
||||
}
|
||||
}
|
||||
for (const [name, entry] of shadowMetrics) {
|
||||
const lastUsed = entry.lastUsedAt ? new Date(entry.lastUsedAt).getTime() : now;
|
||||
if (now - lastUsed > METRICS_TTL_MS) {
|
||||
metrics.delete(name);
|
||||
shadowMetrics.delete(name);
|
||||
}
|
||||
}
|
||||
}, 5 * 60 * 1000); // every 5 minutes
|
||||
_metricsCleanupTimer.unref?.(); // Don't prevent process exit
|
||||
|
||||
/**
|
||||
* Record a combo request result.
|
||||
@@ -217,6 +251,9 @@ export function recordComboRequest(
|
||||
target?: ComboRequestTargetMeta | null;
|
||||
}
|
||||
): void {
|
||||
if (!metrics.has(comboName) && metrics.size >= MAX_METRICS_ENTRIES) {
|
||||
evictOldestMetric(metrics);
|
||||
}
|
||||
if (!metrics.has(comboName)) {
|
||||
metrics.set(comboName, createComboEntry(strategy));
|
||||
}
|
||||
@@ -283,6 +320,9 @@ export function recordComboShadowRequest(
|
||||
target?: ComboRequestTargetMeta | null;
|
||||
}
|
||||
): void {
|
||||
if (!shadowMetrics.has(comboName) && shadowMetrics.size >= MAX_METRICS_ENTRIES) {
|
||||
evictOldestMetric(shadowMetrics);
|
||||
}
|
||||
if (!shadowMetrics.has(comboName)) {
|
||||
shadowMetrics.set(comboName, createShadowEntry());
|
||||
}
|
||||
@@ -396,6 +436,9 @@ export function getAllComboMetrics(): Record<string, ComboMetricsView | null> {
|
||||
* Record detected prompt intent for a combo (used by multilingual routing analytics).
|
||||
*/
|
||||
export function recordComboIntent(comboName: string, intent: string): void {
|
||||
if (!metrics.has(comboName) && metrics.size >= MAX_METRICS_ENTRIES) {
|
||||
evictOldestMetric(metrics);
|
||||
}
|
||||
if (!metrics.has(comboName)) {
|
||||
metrics.set(comboName, createComboEntry("priority"));
|
||||
}
|
||||
@@ -418,6 +461,7 @@ export function resetComboMetrics(comboName: string): void {
|
||||
* Reset all combo metrics.
|
||||
*/
|
||||
export function resetAllComboMetrics(): void {
|
||||
clearInterval(_metricsCleanupTimer);
|
||||
metrics.clear();
|
||||
shadowMetrics.clear();
|
||||
}
|
||||
|
||||
@@ -1821,6 +1821,28 @@ const _antigravityAvailableModelsInflight = new Map<string, Promise<unknown>>();
|
||||
const _antigravityCreditProbeCache = new Map<string, { data: number | null; fetchedAt: number }>();
|
||||
const _antigravityCreditProbeInflight = new Map<string, Promise<number | null>>();
|
||||
|
||||
// ── Proactive TTL purging for module-level caches ──────────────────────────
|
||||
// All 4 data caches only evict on read (passive TTL). This interval proactively
|
||||
// purges stale entries so keys accessed once and never again don't leak memory.
|
||||
// The 2 inflight Maps (availableModelsInflight, creditProbeInflight) self-clean
|
||||
// when the Promise resolves/rejects, so they are NOT touched here.
|
||||
const _usageCacheCleanupTimer = setInterval(() => {
|
||||
const now = Date.now();
|
||||
for (const [key, entry] of _geminiCliSubCache) {
|
||||
if (now - entry.fetchedAt > GEMINI_CLI_CACHE_TTL_MS) _geminiCliSubCache.delete(key);
|
||||
}
|
||||
for (const [key, entry] of _antigravitySubCache) {
|
||||
if (now - entry.fetchedAt > ANTIGRAVITY_CACHE_TTL_MS) _antigravitySubCache.delete(key);
|
||||
}
|
||||
for (const [key, entry] of _antigravityAvailableModelsCache) {
|
||||
if (now - entry.fetchedAt > ANTIGRAVITY_MODELS_CACHE_TTL_MS) _antigravityAvailableModelsCache.delete(key);
|
||||
}
|
||||
for (const [key, entry] of _antigravityCreditProbeCache) {
|
||||
if (now - entry.fetchedAt > ANTIGRAVITY_CREDIT_PROBE_TTL_MS) _antigravityCreditProbeCache.delete(key);
|
||||
}
|
||||
}, 5 * 60 * 1000); // every 5 minutes
|
||||
_usageCacheCleanupTimer.unref?.(); // Don't prevent process exit
|
||||
|
||||
interface AntigravityUsageOptions {
|
||||
forceRefresh?: boolean;
|
||||
}
|
||||
|
||||
@@ -1,6 +1,13 @@
|
||||
#!/bin/sh
|
||||
set -e
|
||||
|
||||
# ── Memory limit override ──────────────────────────────────────────────
|
||||
# If OMNIROUTE_MEMORY_MB is set, build NODE_OPTIONS dynamically so the
|
||||
# user can tune heap size via environment without editing the Dockerfile.
|
||||
if [ -n "$OMNIROUTE_MEMORY_MB" ]; then
|
||||
export NODE_OPTIONS="${NODE_OPTIONS:-} --max-old-space-size=${OMNIROUTE_MEMORY_MB}"
|
||||
fi
|
||||
|
||||
if [ -d "/app/data" ] && [ ! -w "/app/data" ]; then
|
||||
echo "WARNING: /app/data is not writable by the current user (UID $(id -u))."
|
||||
echo "Run this on the Docker host to fix:"
|
||||
|
||||
276
tests/unit/services/combo-metrics-memory.test.ts
Normal file
276
tests/unit/services/combo-metrics-memory.test.ts
Normal file
@@ -0,0 +1,276 @@
|
||||
/**
|
||||
* Tests for comboMetrics memory management — eviction and TTL behavior.
|
||||
*
|
||||
* Verifies that:
|
||||
* - Recording and retrieval work correctly
|
||||
* - MAX_METRICS_ENTRIES (500) cap is enforced via eviction of oldest entries
|
||||
* - Shadow metrics are isolated from production metrics
|
||||
* - resetComboMetrics and resetAllComboMetrics clear state correctly
|
||||
* - getAllComboMetrics returns all entries
|
||||
* - recordComboIntent tracks intent counts
|
||||
*/
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
const mod = await import("../../../open-sse/services/comboMetrics.ts");
|
||||
const {
|
||||
recordComboRequest,
|
||||
recordComboShadowRequest,
|
||||
getComboMetrics,
|
||||
getAllComboMetrics,
|
||||
resetComboMetrics,
|
||||
resetAllComboMetrics,
|
||||
recordComboIntent,
|
||||
} = mod;
|
||||
|
||||
// Reset all state before each test group to avoid cross-test pollution.
|
||||
resetAllComboMetrics();
|
||||
|
||||
// ─── Basic record + retrieve ────────────────────────────────────────────────
|
||||
|
||||
test("recordComboRequest: basic recording and retrieval", () => {
|
||||
resetAllComboMetrics();
|
||||
recordComboRequest("test-combo", "gpt-4", {
|
||||
success: true,
|
||||
latencyMs: 100,
|
||||
});
|
||||
const metrics = getComboMetrics("test-combo");
|
||||
assert.ok(metrics, "metrics should exist for recorded combo");
|
||||
assert.equal(metrics.totalRequests, 1, "should have 1 total request");
|
||||
assert.equal(metrics.totalSuccesses, 1, "should have 1 success");
|
||||
assert.equal(metrics.totalFailures, 0, "should have 0 failures");
|
||||
assert.equal(metrics.successRate, 100, "success rate should be 100%");
|
||||
assert.equal(metrics.avgLatencyMs, 100, "avg latency should be 100ms");
|
||||
});
|
||||
|
||||
test("recordComboRequest: tracks per-model metrics", () => {
|
||||
resetAllComboMetrics();
|
||||
recordComboRequest("combo-1", "gpt-4", { success: true, latencyMs: 100 });
|
||||
recordComboRequest("combo-1", "claude-3", { success: false, latencyMs: 200 });
|
||||
const metrics = getComboMetrics("combo-1");
|
||||
assert.ok(metrics);
|
||||
assert.equal(metrics.totalRequests, 2, "should have 2 total requests");
|
||||
assert.equal(metrics.successRate, 50, "success rate should be 50%");
|
||||
assert.ok(metrics.byModel["gpt-4"], "should have gpt-4 model metrics");
|
||||
assert.ok(metrics.byModel["claude-3"], "should have claude-3 model metrics");
|
||||
assert.equal(metrics.byModel["gpt-4"].successRate, 100);
|
||||
assert.equal(metrics.byModel["claude-3"].successRate, 0);
|
||||
});
|
||||
|
||||
test("recordComboRequest: handles null model string gracefully", () => {
|
||||
resetAllComboMetrics();
|
||||
recordComboRequest("combo-null", null, { success: true, latencyMs: 50 });
|
||||
const metrics = getComboMetrics("combo-null");
|
||||
assert.ok(metrics, "metrics should exist even with null model");
|
||||
assert.equal(metrics.totalRequests, 1);
|
||||
// null model should not create per-model entries
|
||||
assert.equal(Object.keys(metrics.byModel).length, 0, "should have no model entries");
|
||||
});
|
||||
|
||||
test("recordComboRequest: tracks fallback count", () => {
|
||||
resetAllComboMetrics();
|
||||
recordComboRequest("combo-fb", "gpt-4", {
|
||||
success: true,
|
||||
latencyMs: 200,
|
||||
fallbackCount: 2,
|
||||
});
|
||||
const metrics = getComboMetrics("combo-fb");
|
||||
assert.ok(metrics);
|
||||
assert.equal(metrics.totalFallbacks, 2, "should track 2 fallbacks");
|
||||
// fallbackRate = (totalFallbacks / totalRequests) * 100 = (2/1) * 100 = 200
|
||||
assert.equal(metrics.fallbackRate, 200, "fallback rate should be 200% (2 fallbacks on 1 request)");
|
||||
});
|
||||
|
||||
// ─── getComboMetrics returns null for unknown combos ─────────────────────────
|
||||
|
||||
test("getComboMetrics: returns null for unknown combo", () => {
|
||||
const metrics = getComboMetrics("nonexistent-combo-xyz");
|
||||
assert.equal(metrics, null, "unknown combo should return null");
|
||||
});
|
||||
|
||||
// ─── resetComboMetrics ───────────────────────────────────────────────────────
|
||||
|
||||
test("resetComboMetrics: clears a specific combo", () => {
|
||||
resetAllComboMetrics();
|
||||
recordComboRequest("to-keep", "gpt-4", { success: true, latencyMs: 100 });
|
||||
recordComboRequest("to-reset", "gpt-4", { success: true, latencyMs: 100 });
|
||||
resetComboMetrics("to-reset");
|
||||
assert.ok(getComboMetrics("to-keep"), "to-keep should still exist");
|
||||
assert.equal(getComboMetrics("to-reset"), null, "to-reset should be cleared");
|
||||
});
|
||||
|
||||
// ─── resetAllComboMetrics ────────────────────────────────────────────────────
|
||||
|
||||
test("resetAllComboMetrics: clears all production and shadow metrics", () => {
|
||||
recordComboRequest("combo-a", "gpt-4", { success: true, latencyMs: 100 });
|
||||
recordComboShadowRequest("shadow-a", "gpt-4", { success: true, latencyMs: 100 });
|
||||
assert.ok(getComboMetrics("combo-a"), "combo-a should exist before reset");
|
||||
assert.ok(getComboMetrics("shadow-a"), "shadow-a should exist before reset");
|
||||
resetAllComboMetrics();
|
||||
assert.equal(getComboMetrics("combo-a"), null, "combo-a should be cleared");
|
||||
assert.equal(getComboMetrics("shadow-a"), null, "shadow-a should be cleared");
|
||||
});
|
||||
|
||||
// ─── getAllComboMetrics ───────────────────────────────────────────────────────
|
||||
|
||||
test("getAllComboMetrics: returns all production and shadow entries", () => {
|
||||
resetAllComboMetrics();
|
||||
recordComboRequest("combo-x", "gpt-4", { success: true, latencyMs: 100 });
|
||||
recordComboRequest("combo-y", "claude-3", { success: false, latencyMs: 200 });
|
||||
recordComboShadowRequest("shadow-x", "gpt-4", { success: true, latencyMs: 150 });
|
||||
const all = getAllComboMetrics();
|
||||
assert.ok(all["combo-x"], "should include combo-x");
|
||||
assert.ok(all["combo-y"], "should include combo-y");
|
||||
assert.ok(all["shadow-x"], "should include shadow-x from shadow metrics");
|
||||
});
|
||||
|
||||
// ─── Shadow metrics ──────────────────────────────────────────────────────────
|
||||
|
||||
test("recordComboShadowRequest: tracks shadow metrics separately from production", () => {
|
||||
resetAllComboMetrics();
|
||||
recordComboRequest("prod-combo", "gpt-4", { success: true, latencyMs: 100 });
|
||||
recordComboShadowRequest("prod-combo", "claude-3", { success: false, latencyMs: 200 });
|
||||
const metrics = getComboMetrics("prod-combo");
|
||||
assert.ok(metrics);
|
||||
// Production sees only gpt-4
|
||||
assert.equal(metrics.totalRequests, 1, "production should have 1 request");
|
||||
assert.equal(metrics.successRate, 100, "production success rate should be 100%");
|
||||
// Shadow is separate
|
||||
assert.ok(metrics.shadow, "shadow metrics should be present");
|
||||
assert.equal(metrics.shadow.totalRequests, 1, "shadow should have 1 request");
|
||||
assert.equal(metrics.shadow.successRate, 0, "shadow success rate should be 0%");
|
||||
});
|
||||
|
||||
test("recordComboShadowRequest: shadow-only combo returns in getAllComboMetrics", () => {
|
||||
resetAllComboMetrics();
|
||||
recordComboShadowRequest("shadow-only", "gpt-4", { success: true, latencyMs: 50 });
|
||||
const all = getAllComboMetrics();
|
||||
assert.ok(all["shadow-only"], "shadow-only combo should appear in getAllComboMetrics");
|
||||
});
|
||||
|
||||
// ─── recordComboIntent ───────────────────────────────────────────────────────
|
||||
|
||||
test("recordComboIntent: tracks intent counts on existing combo", () => {
|
||||
resetAllComboMetrics();
|
||||
recordComboRequest("intent-combo", "gpt-4", { success: true, latencyMs: 100 });
|
||||
recordComboIntent("intent-combo", "chat");
|
||||
recordComboIntent("intent-combo", "chat");
|
||||
recordComboIntent("intent-combo", "code");
|
||||
const metrics = getComboMetrics("intent-combo");
|
||||
assert.ok(metrics);
|
||||
assert.equal(metrics.intentCounts["chat"], 2, "should count 2 chat intents");
|
||||
assert.equal(metrics.intentCounts["code"], 1, "should count 1 code intent");
|
||||
});
|
||||
|
||||
test("recordComboIntent: creates combo entry if not yet recorded", () => {
|
||||
resetAllComboMetrics();
|
||||
recordComboIntent("new-intent-combo", "search");
|
||||
const metrics = getComboMetrics("new-intent-combo");
|
||||
assert.ok(metrics, "combo should be created by recordComboIntent");
|
||||
assert.equal(metrics.intentCounts["search"], 1);
|
||||
});
|
||||
|
||||
// ─── Eviction: MAX_METRICS_ENTRIES cap ───────────────────────────────────────
|
||||
|
||||
test("eviction: inserting a new combo at capacity evicts the oldest entry", () => {
|
||||
resetAllComboMetrics();
|
||||
const MAX = 500;
|
||||
// Fill to capacity with unique combos
|
||||
for (let i = 0; i < MAX; i++) {
|
||||
recordComboRequest(`fill-${i}`, "gpt-4", { success: true, latencyMs: 10 });
|
||||
}
|
||||
// All 500 should be present
|
||||
assert.ok(getComboMetrics("fill-0"), "first inserted combo should exist at capacity");
|
||||
assert.ok(getComboMetrics(`fill-${MAX - 1}`), "last inserted combo should exist at capacity");
|
||||
|
||||
// Insert one more — should trigger eviction of the oldest entry
|
||||
recordComboRequest("new-after-capacity", "gpt-4", { success: true, latencyMs: 50 });
|
||||
const all = getAllComboMetrics();
|
||||
const totalProduction = Object.keys(all).filter((k) => {
|
||||
const m = all[k];
|
||||
return m && m.totalRequests > 0;
|
||||
}).length;
|
||||
// Map size should not exceed MAX (the new one replaced the oldest)
|
||||
assert.ok(totalProduction <= MAX, `production combos (${totalProduction}) should not exceed cap (${MAX})`);
|
||||
assert.ok(
|
||||
getComboMetrics("new-after-capacity"),
|
||||
"newly inserted combo should exist after eviction"
|
||||
);
|
||||
});
|
||||
|
||||
test("eviction: shadow metrics respect their own MAX_METRICS_ENTRIES cap", () => {
|
||||
resetAllComboMetrics();
|
||||
const MAX = 500;
|
||||
// Fill shadow metrics to capacity
|
||||
for (let i = 0; i < MAX; i++) {
|
||||
recordComboShadowRequest(`shadow-fill-${i}`, "gpt-4", {
|
||||
success: true,
|
||||
latencyMs: 10,
|
||||
});
|
||||
}
|
||||
assert.ok(
|
||||
getComboMetrics("shadow-fill-0"),
|
||||
"first shadow combo should exist at capacity"
|
||||
);
|
||||
|
||||
// Insert one more shadow — should trigger eviction
|
||||
recordComboShadowRequest("shadow-after-capacity", "gpt-4", {
|
||||
success: true,
|
||||
latencyMs: 50,
|
||||
});
|
||||
assert.ok(
|
||||
getComboMetrics("shadow-after-capacity"),
|
||||
"newly inserted shadow combo should exist after eviction"
|
||||
);
|
||||
});
|
||||
|
||||
test("eviction: recordComboIntent respects MAX_METRICS_ENTRIES cap", () => {
|
||||
resetAllComboMetrics();
|
||||
const MAX = 500;
|
||||
// Fill production metrics to capacity
|
||||
for (let i = 0; i < MAX; i++) {
|
||||
recordComboRequest(`intent-fill-${i}`, "gpt-4", { success: true, latencyMs: 10 });
|
||||
}
|
||||
// Adding intent for a NEW combo should trigger eviction
|
||||
recordComboIntent("intent-after-capacity", "chat");
|
||||
assert.ok(
|
||||
getComboMetrics("intent-after-capacity"),
|
||||
"newly created intent combo should exist after eviction"
|
||||
);
|
||||
});
|
||||
|
||||
test("eviction: updating an existing combo at capacity does NOT evict", () => {
|
||||
resetAllComboMetrics();
|
||||
const MAX = 500;
|
||||
// Fill to capacity
|
||||
for (let i = 0; i < MAX; i++) {
|
||||
recordComboRequest(`update-${i}`, "gpt-4", { success: true, latencyMs: 10 });
|
||||
}
|
||||
// Update an existing combo — should NOT trigger eviction since it's not a new key
|
||||
recordComboRequest("update-0", "gpt-4", { success: false, latencyMs: 50 });
|
||||
const metrics = getComboMetrics("update-0");
|
||||
assert.ok(metrics, "updated combo should still exist");
|
||||
assert.equal(metrics.totalRequests, 2, "updated combo should have 2 requests");
|
||||
});
|
||||
|
||||
// ─── Strategy tracking ───────────────────────────────────────────────────────
|
||||
|
||||
test("recordComboRequest: stores the routing strategy", () => {
|
||||
resetAllComboMetrics();
|
||||
recordComboRequest("strat-combo", "gpt-4", {
|
||||
success: true,
|
||||
latencyMs: 100,
|
||||
strategy: "weighted",
|
||||
});
|
||||
const metrics = getComboMetrics("strat-combo");
|
||||
assert.ok(metrics);
|
||||
assert.equal(metrics.strategy, "weighted", "should store the strategy");
|
||||
});
|
||||
|
||||
test("recordComboRequest: defaults strategy to 'priority'", () => {
|
||||
resetAllComboMetrics();
|
||||
recordComboRequest("default-strat", "gpt-4", { success: true, latencyMs: 100 });
|
||||
const metrics = getComboMetrics("default-strat");
|
||||
assert.ok(metrics);
|
||||
assert.equal(metrics.strategy, "priority", "default strategy should be 'priority'");
|
||||
});
|
||||
Reference in New Issue
Block a user