diff --git a/open-sse/services/autoCombo/__tests__/providerDiversity.test.ts b/open-sse/services/autoCombo/__tests__/providerDiversity.test.ts new file mode 100644 index 0000000000..8fa4e2f57a --- /dev/null +++ b/open-sse/services/autoCombo/__tests__/providerDiversity.test.ts @@ -0,0 +1,125 @@ +import { describe, it, beforeEach } from "node:test"; +import assert from "node:assert/strict"; +import { + recordProviderUsage, + calculateDiversityScore, + getProviderDiversityBoost, + getDiversityReport, + resetDiversity, + configureDiversity, +} from "../providerDiversity"; + +describe("providerDiversity", () => { + beforeEach(() => { + resetDiversity(); + }); + + describe("calculateDiversityScore", () => { + it("returns 1.0 when no data is recorded", () => { + assert.equal(calculateDiversityScore(), 1.0); + }); + + it("returns 0.0 when all requests go to one provider", () => { + for (let i = 0; i < 20; i++) { + recordProviderUsage("claude"); + } + assert.equal(calculateDiversityScore(), 0.0); + }); + + it("returns 1.0 for perfectly even distribution across 2 providers", () => { + for (let i = 0; i < 10; i++) { + recordProviderUsage("claude"); + recordProviderUsage("openai"); + } + assert.equal(calculateDiversityScore(), 1.0); + }); + + it("returns value between 0 and 1 for uneven distribution", () => { + for (let i = 0; i < 15; i++) recordProviderUsage("claude"); + for (let i = 0; i < 5; i++) recordProviderUsage("openai"); + + const score = calculateDiversityScore(); + assert.ok(score > 0, "should be > 0 (not single provider)"); + assert.ok(score < 1, "should be < 1 (not perfectly even)"); + }); + + it("higher entropy with more providers", () => { + // 2 providers + resetDiversity(); + for (let i = 0; i < 10; i++) { + recordProviderUsage("claude"); + recordProviderUsage("openai"); + } + const score2 = calculateDiversityScore(); + + // 4 providers (same total requests) + resetDiversity(); + for (let i = 0; i < 5; i++) { + recordProviderUsage("claude"); + recordProviderUsage("openai"); + recordProviderUsage("google"); + recordProviderUsage("together"); + } + const score4 = calculateDiversityScore(); + + // Both should be 1.0 (perfectly distributed within their pool) + assert.equal(score2, 1.0); + assert.equal(score4, 1.0); + }); + }); + + describe("getProviderDiversityBoost", () => { + it("returns 0.5 when no data is recorded", () => { + assert.equal(getProviderDiversityBoost("claude"), 0.5); + }); + + it("returns low boost for heavily used provider", () => { + for (let i = 0; i < 18; i++) recordProviderUsage("claude"); + for (let i = 0; i < 2; i++) recordProviderUsage("openai"); + + const claudeBoost = getProviderDiversityBoost("claude"); + const openaiBoost = getProviderDiversityBoost("openai"); + + assert.ok(claudeBoost < openaiBoost, "heavily used provider should have lower boost"); + assert.ok(claudeBoost < 0.2, "90% used provider should have very low boost"); + assert.ok(openaiBoost > 0.8, "10% used provider should have high boost"); + }); + + it("returns 1.0 for never-used provider", () => { + for (let i = 0; i < 10; i++) recordProviderUsage("claude"); + + const boost = getProviderDiversityBoost("google"); + assert.equal(boost, 1.0); + }); + }); + + describe("getDiversityReport", () => { + it("returns structured report", () => { + recordProviderUsage("claude"); + recordProviderUsage("claude"); + recordProviderUsage("openai"); + + const report = getDiversityReport(); + + assert.equal(report.totalRequests, 3); + assert.ok(report.score > 0); + assert.ok(report.score < 1); + assert.equal(report.providers["claude"].count, 2); + assert.equal(report.providers["openai"].count, 1); + assert.ok(Math.abs(report.providers["claude"].share - 2 / 3) < 0.01); + }); + }); + + describe("window management", () => { + it("respects windowSize limit", () => { + configureDiversity({ windowSize: 10, ttlMs: 3_600_000 }); + + for (let i = 0; i < 20; i++) { + recordProviderUsage("claude"); + } + + const report = getDiversityReport(); + assert.ok(report.totalRequests <= 10, "should not exceed window size"); + }); + }); +}); diff --git a/open-sse/services/autoCombo/providerDiversity.ts b/open-sse/services/autoCombo/providerDiversity.ts new file mode 100644 index 0000000000..eda6076737 --- /dev/null +++ b/open-sse/services/autoCombo/providerDiversity.ts @@ -0,0 +1,170 @@ +/** + * Provider Diversity Tracking via Shannon Entropy + * + * Measures and tracks how evenly distributed requests are across providers. + * A system routing 90% of traffic to one provider has a catastrophic single + * point of failure. This module provides a diversity score [0..1] that can + * be used as a scoring factor in auto-combo selection. + * + * Shannon entropy normalized to [0..1]: + * - 0.0 = all requests go to one provider (maximum risk) + * - 1.0 = perfectly even distribution (minimum risk) + * + * @see https://en.wikipedia.org/wiki/Entropy_(information_theory) + */ + +/** Rolling window entry for provider usage tracking */ +interface UsageEntry { + provider: string; + timestamp: number; +} + +/** Configuration for the diversity tracker */ +export interface DiversityConfig { + /** Maximum entries in the rolling window (default: 200) */ + windowSize: number; + /** Time-to-live in ms for entries — older entries are pruned (default: 1 hour) */ + ttlMs: number; +} + +const DEFAULT_CONFIG: DiversityConfig = { + windowSize: 200, + ttlMs: 3_600_000, // 1 hour +}; + +/** In-memory rolling window of recent provider usage */ +let usageWindow: UsageEntry[] = []; +let config: DiversityConfig = { ...DEFAULT_CONFIG }; + +/** + * Configure the diversity tracker. + */ +export function configureDiversity(userConfig: Partial): void { + config = { ...DEFAULT_CONFIG, ...userConfig }; +} + +/** + * Record that a provider was used for a request. + * Call this after a successful request completes. + */ +export function recordProviderUsage(provider: string): void { + const now = Date.now(); + + usageWindow.push({ provider, timestamp: now }); + + // Prune by window size + if (usageWindow.length > config.windowSize) { + usageWindow = usageWindow.slice(-config.windowSize); + } + + // Prune by TTL + const cutoff = now - config.ttlMs; + usageWindow = usageWindow.filter((e) => e.timestamp >= cutoff); +} + +/** + * Calculate Shannon entropy normalized to [0..1] for the current usage window. + * + * @returns Normalized entropy where 0 = single provider, 1 = perfect distribution + */ +export function calculateDiversityScore(): number { + if (usageWindow.length === 0) return 1.0; // No data = assume diverse + + const now = Date.now(); + const cutoff = now - config.ttlMs; + const recent = usageWindow.filter((e) => e.timestamp >= cutoff); + + if (recent.length === 0) return 1.0; + + // Count occurrences per provider + const counts = new Map(); + for (const entry of recent) { + counts.set(entry.provider, (counts.get(entry.provider) || 0) + 1); + } + + const total = recent.length; + const nUnique = counts.size; + + if (nUnique <= 1) return 0.0; + + // Shannon entropy + let entropy = 0; + for (const count of counts.values()) { + const p = count / total; + entropy -= p * Math.log2(p); + } + + // Normalize by maximum possible entropy + const maxEntropy = Math.log2(nUnique); + return maxEntropy > 0 ? entropy / maxEntropy : 0; +} + +/** + * Get the diversity score for a specific provider. + * Returns a boost value [0..1] where underrepresented providers score higher. + * This can be used as a per-candidate factor in auto-combo scoring. + * + * @param provider - The provider to score + * @returns Diversity boost where 1.0 = never used (maximum boost), 0.0 = most used + */ +export function getProviderDiversityBoost(provider: string): number { + if (usageWindow.length === 0) return 0.5; // No data = neutral + + const now = Date.now(); + const cutoff = now - config.ttlMs; + const recent = usageWindow.filter((e) => e.timestamp >= cutoff); + + if (recent.length === 0) return 0.5; + + const total = recent.length; + const providerCount = recent.filter((e) => e.provider === provider).length; + + // Inverse usage share: providers used less get higher boost + const usageShare = providerCount / total; + return Math.max(0, 1 - usageShare); +} + +/** + * Get a summary of the current provider distribution. + * Useful for dashboard display and debugging. + */ +export function getDiversityReport(): { + score: number; + totalRequests: number; + providers: Record; + windowSize: number; + ttlMs: number; +} { + const now = Date.now(); + const cutoff = now - config.ttlMs; + const recent = usageWindow.filter((e) => e.timestamp >= cutoff); + + const counts = new Map(); + for (const entry of recent) { + counts.set(entry.provider, (counts.get(entry.provider) || 0) + 1); + } + + const providers: Record = {}; + for (const [provider, count] of counts) { + providers[provider] = { + count, + share: recent.length > 0 ? count / recent.length : 0, + }; + } + + return { + score: calculateDiversityScore(), + totalRequests: recent.length, + providers, + windowSize: config.windowSize, + ttlMs: config.ttlMs, + }; +} + +/** + * Reset the diversity tracker. Useful for testing. + */ +export function resetDiversity(): void { + usageWindow = []; + config = { ...DEFAULT_CONFIG }; +}