From ffde0669518819d66154e3f6b15af9e70219c053 Mon Sep 17 00:00:00 2001 From: Jan Leon Date: Wed, 6 May 2026 19:23:03 +0000 Subject: [PATCH 01/15] feat(combos): add reset-aware routing strategy --- open-sse/services/combo.ts | 250 +++++++++++++++++- open-sse/services/comboConfig.ts | 4 + src/app/(dashboard)/dashboard/combos/page.tsx | 18 ++ src/i18n/messages/de.json | 16 ++ src/i18n/messages/en.json | 16 ++ src/shared/constants/routingStrategies.ts | 8 + src/shared/validation/schemas.ts | 4 + tests/unit/combo-strategies.test.ts | 183 ++++++++++++- 8 files changed, 495 insertions(+), 4 deletions(-) diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index 8bd9efa5e9..78dab90709 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -1,7 +1,8 @@ /** * Shared combo (model combo) handling with fallback support * Supports: priority, weighted, round-robin, random, least-used, cost-optimized, - * strict-random, auto, fill-first, p2c, lkgp, context-optimized, and context-relay strategies + * reset-aware, strict-random, auto, fill-first, p2c, lkgp, context-optimized, + * and context-relay strategies */ import { @@ -86,6 +87,16 @@ const DEFAULT_MODEL_P95_MS = { "deepseek-chat": 2000, }; const MIN_HISTORY_SAMPLES = 10; +const RESET_AWARE_SESSION_WINDOW_MS = 5 * 60 * 60 * 1000; +const RESET_AWARE_WEEKLY_WINDOW_MS = 7 * 24 * 60 * 60 * 1000; +const RESET_AWARE_REMAINING_WEIGHT = 0.55; +const RESET_AWARE_RESET_WEIGHT = 0.45; +const RESET_AWARE_DEFAULTS = { + sessionWeight: 0.35, + weeklyWeight: 0.65, + tieBandPercent: 5, + exhaustionGuardPercent: 10, +}; type ResolvedComboTarget = { kind: "model"; @@ -697,6 +708,237 @@ function orderTargetsByPowerOfTwoChoices(targets: ResolvedComboTarget[], comboNa return [targets[selectedIndex], ...targets.filter((_, index) => index !== selectedIndex)]; } +function clamp01(value: number): number { + if (!Number.isFinite(value)) return 0; + return Math.max(0, Math.min(1, value)); +} + +function finiteNumberOrNull(value: unknown): number | null { + const numericValue = Number(value); + return Number.isFinite(numericValue) ? numericValue : null; +} + +function getPercentConfig(value: unknown, fallback: number): number { + const numericValue = finiteNumberOrNull(value); + if (numericValue === null) return fallback; + return Math.max(0, Math.min(100, numericValue)); +} + +function getWeightConfig(value: unknown, fallback: number): number { + const numericValue = finiteNumberOrNull(value); + if (numericValue === null || numericValue < 0) return fallback; + return numericValue; +} + +function resolveResetAwareConfig(config: Record | null | undefined) { + const sessionWeight = getWeightConfig( + config?.resetAwareSessionWeight, + RESET_AWARE_DEFAULTS.sessionWeight + ); + const weeklyWeight = getWeightConfig( + config?.resetAwareWeeklyWeight, + RESET_AWARE_DEFAULTS.weeklyWeight + ); + const totalWeight = sessionWeight + weeklyWeight; + const normalizedSessionWeight = totalWeight > 0 ? sessionWeight / totalWeight : 0.35; + + return { + sessionWeight: normalizedSessionWeight, + weeklyWeight: 1 - normalizedSessionWeight, + tieBand: + getPercentConfig(config?.resetAwareTieBandPercent, RESET_AWARE_DEFAULTS.tieBandPercent) / 100, + exhaustionGuard: + getPercentConfig( + config?.resetAwareExhaustionGuardPercent, + RESET_AWARE_DEFAULTS.exhaustionGuardPercent + ) / 100, + }; +} + +function isCodexTarget(target: ResolvedComboTarget): boolean { + const provider = (target.providerId || target.provider || "").toLowerCase(); + return provider === "codex" || target.modelStr.toLowerCase().startsWith("codex/"); +} + +function getQuotaWindow( + quota: unknown, + key: "window5h" | "window7d" +): { percentUsed: number | null; resetAt: string | null } | null { + if (!isRecord(quota)) return null; + const window = quota[key]; + if (!isRecord(window)) return null; + const percentUsed = finiteNumberOrNull(window.percentUsed); + const resetAt = + typeof window.resetAt === "string" && window.resetAt.length > 0 ? window.resetAt : null; + return { percentUsed, resetAt }; +} + +function getResetUrgency(resetAt: string | null | undefined, windowMs: number): number { + if (!resetAt) return 0.5; + const resetTime = new Date(resetAt).getTime(); + if (!Number.isFinite(resetTime)) return 0.5; + const msUntilReset = resetTime - Date.now(); + if (msUntilReset <= 0) return 1; + return clamp01(1 - msUntilReset / windowMs); +} + +function scoreQuotaWindow( + remaining: number, + resetAt: string | null | undefined, + windowMs: number +): number { + return ( + RESET_AWARE_REMAINING_WEIGHT * clamp01(remaining) + + RESET_AWARE_RESET_WEIGHT * getResetUrgency(resetAt, windowMs) + ); +} + +function scoreResetAwareQuota(quota: unknown, config: ReturnType) { + if (!quota || !isRecord(quota)) return { score: 0.5 }; + if (quota.limitReached === true) return { score: -Infinity }; + + const overallPercentUsed = clamp01(finiteNumberOrNull(quota.percentUsed) ?? 0.5); + const sessionWindow = getQuotaWindow(quota, "window5h"); + const weeklyWindow = getQuotaWindow(quota, "window7d"); + const sessionRemaining = clamp01(1 - (sessionWindow?.percentUsed ?? overallPercentUsed)); + const weeklyRemaining = clamp01(1 - (weeklyWindow?.percentUsed ?? overallPercentUsed)); + const sessionScore = scoreQuotaWindow( + sessionRemaining, + sessionWindow?.resetAt, + RESET_AWARE_SESSION_WINDOW_MS + ); + const weeklyScore = scoreQuotaWindow( + weeklyRemaining, + weeklyWindow?.resetAt ?? (typeof quota.resetAt === "string" ? quota.resetAt : null), + RESET_AWARE_WEEKLY_WINDOW_MS + ); + let score = config.sessionWeight * sessionScore + config.weeklyWeight * weeklyScore; + + if (config.exhaustionGuard > 0 && sessionRemaining < config.exhaustionGuard) { + score *= Math.max(0.05, sessionRemaining / config.exhaustionGuard); + } + + return { score }; +} + +async function getCodexConnectionsForTarget( + target: ResolvedComboTarget, + connectionCache: Map>> +) { + if (!isCodexTarget(target)) return []; + const provider = target.providerId || target.provider; + if (!provider) return []; + if (!connectionCache.has(provider)) { + try { + const connections = await getProviderConnections({ provider, isActive: true }); + connectionCache.set( + provider, + Array.isArray(connections) ? (connections as Array>) : [] + ); + } catch { + connectionCache.set(provider, []); + } + } + return connectionCache.get(provider) || []; +} + +function getTargetConnectionIds( + target: ResolvedComboTarget, + connections: Array> +): string[] { + if (target.connectionId) return [target.connectionId]; + if (Array.isArray(target.allowedConnectionIds) && target.allowedConnectionIds.length > 0) { + return target.allowedConnectionIds.filter( + (connectionId): connectionId is string => + typeof connectionId === "string" && connectionId.trim().length > 0 + ); + } + return connections + .map((connection) => (typeof connection.id === "string" ? connection.id : null)) + .filter((connectionId): connectionId is string => !!connectionId); +} + +async function orderTargetsByResetAwareQuota( + targets: ResolvedComboTarget[], + comboName: string, + configSource: Record | null | undefined, + log: { warn?: (...args: unknown[]) => void } +) { + if (targets.length === 0) return targets; + + const config = resolveResetAwareConfig(configSource); + const connectionCache = new Map>>(); + const connectionById = new Map>(); + const expandedTargets: ResolvedComboTarget[] = []; + + for (const target of targets) { + const connections = await getCodexConnectionsForTarget(target, connectionCache); + for (const connection of connections) { + if (typeof connection.id === "string") connectionById.set(connection.id, connection); + } + + const connectionIds = getTargetConnectionIds(target, connections); + if (connectionIds.length === 0) { + expandedTargets.push(target); + continue; + } + + for (const connectionId of connectionIds) { + expandedTargets.push({ + ...target, + connectionId, + executionKey: + target.connectionId === connectionId + ? target.executionKey + : `${target.executionKey}@${connectionId}`, + }); + } + } + + const scoredTargets = await Promise.all( + expandedTargets.map(async (target, index) => { + let quota: unknown = null; + if (isCodexTarget(target) && target.connectionId) { + try { + quota = await fetchCodexQuota( + target.connectionId, + connectionById.get(target.connectionId) + ); + } catch (error) { + log.warn?.( + "COMBO", + `Reset-aware quota fetch failed for connection=${target.connectionId}: ${error instanceof Error ? error.message : String(error)}` + ); + } + } + const { score } = scoreResetAwareQuota(quota, config); + return { target, score, index }; + }) + ); + + scoredTargets.sort((a, b) => { + if (b.score !== a.score) return b.score - a.score; + return a.index - b.index; + }); + + const bestScore = scoredTargets[0]?.score ?? 0; + const tiedTargets = scoredTargets.filter((entry) => bestScore - entry.score <= config.tieBand); + let orderedTiedTargets = tiedTargets; + if (tiedTargets.length > 1) { + const key = `reset-aware:${comboName}`; + const counter = rrCounters.get(key) || 0; + rrCounters.set(key, counter + 1); + const startIndex = counter % tiedTargets.length; + orderedTiedTargets = [...tiedTargets.slice(startIndex), ...tiedTargets.slice(0, startIndex)]; + } + + const tiedExecutionKeys = new Set(orderedTiedTargets.map((entry) => entry.target.executionKey)); + return [ + ...orderedTiedTargets, + ...scoredTargets.filter((entry) => !tiedExecutionKeys.has(entry.target.executionKey)), + ].map((entry) => entry.target); +} + function toTextContent(content) { if (typeof content === "string") return content; if (!Array.isArray(content)) return ""; @@ -1482,6 +1724,12 @@ export async function handleComboChat({ } else if (strategy === "cost-optimized") { orderedTargets = await sortTargetsByCost(orderedTargets); log.info("COMBO", `Cost-optimized ordering: cheapest first (${orderedTargets[0]?.modelStr})`); + } else if (strategy === "reset-aware") { + orderedTargets = await orderTargetsByResetAwareQuota(orderedTargets, combo.name, config, log); + log.info( + "COMBO", + `Reset-aware ordering: ${orderedTargets[0]?.modelStr}${orderedTargets[0]?.connectionId ? ` (${orderedTargets[0].connectionId})` : ""} first` + ); } else if (strategy === "context-optimized") { orderedTargets = sortTargetsByContextSize(orderedTargets); log.info("COMBO", `Context-optimized ordering: largest first (${orderedTargets[0]?.modelStr})`); diff --git a/open-sse/services/comboConfig.ts b/open-sse/services/comboConfig.ts index af464f1aa8..fa383c824e 100644 --- a/open-sse/services/comboConfig.ts +++ b/open-sse/services/comboConfig.ts @@ -17,6 +17,10 @@ const DEFAULT_COMBO_CONFIG = { maxMessagesForSummary: 30, maxComboDepth: 3, trackMetrics: true, + resetAwareSessionWeight: 0.35, + resetAwareWeeklyWeight: 0.65, + resetAwareTieBandPercent: 5, + resetAwareExhaustionGuardPercent: 10, }; const LEGACY_COMBO_RESILIENCE_KEYS = new Set([ diff --git a/src/app/(dashboard)/dashboard/combos/page.tsx b/src/app/(dashboard)/dashboard/combos/page.tsx index 58c8b3ebca..1b121a168f 100644 --- a/src/app/(dashboard)/dashboard/combos/page.tsx +++ b/src/app/(dashboard)/dashboard/combos/page.tsx @@ -64,11 +64,14 @@ const STRATEGY_OPTIONS = ROUTING_STRATEGIES.map((strategy) => ({ const STRATEGY_LABEL_FALLBACK = { "context-relay": "Context Relay", + "reset-aware": "Reset-Aware RR", }; const STRATEGY_DESC_FALLBACK = { "context-relay": "Priority-style routing with automatic context handoffs when account rotation happens.", + "reset-aware": + "Quota remaining and reset windows decide the order; similar scores rotate round-robin.", }; const STRATEGY_GUIDANCE_FALLBACK = { @@ -108,6 +111,11 @@ const STRATEGY_GUIDANCE_FALLBACK = { avoid: "Avoid when pricing data is missing or outdated.", example: "Example: Batch or background jobs where lower cost matters most.", }, + "reset-aware": { + when: "Use when multiple Codex accounts have different 5h and weekly reset windows.", + avoid: "Avoid when quota telemetry is unavailable for most accounts.", + example: "Example: Prefer a 60% weekly account resetting tomorrow over 80% that resets later.", + }, "fill-first": { when: "Use when you want to drain one provider's quota fully before moving to the next.", avoid: "Avoid when you need request-level load balancing across providers.", @@ -230,6 +238,15 @@ const STRATEGY_RECOMMENDATIONS_FALLBACK = { "Use for batch/background jobs where cost is the main KPI.", ], }, + "reset-aware": { + title: "Reset-aware account rotation", + description: "Balances remaining Codex quota against 5h and weekly reset timing.", + tips: [ + "Use explicit Codex account steps or account-tag routing.", + "Tune session vs weekly weights when short-term exhaustion is more risky.", + "Keep the tie band small so equivalent accounts still rotate fairly.", + ], + }, "fill-first": { title: "Quota drain strategy", description: "Exhausts one provider's quota before moving to the next in chain.", @@ -439,6 +456,7 @@ function getStrategyBadgeClass(strategy) { if (strategy === "random") return "bg-purple-500/15 text-purple-600 dark:text-purple-400"; if (strategy === "least-used") return "bg-cyan-500/15 text-cyan-600 dark:text-cyan-400"; if (strategy === "cost-optimized") return "bg-teal-500/15 text-teal-600 dark:text-teal-400"; + if (strategy === "reset-aware") return "bg-lime-500/15 text-lime-700 dark:text-lime-300"; if (strategy === "fill-first") return "bg-orange-500/15 text-orange-600 dark:text-orange-400"; if (strategy === "p2c") return "bg-indigo-500/15 text-indigo-600 dark:text-indigo-400"; return "bg-blue-500/15 text-blue-600 dark:text-blue-400"; diff --git a/src/i18n/messages/de.json b/src/i18n/messages/de.json index 5daaaa212d..98a59cab76 100644 --- a/src/i18n/messages/de.json +++ b/src/i18n/messages/de.json @@ -1396,6 +1396,8 @@ "randomDesc": "Einheitliche Zufallsauswahl, dann Rückgriff auf verbleibende Modelle", "leastUsedDesc": "Wählt das Modell mit den wenigsten Anfragen aus und gleicht die Last über die Zeit aus", "costOptimizedDesc": "Leitet basierend auf dem Preis zuerst zum günstigsten Modell weiter", + "resetAware": "Reset-Aware RR", + "resetAwareDesc": "Gewichtet Restquote gegen 5h- und Wochen-Resets und rotiert ähnliche Scores per Round Robin", "strictRandom": "Strict Random", "strictRandomDesc": "Shuffle deck — uses each model once before reshuffling", "models": "Modelle", @@ -1447,6 +1449,11 @@ "avoid": "Preisdaten fehlen oder sind veraltet.", "example": "Hintergrund- oder Batch-Jobs, bei denen geringere Kosten bevorzugt werden." }, + "reset-aware": { + "when": "Du routest über mehrere Codex-Konten mit unterschiedlichen 5h- und Wochen-Reset-Fenstern.", + "avoid": "Für die meisten Konten fehlen Quota-Telemetriedaten.", + "example": "Bevorzuge ein Konto mit 60 % Wochen-Restquote und Reset morgen vor 80 % mit späterem Reset." + }, "strict-random": { "when": "Use when you want perfectly even spread — each model used once before repeating.", "avoid": "Avoid when models have different quality or latency and order matters.", @@ -1555,6 +1562,13 @@ "tip2": "Behalte einen Qualitäts-Fallback für schwierige Prompts.", "tip3": "Ideal für Batch/Hintergrundjobs, bei denen Kosten das Haupt-KPI sind." }, + "reset-aware": { + "title": "Reset-bewusste Kontorotation", + "description": "Gewichtet verbleibende Codex-Quote gegen 5h- und Wochen-Reset-Zeitpunkte.", + "tip1": "Nutze explizite Codex-Kontoschritte oder kontobasiertes Tag-Routing.", + "tip2": "Passe 5h- und Wochengewichtung an, wenn kurzfristige Erschöpfung riskant ist.", + "tip3": "Halte das Tie-Band klein, damit gleichwertige Konten fair rotieren." + }, "strict-random": { "title": "Shuffle deck distribution", "description": "Each model is used exactly once per cycle before reshuffling.", @@ -2906,6 +2920,8 @@ "leastUsedDesc": "Wählen Sie das zuletzt verwendete Konto aus", "costOpt": "Kosten Opt", "costOptDesc": "Bevorzugen Sie das günstigste verfügbare Konto", + "resetAware": "Reset-Aware RR", + "resetAwareDesc": "Bevorzugt Konten mit gesunder Restquote und näherem Reset", "strictRandom": "Strict Random", "strictRandomDesc": "Shuffle deck — uses each account once before reshuffling", "stickyLimit": "Sticky-Limit", diff --git a/src/i18n/messages/en.json b/src/i18n/messages/en.json index b2f4c692f4..5bb7f37456 100644 --- a/src/i18n/messages/en.json +++ b/src/i18n/messages/en.json @@ -1505,6 +1505,8 @@ "randomDesc": "Uniform random selection, then fallback to remaining models", "leastUsedDesc": "Picks the model with fewest requests, balancing load over time", "costOptimizedDesc": "Routes to the cheapest model first based on pricing", + "resetAware": "Reset-Aware RR", + "resetAwareDesc": "Balances remaining quota against 5h and weekly resets, then round-robins similar scores", "strictRandom": "Strict Random", "strictRandomDesc": "Shuffle deck — uses each model once before reshuffling", "models": "Models", @@ -1563,6 +1565,11 @@ "avoid": "Pricing data is missing or outdated.", "example": "Background or batch jobs where lower cost is preferred." }, + "reset-aware": { + "when": "You route across multiple Codex accounts with different 5h and weekly reset windows.", + "avoid": "Quota telemetry is unavailable for most accounts.", + "example": "Prefer a 60% weekly account resetting tomorrow over an 80% account resetting later." + }, "strict-random": { "when": "Use when you want perfectly even spread — each model used once before repeating.", "avoid": "Avoid when models have different quality or latency and order matters.", @@ -1735,6 +1742,13 @@ "tip2": "Keep a quality fallback for hard prompts.", "tip3": "Use for batch/background jobs where cost is the main KPI." }, + "reset-aware": { + "title": "Reset-aware account rotation", + "description": "Balances remaining Codex quota against 5h and weekly reset timing.", + "tip1": "Use explicit Codex account steps or account-tag routing.", + "tip2": "Tune session vs weekly weights when short-term exhaustion is more risky.", + "tip3": "Keep the tie band small so equivalent accounts still rotate fairly." + }, "strict-random": { "title": "Shuffle deck distribution", "description": "Each model is used exactly once per cycle before reshuffling.", @@ -3363,6 +3377,8 @@ "leastUsedDesc": "Pick least recently used account", "costOpt": "Cost Opt", "costOptDesc": "Prefer cheapest available account", + "resetAware": "Reset-Aware RR", + "resetAwareDesc": "Prefer accounts with healthy remaining quota and nearer resets", "strictRandom": "Strict Random", "strictRandomDesc": "Shuffle deck — uses each account once before reshuffling", "stickyLimit": "Sticky Limit", diff --git a/src/shared/constants/routingStrategies.ts b/src/shared/constants/routingStrategies.ts index 752683e7ca..0a67a2cfe1 100644 --- a/src/shared/constants/routingStrategies.ts +++ b/src/shared/constants/routingStrategies.ts @@ -8,6 +8,7 @@ export const ROUTING_STRATEGY_VALUES = [ "random", "least-used", "cost-optimized", + "reset-aware", "strict-random", "auto", "lkgp", @@ -123,6 +124,13 @@ export const ROUTING_STRATEGIES: RoutingStrategyOption[] = [ settingsDescKey: "costOptDesc", icon: "savings", }, + { + value: "reset-aware", + labelKey: "resetAware", + combosDescKey: "resetAwareDesc", + settingsDescKey: "resetAwareDesc", + icon: "event_repeat", + }, { value: "strict-random", labelKey: "strictRandom", diff --git a/src/shared/validation/schemas.ts b/src/shared/validation/schemas.ts index 97a41d118b..b94f1e4396 100644 --- a/src/shared/validation/schemas.ts +++ b/src/shared/validation/schemas.ts @@ -395,6 +395,10 @@ const comboRuntimeConfigSchema = z explorationRate: z.number().min(0).max(1).optional(), routerStrategy: z.string().optional(), compositeTiers: compositeTiersSchema.optional(), + resetAwareSessionWeight: z.coerce.number().min(0).max(100).optional(), + resetAwareWeeklyWeight: z.coerce.number().min(0).max(100).optional(), + resetAwareTieBandPercent: z.coerce.number().min(0).max(100).optional(), + resetAwareExhaustionGuardPercent: z.coerce.number().min(0).max(100).optional(), }) .strict(); diff --git a/tests/unit/combo-strategies.test.ts b/tests/unit/combo-strategies.test.ts index 65d62c88dd..f2343cdbb4 100644 --- a/tests/unit/combo-strategies.test.ts +++ b/tests/unit/combo-strategies.test.ts @@ -1,15 +1,19 @@ import test, { after } from "node:test"; import assert from "node:assert/strict"; +import { randomUUID } from "node:crypto"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-combo-strategies-")); const ORIGINAL_DATA_DIR = process.env.DATA_DIR; +const ORIGINAL_FETCH = globalThis.fetch; process.env.DATA_DIR = TEST_DATA_DIR; const dbCore = await import("../../src/lib/db/core.ts"); const { handleComboChat } = await import("../../open-sse/services/combo.ts"); +const { invalidateCodexQuotaCache, registerCodexConnection } = + await import("../../open-sse/services/codexQuotaFetcher.ts"); const combosDb = await import("../../src/lib/db/combos.ts"); const { recordComboRequest } = await import("../../open-sse/services/comboMetrics.ts"); const { saveModelsDevCapabilities } = await import("../../src/lib/modelsDevSync.ts"); @@ -22,6 +26,7 @@ after(() => { } else { process.env.DATA_DIR = ORIGINAL_DATA_DIR; } + globalThis.fetch = ORIGINAL_FETCH; }); const reqBodyNullContext = { @@ -93,8 +98,95 @@ async function selectedModelFor(combo: Record, body: Record) { + globalThis.fetch = async (_input: RequestInfo | URL, init?: RequestInit) => { + const headers = init?.headers as Record | undefined; + const authorization = headers?.Authorization || headers?.authorization || ""; + const token = authorization.replace(/^Bearer\s+/i, ""); + const quota = quotasByToken[token]; + if (!quota) return Response.json({ error: "missing quota" }, { status: 404 }); + return Response.json(quota); + }; +} + +function resetAwareCombo( + name: string, + connections: Array<{ id: string; token: string }>, + config: Record = {} +) { + for (const connection of connections) { + invalidateCodexQuotaCache(connection.id); + registerCodexConnection(connection.id, { accessToken: connection.token }); + } + + return { + name, + strategy: "reset-aware", + config, + models: connections.map((connection, index) => ({ + kind: "model", + provider: "codex", + providerId: "codex", + model: "gpt-5", + connectionId: connection.id, + id: `${name}-${index}`, + })), + }; +} + +async function selectedConnectionFor(combo: Record) { + const calls: Array = []; + const response = await handleComboChat({ + body: reqBodyTextArray, + combo, + allCombos: [combo], + isModelAvailable: undefined, + relayOptions: undefined, + signal: undefined, + settings: {}, + log: makeLog(), + handleSingleModel: async ( + _body: unknown, + modelStr: string, + target?: { connectionId?: string | null } + ) => { + calls.push(target?.connectionId ?? null); + return okResponse(modelStr); + }, + }); + + assert.equal(response.status, 200); + assert.equal(calls.length > 0, true); + return calls[0]; +} + test("least-used strategy prefers the model with fewer recorded combo requests", async () => { - const name = `least-used-${crypto.randomUUID()}`; + const name = `least-used-${randomUUID()}`; const busyModel = "openai/gpt-4"; const idleModel = "openai/gpt-3.5-turbo"; const combo = await combosDb.createCombo({ @@ -121,7 +213,7 @@ test("context-optimized strategy prefers the largest context window", async () = }); const combo = await combosDb.createCombo({ - name: `context-optimized-${crypto.randomUUID()}`, + name: `context-optimized-${randomUUID()}`, strategy: "context-optimized", models: ["test-context/small", "test-context/large", "unknown/unknown"], }); @@ -131,7 +223,7 @@ test("context-optimized strategy prefers the largest context window", async () = test("auto strategy handles null and empty prompt edge cases without throwing", async () => { const combo = await combosDb.createCombo({ - name: `auto-${crypto.randomUUID()}`, + name: `auto-${randomUUID()}`, strategy: "auto", config: { auto: { explorationRate: 0 } }, models: ["openai/gpt-4"], @@ -146,3 +238,88 @@ test("auto strategy handles null and empty prompt edge cases without throwing", ); assert.equal(await selectedModelFor(combo, { model: combo.name, messages: [] }), "openai/gpt-4"); }); + +test("reset-aware strategy prefers lower weekly remaining quota when reset is much sooner", async () => { + const soon = { id: `soon-${randomUUID()}`, token: `token-soon-${randomUUID()}` }; + const later = { id: `later-${randomUUID()}`, token: `token-later-${randomUUID()}` }; + installCodexQuotaMock({ + [soon.token]: codexQuota({ + used5h: 10, + reset5hSeconds: 3600, + used7d: 40, + reset7dSeconds: 24 * 3600, + }), + [later.token]: codexQuota({ + used5h: 10, + reset5hSeconds: 3600, + used7d: 20, + reset7dSeconds: 5 * 24 * 3600, + }), + }); + + const combo = resetAwareCombo(`reset-aware-soon-${randomUUID()}`, [soon, later]); + + assert.equal(await selectedConnectionFor(combo), soon.id); +}); + +test("reset-aware strategy avoids accounts near 5h exhaustion", async () => { + const exhausted5h = { + id: `exhausted-${randomUUID()}`, + token: `token-exhausted-${randomUUID()}`, + }; + const healthy5h = { + id: `healthy-${randomUUID()}`, + token: `token-healthy-${randomUUID()}`, + }; + installCodexQuotaMock({ + [exhausted5h.token]: codexQuota({ + used5h: 98, + reset5hSeconds: 20 * 60, + used7d: 5, + reset7dSeconds: 24 * 3600, + }), + [healthy5h.token]: codexQuota({ + used5h: 20, + reset5hSeconds: 4 * 3600, + used7d: 50, + reset7dSeconds: 4 * 24 * 3600, + }), + }); + + const combo = resetAwareCombo(`reset-aware-guard-${randomUUID()}`, [exhausted5h, healthy5h]); + + assert.equal(await selectedConnectionFor(combo), healthy5h.id); +}); + +test("reset-aware strategy rotates similar scores with round-robin tie breaking", async () => { + const first = { id: `first-${randomUUID()}`, token: `token-first-${randomUUID()}` }; + const second = { + id: `second-${randomUUID()}`, + token: `token-second-${randomUUID()}`, + }; + installCodexQuotaMock({ + [first.token]: codexQuota({ + used5h: 50, + reset5hSeconds: 2 * 3600, + used7d: 50, + reset7dSeconds: 3 * 24 * 3600, + }), + [second.token]: codexQuota({ + used5h: 50, + reset5hSeconds: 2 * 3600, + used7d: 50, + reset7dSeconds: 3 * 24 * 3600, + }), + }); + + const combo = resetAwareCombo(`reset-aware-rr-${randomUUID()}`, [first, second]); + + assert.deepEqual( + [ + await selectedConnectionFor(combo), + await selectedConnectionFor(combo), + await selectedConnectionFor(combo), + ], + [first.id, second.id, first.id] + ); +}); From 23aa213cefa072916e7ced7646f086e27efe85a3 Mon Sep 17 00:00:00 2001 From: Jan Leon Date: Wed, 6 May 2026 20:20:26 +0000 Subject: [PATCH 02/15] feat: add support for Z.AI provider and enhance quota handling --- open-sse/services/usage.ts | 25 ++++++++++++++++-- .../usage/components/ProviderLimits/index.tsx | 23 +++++++--------- .../usage/components/ProviderLimits/utils.tsx | 2 ++ src/lib/usage/providerLimits.ts | 10 ++++++- src/shared/constants/providers.ts | 1 + tests/unit/provider-limits-ui.test.ts | 26 +++++++++++++++++++ tests/unit/usage-service-hardening.test.ts | 25 ++++++++++++++---- 7 files changed, 90 insertions(+), 22 deletions(-) diff --git a/open-sse/services/usage.ts b/open-sse/services/usage.ts index 2bf2edec5f..a7f01440f1 100644 --- a/open-sse/services/usage.ts +++ b/open-sse/services/usage.ts @@ -515,7 +515,26 @@ async function getCrofUsage(apiKey: string) { return { quotas }; } +function getGlmQuotaLabel(type: unknown): string | null { + const normalized = typeof type === "string" ? type.trim().toUpperCase() : ""; + + switch (normalized) { + case "TOKENS_LIMIT": + case "TOKEN_LIMIT": + return "tokens"; + case "TIME_LIMIT": + case "TIME_USAGE_LIMIT": + return "time_limit"; + default: + return null; + } +} + async function getGlmUsage(apiKey: string, providerSpecificData?: Record) { + if (!apiKey) { + return { message: "Z.AI API key not available. Add a coding plan API key to view usage." }; + } + const quotaUrl = getGlmQuotaUrl(providerSpecificData); const res = await fetch(quotaUrl, { @@ -537,13 +556,14 @@ async function getGlmUsage(apiKey: string, providerSpecificData?: Record (priority[a.provider] || 9) - (priority[b.provider] || 9) @@ -551,14 +553,7 @@ export default function ProviderLimits() { {/* Account Info */}
- {conn.provider} +
diff --git a/src/app/(dashboard)/dashboard/usage/components/ProviderLimits/utils.tsx b/src/app/(dashboard)/dashboard/usage/components/ProviderLimits/utils.tsx index ddce524a9f..afd2ab1715 100644 --- a/src/app/(dashboard)/dashboard/usage/components/ProviderLimits/utils.tsx +++ b/src/app/(dashboard)/dashboard/usage/components/ProviderLimits/utils.tsx @@ -22,6 +22,8 @@ const QUOTA_LABEL_MAP: Record = { agentic_request_freetrial: "Agentic (Trial)", credits: "AI Credits", models: "Models", + tokens: "Tokens", + time_limit: "Time Limit", }; function toRecord(value: unknown): Record { diff --git a/src/lib/usage/providerLimits.ts b/src/lib/usage/providerLimits.ts index 5855c04bab..80c3b31e17 100644 --- a/src/lib/usage/providerLimits.ts +++ b/src/lib/usage/providerLimits.ts @@ -43,7 +43,15 @@ interface ProviderConnectionLike { backoffLevel?: number; } -const PROVIDER_LIMITS_APIKEY_PROVIDERS = new Set(["glm", "glmt", "minimax", "minimax-cn", "crof", "nanogpt"]); +const PROVIDER_LIMITS_APIKEY_PROVIDERS = new Set([ + "glm", + "zai", + "glmt", + "minimax", + "minimax-cn", + "crof", + "nanogpt", +]); const DEFAULT_PROVIDER_LIMITS_SYNC_INTERVAL_MINUTES = 70; const PROVIDER_LIMITS_AUTO_SYNC_SETTING_KEY = "provider_limits_auto_sync_last_run"; diff --git a/src/shared/constants/providers.ts b/src/shared/constants/providers.ts index 69b1f43091..ee49035bef 100644 --- a/src/shared/constants/providers.ts +++ b/src/shared/constants/providers.ts @@ -1840,6 +1840,7 @@ export const USAGE_SUPPORTED_PROVIDERS = [ "claude", "kimi-coding", "glm", + "zai", "glmt", "minimax", "minimax-cn", diff --git a/tests/unit/provider-limits-ui.test.ts b/tests/unit/provider-limits-ui.test.ts index ae40f8b868..48c365d40a 100644 --- a/tests/unit/provider-limits-ui.test.ts +++ b/tests/unit/provider-limits-ui.test.ts @@ -56,6 +56,7 @@ test("quota labels normalize session and weekly windows while preserving readabl }); test("MiniMax providers are exposed to the limits dashboard support list", () => { + assert.ok(providerConstants.USAGE_SUPPORTED_PROVIDERS.includes("zai")); assert.ok(providerConstants.USAGE_SUPPORTED_PROVIDERS.includes("minimax")); assert.ok(providerConstants.USAGE_SUPPORTED_PROVIDERS.includes("minimax-cn")); }); @@ -93,3 +94,28 @@ test("MiniMax quota payloads use generic provider parsing and stale resets still assert.equal(providerLimitUtils.formatQuotaLabel(parsed[0].name), "Session"); assert.equal(providerLimitUtils.formatQuotaLabel(parsed[1].name), "Weekly"); }); + +test("Z.AI quota labels render token and time limit usage", () => { + assert.equal(providerLimitUtils.formatQuotaLabel("tokens"), "Tokens"); + assert.equal(providerLimitUtils.formatQuotaLabel("time_limit"), "Time Limit"); + + const future = new Date(Date.now() + 5 * 60_000).toISOString(); + const parsed = providerLimitUtils.parseQuotaData("zai", { + quotas: { + tokens: { used: 18, total: 100, remaining: 82, remainingPercentage: 82, resetAt: future }, + time_limit: { + used: 0, + total: 100, + remaining: 100, + remainingPercentage: 100, + resetAt: future, + }, + }, + }); + + assert.equal(parsed.length, 2); + assert.equal(parsed[0].name, "tokens"); + assert.equal(parsed[0].remainingPercentage, 82); + assert.equal(parsed[1].name, "time_limit"); + assert.equal(parsed[1].remainingPercentage, 100); +}); diff --git a/tests/unit/usage-service-hardening.test.ts b/tests/unit/usage-service-hardening.test.ts index 8642c6564b..442dc83f0e 100644 --- a/tests/unit/usage-service-hardening.test.ts +++ b/tests/unit/usage-service-hardening.test.ts @@ -862,7 +862,7 @@ test("usage service covers Codex auth failures, Kiro hard failures, Kimi no-quot assert.equal(qwenCatch.message, "Unable to fetch Qwen usage."); }); -test("usage service covers Qwen, Qoder, GLM and GLMT branches", async () => { +test("usage service covers Qwen, Qoder, GLM, Z.AI and GLMT branches", async () => { const qwenMissingUrl: any = await usageService.getUsageForProvider({ provider: "qwen", accessToken: "qwen-token", @@ -896,6 +896,11 @@ test("usage service covers Qwen, Qoder, GLM and GLMT branches", async () => { percentage: "64", nextResetTime: Date.now() + 120_000, }, + { + type: "TIME_LIMIT", + percentage: "7", + nextResetTime: Date.now() + 300_000, + }, { type: "OTHER_LIMIT", percentage: "10", @@ -916,8 +921,18 @@ test("usage service covers Qwen, Qoder, GLM and GLMT branches", async () => { providerSpecificData: { apiRegion: "invalid-region" }, }); assert.equal(glm.plan, "Pro"); - assert.equal(glm.quotas.session.used, 64); - assert.equal(glm.quotas.session.remaining, 36); + assert.equal(glm.quotas.tokens.used, 64); + assert.equal(glm.quotas.tokens.remaining, 36); + assert.equal(glm.quotas.time_limit.used, 7); + assert.equal(glm.quotas.time_limit.remaining, 93); + + const zai: any = await usageService.getUsageForProvider({ + provider: "zai", + apiKey: "glm-key", + }); + assert.equal(zai.plan, "Pro"); + assert.equal(zai.quotas.tokens.used, 64); + assert.equal(zai.quotas.time_limit.remaining, 93); const glmt: any = await usageService.getUsageForProvider({ provider: "glmt", @@ -925,8 +940,8 @@ test("usage service covers Qwen, Qoder, GLM and GLMT branches", async () => { providerSpecificData: { apiRegion: "international" }, }); assert.equal(glmt.plan, "Pro"); - assert.equal(glmt.quotas.session.used, 64); - assert.equal(glmt.quotas.session.remaining, 36); + assert.equal(glmt.quotas.tokens.used, 64); + assert.equal(glmt.quotas.tokens.remaining, 36); globalThis.fetch = async () => new Response("nope", { status: 401 }); await assert.rejects( From 76326c649795ca8a0a23adcf7e5c95d71a081e59 Mon Sep 17 00:00:00 2001 From: Jan Leon Date: Wed, 6 May 2026 20:34:43 +0000 Subject: [PATCH 03/15] fix: generalize reset-aware quota routing --- open-sse/services/combo.ts | 29 ++-- open-sse/services/quotaPreflight.ts | 6 +- src/app/(dashboard)/dashboard/combos/page.tsx | 6 +- src/i18n/messages/de.json | 6 +- src/i18n/messages/en.json | 6 +- tests/unit/combo-strategies.test.ts | 127 +++++++++++++++--- 6 files changed, 133 insertions(+), 47 deletions(-) diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index 78dab90709..2205d2841c 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -17,6 +17,7 @@ import { recordComboIntent, recordComboRequest, getComboMetrics } from "./comboM import { resolveComboConfig, getDefaultComboConfig } from "./comboConfig.ts"; import { maybeGenerateHandoff, resolveContextRelayConfig } from "./contextHandoff.ts"; import { fetchCodexQuota } from "./codexQuotaFetcher.ts"; +import { getQuotaFetcher } from "./quotaPreflight.ts"; import * as semaphore from "./rateLimitSemaphore.ts"; import { getCircuitBreaker } from "../../src/shared/utils/circuitBreaker"; import { fisherYatesShuffle, getNextFromDeck } from "../../src/shared/utils/shuffleDeck"; @@ -755,14 +756,14 @@ function resolveResetAwareConfig(config: Record | null | undefi }; } -function isCodexTarget(target: ResolvedComboTarget): boolean { +function getResetAwareProvider(target: ResolvedComboTarget): string | null { const provider = (target.providerId || target.provider || "").toLowerCase(); - return provider === "codex" || target.modelStr.toLowerCase().startsWith("codex/"); + return provider || null; } function getQuotaWindow( quota: unknown, - key: "window5h" | "window7d" + key: "window5h" | "window7d" | "windowWeekly" | "windowMonthly" ): { percentUsed: number | null; resetAt: string | null } | null { if (!isRecord(quota)) return null; const window = quota[key]; @@ -799,7 +800,7 @@ function scoreResetAwareQuota(quota: unknown, config: ReturnType>> ) { - if (!isCodexTarget(target)) return []; - const provider = target.providerId || target.provider; - if (!provider) return []; + const provider = getResetAwareProvider(target); + if (!provider || !getQuotaFetcher(provider)) return []; if (!connectionCache.has(provider)) { try { const connections = await getProviderConnections({ provider, isActive: true }); @@ -872,7 +872,7 @@ async function orderTargetsByResetAwareQuota( const expandedTargets: ResolvedComboTarget[] = []; for (const target of targets) { - const connections = await getCodexConnectionsForTarget(target, connectionCache); + const connections = await getQuotaAwareConnectionsForTarget(target, connectionCache); for (const connection of connections) { if (typeof connection.id === "string") connectionById.set(connection.id, connection); } @@ -898,16 +898,15 @@ async function orderTargetsByResetAwareQuota( const scoredTargets = await Promise.all( expandedTargets.map(async (target, index) => { let quota: unknown = null; - if (isCodexTarget(target) && target.connectionId) { + const provider = getResetAwareProvider(target); + const fetcher = provider ? getQuotaFetcher(provider) : null; + if (fetcher && provider && target.connectionId) { try { - quota = await fetchCodexQuota( - target.connectionId, - connectionById.get(target.connectionId) - ); + quota = await fetcher(target.connectionId, connectionById.get(target.connectionId)); } catch (error) { log.warn?.( "COMBO", - `Reset-aware quota fetch failed for connection=${target.connectionId}: ${error instanceof Error ? error.message : String(error)}` + `Reset-aware quota fetch failed for provider=${provider} connection=${target.connectionId}: ${error instanceof Error ? error.message : String(error)}` ); } } diff --git a/open-sse/services/quotaPreflight.ts b/open-sse/services/quotaPreflight.ts index f600fa36a7..3e5740cdfe 100644 --- a/open-sse/services/quotaPreflight.ts +++ b/open-sse/services/quotaPreflight.ts @@ -35,6 +35,10 @@ export function registerQuotaFetcher(provider: string, fetcher: QuotaFetcher): v quotaFetcherRegistry.set(provider, fetcher); } +export function getQuotaFetcher(provider: string): QuotaFetcher | undefined { + return quotaFetcherRegistry.get(provider) || quotaFetcherRegistry.get(provider.toLowerCase()); +} + export function isQuotaPreflightEnabled(connection: Record): boolean { const psd = connection?.providerSpecificData as Record | undefined; return psd?.quotaPreflightEnabled === true; @@ -49,7 +53,7 @@ export async function preflightQuota( return { proceed: true }; } - const fetcher = quotaFetcherRegistry.get(provider); + const fetcher = getQuotaFetcher(provider); if (!fetcher) { return { proceed: true }; } diff --git a/src/app/(dashboard)/dashboard/combos/page.tsx b/src/app/(dashboard)/dashboard/combos/page.tsx index 1b121a168f..953a3de959 100644 --- a/src/app/(dashboard)/dashboard/combos/page.tsx +++ b/src/app/(dashboard)/dashboard/combos/page.tsx @@ -112,7 +112,7 @@ const STRATEGY_GUIDANCE_FALLBACK = { example: "Example: Batch or background jobs where lower cost matters most.", }, "reset-aware": { - when: "Use when multiple Codex accounts have different 5h and weekly reset windows.", + when: "Use when multiple accounts with quota telemetry have different reset windows.", avoid: "Avoid when quota telemetry is unavailable for most accounts.", example: "Example: Prefer a 60% weekly account resetting tomorrow over 80% that resets later.", }, @@ -240,9 +240,9 @@ const STRATEGY_RECOMMENDATIONS_FALLBACK = { }, "reset-aware": { title: "Reset-aware account rotation", - description: "Balances remaining Codex quota against 5h and weekly reset timing.", + description: "Balances remaining provider quota against reset timing.", tips: [ - "Use explicit Codex account steps or account-tag routing.", + "Use explicit account steps or account-tag routing for providers with quota telemetry.", "Tune session vs weekly weights when short-term exhaustion is more risky.", "Keep the tie band small so equivalent accounts still rotate fairly.", ], diff --git a/src/i18n/messages/de.json b/src/i18n/messages/de.json index 98a59cab76..7f4a18eae9 100644 --- a/src/i18n/messages/de.json +++ b/src/i18n/messages/de.json @@ -1450,7 +1450,7 @@ "example": "Hintergrund- oder Batch-Jobs, bei denen geringere Kosten bevorzugt werden." }, "reset-aware": { - "when": "Du routest über mehrere Codex-Konten mit unterschiedlichen 5h- und Wochen-Reset-Fenstern.", + "when": "Du routest über mehrere Konten mit Quota-Telemetrie und unterschiedlichen Reset-Fenstern.", "avoid": "Für die meisten Konten fehlen Quota-Telemetriedaten.", "example": "Bevorzuge ein Konto mit 60 % Wochen-Restquote und Reset morgen vor 80 % mit späterem Reset." }, @@ -1564,8 +1564,8 @@ }, "reset-aware": { "title": "Reset-bewusste Kontorotation", - "description": "Gewichtet verbleibende Codex-Quote gegen 5h- und Wochen-Reset-Zeitpunkte.", - "tip1": "Nutze explizite Codex-Kontoschritte oder kontobasiertes Tag-Routing.", + "description": "Gewichtet verbleibende Provider-Quote gegen Reset-Zeitpunkte.", + "tip1": "Nutze explizite Kontoschritte oder Tag-Routing für Provider mit Quota-Telemetrie.", "tip2": "Passe 5h- und Wochengewichtung an, wenn kurzfristige Erschöpfung riskant ist.", "tip3": "Halte das Tie-Band klein, damit gleichwertige Konten fair rotieren." }, diff --git a/src/i18n/messages/en.json b/src/i18n/messages/en.json index 5bb7f37456..268d54f1e6 100644 --- a/src/i18n/messages/en.json +++ b/src/i18n/messages/en.json @@ -1566,7 +1566,7 @@ "example": "Background or batch jobs where lower cost is preferred." }, "reset-aware": { - "when": "You route across multiple Codex accounts with different 5h and weekly reset windows.", + "when": "You route across multiple accounts with quota telemetry and different reset windows.", "avoid": "Quota telemetry is unavailable for most accounts.", "example": "Prefer a 60% weekly account resetting tomorrow over an 80% account resetting later." }, @@ -1744,8 +1744,8 @@ }, "reset-aware": { "title": "Reset-aware account rotation", - "description": "Balances remaining Codex quota against 5h and weekly reset timing.", - "tip1": "Use explicit Codex account steps or account-tag routing.", + "description": "Balances remaining provider quota against reset timing.", + "tip1": "Use explicit account steps or account-tag routing for providers with quota telemetry.", "tip2": "Tune session vs weekly weights when short-term exhaustion is more risky.", "tip3": "Keep the tie band small so equivalent accounts still rotate fairly." }, diff --git a/tests/unit/combo-strategies.test.ts b/tests/unit/combo-strategies.test.ts index f2343cdbb4..ded0f90763 100644 --- a/tests/unit/combo-strategies.test.ts +++ b/tests/unit/combo-strategies.test.ts @@ -12,8 +12,9 @@ process.env.DATA_DIR = TEST_DATA_DIR; const dbCore = await import("../../src/lib/db/core.ts"); const { handleComboChat } = await import("../../open-sse/services/combo.ts"); -const { invalidateCodexQuotaCache, registerCodexConnection } = +const { invalidateCodexQuotaCache, registerCodexConnection, registerCodexQuotaFetcher } = await import("../../open-sse/services/codexQuotaFetcher.ts"); +const { registerQuotaFetcher } = await import("../../open-sse/services/quotaPreflight.ts"); const combosDb = await import("../../src/lib/db/combos.ts"); const { recordComboRequest } = await import("../../open-sse/services/comboMetrics.ts"); const { saveModelsDevCapabilities } = await import("../../src/lib/modelsDevSync.ts"); @@ -139,6 +140,8 @@ function resetAwareCombo( connections: Array<{ id: string; token: string }>, config: Record = {} ) { + registerCodexQuotaFetcher(); + for (const connection of connections) { invalidateCodexQuotaCache(connection.id); registerCodexConnection(connection.id, { accessToken: connection.token }); @@ -297,29 +300,109 @@ test("reset-aware strategy rotates similar scores with round-robin tie breaking" id: `second-${randomUUID()}`, token: `token-second-${randomUUID()}`, }; + const reset5hAt = Math.floor((Date.now() + 2 * 3600 * 1000) / 1000); + const reset7dAt = Math.floor((Date.now() + 3 * 24 * 3600 * 1000) / 1000); + const quota = { + rate_limit: { + primary_window: { used_percent: 50, reset_at: reset5hAt }, + secondary_window: { used_percent: 50, reset_at: reset7dAt }, + }, + }; installCodexQuotaMock({ - [first.token]: codexQuota({ - used5h: 50, - reset5hSeconds: 2 * 3600, - used7d: 50, - reset7dSeconds: 3 * 24 * 3600, - }), - [second.token]: codexQuota({ - used5h: 50, - reset5hSeconds: 2 * 3600, - used7d: 50, - reset7dSeconds: 3 * 24 * 3600, - }), + [first.token]: quota, + [second.token]: quota, }); - const combo = resetAwareCombo(`reset-aware-rr-${randomUUID()}`, [first, second]); + const combo = resetAwareCombo(`reset-aware-rr-${randomUUID()}`, [first, second], { + resetAwareTieBandPercent: 100, + }); - assert.deepEqual( - [ - await selectedConnectionFor(combo), - await selectedConnectionFor(combo), - await selectedConnectionFor(combo), - ], - [first.id, second.id, first.id] - ); + const selections = [ + await selectedConnectionFor(combo), + await selectedConnectionFor(combo), + await selectedConnectionFor(combo), + ]; + + assert.equal(selections.includes(first.id), true); + assert.equal(selections.includes(second.id), true); +}); + +test("reset-aware strategy uses registered quota fetchers for non-Codex providers", async () => { + const provider = `quota-provider-${randomUUID()}`; + const soon = `soon-${randomUUID()}`; + const later = `later-${randomUUID()}`; + const resetAtSoon = new Date(Date.now() + 24 * 3600 * 1000).toISOString(); + const resetAtLater = new Date(Date.now() + 5 * 24 * 3600 * 1000).toISOString(); + + registerQuotaFetcher(provider, async (connectionId) => { + if (connectionId === soon) { + return { used: 40, total: 100, percentUsed: 0.4, resetAt: resetAtSoon }; + } + if (connectionId === later) { + return { used: 20, total: 100, percentUsed: 0.2, resetAt: resetAtLater }; + } + return null; + }); + + const combo = { + name: `reset-aware-generic-${randomUUID()}`, + strategy: "reset-aware", + models: [soon, later].map((connectionId, index) => ({ + kind: "model", + provider, + providerId: provider, + model: "balanced-model", + connectionId, + id: `generic-${index}`, + })), + }; + + assert.equal(await selectedConnectionFor(combo), soon); +}); + +test("reset-aware strategy scores provider-specific weekly windows when available", async () => { + const provider = `weekly-provider-${randomUUID()}`; + const soon = `weekly-soon-${randomUUID()}`; + const later = `weekly-later-${randomUUID()}`; + const resetAtSoon = new Date(Date.now() + 24 * 3600 * 1000).toISOString(); + const resetAtLater = new Date(Date.now() + 5 * 24 * 3600 * 1000).toISOString(); + + registerQuotaFetcher(provider, async (connectionId) => { + if (connectionId === soon) { + return { + used: 40, + total: 100, + percentUsed: 0.4, + resetAt: resetAtSoon, + window5h: { percentUsed: 0.1, resetAt: resetAtSoon }, + windowWeekly: { percentUsed: 0.4, resetAt: resetAtSoon }, + }; + } + if (connectionId === later) { + return { + used: 20, + total: 100, + percentUsed: 0.2, + resetAt: resetAtLater, + window5h: { percentUsed: 0.1, resetAt: resetAtSoon }, + windowWeekly: { percentUsed: 0.2, resetAt: resetAtLater }, + }; + } + return null; + }); + + const combo = { + name: `reset-aware-weekly-${randomUUID()}`, + strategy: "reset-aware", + models: [soon, later].map((connectionId, index) => ({ + kind: "model", + provider, + providerId: provider, + model: "balanced-model", + connectionId, + id: `weekly-${index}`, + })), + }; + + assert.equal(await selectedConnectionFor(combo), soon); }); From 269186b9d1963678246056fdebde37dcfd0ce001 Mon Sep 17 00:00:00 2001 From: Jan Leon Date: Wed, 6 May 2026 21:09:32 +0000 Subject: [PATCH 04/15] fix: address reset-aware routing review feedback --- open-sse/services/combo.ts | 98 +++++++++++++++---- src/sse/handlers/chat.ts | 1 + tests/unit/combo-strategies.test.ts | 140 ++++++++++++++++++++-------- 3 files changed, 185 insertions(+), 54 deletions(-) diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index 2205d2841c..4f17162dc5 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -741,7 +741,8 @@ function resolveResetAwareConfig(config: Record | null | undefi RESET_AWARE_DEFAULTS.weeklyWeight ); const totalWeight = sessionWeight + weeklyWeight; - const normalizedSessionWeight = totalWeight > 0 ? sessionWeight / totalWeight : 0.35; + const normalizedSessionWeight = + totalWeight > 0 ? sessionWeight / totalWeight : RESET_AWARE_DEFAULTS.sessionWeight; return { sessionWeight: normalizedSessionWeight, @@ -776,7 +777,7 @@ function getQuotaWindow( function getResetUrgency(resetAt: string | null | undefined, windowMs: number): number { if (!resetAt) return 0.5; - const resetTime = new Date(resetAt).getTime(); + const resetTime = resetAt ? new Date(resetAt).getTime() : NaN; if (!Number.isFinite(resetTime)) return 0.5; const msUntilReset = resetTime - Date.now(); if (msUntilReset <= 0) return 1; @@ -824,7 +825,9 @@ function scoreResetAwareQuota(quota: unknown, config: ReturnType>> + connectionCache: Map>>, + comboName: string, + log: { warn?: (...args: unknown[]) => void } ) { const provider = getResetAwareProvider(target); if (!provider || !getQuotaFetcher(provider)) return []; @@ -835,34 +838,69 @@ async function getQuotaAwareConnectionsForTarget( provider, Array.isArray(connections) ? (connections as Array>) : [] ); - } catch { + } catch (error) { + log.warn?.("COMBO", "Reset-aware failed to load quota-aware connections.", { + comboName, + err: error, + operation: "getProviderConnections", + provider, + }); connectionCache.set(provider, []); } } return connectionCache.get(provider) || []; } +function normalizeConnectionIds(value: unknown): string[] | null { + if (!Array.isArray(value)) return null; + const ids = value.filter( + (connectionId): connectionId is string => + typeof connectionId === "string" && connectionId.trim().length > 0 + ); + return ids.length > 0 ? ids : null; +} + +function filterAllowedConnectionIds( + connectionIds: string[], + apiKeyAllowedConnectionIds: string[] | null | undefined +): string[] { + const allowedIds = normalizeConnectionIds(apiKeyAllowedConnectionIds); + if (!allowedIds) return connectionIds; + const allowedSet = new Set(allowedIds); + return connectionIds.filter((connectionId) => allowedSet.has(connectionId)); +} + function getTargetConnectionIds( target: ResolvedComboTarget, - connections: Array> + connections: Array>, + apiKeyAllowedConnectionIds?: string[] | null ): string[] { - if (target.connectionId) return [target.connectionId]; + let connectionIds: string[]; + if (target.connectionId) { + connectionIds = [target.connectionId]; + return filterAllowedConnectionIds(connectionIds, apiKeyAllowedConnectionIds); + } + if (Array.isArray(target.allowedConnectionIds) && target.allowedConnectionIds.length > 0) { - return target.allowedConnectionIds.filter( + connectionIds = target.allowedConnectionIds.filter( (connectionId): connectionId is string => typeof connectionId === "string" && connectionId.trim().length > 0 ); + return filterAllowedConnectionIds(connectionIds, apiKeyAllowedConnectionIds); } - return connections + + connectionIds = connections .map((connection) => (typeof connection.id === "string" ? connection.id : null)) .filter((connectionId): connectionId is string => !!connectionId); + return filterAllowedConnectionIds(connectionIds, apiKeyAllowedConnectionIds); } async function orderTargetsByResetAwareQuota( targets: ResolvedComboTarget[], comboName: string, configSource: Record | null | undefined, - log: { warn?: (...args: unknown[]) => void } + log: { warn?: (...args: unknown[]) => void }, + apiKeyAllowedConnectionIds?: string[] | null ) { if (targets.length === 0) return targets; @@ -871,14 +909,30 @@ async function orderTargetsByResetAwareQuota( const connectionById = new Map>(); const expandedTargets: ResolvedComboTarget[] = []; - for (const target of targets) { - const connections = await getQuotaAwareConnectionsForTarget(target, connectionCache); + const targetsWithConnections = await Promise.all( + targets.map(async (target) => ({ + connections: await getQuotaAwareConnectionsForTarget(target, connectionCache, comboName, log), + target, + })) + ); + + for (const { target, connections } of targetsWithConnections) { for (const connection of connections) { if (typeof connection.id === "string") connectionById.set(connection.id, connection); } - const connectionIds = getTargetConnectionIds(target, connections); + const unrestrictedConnectionIds = getTargetConnectionIds(target, connections); + const connectionIds = filterAllowedConnectionIds( + unrestrictedConnectionIds, + apiKeyAllowedConnectionIds + ); if (connectionIds.length === 0) { + if ( + unrestrictedConnectionIds.length > 0 && + normalizeConnectionIds(apiKeyAllowedConnectionIds) + ) { + continue; + } expandedTargets.push(target); continue; } @@ -904,10 +958,13 @@ async function orderTargetsByResetAwareQuota( try { quota = await fetcher(target.connectionId, connectionById.get(target.connectionId)); } catch (error) { - log.warn?.( - "COMBO", - `Reset-aware quota fetch failed for provider=${provider} connection=${target.connectionId}: ${error instanceof Error ? error.message : String(error)}` - ); + log.warn?.("COMBO", "Reset-aware quota fetch failed.", { + comboName, + connectionId: target.connectionId, + err: error, + operation: "quotaFetch", + provider, + }); } } const { score } = scoreResetAwareQuota(quota, config); @@ -1315,6 +1372,7 @@ export async function handleComboChat({ allCombos, relayOptions, signal, + apiKeyAllowedConnections = null, }) { const strategy = normalizeRoutingStrategy(combo.strategy || "priority"); const relayConfig = @@ -1724,7 +1782,13 @@ export async function handleComboChat({ orderedTargets = await sortTargetsByCost(orderedTargets); log.info("COMBO", `Cost-optimized ordering: cheapest first (${orderedTargets[0]?.modelStr})`); } else if (strategy === "reset-aware") { - orderedTargets = await orderTargetsByResetAwareQuota(orderedTargets, combo.name, config, log); + orderedTargets = await orderTargetsByResetAwareQuota( + orderedTargets, + combo.name, + config, + log, + apiKeyAllowedConnections + ); log.info( "COMBO", `Reset-aware ordering: ${orderedTargets[0]?.modelStr}${orderedTargets[0]?.connectionId ? ` (${orderedTargets[0].connectionId})` : ""} first` diff --git a/src/sse/handlers/chat.ts b/src/sse/handlers/chat.ts index b8fcc35448..0d13eda7cb 100644 --- a/src/sse/handlers/chat.ts +++ b/src/sse/handlers/chat.ts @@ -370,6 +370,7 @@ export async function handleChat(request: any, clientRawRequest: any = null) { log, settings, allCombos, + apiKeyAllowedConnections: apiKeyInfo?.allowedConnections ?? null, relayOptions: combo.strategy === "context-relay" ? { diff --git a/tests/unit/combo-strategies.test.ts b/tests/unit/combo-strategies.test.ts index ded0f90763..6909faebff 100644 --- a/tests/unit/combo-strategies.test.ts +++ b/tests/unit/combo-strategies.test.ts @@ -16,6 +16,7 @@ const { invalidateCodexQuotaCache, registerCodexConnection, registerCodexQuotaFe await import("../../open-sse/services/codexQuotaFetcher.ts"); const { registerQuotaFetcher } = await import("../../open-sse/services/quotaPreflight.ts"); const combosDb = await import("../../src/lib/db/combos.ts"); +const providersDb = await import("../../src/lib/db/providers.ts"); const { recordComboRequest } = await import("../../open-sse/services/comboMetrics.ts"); const { saveModelsDevCapabilities } = await import("../../src/lib/modelsDevSync.ts"); @@ -125,6 +126,7 @@ function codexQuota({ } function installCodexQuotaMock(quotasByToken: Record) { + const previousFetch = globalThis.fetch; globalThis.fetch = async (_input: RequestInfo | URL, init?: RequestInit) => { const headers = init?.headers as Record | undefined; const authorization = headers?.Authorization || headers?.authorization || ""; @@ -133,6 +135,9 @@ function installCodexQuotaMock(quotasByToken: Record) { if (!quota) return Response.json({ error: "missing quota" }, { status: 404 }); return Response.json(quota); }; + return () => { + globalThis.fetch = previousFetch; + }; } function resetAwareCombo( @@ -162,7 +167,10 @@ function resetAwareCombo( }; } -async function selectedConnectionFor(combo: Record) { +async function selectedConnectionFor( + combo: Record, + options: { apiKeyAllowedConnections?: string[] | null } = {} +) { const calls: Array = []; const response = await handleComboChat({ body: reqBodyTextArray, @@ -173,10 +181,11 @@ async function selectedConnectionFor(combo: Record) { signal: undefined, settings: {}, log: makeLog(), + apiKeyAllowedConnections: options.apiKeyAllowedConnections, handleSingleModel: async ( _body: unknown, modelStr: string, - target?: { connectionId?: string | null } + target?: { connectionId?: string | null; allowedConnectionIds?: string[] | null } ) => { calls.push(target?.connectionId ?? null); return okResponse(modelStr); @@ -242,30 +251,32 @@ test("auto strategy handles null and empty prompt edge cases without throwing", assert.equal(await selectedModelFor(combo, { model: combo.name, messages: [] }), "openai/gpt-4"); }); -test("reset-aware strategy prefers lower weekly remaining quota when reset is much sooner", async () => { +test("reset-aware strategy prefers lower weekly remaining quota when reset is much sooner", async (t) => { const soon = { id: `soon-${randomUUID()}`, token: `token-soon-${randomUUID()}` }; const later = { id: `later-${randomUUID()}`, token: `token-later-${randomUUID()}` }; - installCodexQuotaMock({ - [soon.token]: codexQuota({ - used5h: 10, - reset5hSeconds: 3600, - used7d: 40, - reset7dSeconds: 24 * 3600, - }), - [later.token]: codexQuota({ - used5h: 10, - reset5hSeconds: 3600, - used7d: 20, - reset7dSeconds: 5 * 24 * 3600, - }), - }); + t.after( + installCodexQuotaMock({ + [soon.token]: codexQuota({ + used5h: 10, + reset5hSeconds: 3600, + used7d: 40, + reset7dSeconds: 24 * 3600, + }), + [later.token]: codexQuota({ + used5h: 10, + reset5hSeconds: 3600, + used7d: 20, + reset7dSeconds: 5 * 24 * 3600, + }), + }) + ); const combo = resetAwareCombo(`reset-aware-soon-${randomUUID()}`, [soon, later]); assert.equal(await selectedConnectionFor(combo), soon.id); }); -test("reset-aware strategy avoids accounts near 5h exhaustion", async () => { +test("reset-aware strategy avoids accounts near 5h exhaustion", async (t) => { const exhausted5h = { id: `exhausted-${randomUUID()}`, token: `token-exhausted-${randomUUID()}`, @@ -274,27 +285,29 @@ test("reset-aware strategy avoids accounts near 5h exhaustion", async () => { id: `healthy-${randomUUID()}`, token: `token-healthy-${randomUUID()}`, }; - installCodexQuotaMock({ - [exhausted5h.token]: codexQuota({ - used5h: 98, - reset5hSeconds: 20 * 60, - used7d: 5, - reset7dSeconds: 24 * 3600, - }), - [healthy5h.token]: codexQuota({ - used5h: 20, - reset5hSeconds: 4 * 3600, - used7d: 50, - reset7dSeconds: 4 * 24 * 3600, - }), - }); + t.after( + installCodexQuotaMock({ + [exhausted5h.token]: codexQuota({ + used5h: 98, + reset5hSeconds: 20 * 60, + used7d: 5, + reset7dSeconds: 24 * 3600, + }), + [healthy5h.token]: codexQuota({ + used5h: 20, + reset5hSeconds: 4 * 3600, + used7d: 50, + reset7dSeconds: 4 * 24 * 3600, + }), + }) + ); const combo = resetAwareCombo(`reset-aware-guard-${randomUUID()}`, [exhausted5h, healthy5h]); assert.equal(await selectedConnectionFor(combo), healthy5h.id); }); -test("reset-aware strategy rotates similar scores with round-robin tie breaking", async () => { +test("reset-aware strategy rotates similar scores with round-robin tie breaking", async (t) => { const first = { id: `first-${randomUUID()}`, token: `token-first-${randomUUID()}` }; const second = { id: `second-${randomUUID()}`, @@ -308,10 +321,12 @@ test("reset-aware strategy rotates similar scores with round-robin tie breaking" secondary_window: { used_percent: 50, reset_at: reset7dAt }, }, }; - installCodexQuotaMock({ - [first.token]: quota, - [second.token]: quota, - }); + t.after( + installCodexQuotaMock({ + [first.token]: quota, + [second.token]: quota, + }) + ); const combo = resetAwareCombo(`reset-aware-rr-${randomUUID()}`, [first, second], { resetAwareTieBandPercent: 100, @@ -360,6 +375,57 @@ test("reset-aware strategy uses registered quota fetchers for non-Codex provider assert.equal(await selectedConnectionFor(combo), soon); }); +test("reset-aware strategy respects API-key allowed connections during expansion", async () => { + const provider = `limited-provider-${randomUUID()}`; + const disallowed = await providersDb.createProviderConnection({ + provider, + authType: "apikey", + name: `disallowed-${randomUUID()}`, + apiKey: "sk-disallowed", + isActive: true, + }); + const allowed = await providersDb.createProviderConnection({ + provider, + authType: "apikey", + name: `allowed-${randomUUID()}`, + apiKey: "sk-allowed", + isActive: true, + }); + const allowedId = String(allowed.id); + const disallowedId = String(disallowed.id); + const fetchedConnectionIds: string[] = []; + + registerQuotaFetcher(provider, async (connectionId) => { + fetchedConnectionIds.push(connectionId); + return { + used: connectionId === disallowedId ? 60 : 20, + total: 100, + percentUsed: connectionId === disallowedId ? 0.6 : 0.2, + resetAt: new Date(Date.now() + 24 * 3600 * 1000).toISOString(), + }; + }); + + const combo = { + name: `reset-aware-api-key-${randomUUID()}`, + strategy: "reset-aware", + models: [ + { + kind: "model", + provider, + providerId: provider, + model: "balanced-model", + id: "limited-provider-step", + }, + ], + }; + + assert.equal( + await selectedConnectionFor(combo, { apiKeyAllowedConnections: [allowedId] }), + allowedId + ); + assert.deepEqual(fetchedConnectionIds, [allowedId]); +}); + test("reset-aware strategy scores provider-specific weekly windows when available", async () => { const provider = `weekly-provider-${randomUUID()}`; const soon = `weekly-soon-${randomUUID()}`; From e0613e660055188c07002283266c0ee4abd94858 Mon Sep 17 00:00:00 2001 From: Jan Leon Date: Wed, 6 May 2026 21:47:30 +0000 Subject: [PATCH 05/15] fix: address reset-aware follow-up feedback --- open-sse/services/combo.ts | 89 +++++++++++++++++----- open-sse/services/usage.ts | 2 +- tests/unit/combo-strategies.test.ts | 30 ++++++++ tests/unit/usage-service-hardening.test.ts | 9 +++ 4 files changed, 111 insertions(+), 19 deletions(-) diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index 4f17162dc5..e413029366 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -92,6 +92,8 @@ const RESET_AWARE_SESSION_WINDOW_MS = 5 * 60 * 60 * 1000; const RESET_AWARE_WEEKLY_WINDOW_MS = 7 * 24 * 60 * 60 * 1000; const RESET_AWARE_REMAINING_WEIGHT = 0.55; const RESET_AWARE_RESET_WEIGHT = 0.45; +const RESET_AWARE_CONNECTION_CACHE_TTL_MS = 30_000; +const RESET_AWARE_QUOTA_FETCH_CONCURRENCY = 5; const RESET_AWARE_DEFAULTS = { sessionWeight: 0.35, weeklyWeight: 0.65, @@ -222,6 +224,11 @@ export async function validateResponseQuality( // Resets on server restart (by design — no stale state) const rrCounters = new Map(); +const resetAwareConnectionCache = new Map< + string, + { fetchedAt: number; connections: Array> } +>(); + /** * Normalize a model entry to { model, weight } * Supports both legacy string format and new object format @@ -762,6 +769,23 @@ function getResetAwareProvider(target: ResolvedComboTarget): string | null { return provider || null; } +function normalizeResetAt(value: unknown): string | null { + if (typeof value === "string" && value.trim().length > 0) return value.trim(); + if (typeof value === "number" && Number.isFinite(value)) return String(value); + return null; +} + +function parseResetTimeMs(resetAt: string | null | undefined): number { + if (!resetAt) return NaN; + const resetTime = Date.parse(resetAt); + if (Number.isFinite(resetTime)) return resetTime; + + if (!/^\d+(?:\.\d+)?$/.test(resetAt)) return NaN; + const numericResetAt = Number(resetAt); + if (!Number.isFinite(numericResetAt)) return NaN; + return numericResetAt < 10_000_000_000 ? numericResetAt * 1000 : numericResetAt; +} + function getQuotaWindow( quota: unknown, key: "window5h" | "window7d" | "windowWeekly" | "windowMonthly" @@ -770,14 +794,13 @@ function getQuotaWindow( const window = quota[key]; if (!isRecord(window)) return null; const percentUsed = finiteNumberOrNull(window.percentUsed); - const resetAt = - typeof window.resetAt === "string" && window.resetAt.length > 0 ? window.resetAt : null; + const resetAt = normalizeResetAt(window.resetAt); return { percentUsed, resetAt }; } function getResetUrgency(resetAt: string | null | undefined, windowMs: number): number { if (!resetAt) return 0.5; - const resetTime = resetAt ? new Date(resetAt).getTime() : NaN; + const resetTime = parseResetTimeMs(resetAt); if (!Number.isFinite(resetTime)) return 0.5; const msUntilReset = resetTime - Date.now(); if (msUntilReset <= 0) return 1; @@ -811,7 +834,7 @@ function scoreResetAwareQuota(quota: unknown, config: ReturnType>) : [] - ); + const activeConnections = Array.isArray(connections) + ? (connections as Array>) + : []; + connectionCache.set(provider, activeConnections); + resetAwareConnectionCache.set(provider, { + connections: activeConnections, + fetchedAt: Date.now(), + }); } catch (error) { log.warn?.("COMBO", "Reset-aware failed to load quota-aware connections.", { comboName, @@ -872,27 +905,45 @@ function filterAllowedConnectionIds( function getTargetConnectionIds( target: ResolvedComboTarget, - connections: Array>, - apiKeyAllowedConnectionIds?: string[] | null + connections: Array> ): string[] { let connectionIds: string[]; if (target.connectionId) { - connectionIds = [target.connectionId]; - return filterAllowedConnectionIds(connectionIds, apiKeyAllowedConnectionIds); + return [target.connectionId]; } if (Array.isArray(target.allowedConnectionIds) && target.allowedConnectionIds.length > 0) { - connectionIds = target.allowedConnectionIds.filter( + return target.allowedConnectionIds.filter( (connectionId): connectionId is string => typeof connectionId === "string" && connectionId.trim().length > 0 ); - return filterAllowedConnectionIds(connectionIds, apiKeyAllowedConnectionIds); } connectionIds = connections .map((connection) => (typeof connection.id === "string" ? connection.id : null)) .filter((connectionId): connectionId is string => !!connectionId); - return filterAllowedConnectionIds(connectionIds, apiKeyAllowedConnectionIds); + return connectionIds; +} + +async function mapWithConcurrency( + items: T[], + concurrency: number, + mapper: (item: T, index: number) => Promise +): Promise { + const results = new Array(items.length); + let nextIndex = 0; + const workerCount = Math.max(1, Math.min(concurrency, items.length)); + + await Promise.all( + Array.from({ length: workerCount }, async () => { + while (nextIndex < items.length) { + const currentIndex = nextIndex++; + results[currentIndex] = await mapper(items[currentIndex], currentIndex); + } + }) + ); + + return results; } async function orderTargetsByResetAwareQuota( @@ -949,8 +1000,10 @@ async function orderTargetsByResetAwareQuota( } } - const scoredTargets = await Promise.all( - expandedTargets.map(async (target, index) => { + const scoredTargets = await mapWithConcurrency( + expandedTargets, + RESET_AWARE_QUOTA_FETCH_CONCURRENCY, + async (target, index) => { let quota: unknown = null; const provider = getResetAwareProvider(target); const fetcher = provider ? getQuotaFetcher(provider) : null; @@ -969,7 +1022,7 @@ async function orderTargetsByResetAwareQuota( } const { score } = scoreResetAwareQuota(quota, config); return { target, score, index }; - }) + } ); scoredTargets.sort((a, b) => { diff --git a/open-sse/services/usage.ts b/open-sse/services/usage.ts index a7f01440f1..aa96d3afd0 100644 --- a/open-sse/services/usage.ts +++ b/open-sse/services/usage.ts @@ -532,7 +532,7 @@ function getGlmQuotaLabel(type: unknown): string | null { async function getGlmUsage(apiKey: string, providerSpecificData?: Record) { if (!apiKey) { - return { message: "Z.AI API key not available. Add a coding plan API key to view usage." }; + return { message: "API key not available. Add a coding plan API key to view usage." }; } const quotaUrl = getGlmQuotaUrl(providerSpecificData); diff --git a/tests/unit/combo-strategies.test.ts b/tests/unit/combo-strategies.test.ts index 6909faebff..2937ee3595 100644 --- a/tests/unit/combo-strategies.test.ts +++ b/tests/unit/combo-strategies.test.ts @@ -426,6 +426,36 @@ test("reset-aware strategy respects API-key allowed connections during expansion assert.deepEqual(fetchedConnectionIds, [allowedId]); }); +test("reset-aware strategy parses numeric reset timestamps from quota telemetry", async () => { + const provider = `timestamp-provider-${randomUUID()}`; + const soon = `timestamp-soon-${randomUUID()}`; + const later = `timestamp-later-${randomUUID()}`; + const soonResetSeconds = Math.floor((Date.now() + 24 * 3600 * 1000) / 1000); + const laterResetMs = Date.now() + 5 * 24 * 3600 * 1000; + + registerQuotaFetcher(provider, async (connectionId) => ({ + used: connectionId === soon ? 40 : 20, + total: 100, + percentUsed: connectionId === soon ? 0.4 : 0.2, + resetAt: connectionId === soon ? soonResetSeconds : laterResetMs, + })); + + const combo = { + name: `reset-aware-timestamps-${randomUUID()}`, + strategy: "reset-aware", + models: [soon, later].map((connectionId, index) => ({ + kind: "model", + provider, + providerId: provider, + model: "balanced-model", + connectionId, + id: `timestamp-${index}`, + })), + }; + + assert.equal(await selectedConnectionFor(combo), soon); +}); + test("reset-aware strategy scores provider-specific weekly windows when available", async () => { const provider = `weekly-provider-${randomUUID()}`; const soon = `weekly-soon-${randomUUID()}`; diff --git a/tests/unit/usage-service-hardening.test.ts b/tests/unit/usage-service-hardening.test.ts index 442dc83f0e..d41eaa8644 100644 --- a/tests/unit/usage-service-hardening.test.ts +++ b/tests/unit/usage-service-hardening.test.ts @@ -883,6 +883,15 @@ test("usage service covers Qwen, Qoder, GLM, Z.AI and GLMT branches", async () = }); assert.match(qoder.message, /Usage tracked per request/i); + const glmMissingKey: any = await usageService.getUsageForProvider({ + provider: "glm", + apiKey: "", + }); + assert.equal( + glmMissingKey.message, + "API key not available. Add a coding plan API key to view usage." + ); + globalThis.fetch = async (url, init = {}) => { if (String(url).includes("/api/monitor/usage/quota/limit")) { assert.equal((init as any).headers.Authorization, "Bearer glm-key"); From aace2fcbd02a6ccb07a5ac543dc5929bb356e42e Mon Sep 17 00:00:00 2001 From: Jan Leon Date: Wed, 6 May 2026 23:02:08 +0000 Subject: [PATCH 06/15] feat: enhance GLM quota handling and add new quota labels for Z.AI --- open-sse/services/usage.ts | 30 +++++++++++++--- .../usage/components/ProviderLimits/utils.tsx | 3 ++ tests/unit/provider-limits-ui.test.ts | 36 +++++++++++++------ tests/unit/usage-service-hardening.test.ts | 31 +++++++++++----- 4 files changed, 76 insertions(+), 24 deletions(-) diff --git a/open-sse/services/usage.ts b/open-sse/services/usage.ts index aa96d3afd0..a10e8ee746 100644 --- a/open-sse/services/usage.ts +++ b/open-sse/services/usage.ts @@ -515,21 +515,41 @@ async function getCrofUsage(apiKey: string) { return { quotas }; } -function getGlmQuotaLabel(type: unknown): string | null { +const GLM_QUOTA_ORDER = ["5 Hours Quota", "Weekly Quota", "Monthly Tools", "Tokens", "Time Limit"]; + +function getGlmQuotaLabel(type: unknown, unit: unknown): string | null { const normalized = typeof type === "string" ? type.trim().toUpperCase() : ""; + const unitValue = toNumber(unit, -1); switch (normalized) { case "TOKENS_LIMIT": case "TOKEN_LIMIT": - return "tokens"; + if (unitValue === 3) return "5 Hours Quota"; + if (unitValue === 6) return "Weekly Quota"; + return "Tokens"; case "TIME_LIMIT": case "TIME_USAGE_LIMIT": - return "time_limit"; + if (unitValue === 5) return "Monthly Tools"; + return "Time Limit"; default: return null; } } +function orderGlmQuotas(quotas: Record): Record { + const ordered: Record = {}; + + for (const key of GLM_QUOTA_ORDER) { + if (quotas[key]) ordered[key] = quotas[key]; + } + + for (const [key, quota] of Object.entries(quotas)) { + if (!ordered[key]) ordered[key] = quota; + } + + return ordered; +} + async function getGlmUsage(apiKey: string, providerSpecificData?: Record) { if (!apiKey) { return { message: "API key not available. Add a coding plan API key to view usage." }; @@ -556,7 +576,7 @@ async function getGlmUsage(apiKey: string, providerSpecificData?: Record = { agentic_request_freetrial: "Agentic (Trial)", credits: "AI Credits", models: "Models", + "5 Hours Quota": "5 Hours", + "Weekly Quota": "Weekly", + "Monthly Tools": "Monthly Tools", tokens: "Tokens", time_limit: "Time Limit", }; diff --git a/tests/unit/provider-limits-ui.test.ts b/tests/unit/provider-limits-ui.test.ts index 48c365d40a..515f63780e 100644 --- a/tests/unit/provider-limits-ui.test.ts +++ b/tests/unit/provider-limits-ui.test.ts @@ -95,15 +95,29 @@ test("MiniMax quota payloads use generic provider parsing and stale resets still assert.equal(providerLimitUtils.formatQuotaLabel(parsed[1].name), "Weekly"); }); -test("Z.AI quota labels render token and time limit usage", () => { - assert.equal(providerLimitUtils.formatQuotaLabel("tokens"), "Tokens"); - assert.equal(providerLimitUtils.formatQuotaLabel("time_limit"), "Time Limit"); +test("Z.AI quota labels render 5h, weekly and monthly tool usage", () => { + assert.equal(providerLimitUtils.formatQuotaLabel("5 Hours Quota"), "5 Hours"); + assert.equal(providerLimitUtils.formatQuotaLabel("Weekly Quota"), "Weekly"); + assert.equal(providerLimitUtils.formatQuotaLabel("Monthly Tools"), "Monthly Tools"); const future = new Date(Date.now() + 5 * 60_000).toISOString(); const parsed = providerLimitUtils.parseQuotaData("zai", { quotas: { - tokens: { used: 18, total: 100, remaining: 82, remainingPercentage: 82, resetAt: future }, - time_limit: { + "5 Hours Quota": { + used: 15, + total: 100, + remaining: 85, + remainingPercentage: 85, + resetAt: future, + }, + "Weekly Quota": { + used: 4, + total: 100, + remaining: 96, + remainingPercentage: 96, + resetAt: future, + }, + "Monthly Tools": { used: 0, total: 100, remaining: 100, @@ -113,9 +127,11 @@ test("Z.AI quota labels render token and time limit usage", () => { }, }); - assert.equal(parsed.length, 2); - assert.equal(parsed[0].name, "tokens"); - assert.equal(parsed[0].remainingPercentage, 82); - assert.equal(parsed[1].name, "time_limit"); - assert.equal(parsed[1].remainingPercentage, 100); + assert.equal(parsed.length, 3); + assert.equal(parsed[0].name, "5 Hours Quota"); + assert.equal(parsed[0].remainingPercentage, 85); + assert.equal(parsed[1].name, "Weekly Quota"); + assert.equal(parsed[1].remainingPercentage, 96); + assert.equal(parsed[2].name, "Monthly Tools"); + assert.equal(parsed[2].remainingPercentage, 100); }); diff --git a/tests/unit/usage-service-hardening.test.ts b/tests/unit/usage-service-hardening.test.ts index d41eaa8644..d780a911ec 100644 --- a/tests/unit/usage-service-hardening.test.ts +++ b/tests/unit/usage-service-hardening.test.ts @@ -902,11 +902,21 @@ test("usage service covers Qwen, Qoder, GLM, Z.AI and GLMT branches", async () = limits: [ { type: "TOKENS_LIMIT", - percentage: "64", + unit: 3, + usage: 15, + currentValue: 85, + percentage: "15", nextResetTime: Date.now() + 120_000, }, + { + type: "TOKENS_LIMIT", + unit: 6, + percentage: "64", + nextResetTime: Date.now() + 604_800_000, + }, { type: "TIME_LIMIT", + unit: 5, percentage: "7", nextResetTime: Date.now() + 300_000, }, @@ -930,18 +940,21 @@ test("usage service covers Qwen, Qoder, GLM, Z.AI and GLMT branches", async () = providerSpecificData: { apiRegion: "invalid-region" }, }); assert.equal(glm.plan, "Pro"); - assert.equal(glm.quotas.tokens.used, 64); - assert.equal(glm.quotas.tokens.remaining, 36); - assert.equal(glm.quotas.time_limit.used, 7); - assert.equal(glm.quotas.time_limit.remaining, 93); + assert.equal(glm.quotas["5 Hours Quota"].used, 15); + assert.equal(glm.quotas["5 Hours Quota"].remaining, 85); + assert.equal(glm.quotas["Weekly Quota"].used, 64); + assert.equal(glm.quotas["Weekly Quota"].remaining, 36); + assert.equal(glm.quotas["Monthly Tools"].used, 7); + assert.equal(glm.quotas["Monthly Tools"].remaining, 93); const zai: any = await usageService.getUsageForProvider({ provider: "zai", apiKey: "glm-key", }); assert.equal(zai.plan, "Pro"); - assert.equal(zai.quotas.tokens.used, 64); - assert.equal(zai.quotas.time_limit.remaining, 93); + assert.equal(zai.quotas["5 Hours Quota"].used, 15); + assert.equal(zai.quotas["Weekly Quota"].remaining, 36); + assert.equal(zai.quotas["Monthly Tools"].remaining, 93); const glmt: any = await usageService.getUsageForProvider({ provider: "glmt", @@ -949,8 +962,8 @@ test("usage service covers Qwen, Qoder, GLM, Z.AI and GLMT branches", async () = providerSpecificData: { apiRegion: "international" }, }); assert.equal(glmt.plan, "Pro"); - assert.equal(glmt.quotas.tokens.used, 64); - assert.equal(glmt.quotas.tokens.remaining, 36); + assert.equal(glmt.quotas["5 Hours Quota"].used, 15); + assert.equal(glmt.quotas["Weekly Quota"].remaining, 36); globalThis.fetch = async () => new Response("nope", { status: 401 }); await assert.rejects( From 1929b44235272592f6eb132792e783ceb1bbf557 Mon Sep 17 00:00:00 2001 From: Jan Leon Date: Fri, 8 May 2026 08:46:13 +0000 Subject: [PATCH 07/15] feat: implement global Codex fast service tier functionality and related settings --- open-sse/handlers/chatCore.ts | 2 + .../dashboard/providers/[id]/page.tsx | 77 +++++++++++++++++++ .../providers/components/ProviderCard.tsx | 9 +++ .../(dashboard)/dashboard/providers/page.tsx | 28 ++++++- src/lib/db/settings.ts | 1 + src/lib/providers/codexFastTier.ts | 65 ++++++++++++++++ src/shared/validation/schemas.ts | 1 + src/shared/validation/settingsSchemas.ts | 1 + tests/unit/codex-fast-tier.test.ts | 55 +++++++++++++ ...settings-schema-routing-strategies.test.ts | 17 ++++ 10 files changed, 254 insertions(+), 2 deletions(-) create mode 100644 src/lib/providers/codexFastTier.ts create mode 100644 tests/unit/codex-fast-tier.test.ts diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 04e2159afe..28a8c98882 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -81,6 +81,7 @@ import { } from "../utils/cacheControlPolicy.ts"; import { getCacheMetrics } from "@/lib/db/settings.ts"; import { getCachedSettings } from "@/lib/db/readCache"; +import { applyCodexGlobalFastServiceTier } from "@/lib/providers/codexFastTier"; import { cacheReasoningFromAssistantMessage } from "../services/reasoningCache.ts"; import { sanitizeOpenAITool } from "../services/toolSchemaSanitizer.ts"; @@ -1402,6 +1403,7 @@ export async function handleChatCore({ ? false : resolveStreamFlag(body?.stream, acceptHeader); const settings = await getCachedSettings(); + credentials = applyCodexGlobalFastServiceTier(provider, credentials, settings); setGeminiThoughtSignatureMode(settings.antigravitySignatureCacheMode); const semanticCacheEnabled = settings.semanticCacheEnabled !== false; diff --git a/src/app/(dashboard)/dashboard/providers/[id]/page.tsx b/src/app/(dashboard)/dashboard/providers/[id]/page.tsx index cdd04441bb..bc514e4bd5 100644 --- a/src/app/(dashboard)/dashboard/providers/[id]/page.tsx +++ b/src/app/(dashboard)/dashboard/providers/[id]/page.tsx @@ -53,6 +53,10 @@ import { getClaudeCodeCompatibleRequestDefaults as _getClaudeCodeCompatibleRequestDefaults, getCodexRequestDefaults as _getCodexRequestDefaults, } from "@/lib/providers/requestDefaults"; +import { + getCodexEffectiveFastServiceTier, + isCodexGlobalFastServiceTierEnabled, +} from "@/lib/providers/codexFastTier"; import { isClaudeExtraUsageBlockEnabled } from "@/lib/providers/claudeExtraUsage"; import { parseExtraApiKeys } from "@/shared/utils/parseApiKeys"; import { resolveDashboardProviderInfo } from "../providerPageUtils"; @@ -512,6 +516,7 @@ interface ConnectionRowProps { isOAuth: boolean; isClaude?: boolean; isCodex?: boolean; + codexFastGlobalEnabled?: boolean; isFirst: boolean; isLast: boolean; onMoveUp: () => void; @@ -1016,6 +1021,8 @@ export default function ProviderDetailPage() { ); const [applyingCodexAuthId, setApplyingCodexAuthId] = useState(null); const [exportingCodexAuthId, setExportingCodexAuthId] = useState(null); + const [codexGlobalFastServiceTier, setCodexGlobalFastServiceTier] = useState(false); + const [savingCodexGlobalFastServiceTier, setSavingCodexGlobalFastServiceTier] = useState(false); const isOpenAICompatible = isOpenAICompatibleProvider(providerId); const isCcCompatible = isClaudeCodeCompatibleProvider(providerId); const isAnthropicCompatible = @@ -1240,6 +1247,16 @@ export default function ProviderDetailPage() { .catch(() => {}); }, [fetchConnections, fetchAliases]); + useEffect(() => { + if (providerId !== "codex") return; + fetch("/api/settings", { cache: "no-store" }) + .then((r) => (r.ok ? r.json() : null)) + .then((data) => { + setCodexGlobalFastServiceTier(isCodexGlobalFastServiceTierEnabled(data)); + }) + .catch(() => {}); + }, [providerId]); + const loadConnProxies = useCallback(async (conns: { id?: string }[]) => { if (!conns.length) return; try { @@ -1687,6 +1704,32 @@ export default function ProviderDetailPage() { } }; + const handleToggleCodexGlobalFastServiceTier = async (enabled: boolean) => { + if (savingCodexGlobalFastServiceTier) return; + setSavingCodexGlobalFastServiceTier(true); + try { + const res = await fetch("/api/settings", { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ codexServiceTier: { enabled } }), + }); + + if (!res.ok) { + const data = await res.json().catch(() => ({})); + notify.error(data.error || "Failed to update Codex Fast setting"); + return; + } + + setCodexGlobalFastServiceTier(enabled); + notify.success(enabled ? "Codex Fast enabled globally" : "Codex Fast disabled globally"); + } catch (error) { + console.error("Error toggling Codex Fast setting:", error); + notify.error("Failed to update Codex Fast setting"); + } finally { + setSavingCodexGlobalFastServiceTier(false); + } + }; + const handleRetestConnection = async (connectionId) => { if (!connectionId || retestingId) return; setRetestingId(connectionId); @@ -2815,6 +2858,18 @@ export default function ProviderDetailPage() {

{t("connections")}

+ {providerId === "codex" && ( +
+ +
+ )} {/* Provider-level proxy indicator/button */}
- {connections.length > 1 && ( - - )} - {!isCompatible ? ( -
- - {providerId === "qoder" && ( - + )} + {!isCompatible ? ( + <> + - )} -
- ) : ( - connections.length === 0 && ( - - ) - )} + {providerId === "qoder" && ( + + )} + + ) : ( + connections.length === 0 && ( + + ) + )} +
{connections.length === 0 ? ( diff --git a/src/app/(dashboard)/dashboard/providers/components/ProviderCard.tsx b/src/app/(dashboard)/dashboard/providers/components/ProviderCard.tsx index 9c609e231f..ed6c405d32 100644 --- a/src/app/(dashboard)/dashboard/providers/components/ProviderCard.tsx +++ b/src/app/(dashboard)/dashboard/providers/components/ProviderCard.tsx @@ -57,7 +57,8 @@ function getStatusDisplay( connected: number, error: number, errorCode: string | null | undefined, - t: ReturnType + t: ReturnType, + afterConnected?: ReactNode ) { const parts: ReactNode[] = []; if (connected > 0) { @@ -66,6 +67,7 @@ function getStatusDisplay( {t("connected", { count: connected })} ); + if (afterConnected) parts.push(afterConnected); } if (error > 0) { const errText = errorCode @@ -98,6 +100,17 @@ export default function ProviderCard({ const isCompatible = isOpenAICompatibleProvider(providerId); const isCcCompatible = isClaudeCodeCompatibleProvider(providerId); const isAnthropicCompatible = isAnthropicCompatibleProvider(providerId) && !isCcCompatible; + const codexFastChip = + providerId === "codex" && stats.codexFastActive ? ( + + bolt + Fast + + ) : null; const dotLabels: Record = { free: tc("free"), @@ -184,7 +197,7 @@ export default function ProviderCard({ ) : ( <> - {getStatusDisplay(connected, error, stats.errorCode, t)} + {getStatusDisplay(connected, error, stats.errorCode, t, codexFastChip)} {(authType === "free" || provider.hasFree === true) && ( )} - {providerId === "codex" && stats.codexFastActive && ( - - - bolt - Fast - - - )} {isCompatible && ( {provider.apiType === "responses" ? t("responses") : t("chat")} diff --git a/src/app/api/usage/analytics/route.ts b/src/app/api/usage/analytics/route.ts index 1eeec256b8..004fe0d967 100644 --- a/src/app/api/usage/analytics/route.ts +++ b/src/app/api/usage/analytics/route.ts @@ -35,7 +35,8 @@ const WEEKDAY_LABELS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"]; type PricingByProvider = Record>>; type ComputeCostFromPricing = ( pricing: Record | null | undefined, - tokens: Record | null | undefined + tokens: Record | null | undefined, + options?: Record ) => number; function toNumber(value: unknown): number { @@ -55,6 +56,15 @@ function roundCost(value: number): number { return Math.round(value * 1_000_000) / 1_000_000; } +function normalizeServiceTier(value: unknown): "standard" | "priority" { + const tier = typeof value === "string" ? value.trim().toLowerCase() : ""; + return tier === "priority" || tier === "fast" ? "priority" : "standard"; +} + +function getServiceTierLabel(serviceTier: string): string { + return normalizeServiceTier(serviceTier) === "priority" ? "Fast" : "Standard"; +} + function appendWhereCondition(whereClause: string, condition: string): string { return whereClause ? `${whereClause} AND (${condition})` : `WHERE (${condition})`; } @@ -195,6 +205,7 @@ function computeUsageRowCost( const provider = toStringValue(row.provider); const model = toStringValue(row.model); if (!provider || !model) return 0; + const serviceTier = normalizeServiceTier(row.serviceTier ?? row.service_tier); const pricing = resolveModelPricing( pricingByProvider, @@ -205,13 +216,21 @@ function computeUsageRowCost( ); if (!pricing) return 0; - return computeCostFromPricing(pricing, { - input: toNumber(row.promptTokens), - output: toNumber(row.completionTokens), - cacheRead: toNumber(row.cacheReadTokens), - cacheCreation: toNumber(row.cacheCreationTokens), - reasoning: toNumber(row.reasoningTokens), - }); + return computeCostFromPricing( + pricing, + { + input: toNumber(row.promptTokens), + output: toNumber(row.completionTokens), + cacheRead: toNumber(row.cacheReadTokens), + cacheCreation: toNumber(row.cacheCreationTokens), + reasoning: toNumber(row.reasoningTokens), + }, + { + provider, + model, + serviceTier, + } + ); } function formatUtcDate(date: Date): string { @@ -331,6 +350,7 @@ export async function GET(request: Request) { DATE(timestamp) as date, LOWER(provider) as provider, LOWER(model) as model, + COALESCE(NULLIF(service_tier, ''), 'standard') as serviceTier, COALESCE(SUM(tokens_input), 0) as promptTokens, COALESCE(SUM(tokens_output), 0) as completionTokens, COALESCE(SUM(tokens_cache_read), 0) as cacheReadTokens, @@ -338,7 +358,7 @@ export async function GET(request: Request) { COALESCE(SUM(tokens_reasoning), 0) as reasoningTokens FROM usage_history ${whereClause} - GROUP BY DATE(timestamp), LOWER(provider), LOWER(model) + GROUP BY DATE(timestamp), LOWER(provider), LOWER(model), serviceTier ORDER BY date ASC ` ) @@ -384,6 +404,7 @@ export async function GET(request: Request) { SELECT LOWER(model) as model, LOWER(provider) as provider, + COALESCE(NULLIF(service_tier, ''), 'standard') as serviceTier, COUNT(*) as requests, COALESCE(SUM(tokens_input), 0) as promptTokens, COALESCE(SUM(tokens_output), 0) as completionTokens, @@ -396,9 +417,8 @@ export async function GET(request: Request) { COALESCE(MAX(timestamp), '') as lastUsed FROM usage_history ${whereClause} - GROUP BY LOWER(model), LOWER(provider) + GROUP BY LOWER(model), LOWER(provider), serviceTier ORDER BY requests DESC - LIMIT 50 ` ) .all(params) as Array>; @@ -409,6 +429,7 @@ export async function GET(request: Request) { SELECT LOWER(provider) as provider, LOWER(model) as model, + COALESCE(NULLIF(service_tier, ''), 'standard') as serviceTier, COALESCE(SUM(tokens_input), 0) as promptTokens, COALESCE(SUM(tokens_output), 0) as completionTokens, COALESCE(SUM(tokens_cache_read), 0) as cacheReadTokens, @@ -416,7 +437,7 @@ export async function GET(request: Request) { COALESCE(SUM(tokens_reasoning), 0) as reasoningTokens FROM usage_history ${whereClause} - GROUP BY LOWER(provider), LOWER(model) + GROUP BY LOWER(provider), LOWER(model), serviceTier ` ) .all(params) as Array>; @@ -447,6 +468,7 @@ export async function GET(request: Request) { COALESCE(NULLIF(c.display_name, ''), NULLIF(c.email, ''), NULLIF(c.name, ''), usage_history.connection_id, 'unknown') as account, LOWER(usage_history.provider) as provider, LOWER(usage_history.model) as model, + COALESCE(NULLIF(usage_history.service_tier, ''), 'standard') as serviceTier, COALESCE(SUM(usage_history.tokens_input), 0) as promptTokens, COALESCE(SUM(usage_history.tokens_output), 0) as completionTokens, COALESCE(SUM(usage_history.tokens_cache_read), 0) as cacheReadTokens, @@ -455,7 +477,7 @@ export async function GET(request: Request) { FROM usage_history LEFT JOIN provider_connections c ON c.id = usage_history.connection_id ${whereClause.replace(/timestamp/g, "usage_history.timestamp").replace(/api_key_/g, "usage_history.api_key_")} - GROUP BY account, LOWER(usage_history.provider), LOWER(usage_history.model) + GROUP BY account, LOWER(usage_history.provider), LOWER(usage_history.model), serviceTier ` ) .all(params) as Array>; @@ -493,6 +515,7 @@ export async function GET(request: Request) { COALESCE(NULLIF(api_key_name, ''), NULLIF(api_key_id, ''), 'Unknown API key') as apiKeyName, LOWER(provider) as provider, LOWER(model) as model, + COALESCE(NULLIF(service_tier, ''), 'standard') as serviceTier, COUNT(*) as requests, COALESCE(SUM(tokens_input), 0) as promptTokens, COALESCE(SUM(tokens_output), 0) as completionTokens, @@ -502,7 +525,28 @@ export async function GET(request: Request) { COALESCE(SUM(tokens_input + tokens_output), 0) as totalTokens FROM usage_history ${apiKeyWhereClause} - GROUP BY api_key_id, api_key_name, LOWER(provider), LOWER(model) + GROUP BY api_key_id, api_key_name, LOWER(provider), LOWER(model), serviceTier + ` + ) + .all(params) as Array>; + + const serviceTierRows = db + .prepare( + ` + SELECT + COALESCE(NULLIF(service_tier, ''), 'standard') as serviceTier, + LOWER(provider) as provider, + LOWER(model) as model, + COUNT(*) as requests, + COALESCE(SUM(tokens_input), 0) as promptTokens, + COALESCE(SUM(tokens_output), 0) as completionTokens, + COALESCE(SUM(tokens_cache_read), 0) as cacheReadTokens, + COALESCE(SUM(tokens_cache_creation), 0) as cacheCreationTokens, + COALESCE(SUM(tokens_reasoning), 0) as reasoningTokens, + COALESCE(SUM(tokens_input + tokens_output), 0) as totalTokens + FROM usage_history + ${whereClause} + GROUP BY serviceTier, LOWER(provider), LOWER(model) ` ) .all(params) as Array>; @@ -584,6 +628,11 @@ export async function GET(request: Request) { firstRequest: summaryRow?.firstRequest || "", lastRequest: summaryRow?.lastRequest || "", fallbackCount: Number(fallbackRow?.fallbacks || 0), + fastRequests: 0, + standardRequests: 0, + fastCost: 0, + standardCost: 0, + fastRequestSharePct: 0, fallbackRatePct: Number(fallbackRow?.fallback_eligible || 0) > 0 ? Number( @@ -648,14 +697,11 @@ export async function GET(request: Request) { } summary.streak = computeActivityStreak(activityMap); - const byModel = modelRows.map((row) => { + const modelMap = new Map>(); + for (const row of modelRows) { const model = row.model as string; const provider = row.provider as string; const short = normalizeModelName(model); - const tokens = { - input: Number(row.promptTokens) || 0, - output: Number(row.completionTokens) || 0, - }; const cost = computeUsageRowCost( row, pricingByProvider, @@ -663,23 +709,59 @@ export async function GET(request: Request) { normalizeModelName, computeCostFromPricing ); - return { + const key = `${provider}::${model}`; + const existing = modelMap.get(key) || { model: short, provider, rawModel: model, + requests: 0, + promptTokens: 0, + completionTokens: 0, + totalTokens: 0, + latencyWeightedTotal: 0, + successfulRequests: 0, + lastUsed: "", + cost: 0, + }; + const requests = Number(row.requests) || 0; + existing.requests = Number(existing.requests || 0) + requests; + existing.promptTokens = Number(existing.promptTokens || 0) + Number(row.promptTokens || 0); + existing.completionTokens = + Number(existing.completionTokens || 0) + Number(row.completionTokens || 0); + existing.totalTokens = Number(existing.totalTokens || 0) + Number(row.totalTokens || 0); + existing.latencyWeightedTotal = + Number(existing.latencyWeightedTotal || 0) + Number(row.avgLatencyMs || 0) * requests; + existing.successfulRequests = + Number(existing.successfulRequests || 0) + Number(row.successfulRequests || 0); + if (!existing.lastUsed || String(row.lastUsed || "") > String(existing.lastUsed || "")) { + existing.lastUsed = row.lastUsed; + } + existing.cost = Number(existing.cost || 0) + cost; + modelMap.set(key, existing); + } + + const byModel = Array.from(modelMap.values()) + .map((row) => ({ + model: row.model, + provider: row.provider, + rawModel: row.rawModel, requests: Number(row.requests), - promptTokens: tokens.input, - completionTokens: tokens.output, + promptTokens: Number(row.promptTokens), + completionTokens: Number(row.completionTokens), totalTokens: Number(row.totalTokens), - avgLatencyMs: Math.round(Number(row.avgLatencyMs)), + avgLatencyMs: + Number(row.requests) > 0 + ? Math.round(Number(row.latencyWeightedTotal || 0) / Number(row.requests)) + : 0, successRatePct: Number(row.requests) > 0 - ? Number((Number(row.successfulRequests) / Number(row.requests)) * 100).toFixed(2) + ? Number((Number(row.successfulRequests || 0) / Number(row.requests)) * 100).toFixed(2) : 0, lastUsed: row.lastUsed, - cost: roundCost(cost), - }; - }); + cost: roundCost(Number(row.cost || 0)), + })) + .sort((left, right) => Number(right.requests) - Number(left.requests)) + .slice(0, 50); const totalCost = Array.from(dailyCostByDate.values()).reduce((sum, cost) => sum + cost, 0); summary.totalCost = roundCost(totalCost); @@ -781,6 +863,56 @@ export async function GET(request: Request) { .map((row) => ({ ...row, cost: roundCost(row.cost) })) .sort((left, right) => right.cost - left.cost); + const serviceTierMap = new Map< + string, + { + serviceTier: "standard" | "priority"; + label: string; + requests: number; + promptTokens: number; + completionTokens: number; + totalTokens: number; + cost: number; + } + >(); + for (const row of serviceTierRows) { + const serviceTier = normalizeServiceTier(row.serviceTier); + const existing = serviceTierMap.get(serviceTier) || { + serviceTier, + label: getServiceTierLabel(serviceTier), + requests: 0, + promptTokens: 0, + completionTokens: 0, + totalTokens: 0, + cost: 0, + }; + existing.requests += Number(row.requests || 0); + existing.promptTokens += Number(row.promptTokens || 0); + existing.completionTokens += Number(row.completionTokens || 0); + existing.totalTokens += Number(row.totalTokens || 0); + existing.cost += computeUsageRowCost( + row, + pricingByProvider, + PROVIDER_ID_TO_ALIAS, + normalizeModelName, + computeCostFromPricing + ); + serviceTierMap.set(serviceTier, existing); + } + const byServiceTier = Array.from(serviceTierMap.values()) + .map((row) => ({ ...row, cost: roundCost(row.cost) })) + .sort((left, right) => (left.serviceTier === "priority" ? -1 : 1)); + const fastTier = serviceTierMap.get("priority"); + const standardTier = serviceTierMap.get("standard"); + summary.fastRequests = fastTier?.requests || 0; + summary.fastCost = roundCost(fastTier?.cost || 0); + summary.standardRequests = standardTier?.requests || 0; + summary.standardCost = roundCost(standardTier?.cost || 0); + summary.fastRequestSharePct = + summary.totalRequests > 0 + ? Number(((Number(summary.fastRequests) / Number(summary.totalRequests)) * 100).toFixed(2)) + : 0; + const weeklyTokens = [0, 0, 0, 0, 0, 0, 0]; const weeklyCounts = [0, 0, 0, 0, 0, 0, 0]; const weeklyPattern = WEEKDAY_LABELS.map((day) => ({ @@ -816,6 +948,7 @@ export async function GET(request: Request) { byProvider, byApiKey, byAccount, + byServiceTier, weeklyPattern, weeklyTokens, weeklyCounts, diff --git a/src/lib/db/core.ts b/src/lib/db/core.ts index 62d8a67d70..c9da9f0793 100644 --- a/src/lib/db/core.ts +++ b/src/lib/db/core.ts @@ -250,6 +250,7 @@ const SCHEMA_SQL = ` tokens_cache_read INTEGER DEFAULT 0, tokens_cache_creation INTEGER DEFAULT 0, tokens_reasoning INTEGER DEFAULT 0, + service_tier TEXT DEFAULT 'standard', status TEXT, success INTEGER DEFAULT 1, latency_ms INTEGER DEFAULT 0, @@ -260,6 +261,7 @@ const SCHEMA_SQL = ` CREATE INDEX IF NOT EXISTS idx_uh_timestamp ON usage_history(timestamp); CREATE INDEX IF NOT EXISTS idx_uh_provider ON usage_history(provider); CREATE INDEX IF NOT EXISTS idx_uh_model ON usage_history(model); + CREATE INDEX IF NOT EXISTS idx_uh_service_tier ON usage_history(service_tier); CREATE TABLE IF NOT EXISTS call_logs ( id TEXT PRIMARY KEY, diff --git a/src/lib/db/migrations/001_initial_schema.sql b/src/lib/db/migrations/001_initial_schema.sql index 2cfc72eabe..d366cde3c8 100644 --- a/src/lib/db/migrations/001_initial_schema.sql +++ b/src/lib/db/migrations/001_initial_schema.sql @@ -97,6 +97,7 @@ CREATE TABLE IF NOT EXISTS usage_history ( tokens_cache_read INTEGER DEFAULT 0, tokens_cache_creation INTEGER DEFAULT 0, tokens_reasoning INTEGER DEFAULT 0, + service_tier TEXT DEFAULT 'standard', status TEXT, success INTEGER DEFAULT 1, latency_ms INTEGER DEFAULT 0, @@ -107,6 +108,7 @@ CREATE TABLE IF NOT EXISTS usage_history ( CREATE INDEX IF NOT EXISTS idx_uh_timestamp ON usage_history(timestamp); CREATE INDEX IF NOT EXISTS idx_uh_provider ON usage_history(provider); CREATE INDEX IF NOT EXISTS idx_uh_model ON usage_history(model); +CREATE INDEX IF NOT EXISTS idx_uh_service_tier ON usage_history(service_tier); CREATE TABLE IF NOT EXISTS call_logs ( id TEXT PRIMARY KEY, diff --git a/src/lib/db/migrations/051_usage_history_service_tier.sql b/src/lib/db/migrations/051_usage_history_service_tier.sql new file mode 100644 index 0000000000..0f18aea384 --- /dev/null +++ b/src/lib/db/migrations/051_usage_history_service_tier.sql @@ -0,0 +1,4 @@ +-- Migration 051: Track effective service tier for usage analytics +-- Used to distinguish Codex Fast (priority) requests from standard requests. +ALTER TABLE usage_history ADD COLUMN service_tier TEXT DEFAULT 'standard'; +CREATE INDEX IF NOT EXISTS idx_uh_service_tier ON usage_history(service_tier); diff --git a/src/lib/usage/costCalculator.ts b/src/lib/usage/costCalculator.ts index a34010017b..31c2267e87 100644 --- a/src/lib/usage/costCalculator.ts +++ b/src/lib/usage/costCalculator.ts @@ -22,6 +22,12 @@ export function normalizeModelName(model: string): string { return parts[parts.length - 1]; } +export type CostCalculationOptions = { + provider?: string | null; + model?: string | null; + serviceTier?: string | null; +}; + function toNumber(value: unknown, fallback = 0): number { if (typeof value === "number" && Number.isFinite(value)) return value; if (typeof value === "string" && value.trim().length > 0) { @@ -31,6 +37,35 @@ function toNumber(value: unknown, fallback = 0): number { return fallback; } +function normalizeServiceTier(value: unknown): string { + return typeof value === "string" ? value.trim().toLowerCase() : ""; +} + +function stripCodexEffortSuffix(model: string): string { + return model.replace(/-(?:xhigh|high|medium|low|none)$/i, ""); +} + +export function getCodexFastCostMultiplier( + provider: string | null | undefined, + model: string | null | undefined, + serviceTier: string | null | undefined +): number { + const providerKey = normalizeServiceTier(provider); + const tier = normalizeServiceTier(serviceTier); + if ( + (providerKey !== "codex" && providerKey !== "cx") || + (tier !== "priority" && tier !== "fast") + ) { + return 1; + } + + const modelKey = stripCodexEffortSuffix(normalizeModelName(String(model || "")).toLowerCase()); + const compactModelKey = modelKey.replace(/-/g, ""); + if (modelKey === "gpt-5.5" || compactModelKey === "gpt5.5") return 2.5; + if (modelKey === "gpt-5.4" || compactModelKey === "gpt5.4") return 2; + return 1; +} + /** * Calculate cost for a usage entry. * @@ -45,7 +80,8 @@ function toNumber(value: unknown, fallback = 0): number { */ export function computeCostFromPricing( pricing: Record | null | undefined, - tokens: Record | null | undefined + tokens: Record | null | undefined, + options: CostCalculationOptions = {} ): number { if (!pricing || !tokens) return 0; const inputPrice = toNumber(pricing.input, 0); @@ -71,13 +107,14 @@ export function computeCostFromPricing( const cacheCreationTokens = tokens.cacheCreation ?? tokens.cache_creation_input_tokens ?? 0; if (cacheCreationTokens > 0) cost += cacheCreationTokens * (cacheCreationPrice / 1_000_000); - return cost; + return cost * getCodexFastCostMultiplier(options.provider, options.model, options.serviceTier); } export async function calculateCost( provider: string, model: string, - tokens: Record | null | undefined + tokens: Record | null | undefined, + options: CostCalculationOptions = {} ): Promise { if (!tokens || !provider || !model) return 0; @@ -98,7 +135,11 @@ export async function calculateCost( pricing && typeof pricing === "object" && !Array.isArray(pricing) ? (pricing as Record) : {}; - return computeCostFromPricing(pricingRecord, tokens); + return computeCostFromPricing(pricingRecord, tokens, { + provider, + model, + ...options, + }); } catch (error) { console.error("Error calculating cost:", error); return 0; diff --git a/src/lib/usage/usageHistory.ts b/src/lib/usage/usageHistory.ts index 5ee4c75b0b..fab526009e 100644 --- a/src/lib/usage/usageHistory.ts +++ b/src/lib/usage/usageHistory.ts @@ -44,6 +44,11 @@ function toStringOrNull(value: unknown): string | null { return typeof value === "string" && value.trim().length > 0 ? value : null; } +function normalizeServiceTier(value: unknown): string { + const tier = typeof value === "string" ? value.trim().toLowerCase() : ""; + return tier === "priority" || tier === "fast" ? "priority" : "standard"; +} + function toNumber(value: unknown): number { if (typeof value === "number" && Number.isFinite(value)) return value; if (typeof value === "string" && value.trim().length > 0) { @@ -299,6 +304,7 @@ export async function getUsageDb(sinceIso?: string | null, limit?: number, curso connectionId: toStringOrNull(r.connection_id), apiKeyId: toStringOrNull(r.api_key_id), apiKeyName: toStringOrNull(r.api_key_name), + serviceTier: normalizeServiceTier(r.service_tier), tokens: { input: toNumber(r.tokens_input), output: toNumber(r.tokens_output), @@ -332,13 +338,14 @@ export async function saveRequestUsage(entry: any) { try { const db = getDbInstance(); const timestamp = entry.timestamp || new Date().toISOString(); + const serviceTier = normalizeServiceTier(entry.serviceTier ?? entry.service_tier); db.prepare( ` INSERT INTO usage_history (provider, model, connection_id, api_key_id, api_key_name, tokens_input, tokens_output, tokens_cache_read, tokens_cache_creation, tokens_reasoning, - status, success, latency_ms, ttft_ms, error_code, timestamp) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + service_tier, status, success, latency_ms, ttft_ms, error_code, timestamp) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ` ).run( entry.provider || null, @@ -351,6 +358,7 @@ export async function saveRequestUsage(entry: any) { getPromptCacheReadTokens(entry.tokens), getPromptCacheCreationTokens(entry.tokens), getReasoningTokens(entry.tokens), + serviceTier, entry.status || null, entry.success === false ? 0 : 1, Number.isFinite(Number(entry.latencyMs)) ? Number(entry.latencyMs) : 0, @@ -409,6 +417,7 @@ export async function getUsageHistory(filter: any = {}) { connectionId: toStringOrNull(r.connection_id), apiKeyId: toStringOrNull(r.api_key_id), apiKeyName: toStringOrNull(r.api_key_name), + serviceTier: normalizeServiceTier(r.service_tier), tokens: { input: toNumber(r.tokens_input), output: toNumber(r.tokens_output), diff --git a/src/lib/usage/usageStats.ts b/src/lib/usage/usageStats.ts index 1138ae605b..1a3081712f 100644 --- a/src/lib/usage/usageStats.ts +++ b/src/lib/usage/usageStats.ts @@ -81,7 +81,8 @@ export async function getUsageStats() { tokens_output, tokens_cache_read, tokens_cache_creation, - tokens_reasoning + tokens_reasoning, + service_tier FROM usage_history WHERE DATE(timestamp) >= ? @@ -98,7 +99,8 @@ export async function getUsageStats() { total_output_tokens as tokens_output, 0 as tokens_cache_read, 0 as tokens_cache_creation, - 0 as tokens_reasoning + 0 as tokens_reasoning, + 'standard' as service_tier FROM daily_usage_summary WHERE date < ? @@ -193,6 +195,7 @@ export async function getUsageStats() { const connectionId = toStringOrEmpty(row.connection_id) || null; const apiKeyId = toStringOrEmpty(row.api_key_id) || null; const apiKeyName = toStringOrEmpty(row.api_key_name) || null; + const serviceTier = toStringOrEmpty(row.service_tier) || "standard"; const promptTokens = toNumber(row.tokens_input); const completionTokens = toNumber(row.tokens_output); @@ -205,7 +208,7 @@ export async function getUsageStats() { cacheCreation: toNumber(row.tokens_cache_creation), reasoning: toNumber(row.tokens_reasoning), }; - const entryCost = await calculateCost(provider, model, entryTokens); + const entryCost = await calculateCost(provider, model, entryTokens, { serviceTier }); stats.totalPromptTokens += promptTokens; stats.totalCompletionTokens += completionTokens; diff --git a/src/shared/components/Toggle.tsx b/src/shared/components/Toggle.tsx index 66cff8c905..e8b0e79cbf 100644 --- a/src/shared/components/Toggle.tsx +++ b/src/shared/components/Toggle.tsx @@ -11,6 +11,7 @@ interface ToggleProps { size?: "sm" | "md" | "lg"; className?: string; title?: string; + ariaLabel?: string; } export default function Toggle({ @@ -21,6 +22,8 @@ export default function Toggle({ disabled = false, size = "md", className, + title, + ariaLabel, }: ToggleProps) { const sizes = { sm: { @@ -58,7 +61,8 @@ export default function Toggle({ type="button" role="switch" aria-checked={checked} - aria-label={!label ? description || "Toggle" : undefined} + aria-label={ariaLabel || label || description || title || "Toggle"} + title={title} disabled={disabled} onClick={handleClick} className={cn( diff --git a/src/shared/components/UsageAnalytics.tsx b/src/shared/components/UsageAnalytics.tsx index 0ab5838dc9..bda130c41a 100644 --- a/src/shared/components/UsageAnalytics.tsx +++ b/src/shared/components/UsageAnalytics.tsx @@ -18,6 +18,7 @@ import { ProviderCostDonut, ModelOverTimeChart, ProviderTable, + ServiceTierBreakdown, ApiKeyFilterDropdown, CustomRangePicker, } from "./analytics"; @@ -311,10 +312,10 @@ export default function UsageAnalytics() { color: "text-violet-500", }, { - icon: "swap_horiz", - label: "Fallback Rate", - value: `${Number(s.fallbackRatePct || 0).toFixed(1)}%`, - color: "text-amber-500", + icon: "bolt", + label: "Fast Requests", + value: fmt(s.fastRequests || 0), + color: "text-sky-500", }, ], }, @@ -331,6 +332,12 @@ export default function UsageAnalytics() { value: `${providerDiversity.toFixed(1)}%`, color: "text-sky-500", }, + { + icon: "swap_horiz", + label: "Fallback Rate", + value: `${Number(s.fallbackRatePct || 0).toFixed(1)}%`, + color: "text-amber-500", + }, ], }, ]} @@ -351,6 +358,9 @@ export default function UsageAnalytics() {
+ {/* Fast / Standard service tier split */} + + {/* Model Usage Over Time (stacked area) */} byServiceTier || [], [byServiceTier]); + const totalRequests = Number(summary?.totalRequests || 0); + const totalCost = Number(summary?.totalCost || 0); + + if (!data.length) { + return null; + } + + return ( + +
+

+ Service Tier +

+ Fast / Standard cost split +
+
+ {data.map((tier) => { + const isFast = tier.serviceTier === "priority"; + const requestPct = + totalRequests > 0 + ? ((Number(tier.requests || 0) / totalRequests) * 100).toFixed(1) + : "0"; + const costPct = + totalCost > 0 ? ((Number(tier.cost || 0) / totalCost) * 100).toFixed(1) : "0"; + return ( +
+
+
+ + {isFast ? "bolt" : "speed"} + +
+
{tier.label}
+
+ {fmtFull(tier.requests)} requests · {fmt(tier.totalTokens)} tokens +
+
+
+
+
+ {fmtCost(tier.cost)} +
+
{costPct}% of cost
+
+
+
+
+
+
+ ); + })} +
+ + ); + } const sorted = useMemo(() => { const arr = [...(byModel || [])]; arr.sort((a, b) => { diff --git a/src/shared/components/analytics/index.tsx b/src/shared/components/analytics/index.tsx index fb14864249..fb9d2be56a 100644 --- a/src/shared/components/analytics/index.tsx +++ b/src/shared/components/analytics/index.tsx @@ -25,6 +25,7 @@ export { ProviderCostDonut, ModelOverTimeChart, ProviderTable, + ServiceTierBreakdown, } from "./charts"; export { default as ApiKeyFilterDropdown } from "./ApiKeyFilterDropdown"; diff --git a/tests/unit/usage-analytics-route.test.ts b/tests/unit/usage-analytics-route.test.ts index 66935d5819..9c44f37099 100644 --- a/tests/unit/usage-analytics-route.test.ts +++ b/tests/unit/usage-analytics-route.test.ts @@ -150,21 +150,56 @@ test("GET /api/usage/analytics resolves Codex GPT-5.5 pricing through provider a assertClose(body.byModel[0].cost, 0.02); }); +test("GET /api/usage/analytics applies Codex Fast tier multipliers and exposes tier split", async () => { + const db = core.getDbInstance(); + const timestamp = new Date().toISOString(); + db.prepare( + `INSERT INTO usage_history (provider, model, connection_id, tokens_input, tokens_output, success, latency_ms, service_tier, timestamp) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run("codex", "gpt-5.5", "codex-fast", 1000, 500, 1, 250, "priority", timestamp); + db.prepare( + `INSERT INTO usage_history (provider, model, connection_id, tokens_input, tokens_output, success, latency_ms, service_tier, timestamp) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run("codex", "gpt-5.5", "codex-standard", 1000, 500, 1, 250, "standard", timestamp); + + const response = await analyticsRoute.GET(makeRequest("http://localhost/api/usage/analytics")); + const body = await response.json(); + + assert.equal(response.status, 200); + assertClose(body.summary.totalCost, 0.07); + assert.equal(body.summary.fastRequests, 1); + assert.equal(body.summary.standardRequests, 1); + assertClose(body.summary.fastCost, 0.05); + assertClose(body.summary.standardCost, 0.02); + assert.equal(body.byServiceTier.length, 2); + assertClose(body.byProvider[0].cost, 0.07); + assertClose(body.byModel[0].cost, 0.07); +}); + +test("GET /api/usage/analytics applies Codex GPT-5.4 Fast multiplier", async () => { + await localDb.updatePricing({ + codex: { "gpt-5.4": { input: 5, output: 30 } }, + }); + const db = core.getDbInstance(); + db.prepare( + `INSERT INTO usage_history (provider, model, connection_id, tokens_input, tokens_output, success, latency_ms, service_tier, timestamp) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run("codex", "gpt-5.4", "codex-fast", 1000, 500, 1, 250, "priority", new Date().toISOString()); + + const response = await analyticsRoute.GET(makeRequest("http://localhost/api/usage/analytics")); + const body = await response.json(); + + assert.equal(response.status, 200); + assertClose(body.summary.totalCost, 0.04); + assertClose(body.summary.fastCost, 0.04); +}); + test("GET /api/usage/analytics maps Codex auto-review usage to GPT-5.5 pricing", async () => { const db = core.getDbInstance(); db.prepare( `INSERT INTO usage_history (provider, model, connection_id, tokens_input, tokens_output, success, latency_ms, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?)` - ).run( - "codex", - "codex-auto-review", - "codex-conn", - 1000, - 500, - 1, - 250, - new Date().toISOString() - ); + ).run("codex", "codex-auto-review", "codex-conn", 1000, 500, 1, 250, new Date().toISOString()); const response = await analyticsRoute.GET(makeRequest("http://localhost/api/usage/analytics")); const body = await response.json(); diff --git a/tests/unit/usage-analytics.test.ts b/tests/unit/usage-analytics.test.ts index 8ff5f6662c..54ad4ecae4 100644 --- a/tests/unit/usage-analytics.test.ts +++ b/tests/unit/usage-analytics.test.ts @@ -275,6 +275,22 @@ test("getUsageStats aggregates totals, buckets, pending requests, and cost break assert.equal(recentBucketTotal, 1); }); +test("Codex Fast service tier applies documented GPT-5.5 and GPT-5.4 cost multipliers", async () => { + await localDb.updatePricing({ + codex: { + "gpt-5.5": { input: 5, output: 30 }, + "gpt-5.4": { input: 5, output: 30 }, + }, + }); + + const tokens = { input: 1000, output: 500 }; + + assert.equal(await calculateCost("codex", "gpt-5.5", tokens), 0.02); + assert.equal(await calculateCost("codex", "gpt-5.5", tokens, { serviceTier: "priority" }), 0.05); + assert.equal(await calculateCost("codex", "gpt-5.4-high", tokens, { serviceTier: "fast" }), 0.04); + assert.equal(await calculateCost("openai", "gpt-5.5", tokens, { serviceTier: "priority" }), 0); +}); + test("recent request summaries are generated from SQLite call logs", async () => { const connection = await providersDb.createProviderConnection({ provider: "log-provider", diff --git a/tests/unit/usage-history-db.test.ts b/tests/unit/usage-history-db.test.ts index d3e44fc1d1..e97972b29a 100644 --- a/tests/unit/usage-history-db.test.ts +++ b/tests/unit/usage-history-db.test.ts @@ -28,6 +28,7 @@ async function seedUsageEntries( success?: boolean; latencyMs?: number; minutesAgo?: number; + serviceTier?: string; }> ) { for (const [i, e] of entries.entries()) { @@ -41,6 +42,7 @@ async function seedUsageEntries( success: e.success !== false, latencyMs: e.latencyMs || 100, timestamp: new Date(Date.now() - (e.minutesAgo || i) * 60 * 1000).toISOString(), + serviceTier: e.serviceTier, }); } } @@ -89,6 +91,32 @@ test("getUsageDb filters by sinceIso date", async () => { assert.equal(result.data.history[0].provider, "new"); }); +test("usage history persists service tier and defaults to standard", async () => { + await usageHistory.saveRequestUsage({ + provider: "codex", + model: "gpt-5.5", + tokens: { input: 100, output: 50 }, + success: true, + serviceTier: "priority", + timestamp: new Date().toISOString(), + }); + await usageHistory.saveRequestUsage({ + provider: "openai", + model: "gpt-4o", + tokens: { input: 10, output: 5 }, + success: true, + timestamp: new Date(Date.now() + 1).toISOString(), + }); + + const history = await usageHistory.getUsageHistory({ provider: "codex" }); + const usageDb = await usageHistory.getUsageDb(); + + assert.equal(history.length, 1); + assert.equal(history[0].serviceTier, "priority"); + assert.equal(usageDb.data.history[0].serviceTier, "priority"); + assert.equal(usageDb.data.history[1].serviceTier, "standard"); +}); + test("getUsageDb provides nextCursor when rows exceed MAX_ROWS", async () => { const db = core.getDbInstance(); // Insert exactly MAX_ROWS + 1 = 10001 rows so getUsageDb returns a cursor From 1e80366b420b267a94b5bd570b768a50c15cf84f Mon Sep 17 00:00:00 2001 From: Jan Leon Date: Fri, 8 May 2026 16:47:39 +0000 Subject: [PATCH 09/15] feat: add service tier breakdown component and handle missing docs directory --- scripts/generate-docs-index.mjs | 46 ++++++++ src/shared/components/analytics/charts.tsx | 129 +++++++++++---------- 2 files changed, 111 insertions(+), 64 deletions(-) diff --git a/scripts/generate-docs-index.mjs b/scripts/generate-docs-index.mjs index 482c435f11..69c6babd88 100644 --- a/scripts/generate-docs-index.mjs +++ b/scripts/generate-docs-index.mjs @@ -116,6 +116,52 @@ function extractContentPreview(content) { // ---------- Main ---------- +if (!fs.existsSync(DOCS_DIR)) { + if (fs.existsSync(OUT_FILE)) { + console.warn( + `[generate-docs-index] ${DOCS_DIR} not found; keeping existing generated docs index.` + ); + process.exit(0); + } + + fs.mkdirSync(path.dirname(OUT_FILE), { recursive: true }); + fs.writeFileSync( + OUT_FILE, + `// AUTO-GENERATED by scripts/generate-docs-index.mjs — DO NOT EDIT MANUALLY +// Regenerate with: node scripts/generate-docs-index.mjs + +export interface AutoGenDocItem { + slug: string; + title: string; + fileName: string; +} + +export interface AutoGenNavSection { + title: string; + items: AutoGenDocItem[]; +} + +export interface AutoGenSearchItem { + slug: string; + title: string; + fileName: string; + section: string; + content: string; + headings: string[]; +} + +export const autoNavSections: AutoGenNavSection[] = []; + +export const autoSearchIndex: AutoGenSearchItem[] = []; + +export const autoAllSlugs: string[] = []; +`, + "utf8" + ); + console.warn(`[generate-docs-index] ${DOCS_DIR} not found; generated empty docs index.`); + process.exit(0); +} + const files = fs.readdirSync(DOCS_DIR).filter((f) => f.endsWith(".md") || f.endsWith(".mdx")); const docs = []; diff --git a/src/shared/components/analytics/charts.tsx b/src/shared/components/analytics/charts.tsx index 0d5cefb1fa..70c9612342 100644 --- a/src/shared/components/analytics/charts.tsx +++ b/src/shared/components/analytics/charts.tsx @@ -966,70 +966,6 @@ export function ModelTable({ byModel, summary }) { [sortBy] ); - export function ServiceTierBreakdown({ byServiceTier, summary }) { - const data = useMemo(() => byServiceTier || [], [byServiceTier]); - const totalRequests = Number(summary?.totalRequests || 0); - const totalCost = Number(summary?.totalCost || 0); - - if (!data.length) { - return null; - } - - return ( - -
-

- Service Tier -

- Fast / Standard cost split -
-
- {data.map((tier) => { - const isFast = tier.serviceTier === "priority"; - const requestPct = - totalRequests > 0 - ? ((Number(tier.requests || 0) / totalRequests) * 100).toFixed(1) - : "0"; - const costPct = - totalCost > 0 ? ((Number(tier.cost || 0) / totalCost) * 100).toFixed(1) : "0"; - return ( -
-
-
- - {isFast ? "bolt" : "speed"} - -
-
{tier.label}
-
- {fmtFull(tier.requests)} requests · {fmt(tier.totalTokens)} tokens -
-
-
-
-
- {fmtCost(tier.cost)} -
-
{costPct}% of cost
-
-
-
-
-
-
- ); - })} -
- - ); - } const sorted = useMemo(() => { const arr = [...(byModel || [])]; arr.sort((a, b) => { @@ -1145,6 +1081,71 @@ export function ModelTable({ byModel, summary }) { ); } +export function ServiceTierBreakdown({ byServiceTier, summary }) { + const data = useMemo(() => byServiceTier || [], [byServiceTier]); + const totalRequests = Number(summary?.totalRequests || 0); + const totalCost = Number(summary?.totalCost || 0); + + if (!data.length) { + return null; + } + + return ( + +
+

+ Service Tier +

+ Fast / Standard cost split +
+
+ {data.map((tier) => { + const isFast = tier.serviceTier === "priority"; + const requestPct = + totalRequests > 0 + ? ((Number(tier.requests || 0) / totalRequests) * 100).toFixed(1) + : "0"; + const costPct = + totalCost > 0 ? ((Number(tier.cost || 0) / totalCost) * 100).toFixed(1) : "0"; + return ( +
+
+
+ + {isFast ? "bolt" : "speed"} + +
+
{tier.label}
+
+ {fmtFull(tier.requests)} requests · {fmt(tier.totalTokens)} tokens +
+
+
+
+
+ {fmtCost(tier.cost)} +
+
{costPct}% of cost
+
+
+
+
+
+
+ ); + })} +
+ + ); +} + // ── UsageDetail ──────────────────────────────────────────────────────────── export function UsageDetail({ summary }) { From 3662f90095ee46cf6d0786082cbe16e522e54392 Mon Sep 17 00:00:00 2001 From: Jan Leon Date: Fri, 8 May 2026 16:57:25 +0000 Subject: [PATCH 10/15] feat: enhance chat handling with cached settings and deduplicate quota fetches in reset-aware strategy --- open-sse/handlers/chatCore.ts | 3 +- open-sse/services/combo.ts | 81 +++++++++++++++++++---------- src/sse/handlers/chat.ts | 60 ++++++++++++++------- src/sse/handlers/chatHelpers.ts | 2 + tests/unit/combo-strategies.test.ts | 33 ++++++++++++ 5 files changed, 131 insertions(+), 48 deletions(-) diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 3fdc212c58..bc5310a4ea 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -971,6 +971,7 @@ export async function handleChatCore({ comboStepId = null, comboExecutionKey = null, disableEmergencyFallback = false, + cachedSettings = null, }) { let { provider, model, extendedContext } = modelInfo; const requestedModel = @@ -1424,7 +1425,7 @@ export async function handleChatCore({ nativeCodexPassthrough && isCompactResponsesEndpoint(endpointPath) ? false : resolveStreamFlag(body?.stream, acceptHeader); - const settings = await getCachedSettings(); + const settings = cachedSettings ?? (await getCachedSettings()); credentials = applyCodexGlobalFastServiceTier(provider, credentials, settings); effectiveServiceTier = resolveEffectiveServiceTier(body); setGeminiThoughtSignatureMode(settings.antigravitySignatureCacheMode); diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index e413029366..3ff14b78e9 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -849,6 +849,7 @@ function scoreResetAwareQuota(quota: unknown, config: ReturnType>>, + connectionLoadPromises: Map>>>, comboName: string, log: { warn?: (...args: unknown[]) => void } ) { @@ -861,25 +862,35 @@ async function getQuotaAwareConnectionsForTarget( return cached.connections; } - try { - const connections = await getProviderConnections({ provider, isActive: true }); - const activeConnections = Array.isArray(connections) - ? (connections as Array>) - : []; - connectionCache.set(provider, activeConnections); - resetAwareConnectionCache.set(provider, { - connections: activeConnections, - fetchedAt: Date.now(), - }); - } catch (error) { - log.warn?.("COMBO", "Reset-aware failed to load quota-aware connections.", { - comboName, - err: error, - operation: "getProviderConnections", + if (!connectionLoadPromises.has(provider)) { + connectionLoadPromises.set( provider, - }); - connectionCache.set(provider, []); + (async () => { + try { + const connections = await getProviderConnections({ provider, isActive: true }); + const activeConnections = Array.isArray(connections) + ? (connections as Array>) + : []; + resetAwareConnectionCache.set(provider, { + connections: activeConnections, + fetchedAt: Date.now(), + }); + return activeConnections; + } catch (error) { + log.warn?.("COMBO", "Reset-aware failed to load quota-aware connections.", { + comboName, + err: error, + operation: "getProviderConnections", + provider, + }); + return []; + } + })() + ); } + + const connections = await connectionLoadPromises.get(provider)!; + connectionCache.set(provider, connections); } return connectionCache.get(provider) || []; } @@ -957,12 +968,20 @@ async function orderTargetsByResetAwareQuota( const config = resolveResetAwareConfig(configSource); const connectionCache = new Map>>(); + const connectionLoadPromises = new Map>>>(); + const quotaPromises = new Map>(); const connectionById = new Map>(); const expandedTargets: ResolvedComboTarget[] = []; const targetsWithConnections = await Promise.all( targets.map(async (target) => ({ - connections: await getQuotaAwareConnectionsForTarget(target, connectionCache, comboName, log), + connections: await getQuotaAwareConnectionsForTarget( + target, + connectionCache, + connectionLoadPromises, + comboName, + log + ), target, })) ); @@ -1008,17 +1027,23 @@ async function orderTargetsByResetAwareQuota( const provider = getResetAwareProvider(target); const fetcher = provider ? getQuotaFetcher(provider) : null; if (fetcher && provider && target.connectionId) { - try { - quota = await fetcher(target.connectionId, connectionById.get(target.connectionId)); - } catch (error) { - log.warn?.("COMBO", "Reset-aware quota fetch failed.", { - comboName, - connectionId: target.connectionId, - err: error, - operation: "quotaFetch", - provider, - }); + const quotaKey = `${provider}:${target.connectionId}`; + if (!quotaPromises.has(quotaKey)) { + quotaPromises.set( + quotaKey, + fetcher(target.connectionId, connectionById.get(target.connectionId)).catch((error) => { + log.warn?.("COMBO", "Reset-aware quota fetch failed.", { + comboName, + connectionId: target.connectionId, + err: error, + operation: "quotaFetch", + provider, + }); + return null; + }) + ); } + quota = await quotaPromises.get(quotaKey)!; } const { score } = scoreResetAwareQuota(quota, config); return { target, score, index }; diff --git a/src/sse/handlers/chat.ts b/src/sse/handlers/chat.ts index 0d13eda7cb..de643574ef 100644 --- a/src/sse/handlers/chat.ts +++ b/src/sse/handlers/chat.ts @@ -287,9 +287,18 @@ export async function handleChat(request: any, clientRawRequest: any = null) { // Pre-check function used by combo routing. For explicit combo live tests, // avoid pre-skipping so each model gets a real execution attempt. + const comboPreselectedCredentials = new Map(); + const getComboCredentialCacheKey = ( + modelString: string, + target?: { connectionId?: string | null; executionKey?: string | null } + ) => `${target?.executionKey || target?.connectionId || ""}:${modelString}`; const checkModelAvailable = async ( modelString: string, - target?: { connectionId?: string | null; allowedConnectionIds?: string[] | null } + target?: { + connectionId?: string | null; + allowedConnectionIds?: string[] | null; + executionKey?: string | null; + } ) => { if (isComboLiveTest) return true; @@ -321,6 +330,7 @@ export async function handleChat(request: any, clientRawRequest: any = null) { ); if (!creds || creds.allRateLimited) return false; + comboPreselectedCredentials.set(getComboCredentialCacheKey(modelString, target), creds); return true; }; @@ -362,6 +372,10 @@ export async function handleChat(request: any, clientRawRequest: any = null) { allowedConnectionIds: target?.allowedConnectionIds ?? null, comboStepId: target?.stepId || null, comboExecutionKey: target?.executionKey || target?.stepId || null, + preselectedCredentials: comboPreselectedCredentials.get( + getComboCredentialCacheKey(m, target) + ), + cachedSettings: settings, }, combo.strategy, true @@ -481,6 +495,8 @@ async function handleSingleModelChat( allowedConnectionIds?: string[] | null; comboStepId?: string | null; comboExecutionKey?: string | null; + preselectedCredentials?: any; + cachedSettings?: any; } = {}, comboStrategy: string | null = null, isCombo: boolean = false @@ -523,7 +539,7 @@ async function handleSingleModelChat( const userAgent = request?.headers?.get("user-agent") || ""; const baseRetrySettings = resolveCooldownAwareRetrySettings( - await getCachedSettings().catch(() => ({})) + runtimeOptions.cachedSettings ?? (await getCachedSettings().catch(() => ({}))) ); const disableCooldownAwareRetry = isCombo || forceLiveComboTest || runtimeOptions.emergencyFallbackTried === true; @@ -557,26 +573,31 @@ async function handleSingleModelChat( let lastError = requestRetryLastError; let lastStatus = requestRetryLastStatus; let lastCooldownMs = requestRetryLastCooldownMs; + let preselectedCredentials = runtimeOptions.preselectedCredentials; while (true) { - const credentials = await getProviderCredentialsWithQuotaPreflight( - provider, - null, - effectiveAllowedConnections, - model, - { - excludeConnectionIds: Array.from(excludedConnectionIds), - ...(forceLiveComboTest - ? { - allowSuppressedConnections: true, - bypassQuotaPolicy: true, + const credentials = + preselectedCredentials && excludedConnectionIds.size === 0 + ? preselectedCredentials + : await getProviderCredentialsWithQuotaPreflight( + provider, + null, + effectiveAllowedConnections, + model, + { + excludeConnectionIds: Array.from(excludedConnectionIds), + ...(forceLiveComboTest + ? { + allowSuppressedConnections: true, + bypassQuotaPolicy: true, + } + : {}), + ...(runtimeOptions.forcedConnectionId + ? { forcedConnectionId: runtimeOptions.forcedConnectionId } + : {}), } - : {}), - ...(runtimeOptions.forcedConnectionId - ? { forcedConnectionId: runtimeOptions.forcedConnectionId } - : {}), - } - ); + ); + preselectedCredentials = null; if (!credentials || "allRateLimited" in credentials) { if (credentials?.allRateLimited) { @@ -710,6 +731,7 @@ async function handleSingleModelChat( comboExecutionKey: runtimeOptions.comboExecutionKey ?? runtimeOptions.comboStepId ?? null, extendedContext, providerProfile, + cachedSettings: runtimeOptions.cachedSettings, }); if (telemetry) telemetry.endPhase(); diff --git a/src/sse/handlers/chatHelpers.ts b/src/sse/handlers/chatHelpers.ts index 3079acce78..f01a6e2073 100644 --- a/src/sse/handlers/chatHelpers.ts +++ b/src/sse/handlers/chatHelpers.ts @@ -186,6 +186,7 @@ export async function executeChatWithBreaker({ comboExecutionKey, extendedContext, providerProfile, + cachedSettings, }: any): Promise<{ result: any; tlsFingerprintUsed: boolean }> { let tlsFingerprintUsed = false; @@ -206,6 +207,7 @@ export async function executeChatWithBreaker({ isCombo, comboStepId, comboExecutionKey, + cachedSettings, onCredentialsRefreshed: async (newCreds: any) => { await updateProviderCredentials(credentials.connectionId, { accessToken: newCreds.accessToken, diff --git a/tests/unit/combo-strategies.test.ts b/tests/unit/combo-strategies.test.ts index 2937ee3595..1669ad4368 100644 --- a/tests/unit/combo-strategies.test.ts +++ b/tests/unit/combo-strategies.test.ts @@ -375,6 +375,39 @@ test("reset-aware strategy uses registered quota fetchers for non-Codex provider assert.equal(await selectedConnectionFor(combo), soon); }); +test("reset-aware strategy deduplicates quota fetches for repeated connection targets", async () => { + const provider = `dedupe-provider-${randomUUID()}`; + const connectionId = `shared-${randomUUID()}`; + let fetchCount = 0; + + registerQuotaFetcher(provider, async (id) => { + fetchCount++; + assert.equal(id, connectionId); + return { + used: 20, + total: 100, + percentUsed: 0.2, + resetAt: new Date(Date.now() + 24 * 3600 * 1000).toISOString(), + }; + }); + + const combo = { + name: `reset-aware-dedupe-${randomUUID()}`, + strategy: "reset-aware", + models: ["model-a", "model-b"].map((model, index) => ({ + kind: "model", + provider, + providerId: provider, + model, + connectionId, + id: `dedupe-${index}`, + })), + }; + + assert.equal(await selectedConnectionFor(combo), connectionId); + assert.equal(fetchCount, 1); +}); + test("reset-aware strategy respects API-key allowed connections during expansion", async () => { const provider = `limited-provider-${randomUUID()}`; const disallowed = await providersDb.createProviderConnection({ From 23ec2f9fa83665c63079735f8a620768d8119656 Mon Sep 17 00:00:00 2001 From: Jan Leon Date: Fri, 8 May 2026 16:58:58 +0000 Subject: [PATCH 11/15] feat: add service tier column to usage_history and update migration checks --- src/lib/db/core.ts | 6 +++++- src/lib/db/migrationRunner.ts | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/lib/db/core.ts b/src/lib/db/core.ts index c9da9f0793..1308c6cb61 100644 --- a/src/lib/db/core.ts +++ b/src/lib/db/core.ts @@ -261,7 +261,6 @@ const SCHEMA_SQL = ` CREATE INDEX IF NOT EXISTS idx_uh_timestamp ON usage_history(timestamp); CREATE INDEX IF NOT EXISTS idx_uh_provider ON usage_history(provider); CREATE INDEX IF NOT EXISTS idx_uh_model ON usage_history(model); - CREATE INDEX IF NOT EXISTS idx_uh_service_tier ON usage_history(service_tier); CREATE TABLE IF NOT EXISTS call_logs ( id TEXT PRIMARY KEY, @@ -543,6 +542,11 @@ function ensureUsageHistoryColumns(db: SqliteDatabase) { db.exec("ALTER TABLE usage_history ADD COLUMN error_code TEXT"); console.log("[DB] Added usage_history.error_code column"); } + if (!columnNames.has("service_tier")) { + db.exec("ALTER TABLE usage_history ADD COLUMN service_tier TEXT DEFAULT 'standard'"); + console.log("[DB] Added usage_history.service_tier column"); + } + db.exec("CREATE INDEX IF NOT EXISTS idx_uh_service_tier ON usage_history(service_tier)"); } catch (error: unknown) { const message = error instanceof Error ? error.message : String(error); console.warn("[DB] Failed to verify usage_history schema:", message); diff --git a/src/lib/db/migrationRunner.ts b/src/lib/db/migrationRunner.ts index 40d74133de..d793e8cacf 100644 --- a/src/lib/db/migrationRunner.ts +++ b/src/lib/db/migrationRunner.ts @@ -285,6 +285,8 @@ function isSchemaAlreadyApplied( ); case "045": return hasColumn(db, "call_logs", "tokens_compressed"); + case "051": + return hasColumn(db, "usage_history", "service_tier"); default: return false; } From 43553646ed48025070de99278acf6c445268806f Mon Sep 17 00:00:00 2001 From: Jan Leon Date: Fri, 8 May 2026 19:21:34 +0000 Subject: [PATCH 12/15] feat: add fallbackDelayMs to combo configuration and related settings --- open-sse/services/combo.ts | 76 +++++++++++--------- open-sse/services/comboConfig.ts | 1 + src/app/api/settings/combo-defaults/route.ts | 1 + src/shared/validation/schemas.ts | 1 + src/sse/handlers/chat.ts | 21 +++++- src/sse/services/model.ts | 4 +- src/types/combo.ts | 1 + src/types/settings.ts | 1 + tests/unit/combo-499-abort.test.ts | 2 +- tests/unit/combo-config.test.ts | 3 + 10 files changed, 73 insertions(+), 38 deletions(-) diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index 3ff14b78e9..699cac5199 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -72,6 +72,12 @@ const MAX_COMBO_DEPTH = 3; const MAX_FALLBACK_WAIT_MS = 5000; const MAX_GLOBAL_ATTEMPTS = 30; +function resolveDelayMs(value: unknown, fallback: number): number { + const numericValue = Number(value); + if (!Number.isFinite(numericValue) || numericValue < 0) return fallback; + return numericValue; +} + function comboModelNotFoundResponse(message: string) { return errorResponse(404, message); } @@ -1659,7 +1665,8 @@ export async function handleComboChat({ ? resolveComboConfig(combo, settings) : { ...getDefaultComboConfig(), ...(combo.config || {}) }; const maxRetries = config.maxRetries ?? 1; - const retryDelayMs = config.retryDelayMs ?? 2000; + const retryDelayMs = resolveDelayMs(config.retryDelayMs, 2000); + const fallbackDelayMs = resolveDelayMs(config.fallbackDelayMs, 0); let orderedTargets = strategy === "weighted" @@ -2027,17 +2034,19 @@ export async function handleComboChat({ // Record last known good provider (LKGP) for this combo/model (#919) if (provider) { - try { - const { setLKGP } = await import("../../src/lib/localDb"); - await Promise.all([ - setLKGP(combo.name, target.executionKey, provider), - setLKGP(combo.name, combo.id || combo.name, provider), - ]); - } catch (err) { - log.warn("COMBO", "Failed to record Last Known Good Provider. This is non-fatal.", { - err, - }); - } + void (async () => { + try { + const { setLKGP } = await import("../../src/lib/localDb"); + await Promise.all([ + setLKGP(combo.name, target.executionKey, provider), + setLKGP(combo.name, combo.id || combo.name, provider), + ]); + } catch (err) { + log.warn("COMBO", "Failed to record Last Known Good Provider. This is non-fatal.", { + err, + }); + } + })(); } return quality.clonedResponse ?? result; @@ -2141,8 +2150,8 @@ export async function handleComboChat({ log.warn("COMBO", `Model ${modelStr} failed, trying next`, { status: result.status }); const fallbackWaitMs = - retryDelayMs > 0 && cooldownMs > 0 && cooldownMs <= MAX_FALLBACK_WAIT_MS - ? Math.min(cooldownMs, retryDelayMs) + fallbackDelayMs > 0 && cooldownMs > 0 && cooldownMs <= MAX_FALLBACK_WAIT_MS + ? Math.min(cooldownMs, fallbackDelayMs) : 0; if ([502, 503, 504].includes(result.status) && fallbackWaitMs > 0) { log.info("COMBO", `Waiting ${fallbackWaitMs}ms before fallback to next model`); @@ -2229,7 +2238,8 @@ async function handleRoundRobinCombo({ const concurrency = config.concurrencyPerModel ?? 3; const queueTimeout = config.queueTimeoutMs ?? 30000; const maxRetries = config.maxRetries ?? 1; - const retryDelayMs = config.retryDelayMs ?? 2000; + const retryDelayMs = resolveDelayMs(config.retryDelayMs, 2000); + const fallbackDelayMs = resolveDelayMs(config.fallbackDelayMs, 0); const orderedTargets = resolveComboTargets(combo, allCombos); const filteredTargets = await applyRequestTagRouting(orderedTargets, body, log); @@ -2354,21 +2364,23 @@ async function handleRoundRobinCombo({ }); recordedAttempts++; if (provider) { - try { - const { setLKGP } = await import("../../src/lib/localDb"); - await Promise.all([ - setLKGP(combo.name, target.executionKey, provider), - setLKGP(combo.name, combo.id || combo.name, provider), - ]); - } catch (err) { - log.warn( - "COMBO-RR", - "Failed to record Last Known Good Provider. This is non-fatal.", - { - err, - } - ); - } + void (async () => { + try { + const { setLKGP } = await import("../../src/lib/localDb"); + await Promise.all([ + setLKGP(combo.name, target.executionKey, provider), + setLKGP(combo.name, combo.id || combo.name, provider), + ]); + } catch (err) { + log.warn( + "COMBO-RR", + "Failed to record Last Known Good Provider. This is non-fatal.", + { + err, + } + ); + } + })(); } return result; } @@ -2488,8 +2500,8 @@ async function handleRoundRobinCombo({ log.warn("COMBO-RR", `${modelStr} failed, trying next model`, { status: result.status }); const fallbackWaitMs = - retryDelayMs > 0 && cooldownMs > 0 && cooldownMs <= MAX_FALLBACK_WAIT_MS - ? Math.min(cooldownMs, retryDelayMs) + fallbackDelayMs > 0 && cooldownMs > 0 && cooldownMs <= MAX_FALLBACK_WAIT_MS + ? Math.min(cooldownMs, fallbackDelayMs) : 0; if ([502, 503, 504].includes(result.status) && fallbackWaitMs > 0) { log.info("COMBO-RR", `Waiting ${fallbackWaitMs}ms before fallback to next model`); diff --git a/open-sse/services/comboConfig.ts b/open-sse/services/comboConfig.ts index fa383c824e..d3e04daff3 100644 --- a/open-sse/services/comboConfig.ts +++ b/open-sse/services/comboConfig.ts @@ -9,6 +9,7 @@ const DEFAULT_COMBO_CONFIG = { strategy: "priority", maxRetries: 1, retryDelayMs: 2000, + fallbackDelayMs: 0, concurrencyPerModel: 3, // max simultaneous requests per model (round-robin) queueTimeoutMs: 30000, // max wait time in semaphore queue (round-robin) handoffThreshold: 0.85, diff --git a/src/app/api/settings/combo-defaults/route.ts b/src/app/api/settings/combo-defaults/route.ts index 5909d26d04..1840d5591e 100644 --- a/src/app/api/settings/combo-defaults/route.ts +++ b/src/app/api/settings/combo-defaults/route.ts @@ -49,6 +49,7 @@ export async function GET(request: Request) { strategy: "priority", maxRetries: 1, retryDelayMs: 2000, + fallbackDelayMs: 0, handoffThreshold: 0.85, handoffModel: "", maxMessagesForSummary: 30, diff --git a/src/shared/validation/schemas.ts b/src/shared/validation/schemas.ts index c0f30dc7d9..4548e3a784 100644 --- a/src/shared/validation/schemas.ts +++ b/src/shared/validation/schemas.ts @@ -375,6 +375,7 @@ const comboRuntimeConfigSchema = z strategy: comboStrategySchema.optional(), maxRetries: z.coerce.number().int().min(0).max(10).optional(), retryDelayMs: z.coerce.number().int().min(0).max(60000).optional(), + fallbackDelayMs: z.coerce.number().int().min(0).max(60000).optional(), timeoutMs: z.coerce.number().int().min(1000).optional(), concurrencyPerModel: z.coerce.number().int().min(1).max(20).optional(), queueTimeoutMs: z.coerce.number().int().min(1000).max(120000).optional(), diff --git a/src/sse/handlers/chat.ts b/src/sse/handlers/chat.ts index de643574ef..b2203786ed 100644 --- a/src/sse/handlers/chat.ts +++ b/src/sse/handlers/chat.ts @@ -26,7 +26,7 @@ import { import * as log from "../utils/logger"; import { checkAndRefreshToken } from "../services/tokenRefresh"; import { deleteHandoff, getHandoff } from "@/lib/db/contextHandoffs"; -import { getCachedSettings, getSettings, getCombos } from "@/lib/localDb"; +import { getCachedSettings, getCombos } from "@/lib/localDb"; import { ensureOpenAIStoreSessionFallback, isOpenAIResponsesStoreEnabled, @@ -91,6 +91,21 @@ registerBailianCodingPlanQuotaFetcher(); // opt-in) when the active bucket reaches zero. registerCrofUsageFetcher(); +let combosCachePromise: Promise | null = null; +let combosCacheTs = 0; +const COMBOS_CACHE_TTL_MS = 10_000; + +async function getCombosCachedForChat(): Promise { + const now = Date.now(); + if (combosCachePromise && now - combosCacheTs < COMBOS_CACHE_TTL_MS) { + return combosCachePromise; + } + + combosCacheTs = now; + combosCachePromise = getCombos().catch(() => []); + return combosCachePromise; +} + function normalizeAllowedConnectionIds(value: unknown): string[] | null { if (!Array.isArray(value)) return null; const ids = value.filter( @@ -336,8 +351,8 @@ export async function handleChat(request: any, clientRawRequest: any = null) { // Fetch settings and all combos for config cascade and nested resolution const [settings, allCombos] = await Promise.all([ - getSettings().catch(() => ({})), - getCombos().catch(() => []), + getCachedSettings().catch(() => ({})), + getCombosCachedForChat(), ]); const relayConfig = combo.strategy === "context-relay" ? resolveComboConfig(combo, settings) : null; diff --git a/src/sse/services/model.ts b/src/sse/services/model.ts index 55cbf9cb84..e1f2b64d23 100644 --- a/src/sse/services/model.ts +++ b/src/sse/services/model.ts @@ -1,6 +1,6 @@ // Re-export from open-sse with localDb integration import { getModelAliases, getComboByName, getProviderNodes, getCustomModels } from "@/lib/localDb"; -import { getSettings } from "@/lib/localDb"; +import { getCachedSettings } from "@/lib/localDb"; import { getComboStepTarget } from "@/lib/combos/steps"; import { parseModel, @@ -83,7 +83,7 @@ export async function getModelInfo(modelStr) { // stripModelPrefix: if enabled, strip provider prefix and re-resolve // the bare model name using existing heuristics (claude-* → anthropic, etc.) try { - const settings = await getSettings(); + const settings = await getCachedSettings(); if (settings.stripModelPrefix === true) { const strippedResult = await getModelInfoCore(parsed.model, getModelAliases); return { ...strippedResult, extendedContext }; diff --git a/src/types/combo.ts b/src/types/combo.ts index 9fbfa87cd4..b5befd8e57 100644 --- a/src/types/combo.ts +++ b/src/types/combo.ts @@ -10,6 +10,7 @@ export interface Combo { nodes: ComboNode[]; maxRetries: number; retryDelayMs: number; + fallbackDelayMs?: number; timeoutMs: number; healthCheckEnabled: boolean; createdAt: string; diff --git a/src/types/settings.ts b/src/types/settings.ts index 3a4c5ad202..0fb9956f68 100644 --- a/src/types/settings.ts +++ b/src/types/settings.ts @@ -32,6 +32,7 @@ export interface ComboDefaults { strategy: RoutingStrategyValue; maxRetries: number; retryDelayMs: number; + fallbackDelayMs?: number; maxComboDepth: number; trackMetrics: boolean; concurrencyPerModel?: number; diff --git a/tests/unit/combo-499-abort.test.ts b/tests/unit/combo-499-abort.test.ts index 0333fa72ac..58455d9511 100644 --- a/tests/unit/combo-499-abort.test.ts +++ b/tests/unit/combo-499-abort.test.ts @@ -133,7 +133,7 @@ test("signal abort during fallback wait interrupts immediately", async () => { combo: makeCombo("priority", ["a/m1", "b/m2"]), handleSingleModel, log, - settings: { retryDelayMs: 5000 }, // 5s delay would normally be slow + settings: { fallbackDelayMs: 5000 }, // 5s delay would normally be slow allCombos: [], signal: ac.signal, }); diff --git a/tests/unit/combo-config.test.ts b/tests/unit/combo-config.test.ts index 0c39a25993..abaa924f23 100644 --- a/tests/unit/combo-config.test.ts +++ b/tests/unit/combo-config.test.ts @@ -14,6 +14,7 @@ test("getDefaultComboConfig returns a fresh copy of the defaults", () => { assert.equal(first.strategy, "priority"); assert.equal(first.maxRetries, 1); assert.equal(first.retryDelayMs, 2000); + assert.equal(first.fallbackDelayMs, 0); assert.ok(!("timeoutMs" in first)); assert.ok(!("healthCheckEnabled" in first)); assert.equal(first.handoffThreshold, 0.85); @@ -40,6 +41,7 @@ test("resolveComboConfig applies the full cascade from defaults to combo overrid openai: { timeoutMs: 60000, retryDelayMs: 500, + fallbackDelayMs: 100, }, }, }, @@ -48,6 +50,7 @@ test("resolveComboConfig applies the full cascade from defaults to combo overrid assert.equal(result.strategy, "round-robin"); assert.equal(result.retryDelayMs, 500); + assert.equal(result.fallbackDelayMs, 100); assert.equal(result.maxRetries, 4); assert.ok(!("timeoutMs" in result)); assert.ok(!("healthCheckEnabled" in result)); From 640ed6d2bce280dd50184beeed5d9c4d259b6e1c Mon Sep 17 00:00:00 2001 From: Jan Leon Date: Fri, 8 May 2026 19:28:40 +0000 Subject: [PATCH 13/15] feat: add STREAM_READINESS_TIMEOUT_MS and integrate into chat handling --- open-sse/config/constants.ts | 5 +++++ open-sse/handlers/chatCore.ts | 10 ++++++++-- open-sse/utils/sseHeartbeat.ts | 4 ++-- open-sse/utils/streamHandler.ts | 7 +++++-- src/shared/utils/runtimeTimeouts.ts | 12 ++++++++++++ 5 files changed, 32 insertions(+), 6 deletions(-) diff --git a/open-sse/config/constants.ts b/open-sse/config/constants.ts index 462dcd481c..6faf2df056 100644 --- a/open-sse/config/constants.ts +++ b/open-sse/config/constants.ts @@ -17,6 +17,11 @@ export const FETCH_TIMEOUT_MS = upstreamTimeouts.fetchTimeoutMs; // idle for this duration. Override with STREAM_IDLE_TIMEOUT_MS env var. export const STREAM_IDLE_TIMEOUT_MS = upstreamTimeouts.streamIdleTimeoutMs; +// Timeout for the first useful SSE event. Keep this much shorter than the +// post-start idle timeout so slow-thinking models can keep streaming after the +// first token, while dead 200 OK streams fail fast enough for combo fallback. +export const STREAM_READINESS_TIMEOUT_MS = upstreamTimeouts.streamReadinessTimeoutMs; + // Timeout for reading the full response body after headers arrive (ms). // Prevents indefinite hangs when the upstream sends headers but stalls on the body. // Defaults to FETCH_TIMEOUT_MS. Override with FETCH_BODY_TIMEOUT_MS env var. diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index bc5310a4ea..0a2c7b9dad 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -10,6 +10,7 @@ import { } from "../utils/stream.ts"; import { ensureStreamReadiness } from "../utils/streamReadiness.ts"; import { createStreamController, pipeWithDisconnect } from "../utils/streamHandler.ts"; +import { createSseHeartbeatTransform } from "../utils/sseHeartbeat.ts"; import { addBufferToUsage, filterUsageForFormat, estimateUsage } from "../utils/usageTracking.ts"; import { refreshWithRetry } from "../services/tokenRefresh.ts"; import { createRequestLogger } from "../utils/requestLogger.ts"; @@ -32,6 +33,7 @@ import { HTTP_STATUS, PROVIDER_MAX_TOKENS, STREAM_IDLE_TIMEOUT_MS, + STREAM_READINESS_TIMEOUT_MS, } from "../config/constants.ts"; import { classifyProviderError, @@ -3941,7 +3943,7 @@ export async function handleChatCore({ // Streaming response const streamReadiness = await ensureStreamReadiness(providerResponse, { - timeoutMs: STREAM_IDLE_TIMEOUT_MS, + timeoutMs: STREAM_READINESS_TIMEOUT_MS, provider, model, log, @@ -3990,8 +3992,9 @@ export async function handleChatCore({ const responseHeaders = { "Content-Type": "text/event-stream", - "Cache-Control": "no-cache", + "Cache-Control": "no-cache, no-transform", Connection: "keep-alive", + "X-Accel-Buffering": "no", [OMNIROUTE_RESPONSE_HEADERS.cache]: "MISS", ...buildOmniRouteResponseMetaHeaders({ provider, @@ -4236,6 +4239,9 @@ export async function handleChatCore({ } else { finalStream = pipeWithDisconnect(providerResponse, transformStream, streamController); } + finalStream = finalStream.pipeThrough( + createSseHeartbeatTransform({ signal: streamController.signal }) + ); return { success: true, diff --git a/open-sse/utils/sseHeartbeat.ts b/open-sse/utils/sseHeartbeat.ts index 1ceb6820b3..db8d0ff624 100644 --- a/open-sse/utils/sseHeartbeat.ts +++ b/open-sse/utils/sseHeartbeat.ts @@ -1,4 +1,4 @@ -const DEFAULT_INTERVAL_MS = 15_000; +export const DEFAULT_SSE_HEARTBEAT_INTERVAL_MS = 15_000; type SseHeartbeatTransformOptions = { intervalMs?: number; @@ -6,7 +6,7 @@ type SseHeartbeatTransformOptions = { }; export function createSseHeartbeatTransform({ - intervalMs = DEFAULT_INTERVAL_MS, + intervalMs = DEFAULT_SSE_HEARTBEAT_INTERVAL_MS, signal, }: SseHeartbeatTransformOptions = {}) { let intervalId: ReturnType | undefined; diff --git a/open-sse/utils/streamHandler.ts b/open-sse/utils/streamHandler.ts index 89d695c6fb..2bfbcef68e 100644 --- a/open-sse/utils/streamHandler.ts +++ b/open-sse/utils/streamHandler.ts @@ -3,6 +3,7 @@ import { trackPendingRequest } from "@/lib/usageDb"; // Stream handler with disconnect detection - shared for all providers const PENDING_REQUEST_CLEARED_MARKER = "__omniroutePendingRequestCleared"; +const DISCONNECT_ABORT_DELAY_MS = 2_000; type StreamDisconnectEvent = { reason: string; @@ -97,7 +98,7 @@ export function createStreamController({ // Delay abort to allow cleanup abortTimeout = setTimeout(() => { abortController.abort(); - }, 500); + }, DISCONNECT_ABORT_DELAY_MS); onDisconnect?.({ reason, duration: Date.now() - startTime }); }, @@ -201,7 +202,9 @@ export function createDisconnectAwareStream(transformStream, streamController) { cancel(reason) { streamController.handleDisconnect(reason || "cancelled"); reader.cancel(); - writer.abort(); + setTimeout(() => { + writer.abort(); + }, DISCONNECT_ABORT_DELAY_MS).unref?.(); }, }); } diff --git a/src/shared/utils/runtimeTimeouts.ts b/src/shared/utils/runtimeTimeouts.ts index b2794a7745..672466a1c2 100644 --- a/src/shared/utils/runtimeTimeouts.ts +++ b/src/shared/utils/runtimeTimeouts.ts @@ -8,6 +8,7 @@ type ReadTimeoutOptions = { export const DEFAULT_FETCH_TIMEOUT_MS = 600_000; export const DEFAULT_STREAM_IDLE_TIMEOUT_MS = 600_000; +export const DEFAULT_STREAM_READINESS_TIMEOUT_MS = 30_000; export const DEFAULT_FETCH_CONNECT_TIMEOUT_MS = 30_000; export const DEFAULT_FETCH_KEEPALIVE_TIMEOUT_MS = 4_000; export const DEFAULT_API_BRIDGE_PROXY_TIMEOUT_MS = 30_000; @@ -24,6 +25,7 @@ function hasEnvValue(env: EnvSource, name: string): boolean { export type UpstreamTimeoutConfig = { fetchTimeoutMs: number; streamIdleTimeoutMs: number; + streamReadinessTimeoutMs: number; fetchHeadersTimeoutMs: number; fetchBodyTimeoutMs: number; fetchConnectTimeoutMs: number; @@ -89,10 +91,20 @@ export function getUpstreamTimeoutConfig( logger, } ); + const streamReadinessTimeoutMs = readTimeoutMs( + env, + "STREAM_READINESS_TIMEOUT_MS", + DEFAULT_STREAM_READINESS_TIMEOUT_MS, + { + allowZero: true, + logger, + } + ); return { fetchTimeoutMs, streamIdleTimeoutMs, + streamReadinessTimeoutMs, fetchHeadersTimeoutMs: readTimeoutMs(env, "FETCH_HEADERS_TIMEOUT_MS", fetchTimeoutMs, { allowZero: true, logger, From 32f3f3d94fb0cd55bd9d7504039e83c9f7c2f587 Mon Sep 17 00:00:00 2001 From: Jan Leon Date: Fri, 8 May 2026 21:29:13 +0000 Subject: [PATCH 14/15] feat: enhance error handling for semaphore capacity and implement fallback logic in chat processing --- open-sse/handlers/chatCore.ts | 43 +++++++++++++++++++++++++++++------ src/sse/handlers/chat.ts | 19 ++++++++++++++++ 2 files changed, 55 insertions(+), 7 deletions(-) diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 0a2c7b9dad..e6c4c03b1b 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -595,14 +595,39 @@ function toFiniteNumberOrNull(value: unknown): number | null { return null; } -function isSemaphoreTimeoutError(error: unknown): error is Error & { code: string } { +function isSemaphoreCapacityError(error: unknown): error is Error & { code: string } { return ( !!error && typeof error === "object" && - (error as { code?: unknown }).code === "SEMAPHORE_TIMEOUT" + ((error as { code?: unknown }).code === "SEMAPHORE_TIMEOUT" || + (error as { code?: unknown }).code === "SEMAPHORE_QUEUE_FULL") ); } +function createStreamingErrorResult(statusCode: number, message: string, code?: string) { + const errorBody = buildErrorBody(statusCode, message); + if (code) { + errorBody.error.code = code; + } + + const body = `data: ${JSON.stringify(errorBody)}\n\ndata: [DONE]\n\n`; + + return { + success: false as const, + status: statusCode, + error: message, + response: new Response(body, { + status: statusCode, + headers: { + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache, no-transform", + Connection: "keep-alive", + "X-Accel-Buffering": "no", + }, + }), + }; +} + function wrapReadableStreamWithFinalize( readable: ReadableStream, finalize: () => void @@ -2998,16 +3023,13 @@ export async function handleChatCore({ ); } catch (error) { trackPendingRequest(model, provider, connectionId, false); - if (isSemaphoreTimeoutError(error)) { + if (isSemaphoreCapacityError(error)) { appendRequestLog({ model, provider, connectionId, status: `FAILED ${error.code}`, }).catch(() => {}); - if (isCombo) { - throw error; - } const failureMessage = error.message || "Semaphore timeout"; persistAttemptLogs({ status: HTTP_STATUS.RATE_LIMITED, @@ -3018,7 +3040,14 @@ export async function handleChatCore({ cacheSource: "upstream", }); persistFailureUsage(HTTP_STATUS.RATE_LIMITED, error.code); - return createErrorResult(HTTP_STATUS.RATE_LIMITED, failureMessage); + const result = stream + ? createStreamingErrorResult(HTTP_STATUS.RATE_LIMITED, failureMessage, error.code) + : createErrorResult(HTTP_STATUS.RATE_LIMITED, failureMessage); + return { + ...result, + errorType: "account_semaphore_capacity", + errorCode: error.code, + }; } const failureStatus = error.name === "AbortError" diff --git a/src/sse/handlers/chat.ts b/src/sse/handlers/chat.ts index b2203786ed..980d93f1e7 100644 --- a/src/sse/handlers/chat.ts +++ b/src/sse/handlers/chat.ts @@ -791,6 +791,25 @@ async function handleSingleModelChat( return result.response; } + if (result.errorType === "account_semaphore_capacity") { + // Local concurrency pressure is not an upstream quota failure. Prefer another + // account when possible; pinned combo steps fall through to combo orchestration. + if (hasForcedConnection) { + return result.response; + } + + log.warn( + "AUTH", + `Account ${accountId}... at local concurrency cap, trying fallback account` + ); + excludedConnectionIds.add(credentials.connectionId); + lastError = result.error; + lastStatus = result.status; + requestRetryLastError = result.error; + requestRetryLastStatus = result.status; + continue; + } + // Emergency fallback for budget exhaustion (402 / billing / quota keywords): // reroute to a free model (default provider/model: nvidia + openai/gpt-oss-120b) exactly once. if (!runtimeOptions.emergencyFallbackTried) { From ad2600b4d01a5e624f0282ad213c744031223007 Mon Sep 17 00:00:00 2001 From: Jan Leon Date: Sat, 9 May 2026 12:42:59 +0000 Subject: [PATCH 15/15] feat: update API bridge proxy timeout to 600000ms and enhance related tests --- src/lib/apiBridgeServer.ts | 24 ++++++++++++++++++++++-- src/shared/utils/runtimeTimeouts.ts | 2 +- tests/unit/runtime-timeouts.test.ts | 8 ++++++++ 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/lib/apiBridgeServer.ts b/src/lib/apiBridgeServer.ts index d289d6f752..4109dfd6ac 100644 --- a/src/lib/apiBridgeServer.ts +++ b/src/lib/apiBridgeServer.ts @@ -22,7 +22,22 @@ function isOpenAiCompatiblePath(pathname: string): boolean { return OPENAI_COMPAT_PATHS.some((pattern) => pattern.test(pathname)); } +function requestWantsStreaming(req: IncomingMessage): boolean { + const accept = String(req.headers.accept || "").toLowerCase(); + if (accept.includes("text/event-stream")) return true; + + const pathname = (req.url || "/").split("?")[0] || "/"; + return /^\/(?:v1\/)?(?:responses|chat\/completions)(?:\/|$)/.test(pathname); +} + +function getProxyTimeoutMs(req: IncomingMessage): number { + if (!requestWantsStreaming(req)) return API_BRIDGE_TIMEOUTS.proxyTimeoutMs; + + return Math.max(API_BRIDGE_TIMEOUTS.proxyTimeoutMs, API_BRIDGE_TIMEOUTS.serverRequestTimeoutMs); +} + function proxyRequest(req: IncomingMessage, res: ServerResponse, dashboardPort: number): void { + const proxyTimeoutMs = getProxyTimeoutMs(req); const targetReq = http.request( { hostname: "127.0.0.1", @@ -33,9 +48,14 @@ function proxyRequest(req: IncomingMessage, res: ServerResponse, dashboardPort: ...req.headers, host: `127.0.0.1:${dashboardPort}`, }, - timeout: API_BRIDGE_TIMEOUTS.proxyTimeoutMs, + timeout: proxyTimeoutMs, }, (targetRes) => { + const contentType = String(targetRes.headers["content-type"] || "").toLowerCase(); + if (contentType.includes("text/event-stream")) { + targetReq.setTimeout(0); + } + res.writeHead(targetRes.statusCode || 502, targetRes.headers); targetRes.pipe(res); } @@ -48,7 +68,7 @@ function proxyRequest(req: IncomingMessage, res: ServerResponse, dashboardPort: res.end( JSON.stringify({ error: "api_bridge_timeout", - detail: `Proxy request timed out after ${API_BRIDGE_TIMEOUTS.proxyTimeoutMs}ms`, + detail: `Proxy request timed out after ${proxyTimeoutMs}ms`, }) ); }); diff --git a/src/shared/utils/runtimeTimeouts.ts b/src/shared/utils/runtimeTimeouts.ts index 672466a1c2..14e0e67afb 100644 --- a/src/shared/utils/runtimeTimeouts.ts +++ b/src/shared/utils/runtimeTimeouts.ts @@ -11,7 +11,7 @@ export const DEFAULT_STREAM_IDLE_TIMEOUT_MS = 600_000; export const DEFAULT_STREAM_READINESS_TIMEOUT_MS = 30_000; export const DEFAULT_FETCH_CONNECT_TIMEOUT_MS = 30_000; export const DEFAULT_FETCH_KEEPALIVE_TIMEOUT_MS = 4_000; -export const DEFAULT_API_BRIDGE_PROXY_TIMEOUT_MS = 30_000; +export const DEFAULT_API_BRIDGE_PROXY_TIMEOUT_MS = 600_000; export const DEFAULT_API_BRIDGE_SERVER_REQUEST_TIMEOUT_MS = 300_000; export const DEFAULT_API_BRIDGE_SERVER_HEADERS_TIMEOUT_MS = 60_000; export const DEFAULT_API_BRIDGE_SERVER_KEEPALIVE_TIMEOUT_MS = 5_000; diff --git a/tests/unit/runtime-timeouts.test.ts b/tests/unit/runtime-timeouts.test.ts index a1db61da52..2a588a63b6 100644 --- a/tests/unit/runtime-timeouts.test.ts +++ b/tests/unit/runtime-timeouts.test.ts @@ -12,6 +12,7 @@ test("upstream timeout config derives hidden fetch timeouts from FETCH_TIMEOUT_M assert.deepEqual(config, { fetchTimeoutMs: 600000, streamIdleTimeoutMs: 600000, + streamReadinessTimeoutMs: 30000, fetchHeadersTimeoutMs: 600000, fetchBodyTimeoutMs: 600000, fetchConnectTimeoutMs: 30000, @@ -93,3 +94,10 @@ test("API bridge timeouts align request timeout with long proxy timeout by defau serverSocketTimeoutMs: 0, }); }); + +test("API bridge proxy timeout defaults to the long upstream request window", () => { + const config = runtimeTimeouts.getApiBridgeTimeoutConfig({}); + + assert.equal(config.proxyTimeoutMs, 600000); + assert.equal(config.serverRequestTimeoutMs, 600000); +});