diff --git a/config/quality/file-size-baseline.json b/config/quality/file-size-baseline.json index cb897203d3..ec7bed3a78 100644 --- a/config/quality/file-size-baseline.json +++ b/config/quality/file-size-baseline.json @@ -199,7 +199,7 @@ "src/lib/db/providers.ts": 1063, "src/lib/db/proxies.ts": 1057, "src/lib/db/settings.ts": 1149, - "src/lib/db/usageAnalytics.ts": 873, + "src/lib/db/usageAnalytics.ts": 925, "src/lib/evals/evalRunner.ts": 961, "src/lib/memory/retrieval.ts": 1171, "src/lib/modelsDevSync.ts": 934, diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 6f5cbe4035..75db44fa95 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -433,6 +433,7 @@ export async function handleChatCore({ statusCode, errorCode, latencyMs: Date.now() - startTime, + endpoint: endpointPath, }) ).catch(() => {}); }; @@ -3422,6 +3423,7 @@ export async function handleChatCore({ effectiveServiceTier, isCombo, comboStrategy, + endpoint: endpointPath, }); // Translate response to client's expected format (usually OpenAI) @@ -3806,6 +3808,7 @@ export async function handleChatCore({ effectiveServiceTier, isCombo, comboStrategy, + endpoint: endpointPath, }); persistAttemptLogs({ diff --git a/open-sse/handlers/chatCore/failureUsage.ts b/open-sse/handlers/chatCore/failureUsage.ts index fd1860eba9..52f70fbfee 100644 --- a/open-sse/handlers/chatCore/failureUsage.ts +++ b/open-sse/handlers/chatCore/failureUsage.ts @@ -19,6 +19,7 @@ export function buildFailureUsageRecord(opts: { statusCode: number; errorCode: string | null | undefined; latencyMs: number; + endpoint?: string | null | undefined; }) { return { provider: opts.provider || "unknown", @@ -35,5 +36,6 @@ export function buildFailureUsageRecord(opts: { apiKeyName: opts.apiKeyInfo?.name || undefined, serviceTier: opts.effectiveServiceTier, comboStrategy: opts.isCombo ? opts.comboStrategy || undefined : undefined, + endpoint: opts.endpoint || undefined, }; } diff --git a/open-sse/handlers/chatCore/nonStreamingUsageStats.ts b/open-sse/handlers/chatCore/nonStreamingUsageStats.ts index d7514e5d44..13e4a9ae91 100644 --- a/open-sse/handlers/chatCore/nonStreamingUsageStats.ts +++ b/open-sse/handlers/chatCore/nonStreamingUsageStats.ts @@ -27,6 +27,7 @@ export type RecordNonStreamingUsageStatsContext = { effectiveServiceTier: EffectiveServiceTier; isCombo: boolean; comboStrategy: string | null | undefined; + endpoint?: string | null | undefined; }; function logUsageTrace( @@ -55,6 +56,7 @@ function persistUsageRow(usage: object, ctx: RecordNonStreamingUsageStatsContext apiKeyName: apiKeyInfo?.name || undefined, serviceTier: effectiveServiceTier, comboStrategy: ctx.isCombo ? ctx.comboStrategy || undefined : undefined, + endpoint: ctx.endpoint || undefined, }).catch((err) => { console.error("Failed to save usage stats:", err.message); }); diff --git a/open-sse/handlers/chatCore/streamingUsageStats.ts b/open-sse/handlers/chatCore/streamingUsageStats.ts index 0561f8099b..1a48b8d857 100644 --- a/open-sse/handlers/chatCore/streamingUsageStats.ts +++ b/open-sse/handlers/chatCore/streamingUsageStats.ts @@ -27,6 +27,7 @@ export type RecordStreamingUsageStatsContext = { effectiveServiceTier: EffectiveServiceTier; isCombo: boolean; comboStrategy: string | null | undefined; + endpoint?: string | null | undefined; }; function persistStreamingUsageRow(usage: object, ctx: RecordStreamingUsageStatsContext): void { @@ -46,6 +47,7 @@ function persistStreamingUsageRow(usage: object, ctx: RecordStreamingUsageStatsC apiKeyName: ctx.apiKeyInfo?.name || undefined, serviceTier: ctx.effectiveServiceTier, comboStrategy: ctx.isCombo ? ctx.comboStrategy || undefined : undefined, + endpoint: ctx.endpoint || undefined, }).catch((err) => { console.error("Failed to save usage stats:", err.message); }); diff --git a/src/app/api/internal/codex-responses-ws/route.ts b/src/app/api/internal/codex-responses-ws/route.ts index 0e68764444..ddb4760dcf 100644 --- a/src/app/api/internal/codex-responses-ws/route.ts +++ b/src/app/api/internal/codex-responses-ws/route.ts @@ -513,6 +513,7 @@ async function persistResponsesWsCallHistory(body: JsonRecord) { latencyMs: durationMs, timeToFirstTokenMs: durationMs, errorCode, + endpoint: "/v1/responses", }); logProxyEvent({ diff --git a/src/lib/db/migrations/105_usage_history_endpoint.sql b/src/lib/db/migrations/105_usage_history_endpoint.sql new file mode 100644 index 0000000000..55a459b138 --- /dev/null +++ b/src/lib/db/migrations/105_usage_history_endpoint.sql @@ -0,0 +1,6 @@ +-- Migration 105: Add endpoint column to usage_history +-- Tracks the API endpoint path (e.g. /v1/chat/completions, /v1/messages, /v1/responses) +-- so usage analytics can break down activity per endpoint dimension. +-- Backward compatible: existing rows default to NULL; aggregation queries fold NULL into 'unknown'. +ALTER TABLE usage_history ADD COLUMN endpoint TEXT; +CREATE INDEX IF NOT EXISTS idx_uh_endpoint ON usage_history(endpoint); diff --git a/src/lib/db/usageAnalytics.ts b/src/lib/db/usageAnalytics.ts index 2d73c1d92f..1bdf859648 100644 --- a/src/lib/db/usageAnalytics.ts +++ b/src/lib/db/usageAnalytics.ts @@ -821,6 +821,79 @@ export function getPresetCostModelRows( .all(params) as PresetCostModelRow[]; } +// --------------------------------------------------------------------------- +// Endpoint dimension — ported from decolua/9router#152 (thanks @toanalien). +// Reads directly from usage_history (raw rows) so the unified CTE stays +// untouched; matches the pattern used by getAutoRoutingVariantBreakdown. +// --------------------------------------------------------------------------- + +export interface EndpointUsageRow { + endpoint: string; + provider: string; + model: string; + requests: number; + promptTokens: number; + completionTokens: number; + cacheReadTokens: number; + cacheCreationTokens: number; + reasoningTokens: number; + totalTokens: number; + avgLatencyMs: number; + successfulRequests: number; + lastUsed: string; +} + +export interface EndpointUsageParams { + sinceIso?: string | null; + untilIso?: string | null; +} + +/** + * Per-endpoint × provider × model usage aggregates from `usage_history`. + * NULL endpoints fold into the 'unknown' bucket so legacy rows stay visible. + * + * Inspired by decolua/9router#152 (byEndpoint aggregation), reshaped for the + * OmniRoute SQLite schema + analytics conventions. + */ +export function getEndpointUsageRows(params: EndpointUsageParams = {}): EndpointUsageRow[] { + const db = getDbInstance(); + const conditions: string[] = []; + const bind: Record = {}; + if (params.sinceIso) { + conditions.push("timestamp >= @since"); + bind.since = params.sinceIso; + } + if (params.untilIso) { + conditions.push("timestamp <= @until"); + bind.until = params.untilIso; + } + const whereSql = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : ""; + return db + .prepare( + ` + SELECT + COALESCE(NULLIF(endpoint, ''), 'unknown') as endpoint, + LOWER(COALESCE(provider, 'unknown')) as provider, + LOWER(COALESCE(model, 'unknown')) as model, + COUNT(*) as requests, + COALESCE(SUM(tokens_input), 0) as promptTokens, + COALESCE(SUM(tokens_output), 0) as completionTokens, + COALESCE(SUM(tokens_cache_read), 0) as cacheReadTokens, + COALESCE(SUM(tokens_cache_creation), 0) as cacheCreationTokens, + COALESCE(SUM(tokens_reasoning), 0) as reasoningTokens, + COALESCE(SUM(tokens_input + tokens_output), 0) as totalTokens, + COALESCE(AVG(latency_ms), 0) as avgLatencyMs, + COALESCE(SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END), 0) as successfulRequests, + COALESCE(MAX(timestamp), '') as lastUsed + FROM usage_history + ${whereSql} + GROUP BY endpoint, LOWER(COALESCE(provider, 'unknown')), LOWER(COALESCE(model, 'unknown')) + ORDER BY requests DESC + ` + ) + .all(bind) as EndpointUsageRow[]; +} + // --------------------------------------------------------------------------- // Export-JSON backup — /api/settings/export-json // --------------------------------------------------------------------------- diff --git a/src/lib/usage/usageHistory.ts b/src/lib/usage/usageHistory.ts index f22d053e46..0e5ed1fe82 100644 --- a/src/lib/usage/usageHistory.ts +++ b/src/lib/usage/usageHistory.ts @@ -641,8 +641,8 @@ export async function saveRequestUsage(entry: any) { ` INSERT INTO usage_history (provider, model, connection_id, api_key_id, api_key_name, tokens_input, tokens_output, tokens_cache_read, tokens_cache_creation, tokens_reasoning, - service_tier, status, success, latency_ms, ttft_ms, error_code, combo_strategy, timestamp) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + service_tier, status, success, latency_ms, ttft_ms, error_code, combo_strategy, endpoint, timestamp) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ` ).run( entry.provider || null, @@ -666,6 +666,7 @@ export async function saveRequestUsage(entry: any) { : 0, entry.errorCode || null, entry.comboStrategy || entry.combo_strategy || null, + entry.endpoint || null, timestamp ); diff --git a/tests/unit/usage-endpoint-dimension.test.ts b/tests/unit/usage-endpoint-dimension.test.ts new file mode 100644 index 0000000000..9712d92407 --- /dev/null +++ b/tests/unit/usage-endpoint-dimension.test.ts @@ -0,0 +1,125 @@ +/** + * Ported feature regression — decolua/9router#152 (thanks @toanalien). + * + * Covers the new `endpoint` column on usage_history + getEndpointUsageRows() + * aggregation. Asserts: persistence round-trip, NULL → 'unknown' folding, + * per-endpoint grouping, sinceIso filter. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-usage-endpoint-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../src/lib/db/core.ts"); +const usageHistory = await import("../../src/lib/usage/usageHistory.ts"); +const usageAnalytics = await import("../../src/lib/db/usageAnalytics.ts"); + +async function resetStorage() { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); + usageHistory.clearPendingRequests(); +} + +test.beforeEach(async () => { + await resetStorage(); +}); + +test.after(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +test("saveRequestUsage persists endpoint and getEndpointUsageRows groups by endpoint", async () => { + await usageHistory.saveRequestUsage({ + provider: "openai", + model: "gpt-4o-mini", + tokens: { input: 10, output: 5 }, + success: true, + latencyMs: 100, + timestamp: new Date().toISOString(), + endpoint: "/v1/chat/completions", + }); + await usageHistory.saveRequestUsage({ + provider: "openai", + model: "gpt-4o-mini", + tokens: { input: 20, output: 8 }, + success: true, + latencyMs: 200, + timestamp: new Date().toISOString(), + endpoint: "/v1/chat/completions", + }); + await usageHistory.saveRequestUsage({ + provider: "anthropic", + model: "claude-sonnet-4", + tokens: { input: 30, output: 12 }, + success: true, + latencyMs: 300, + timestamp: new Date().toISOString(), + endpoint: "/v1/messages", + }); + + const rows = usageAnalytics.getEndpointUsageRows(); + const byKey = new Map(rows.map((r) => [`${r.endpoint}|${r.provider}|${r.model}`, r])); + + const chat = byKey.get("/v1/chat/completions|openai|gpt-4o-mini"); + assert.ok(chat, "expected /v1/chat/completions row"); + assert.equal(chat.requests, 2); + assert.equal(chat.promptTokens, 30); + assert.equal(chat.completionTokens, 13); + + const messages = byKey.get("/v1/messages|anthropic|claude-sonnet-4"); + assert.ok(messages, "expected /v1/messages row"); + assert.equal(messages.requests, 1); + assert.equal(messages.promptTokens, 30); +}); + +test("getEndpointUsageRows folds NULL endpoint into 'unknown' bucket (backward compat)", async () => { + // Legacy entry: no endpoint field set → stored as NULL. + await usageHistory.saveRequestUsage({ + provider: "openai", + model: "gpt-4o-mini", + tokens: { input: 5, output: 5 }, + success: true, + latencyMs: 50, + timestamp: new Date().toISOString(), + }); + + const rows = usageAnalytics.getEndpointUsageRows(); + const unknown = rows.find((r) => r.endpoint === "unknown"); + assert.ok(unknown, "NULL endpoint should fold into 'unknown'"); + assert.equal(unknown.requests, 1); +}); + +test("getEndpointUsageRows honors sinceIso filter", async () => { + const old = new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString(); + const recent = new Date().toISOString(); + await usageHistory.saveRequestUsage({ + provider: "openai", + model: "gpt-4o-mini", + tokens: { input: 5, output: 5 }, + success: true, + latencyMs: 50, + timestamp: old, + endpoint: "/v1/chat/completions", + }); + await usageHistory.saveRequestUsage({ + provider: "openai", + model: "gpt-4o-mini", + tokens: { input: 5, output: 5 }, + success: true, + latencyMs: 50, + timestamp: recent, + endpoint: "/v1/chat/completions", + }); + + const sinceIso = new Date(Date.now() - 60 * 60 * 1000).toISOString(); + const rows = usageAnalytics.getEndpointUsageRows({ sinceIso }); + const chat = rows.find((r) => r.endpoint === "/v1/chat/completions"); + assert.ok(chat); + assert.equal(chat.requests, 1, "only the recent row should be counted"); +});