From 69c778eb45441b565a39d7a23ccb3db250b48fae Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza <8016841+diegosouzapw@users.noreply.github.com> Date: Fri, 17 Jul 2026 10:40:01 -0300 Subject: [PATCH] feat(api): expose GET /api/usage/model-latency-stats (#6873) (#7218) --- .../features/6873-model-latency-stats-api.md | 1 + .../api/usage/model-latency-stats/route.ts | 61 +++++ src/lib/usage/usageHistory.ts | 25 ++- tests/unit/model-latency-stats-route.test.ts | 210 ++++++++++++++++++ 4 files changed, 292 insertions(+), 5 deletions(-) create mode 100644 changelog.d/features/6873-model-latency-stats-api.md create mode 100644 src/app/api/usage/model-latency-stats/route.ts create mode 100644 tests/unit/model-latency-stats-route.test.ts diff --git a/changelog.d/features/6873-model-latency-stats-api.md b/changelog.d/features/6873-model-latency-stats-api.md new file mode 100644 index 0000000000..1b5b0656e5 --- /dev/null +++ b/changelog.d/features/6873-model-latency-stats-api.md @@ -0,0 +1 @@ +- **feat(api):** new **GET /api/usage/model-latency-stats** management endpoint exposes the existing rolling per-provider/model latency aggregate (avg/p50/p95/p99, success rate) already used internally by auto-combo routing — supports `windowHours`/`minSamples`/`maxRows`/`provider`/`model` filters (#6873). diff --git a/src/app/api/usage/model-latency-stats/route.ts b/src/app/api/usage/model-latency-stats/route.ts new file mode 100644 index 0000000000..bd096d9496 --- /dev/null +++ b/src/app/api/usage/model-latency-stats/route.ts @@ -0,0 +1,61 @@ +import { NextResponse } from "next/server"; +import { z } from "zod"; + +import { requireManagementAuth } from "@/lib/api/requireManagementAuth"; +import { getModelLatencyStats } from "@/lib/usageDb"; +import { buildErrorBody } from "@omniroute/open-sse/utils/error.ts"; + +const querySchema = z.object({ + windowHours: z.coerce + .number() + .positive() + .max(24 * 30) + .optional(), + minSamples: z.coerce.number().int().positive().optional(), + maxRows: z.coerce.number().int().positive().max(50000).optional(), + provider: z.string().trim().min(1).max(64).optional(), + model: z.string().trim().min(1).max(256).optional(), +}); + +export async function GET(request: Request) { + const authError = await requireManagementAuth(request); + if (authError) return authError; + + try { + const { searchParams } = new URL(request.url); + const parsed = querySchema.safeParse({ + windowHours: searchParams.get("windowHours") || undefined, + minSamples: searchParams.get("minSamples") || undefined, + maxRows: searchParams.get("maxRows") || undefined, + provider: searchParams.get("provider") || undefined, + model: searchParams.get("model") || undefined, + }); + + if (!parsed.success) { + return NextResponse.json( + buildErrorBody(400, parsed.error.issues[0]?.message ?? "Invalid query parameters"), + { status: 400 } + ); + } + + const { windowHours, minSamples, maxRows, provider, model } = parsed.data; + const statsByKey = await getModelLatencyStats({ + windowHours, + minSamples, + maxRows, + provider, + model, + }); + + return NextResponse.json({ + entries: Object.values(statsByKey), + windowHours: windowHours ?? 24, + generatedAt: new Date().toISOString(), + }); + } catch (error) { + console.error("[API] GET /api/usage/model-latency-stats error:", error); + return NextResponse.json(buildErrorBody(500, "Failed to build model latency stats"), { + status: 500, + }); + } +} diff --git a/src/lib/usage/usageHistory.ts b/src/lib/usage/usageHistory.ts index a13ffb13f6..b6a019df3a 100644 --- a/src/lib/usage/usageHistory.ts +++ b/src/lib/usage/usageHistory.ts @@ -792,7 +792,13 @@ export interface ModelLatencyStatsEntry { * Used by auto-combo routing to incorporate real-world latency and reliability. */ export async function getModelLatencyStats( - options: { windowHours?: number; minSamples?: number; maxRows?: number } = {} + options: { + windowHours?: number; + minSamples?: number; + maxRows?: number; + provider?: string; + model?: string; + } = {} ): Promise> { const windowHours = Number.isFinite(Number(options.windowHours)) && Number(options.windowHours) > 0 @@ -817,19 +823,28 @@ export async function getModelLatencyStats( latency_ms: number | null; }; + const conditions = ["timestamp >= @sinceIso", "provider IS NOT NULL", "model IS NOT NULL"]; + const queryParams: Record = { sinceIso, maxRows }; + if (options.provider) { + conditions.push("provider = @provider"); + queryParams.provider = options.provider; + } + if (options.model) { + conditions.push("model = @model"); + queryParams.model = options.model; + } + const rows = db .prepare( ` SELECT provider, model, success, latency_ms FROM usage_history - WHERE timestamp >= @sinceIso - AND provider IS NOT NULL - AND model IS NOT NULL + WHERE ${conditions.join(" AND ")} ORDER BY timestamp DESC LIMIT @maxRows ` ) - .all({ sinceIso, maxRows }) as LatencyRow[]; + .all(queryParams) as LatencyRow[]; const grouped = new Map< string, diff --git a/tests/unit/model-latency-stats-route.test.ts b/tests/unit/model-latency-stats-route.test.ts new file mode 100644 index 0000000000..15cbc0650a --- /dev/null +++ b/tests/unit/model-latency-stats-route.test.ts @@ -0,0 +1,210 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { makeManagementSessionRequest } from "../helpers/managementSession.ts"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-model-latency-")); +const ORIGINAL_DATA_DIR = process.env.DATA_DIR; +const ORIGINAL_INITIAL_PASSWORD = process.env.INITIAL_PASSWORD; +const ORIGINAL_JWT_SECRET = process.env.JWT_SECRET; + +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../src/lib/db/core.ts"); +const settingsDb = await import("../../src/lib/db/settings.ts"); +const usageHistory = await import("../../src/lib/usage/usageHistory.ts"); +const route = await import("../../src/app/api/usage/model-latency-stats/route.ts"); + +async function resetStorage() { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +} + +async function enableManagementAuth() { + process.env.INITIAL_PASSWORD = "model-latency-password"; + await settingsDb.updateSettings({ requireLogin: true, password: "" }); +} + +let seedCounter = 0; + +// Each call gets a distinct connectionId + timestamp so the saveRequestUsage +// dedup guard (same-second identity match on provider/model/connection/apiKey/ +// tokens) never collapses two intentionally-distinct seeded rows into one — +// aggregation groups by provider/model only, so connectionId has no effect +// on the assertions below. +async function seedUsage(provider: string, model: string, latencyMs: number, success = true) { + seedCounter += 1; + await usageHistory.saveRequestUsage({ + provider, + model, + success, + latencyMs, + status: success ? "200" : "500", + connectionId: `seed-conn-${seedCounter}`, + timestamp: new Date(Date.now() + seedCounter).toISOString(), + }); +} + +test.beforeEach(async () => { + await resetStorage(); +}); + +test.after(async () => { + await resetStorage(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + + if (ORIGINAL_DATA_DIR === undefined) delete process.env.DATA_DIR; + else process.env.DATA_DIR = ORIGINAL_DATA_DIR; + + if (ORIGINAL_INITIAL_PASSWORD === undefined) delete process.env.INITIAL_PASSWORD; + else process.env.INITIAL_PASSWORD = ORIGINAL_INITIAL_PASSWORD; + + if (ORIGINAL_JWT_SECRET === undefined) delete process.env.JWT_SECRET; + else process.env.JWT_SECRET = ORIGINAL_JWT_SECRET; +}); + +test("model latency stats route requires management auth", async () => { + await enableManagementAuth(); + + const unauthenticated = await route.GET( + new Request("http://localhost/api/usage/model-latency-stats") + ); + assert.equal(unauthenticated.status, 401); +}); + +test("model latency stats route aggregates and returns entries for seeded providers/models", async () => { + await enableManagementAuth(); + await seedUsage("openai", "gpt-4o-mini", 100); + await seedUsage("openai", "gpt-4o-mini", 120); + await seedUsage("anthropic", "claude-3-5-haiku", 200); + await seedUsage("anthropic", "claude-3-5-haiku", 220); + + const response = await route.GET( + await makeManagementSessionRequest("http://localhost/api/usage/model-latency-stats") + ); + assert.equal(response.status, 200); + const body = await response.json(); + + assert.equal(body.windowHours, 24); + assert.ok(typeof body.generatedAt === "string"); + assert.equal(body.entries.length, 2); + + const openaiEntry = body.entries.find((e: { provider: string }) => e.provider === "openai"); + assert.ok(openaiEntry); + assert.equal(openaiEntry.model, "gpt-4o-mini"); + assert.equal(openaiEntry.totalRequests, 2); + assert.equal(openaiEntry.successfulRequests, 2); + assert.equal(openaiEntry.successRate, 1); + assert.equal(openaiEntry.avgLatencyMs, 110); +}); + +test("model latency stats route filters by provider query param", async () => { + await enableManagementAuth(); + await seedUsage("openai", "gpt-4o-mini", 100); + await seedUsage("anthropic", "claude-3-5-haiku", 200); + + const response = await route.GET( + await makeManagementSessionRequest( + "http://localhost/api/usage/model-latency-stats?provider=openai" + ) + ); + assert.equal(response.status, 200); + const body = await response.json(); + + assert.equal(body.entries.length, 1); + assert.equal(body.entries[0].provider, "openai"); +}); + +test("model latency stats route filters by model query param", async () => { + await enableManagementAuth(); + await seedUsage("openai", "gpt-4o-mini", 100); + await seedUsage("openai", "gpt-4o", 150); + + const response = await route.GET( + await makeManagementSessionRequest( + "http://localhost/api/usage/model-latency-stats?model=gpt-4o-mini" + ) + ); + assert.equal(response.status, 200); + const body = await response.json(); + + assert.equal(body.entries.length, 1); + assert.equal(body.entries[0].model, "gpt-4o-mini"); +}); + +test("model latency stats route excludes provider/model pairs below minSamples", async () => { + await enableManagementAuth(); + await seedUsage("openai", "gpt-4o-mini", 100); + await seedUsage("anthropic", "claude-3-5-haiku", 200); + await seedUsage("anthropic", "claude-3-5-haiku", 220); + + const response = await route.GET( + await makeManagementSessionRequest( + "http://localhost/api/usage/model-latency-stats?minSamples=2" + ) + ); + assert.equal(response.status, 200); + const body = await response.json(); + + assert.equal(body.entries.length, 1); + assert.equal(body.entries[0].provider, "anthropic"); +}); + +test("model latency stats route returns 400 with sanitized error body on invalid query params", async () => { + await enableManagementAuth(); + + const response = await route.GET( + await makeManagementSessionRequest( + "http://localhost/api/usage/model-latency-stats?windowHours=-5" + ) + ); + assert.equal(response.status, 400); + const body = await response.json(); + + assert.ok(body.error); + assert.ok(typeof body.error.message === "string"); + assert.ok(!body.error.message.includes("at /")); +}); + +test("model latency stats route returns 400 for maxRows above the allowed cap", async () => { + await enableManagementAuth(); + + const response = await route.GET( + await makeManagementSessionRequest( + "http://localhost/api/usage/model-latency-stats?maxRows=999999999" + ) + ); + assert.equal(response.status, 400); +}); + +test("model latency stats route returns sanitized 500 body when the aggregate throws", async () => { + await enableManagementAuth(); + + // Close the underlying SQLite handle without resetting the module-level + // singleton reference, so the next getDbInstance() call inside the route + // hits a closed connection ("The database connection is not open") and + // the route's catch block has to produce a real sanitized 500 — no + // module-namespace mocking (ESM bindings here are non-writable at runtime + // under node:test) and no fabricated error message. + core.closeDbInstance(); + const db = core.getDbInstance(); + db.close(); + + try { + const response = await route.GET( + await makeManagementSessionRequest("http://localhost/api/usage/model-latency-stats") + ); + assert.equal(response.status, 500); + const body = await response.json(); + + assert.ok(body.error); + assert.ok(typeof body.error.message === "string"); + assert.ok(!body.error.message.includes("at /")); + } finally { + core.resetDbInstance(); + } +});