feat(api): expose GET /api/usage/model-latency-stats (#6873) (#7218)

changelog.d/features/6873-model-latency-stats-api.md

@@ -0,0 +1 @@
+- **feat(api):** new **GET /api/usage/model-latency-stats** management endpoint exposes the existing rolling per-provider/model latency aggregate (avg/p50/p95/p99, success rate) already used internally by auto-combo routing — supports `windowHours`/`minSamples`/`maxRows`/`provider`/`model` filters (#6873).

src/app/api/usage/model-latency-stats/route.ts

@@ -0,0 +1,61 @@
+import { NextResponse } from "next/server";
+import { z } from "zod";
+
+import { requireManagementAuth } from "@/lib/api/requireManagementAuth";
+import { getModelLatencyStats } from "@/lib/usageDb";
+import { buildErrorBody } from "@omniroute/open-sse/utils/error.ts";
+
+const querySchema = z.object({
+  windowHours: z.coerce
+    .number()
+    .positive()
+    .max(24 * 30)
+    .optional(),
+  minSamples: z.coerce.number().int().positive().optional(),
+  maxRows: z.coerce.number().int().positive().max(50000).optional(),
+  provider: z.string().trim().min(1).max(64).optional(),
+  model: z.string().trim().min(1).max(256).optional(),
+});
+
+export async function GET(request: Request) {
+  const authError = await requireManagementAuth(request);
+  if (authError) return authError;
+
+  try {
+    const { searchParams } = new URL(request.url);
+    const parsed = querySchema.safeParse({
+      windowHours: searchParams.get("windowHours") || undefined,
+      minSamples: searchParams.get("minSamples") || undefined,
+      maxRows: searchParams.get("maxRows") || undefined,
+      provider: searchParams.get("provider") || undefined,
+      model: searchParams.get("model") || undefined,
+    });
+
+    if (!parsed.success) {
+      return NextResponse.json(
+        buildErrorBody(400, parsed.error.issues[0]?.message ?? "Invalid query parameters"),
+        { status: 400 }
+      );
+    }
+
+    const { windowHours, minSamples, maxRows, provider, model } = parsed.data;
+    const statsByKey = await getModelLatencyStats({
+      windowHours,
+      minSamples,
+      maxRows,
+      provider,
+      model,
+    });
+
+    return NextResponse.json({
+      entries: Object.values(statsByKey),
+      windowHours: windowHours ?? 24,
+      generatedAt: new Date().toISOString(),
+    });
+  } catch (error) {
+    console.error("[API] GET /api/usage/model-latency-stats error:", error);
+    return NextResponse.json(buildErrorBody(500, "Failed to build model latency stats"), {
+      status: 500,
+    });
+  }
+}

src/lib/usage/usageHistory.ts

@@ -792,7 +792,13 @@ export interface ModelLatencyStatsEntry {
  * Used by auto-combo routing to incorporate real-world latency and reliability.
  */
 export async function getModelLatencyStats(
-  options: { windowHours?: number; minSamples?: number; maxRows?: number } = {}
+  options: {
+    windowHours?: number;
+    minSamples?: number;
+    maxRows?: number;
+    provider?: string;
+    model?: string;
+  } = {}
 ): Promise<Record<string, ModelLatencyStatsEntry>> {
   const windowHours =
     Number.isFinite(Number(options.windowHours)) && Number(options.windowHours) > 0
@@ -817,19 +823,28 @@ export async function getModelLatencyStats(
     latency_ms: number | null;
   };
 
+  const conditions = ["timestamp >= @sinceIso", "provider IS NOT NULL", "model IS NOT NULL"];
+  const queryParams: Record<string, unknown> = { sinceIso, maxRows };
+  if (options.provider) {
+    conditions.push("provider = @provider");
+    queryParams.provider = options.provider;
+  }
+  if (options.model) {
+    conditions.push("model = @model");
+    queryParams.model = options.model;
+  }
+
   const rows = db
     .prepare(
       `
       SELECT provider, model, success, latency_ms
       FROM usage_history
-      WHERE timestamp >= @sinceIso
-        AND provider IS NOT NULL
-        AND model IS NOT NULL
+      WHERE ${conditions.join(" AND ")}
       ORDER BY timestamp DESC
       LIMIT @maxRows
     `
     )
-    .all({ sinceIso, maxRows }) as LatencyRow[];
+    .all(queryParams) as LatencyRow[];
 
   const grouped = new Map<
     string,

tests/unit/model-latency-stats-route.test.ts

@@ -0,0 +1,210 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+import { makeManagementSessionRequest } from "../helpers/managementSession.ts";
+
+const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-model-latency-"));
+const ORIGINAL_DATA_DIR = process.env.DATA_DIR;
+const ORIGINAL_INITIAL_PASSWORD = process.env.INITIAL_PASSWORD;
+const ORIGINAL_JWT_SECRET = process.env.JWT_SECRET;
+
+process.env.DATA_DIR = TEST_DATA_DIR;
+
+const core = await import("../../src/lib/db/core.ts");
+const settingsDb = await import("../../src/lib/db/settings.ts");
+const usageHistory = await import("../../src/lib/usage/usageHistory.ts");
+const route = await import("../../src/app/api/usage/model-latency-stats/route.ts");
+
+async function resetStorage() {
+  core.resetDbInstance();
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+  fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
+}
+
+async function enableManagementAuth() {
+  process.env.INITIAL_PASSWORD = "model-latency-password";
+  await settingsDb.updateSettings({ requireLogin: true, password: "" });
+}
+
+let seedCounter = 0;
+
+// Each call gets a distinct connectionId + timestamp so the saveRequestUsage
+// dedup guard (same-second identity match on provider/model/connection/apiKey/
+// tokens) never collapses two intentionally-distinct seeded rows into one —
+// aggregation groups by provider/model only, so connectionId has no effect
+// on the assertions below.
+async function seedUsage(provider: string, model: string, latencyMs: number, success = true) {
+  seedCounter += 1;
+  await usageHistory.saveRequestUsage({
+    provider,
+    model,
+    success,
+    latencyMs,
+    status: success ? "200" : "500",
+    connectionId: `seed-conn-${seedCounter}`,
+    timestamp: new Date(Date.now() + seedCounter).toISOString(),
+  });
+}
+
+test.beforeEach(async () => {
+  await resetStorage();
+});
+
+test.after(async () => {
+  await resetStorage();
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+
+  if (ORIGINAL_DATA_DIR === undefined) delete process.env.DATA_DIR;
+  else process.env.DATA_DIR = ORIGINAL_DATA_DIR;
+
+  if (ORIGINAL_INITIAL_PASSWORD === undefined) delete process.env.INITIAL_PASSWORD;
+  else process.env.INITIAL_PASSWORD = ORIGINAL_INITIAL_PASSWORD;
+
+  if (ORIGINAL_JWT_SECRET === undefined) delete process.env.JWT_SECRET;
+  else process.env.JWT_SECRET = ORIGINAL_JWT_SECRET;
+});
+
+test("model latency stats route requires management auth", async () => {
+  await enableManagementAuth();
+
+  const unauthenticated = await route.GET(
+    new Request("http://localhost/api/usage/model-latency-stats")
+  );
+  assert.equal(unauthenticated.status, 401);
+});
+
+test("model latency stats route aggregates and returns entries for seeded providers/models", async () => {
+  await enableManagementAuth();
+  await seedUsage("openai", "gpt-4o-mini", 100);
+  await seedUsage("openai", "gpt-4o-mini", 120);
+  await seedUsage("anthropic", "claude-3-5-haiku", 200);
+  await seedUsage("anthropic", "claude-3-5-haiku", 220);
+
+  const response = await route.GET(
+    await makeManagementSessionRequest("http://localhost/api/usage/model-latency-stats")
+  );
+  assert.equal(response.status, 200);
+  const body = await response.json();
+
+  assert.equal(body.windowHours, 24);
+  assert.ok(typeof body.generatedAt === "string");
+  assert.equal(body.entries.length, 2);
+
+  const openaiEntry = body.entries.find((e: { provider: string }) => e.provider === "openai");
+  assert.ok(openaiEntry);
+  assert.equal(openaiEntry.model, "gpt-4o-mini");
+  assert.equal(openaiEntry.totalRequests, 2);
+  assert.equal(openaiEntry.successfulRequests, 2);
+  assert.equal(openaiEntry.successRate, 1);
+  assert.equal(openaiEntry.avgLatencyMs, 110);
+});
+
+test("model latency stats route filters by provider query param", async () => {
+  await enableManagementAuth();
+  await seedUsage("openai", "gpt-4o-mini", 100);
+  await seedUsage("anthropic", "claude-3-5-haiku", 200);
+
+  const response = await route.GET(
+    await makeManagementSessionRequest(
+      "http://localhost/api/usage/model-latency-stats?provider=openai"
+    )
+  );
+  assert.equal(response.status, 200);
+  const body = await response.json();
+
+  assert.equal(body.entries.length, 1);
+  assert.equal(body.entries[0].provider, "openai");
+});
+
+test("model latency stats route filters by model query param", async () => {
+  await enableManagementAuth();
+  await seedUsage("openai", "gpt-4o-mini", 100);
+  await seedUsage("openai", "gpt-4o", 150);
+
+  const response = await route.GET(
+    await makeManagementSessionRequest(
+      "http://localhost/api/usage/model-latency-stats?model=gpt-4o-mini"
+    )
+  );
+  assert.equal(response.status, 200);
+  const body = await response.json();
+
+  assert.equal(body.entries.length, 1);
+  assert.equal(body.entries[0].model, "gpt-4o-mini");
+});
+
+test("model latency stats route excludes provider/model pairs below minSamples", async () => {
+  await enableManagementAuth();
+  await seedUsage("openai", "gpt-4o-mini", 100);
+  await seedUsage("anthropic", "claude-3-5-haiku", 200);
+  await seedUsage("anthropic", "claude-3-5-haiku", 220);
+
+  const response = await route.GET(
+    await makeManagementSessionRequest(
+      "http://localhost/api/usage/model-latency-stats?minSamples=2"
+    )
+  );
+  assert.equal(response.status, 200);
+  const body = await response.json();
+
+  assert.equal(body.entries.length, 1);
+  assert.equal(body.entries[0].provider, "anthropic");
+});
+
+test("model latency stats route returns 400 with sanitized error body on invalid query params", async () => {
+  await enableManagementAuth();
+
+  const response = await route.GET(
+    await makeManagementSessionRequest(
+      "http://localhost/api/usage/model-latency-stats?windowHours=-5"
+    )
+  );
+  assert.equal(response.status, 400);
+  const body = await response.json();
+
+  assert.ok(body.error);
+  assert.ok(typeof body.error.message === "string");
+  assert.ok(!body.error.message.includes("at /"));
+});
+
+test("model latency stats route returns 400 for maxRows above the allowed cap", async () => {
+  await enableManagementAuth();
+
+  const response = await route.GET(
+    await makeManagementSessionRequest(
+      "http://localhost/api/usage/model-latency-stats?maxRows=999999999"
+    )
+  );
+  assert.equal(response.status, 400);
+});
+
+test("model latency stats route returns sanitized 500 body when the aggregate throws", async () => {
+  await enableManagementAuth();
+
+  // Close the underlying SQLite handle without resetting the module-level
+  // singleton reference, so the next getDbInstance() call inside the route
+  // hits a closed connection ("The database connection is not open") and
+  // the route's catch block has to produce a real sanitized 500 — no
+  // module-namespace mocking (ESM bindings here are non-writable at runtime
+  // under node:test) and no fabricated error message.
+  core.closeDbInstance();
+  const db = core.getDbInstance();
+  db.close();
+
+  try {
+    const response = await route.GET(
+      await makeManagementSessionRequest("http://localhost/api/usage/model-latency-stats")
+    );
+    assert.equal(response.status, 500);
+    const body = await response.json();
+
+    assert.ok(body.error);
+    assert.ok(typeof body.error.message === "string");
+    assert.ok(!body.error.message.includes("at /"));
+  } finally {
+    core.resetDbInstance();
+  }
+});