feat(rankings): report what a free provider actually served (#10926)

Validado no worktree combinado: mesmos gates + testes focados verdes. Extensão opt-in bem desenhada sobre #10909 (dimensão de uso real via call_logs). CI vermelho é o base-red já rastreado em #9985.

.gitignore

@@ -291,3 +291,4 @@ docker-compose.yml.bak
 
 # Ad-hoc test sandboxes (never tracked — may contain local DBs)
 /.sandbox/
+.aider*

changelog.d/features/10926-rankings-usage-reliability.md

@@ -0,0 +1 @@
+- **feat(rankings):** free provider rankings can now report what each provider actually served — `reliability.usage` (requests, successes, success rate over a window) behind the opt-in `withUsage`/`usageRange` query parameters, so a provider that answers every call with an error is no longer described as healthy ([#10926](https://github.com/diegosouzapw/OmniRoute/pull/10926))

src/app/api/free-provider-rankings/route.ts

@@ -22,6 +22,11 @@ const QuerySchema = z.object({
   // Additive filters (default off → current behavior). `availableOnly` implies configured.
   configuredOnly: boolParam,
   availableOnly: boolParam,
+  // Opt-in usage reporting: costs one aggregate query, so it is never implicit.
+  withUsage: boolParam,
+  // Rejected rather than silently coerced: a typo must not quietly return a
+  // different window than the caller asked for.
+  usageRange: z.enum(["1h", "24h", "7d", "30d"]).optional(),
 });
 
 export async function OPTIONS() {
@@ -35,6 +40,8 @@ export async function GET(request: NextRequest) {
     limit: url.searchParams.get("limit") || undefined,
     configuredOnly: url.searchParams.get("configuredOnly") || undefined,
     availableOnly: url.searchParams.get("availableOnly") || undefined,
+    withUsage: url.searchParams.get("withUsage") || undefined,
+    usageRange: url.searchParams.get("usageRange") || undefined,
   });
 
   if (!parsed.success) {
@@ -44,10 +51,12 @@ export async function GET(request: NextRequest) {
     );
   }
 
-  const { category, limit, configuredOnly, availableOnly } = parsed.data;
+  const { category, limit, configuredOnly, availableOnly, withUsage, usageRange } = parsed.data;
   const rankings = await computeFreeProviderRankings(category, limit, {
     configuredOnly,
     availableOnly,
+    withUsage,
+    usageRange,
   });
 
   return NextResponse.json({ rankings }, { headers: CORS_HEADERS });

src/lib/db/callLogStats.ts

@@ -25,6 +25,15 @@ export interface ProviderMetricRow {
   lastErrorStatus: number | null;
 }
 
+/** One provider's traffic over a bounded window. See `getProviderUsageSince`. */
+export interface ProviderUsageRow {
+  provider: string;
+  requests: number;
+  successes: number;
+  avgLatencyMs: number | null;
+  lastRequestAt: string | null;
+}
+
 export interface SearchProviderStatRow {
   provider: string;
   requests: number;
@@ -107,6 +116,43 @@ export function getProviderMetrics(): ProviderMetricRow[] {
     .all() as ProviderMetricRow[];
 }
 
+// ---------------------------------------------------------------------------
+// /api/free-provider-rankings — windowed usage aggregate
+// ---------------------------------------------------------------------------
+
+/**
+ * Per-provider usage over a time window: how much traffic a provider actually
+ * served, and how much of it succeeded.
+ *
+ * Deliberately NOT `getProviderMetrics()` with a `since` parameter: that query
+ * carries two correlated subqueries (`lastStatus`, `lastErrorStatus`) which a
+ * ranking never displays, and they dominate its cost — `call_logs` is indexed
+ * on `timestamp` alone, so each correlated pass rescans the whole window per
+ * provider. Here a single bounded `GROUP BY` uses `idx_cl_timestamp` and stops
+ * there. The rules are shared with its neighbour, not the query: same success
+ * definition, same `#10714` guard against providers whose connections are gone.
+ */
+export function getProviderUsageSince(since: string): ProviderUsageRow[] {
+  const db = getDbInstance();
+  return db
+    .prepare(
+      `SELECT
+          c.provider,
+          COUNT(*) as requests,
+          SUM(CASE WHEN c.status >= 200 AND c.status < 400 THEN 1 ELSE 0 END) as successes,
+          ROUND(AVG(c.duration)) as avgLatencyMs,
+          MAX(c.timestamp) as lastRequestAt
+        FROM call_logs c
+        WHERE c.provider IS NOT NULL AND c.provider != '-'
+          AND c.timestamp >= @since
+          AND EXISTS (
+            SELECT 1 FROM provider_connections pc WHERE pc.provider = c.provider
+          )
+        GROUP BY c.provider`
+    )
+    .all({ since }) as ProviderUsageRow[];
+}
+
 // ---------------------------------------------------------------------------
 // /api/search/stats — search provider aggregates + recent entries
 // ---------------------------------------------------------------------------

src/lib/freeProviderRankings.ts

@@ -12,9 +12,14 @@ import { NOAUTH_PROVIDERS, OAUTH_PROVIDERS, APIKEY_PROVIDERS } from "@/shared/co
 import { REGISTRY } from "@omniroute/open-sse/config/providerRegistry";
 import { listModelIntelligence } from "./db/modelIntelligence";
 import { getProviderConnections } from "./db/providers";
+import { getProviderUsageSince, type ProviderUsageRow } from "./db/callLogStats";
 import { getCustomModels } from "./db/models";
 // Type-only: reuse the health vocabulary instead of forking it.
-import type { ProviderHealthState } from "./monitoring/providerHealthMatrix";
+import { RANGE_MS } from "./monitoring/providerHealthMatrix";
+import type {
+  ProviderHealthState,
+  ProviderHealthMatrixRange,
+} from "./monitoring/providerHealthMatrix";
 import type { ProviderAuthType } from "./freeProviderRankingsAuthType";
 
 // Re-exported for backward-compat / same-module ergonomics (#6915) — the
@@ -248,8 +253,31 @@ export interface ProviderReliability {
   }>;
   /** Provider aggregate; absent entirely for providers with no loaded connection. */
   state: ProviderHealthState;
+  /**
+   * What the provider actually served over a window, from `call_logs`. Present
+   * only when the caller asks for it (`withUsage`). Complements `state`, which
+   * describes the connection right now and cannot see a provider that answers
+   * every call with an error.
+   */
+  usage?: ProviderUsage;
 }
 
+export interface ProviderUsage {
+  requests: number;
+  successes: number;
+  /** `null` below `MIN_USAGE_REQUESTS` — too small a sample to state a rate. */
+  successRate: number | null;
+  avgLatencyMs: number | null;
+  lastRequestAt: string | null;
+  windowHours: number;
+}
+
+/**
+ * Below this many requests in the window, no rate is reported: 1 failure out of
+ * 2 calls is not "50% broken", and a provider nobody called is not "0% healthy".
+ */
+const MIN_USAGE_REQUESTS = 5;
+
 /**
  * Options controlling the additive "configured" / "available" filters.
  * Both default off (undefined/false) → output identical to current behavior.
@@ -259,6 +287,14 @@ export interface FreeProviderRankingFilterOptions {
   configuredOnly?: boolean;
   /** Keep only providers that have ≥1 non-exhausted, non-rate-limited connection (implies configured). */
   availableOnly?: boolean;
+  /**
+   * Also report what each provider actually served (`reliability.usage`).
+   * Off by default: it costs one aggregate query over `call_logs`, which a
+   * caller that only needs the ranking should not pay.
+   */
+  withUsage?: boolean;
+  /** Window for `withUsage`. Defaults to `24h`, the health matrix's own default. */
+  usageRange?: ProviderHealthMatrixRange;
 }
 
 /** Group connection states by provider id (shared by filter and reliability attach). */
@@ -381,6 +417,37 @@ export function attachProviderReliability(
   });
 }
 
+/**
+ * Pure enrichment: attach `usage` to the `reliability` of every ranking that has
+ * a row in the windowed aggregate. Rankings without `reliability` (no connection
+ * loaded) are returned unchanged, never mutated.
+ */
+export function attachProviderUsage(
+  rankings: FreeProviderRanking[],
+  usageRows: ProviderUsageRow[],
+  windowHours: number
+): FreeProviderRanking[] {
+  const byProvider = new Map(usageRows.map((row) => [row.provider, row]));
+  return rankings.map((ranking) => {
+    const row = byProvider.get(ranking.id);
+    if (!row || !ranking.reliability) return ranking;
+    return {
+      ...ranking,
+      reliability: {
+        ...ranking.reliability,
+        usage: {
+          requests: row.requests,
+          successes: row.successes,
+          successRate: row.requests >= MIN_USAGE_REQUESTS ? row.successes / row.requests : null,
+          avgLatencyMs: row.avgLatencyMs ?? null,
+          lastRequestAt: row.lastRequestAt ?? null,
+          windowHours,
+        },
+      },
+    };
+  });
+}
+
 /**
  * Compute rankings for free providers based on ELO scores.
  *
@@ -474,6 +541,20 @@ export async function computeFreeProviderRankings(
     // `availableOnly` already drops providers with no healthy connection, so under
     // it `state` is never `down`; `down` needs `configuredOnly` alone.
     filtered = attachProviderReliability(filtered, connections);
+
+    // Third dimension, opt-in: what the provider actually served. `state` above
+    // reads the connection as it stands now and cannot see a provider that
+    // answers every call with an error — only the call log can.
+    if (opts.withUsage) {
+      const range = opts.usageRange ?? "24h";
+      const windowMs = RANGE_MS[range];
+      const since = new Date(Date.now() - windowMs).toISOString();
+      filtered = attachProviderUsage(
+        filtered,
+        getProviderUsageSince(since),
+        windowMs / (60 * 60 * 1000)
+      );
+    }
   }
 
   return filtered.slice(0, limit);

src/lib/monitoring/providerHealthMatrix.ts

@@ -121,7 +121,8 @@ interface CallLogTargetStats {
   lastErrorStatus: number | null;
 }
 
-const RANGE_MS: Record<ProviderHealthMatrixRange, number> = {
+/** Exported so other surfaces reporting over a window use the same scale. */
+export const RANGE_MS: Record<ProviderHealthMatrixRange, number> = {
   "1h": 60 * 60 * 1000,
   "24h": 24 * 60 * 60 * 1000,
   "7d": 7 * 24 * 60 * 60 * 1000,

tests/unit/db-call-log-stats-3500.test.ts

@@ -271,3 +271,97 @@ test("#3500 getSearchProviderCounts — ordered by cnt desc", () => {
     assert.ok(bing.cnt > rare.cnt, "bing cnt > rare_provider cnt");
   }
 });
+
+// ---------------------------------------------------------------------------
+// getProviderUsageSince — windowed usage aggregate for the rankings API
+// ---------------------------------------------------------------------------
+
+const USAGE_CUTOFF = "2025-07-01T00:00:00.000Z";
+const IN_WINDOW = "2025-07-02T12:00:00.000Z";
+const OUT_OF_WINDOW = "2025-06-01T12:00:00.000Z";
+
+function seedConnection(provider: string) {
+  const now = new Date().toISOString();
+  core
+    .getDbInstance()
+    .prepare(
+      `INSERT INTO provider_connections (id, provider, created_at, updated_at) VALUES (?, ?, ?, ?)`
+    )
+    .run(`conn-usage-${provider}`, provider, now, now);
+}
+
+test("getProviderUsageSince — only counts rows inside the window", () => {
+  seedConnection("usage-window");
+  insertCallLog({ provider: "usage-window", status: 200, timestamp: IN_WINDOW });
+  insertCallLog({ provider: "usage-window", status: 200, timestamp: IN_WINDOW });
+  insertCallLog({ provider: "usage-window", status: 200, timestamp: OUT_OF_WINDOW });
+  insertCallLog({ provider: "usage-window", status: 500, timestamp: OUT_OF_WINDOW });
+
+  const row = mod
+    .getProviderUsageSince(USAGE_CUTOFF)
+    .find((r) => r.provider === "usage-window");
+  assert.ok(row, "provider must be present");
+  assert.equal(row.requests, 2, "rows before the cutoff must not be counted");
+  assert.equal(row.successes, 2);
+});
+
+test("getProviderUsageSince — 2xx/3xx count as success, 4xx/5xx do not", () => {
+  seedConnection("usage-status");
+  for (const status of [200, 204, 301, 399]) {
+    insertCallLog({ provider: "usage-status", status, timestamp: IN_WINDOW });
+  }
+  for (const status of [400, 429, 500, 503]) {
+    insertCallLog({ provider: "usage-status", status, timestamp: IN_WINDOW });
+  }
+
+  const row = mod
+    .getProviderUsageSince(USAGE_CUTOFF)
+    .find((r) => r.provider === "usage-status");
+  assert.ok(row);
+  assert.equal(row.requests, 8);
+  assert.equal(row.successes, 4, "same success rule as getProviderMetrics");
+});
+
+test("getProviderUsageSince — a provider with no live connection is excluded (#10714)", () => {
+  // No seedConnection() on purpose: rows exist in call_logs but the provider was deleted.
+  insertCallLog({ provider: "usage-ghost", status: 200, timestamp: IN_WINDOW });
+
+  const rows = mod.getProviderUsageSince(USAGE_CUTOFF);
+  assert.equal(
+    rows.find((r) => r.provider === "usage-ghost"),
+    undefined,
+    "a deleted provider must not resurface from retained logs"
+  );
+});
+
+test("getProviderUsageSince — latency and lastRequestAt are bounded by the window too", () => {
+  seedConnection("usage-latency");
+  insertCallLog({
+    provider: "usage-latency",
+    status: 200,
+    duration: 100,
+    timestamp: IN_WINDOW,
+  });
+  insertCallLog({
+    provider: "usage-latency",
+    status: 200,
+    duration: 900,
+    timestamp: OUT_OF_WINDOW,
+  });
+
+  const row = mod
+    .getProviderUsageSince(USAGE_CUTOFF)
+    .find((r) => r.provider === "usage-latency");
+  assert.ok(row);
+  assert.equal(row.avgLatencyMs, 100, "the out-of-window 900ms row must not weigh in");
+  assert.equal(row.lastRequestAt, IN_WINDOW);
+});
+
+test("getProviderUsageSince — providers '-' and NULL are excluded", () => {
+  seedConnection("-");
+  insertCallLog({ provider: "-", status: 200, timestamp: IN_WINDOW });
+
+  const rows = mod.getProviderUsageSince(USAGE_CUTOFF);
+  assert.equal(rows.find((r) => r.provider === "-"), undefined);
+  assert.equal(rows.find((r) => r.provider === null), undefined);
+});

tests/unit/free-provider-rankings-usage-route.test.ts

@@ -0,0 +1,87 @@
+/**
+ * Contract of the opt-in usage parameters on GET /api/free-provider-rankings.
+ *
+ * Two guarantees are worth a test rather than a reading of the code:
+ *  - an unknown `usageRange` is rejected, never coerced to a default window
+ *    (a typo must not silently answer for a different period);
+ *  - without `withUsage`, the aggregate query over `call_logs` is not issued —
+ *    asserted on a spy, so the opt-in cannot rot into an always-on cost.
+ */
+import test from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import type { NextRequest } from "next/server";
+
+const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omni-rankings-usage-route-"));
+const ORIGINAL_DATA_DIR = process.env.DATA_DIR;
+process.env.DATA_DIR = TEST_DATA_DIR;
+
+const core = await import("../../src/lib/db/core.ts");
+const route = await import("../../src/app/api/free-provider-rankings/route.ts");
+
+function get(query: string): NextRequest {
+  return new Request(`http://localhost/api/free-provider-rankings${query}`) as NextRequest;
+}
+
+test.after(() => {
+  core.resetDbInstance();
+  if (ORIGINAL_DATA_DIR === undefined) delete process.env.DATA_DIR;
+  else process.env.DATA_DIR = ORIGINAL_DATA_DIR;
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+});
+
+test("route: an unknown usageRange is rejected with 400, not coerced", async () => {
+  const res = await route.GET(get("?configuredOnly=1&withUsage=1&usageRange=42h"));
+  assert.equal(res.status, 400);
+  const body = (await res.json()) as { details?: Record<string, unknown> };
+  assert.ok(body.details?.usageRange, "the offending parameter must be named");
+});
+
+test("route: every documented window is accepted", async () => {
+  for (const range of ["1h", "24h", "7d", "30d"]) {
+    const res = await route.GET(get(`?configuredOnly=1&withUsage=1&usageRange=${range}`));
+    assert.equal(res.status, 200, `${range} must be accepted`);
+  }
+});
+
+/**
+ * Counts statements touching `call_logs`. Instrumenting the DB handle rather
+ * than the module export is deliberate: ESM namespaces are sealed (redefining
+ * an export throws), and the invariant worth protecting is "no query hits
+ * call_logs", not "this particular function was not called".
+ */
+function countCallLogQueries(): { stop: () => number } {
+  const db = core.getDbInstance() as { prepare: (sql: string) => unknown };
+  const original = db.prepare.bind(db);
+  let hits = 0;
+  db.prepare = (sql: string) => {
+    if (/from\s+call_logs/i.test(sql)) hits += 1;
+    return original(sql);
+  };
+  return {
+    stop: () => {
+      db.prepare = original;
+      return hits;
+    },
+  };
+}
+
+test("route: without withUsage, call_logs is never queried", async () => {
+  const spy = countCallLogQueries();
+  const res = await route.GET(get("?configuredOnly=1"));
+  const hits = spy.stop();
+
+  assert.equal(res.status, 200);
+  assert.equal(hits, 0, "the default path must not pay for the usage aggregate");
+});
+
+test("route: with withUsage, call_logs is queried exactly once", async () => {
+  const spy = countCallLogQueries();
+  const res = await route.GET(get("?configuredOnly=1&withUsage=1"));
+  const hits = spy.stop();
+
+  assert.equal(res.status, 200);
+  assert.equal(hits, 1, "one aggregate, never one query per provider");
+});

tests/unit/freeProviderRankings-filters.test.ts

@@ -12,6 +12,7 @@ import {
   isProviderUsable,
   filterFreeProviderRankings,
   attachProviderReliability,
+  attachProviderUsage,
   type ConnectionState,
   type FreeProviderRanking,
 } from "../../src/lib/freeProviderRankings.ts";
@@ -266,3 +267,99 @@ test("attachProviderReliability: input rankings are never mutated (pure function
   assert.notEqual(out[0], rankings[0], "returns new objects");
   assert.equal(JSON.stringify(rankings), before, "input untouched");
 });
+
+// ──────────────── attachProviderUsage ────────────────
+
+const WINDOW_HOURS = 24;
+
+function usage(provider: string, requests: number, successes: number) {
+  return {
+    provider,
+    requests,
+    successes,
+    avgLatencyMs: 120,
+    lastRequestAt: "2025-07-02T12:00:00.000Z",
+  };
+}
+
+/** A ranking already carrying #10909's reliability, which `usage` extends. */
+function rankingWithReliability(id: string): FreeProviderRanking {
+  return {
+    ...ranking(id),
+    reliability: {
+      connections: [{ testStatus: "active", rateLimitedUntil: null, state: "healthy" }],
+      state: "healthy",
+    },
+  };
+}
+
+test("attachProviderUsage: fills usage from the windowed aggregate", () => {
+  const out = attachProviderUsage(
+    [rankingWithReliability("alpha")],
+    [usage("alpha", 100, 90)],
+    WINDOW_HOURS
+  );
+  assert.deepEqual(out[0].reliability?.usage, {
+    requests: 100,
+    successes: 90,
+    successRate: 0.9,
+    avgLatencyMs: 120,
+    lastRequestAt: "2025-07-02T12:00:00.000Z",
+    windowHours: 24,
+  });
+});
+
+test("attachProviderUsage: zero requests -> successRate null, never 0", () => {
+  const out = attachProviderUsage(
+    [rankingWithReliability("alpha")],
+    [usage("alpha", 0, 0)],
+    WINDOW_HOURS
+  );
+  // A provider nobody called has no success *rate*; reporting 0 would read as
+  // "always fails" on a brand new provider.
+  assert.equal(out[0].reliability?.usage?.successRate, null);
+  assert.equal(out[0].reliability?.usage?.requests, 0);
+});
+
+test("attachProviderUsage: below MIN_REQUESTS -> successRate null, requests still exposed", () => {
+  const out = attachProviderUsage(
+    [rankingWithReliability("alpha")],
+    [usage("alpha", 2, 1)],
+    WINDOW_HOURS
+  );
+  // 1 failure out of 2 is not "50% broken" — it is too small a sample to say.
+  assert.equal(out[0].reliability?.usage?.successRate, null);
+  assert.equal(out[0].reliability?.usage?.requests, 2);
+  assert.equal(out[0].reliability?.usage?.successes, 1);
+});
+
+test("attachProviderUsage: above the sample floor, all failing -> successRate 0 (not null)", () => {
+  const out = attachProviderUsage(
+    [rankingWithReliability("alpha")],
+    [usage("alpha", 50, 0)],
+    WINDOW_HOURS
+  );
+  // This is the very case the field exists for: null here would hide the outage.
+  assert.equal(out[0].reliability?.usage?.successRate, 0);
+});
+
+test("attachProviderUsage: a provider with no usage row gets no usage field", () => {
+  const out = attachProviderUsage([rankingWithReliability("alpha")], [], WINDOW_HOURS);
+  assert.ok(out[0].reliability, "reliability itself is preserved");
+  assert.equal(out[0].reliability?.usage, undefined);
+});
+
+test("attachProviderUsage: a ranking without reliability is left untouched", () => {
+  const bare = ranking("beta");
+  const out = attachProviderUsage([bare], [usage("beta", 100, 90)], WINDOW_HOURS);
+  assert.deepEqual(out[0], bare, "no connection loaded => nothing to extend");
+});
+
+test("attachProviderUsage: inputs are never mutated", () => {
+  const rankings = [rankingWithReliability("alpha")];
+  const before = JSON.stringify(rankings);
+  const out = attachProviderUsage(rankings, [usage("alpha", 100, 90)], WINDOW_HOURS);
+  assert.notEqual(out[0], rankings[0]);
+  assert.notEqual(out[0].reliability, rankings[0].reliability);
+  assert.equal(JSON.stringify(rankings), before);
+});