From 1f09e2f9b32d7a6fec0e12761d194c2626bc6791 Mon Sep 17 00:00:00 2001 From: Dizzle <112548150+maxmad64bis@users.noreply.github.com> Date: Fri, 21 Aug 2026 13:25:23 +0200 Subject: [PATCH] feat(rankings): report what a free provider actually served (#10926) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Validado no worktree combinado: mesmos gates + testes focados verdes. Extensão opt-in bem desenhada sobre #10909 (dimensão de uso real via call_logs). CI vermelho é o base-red já rastreado em #9985. --- .gitignore | 1 + .../10926-rankings-usage-reliability.md | 1 + src/app/api/free-provider-rankings/route.ts | 11 ++- src/lib/db/callLogStats.ts | 46 +++++++++ src/lib/freeProviderRankings.ts | 83 +++++++++++++++- src/lib/monitoring/providerHealthMatrix.ts | 3 +- tests/unit/db-call-log-stats-3500.test.ts | 94 ++++++++++++++++++ ...free-provider-rankings-usage-route.test.ts | 87 +++++++++++++++++ .../unit/freeProviderRankings-filters.test.ts | 97 +++++++++++++++++++ 9 files changed, 420 insertions(+), 3 deletions(-) create mode 100644 changelog.d/features/10926-rankings-usage-reliability.md create mode 100644 tests/unit/free-provider-rankings-usage-route.test.ts diff --git a/.gitignore b/.gitignore index a21784f4aa..08bceafd36 100644 --- a/.gitignore +++ b/.gitignore @@ -291,3 +291,4 @@ docker-compose.yml.bak # Ad-hoc test sandboxes (never tracked — may contain local DBs) /.sandbox/ +.aider* diff --git a/changelog.d/features/10926-rankings-usage-reliability.md b/changelog.d/features/10926-rankings-usage-reliability.md new file mode 100644 index 0000000000..a79b92b4cf --- /dev/null +++ b/changelog.d/features/10926-rankings-usage-reliability.md @@ -0,0 +1 @@ +- **feat(rankings):** free provider rankings can now report what each provider actually served — `reliability.usage` (requests, successes, success rate over a window) behind the opt-in `withUsage`/`usageRange` query parameters, so a provider that answers every call with an error is no longer described as healthy ([#10926](https://github.com/diegosouzapw/OmniRoute/pull/10926)) diff --git a/src/app/api/free-provider-rankings/route.ts b/src/app/api/free-provider-rankings/route.ts index e493e179a0..96bbd53f8c 100644 --- a/src/app/api/free-provider-rankings/route.ts +++ b/src/app/api/free-provider-rankings/route.ts @@ -22,6 +22,11 @@ const QuerySchema = z.object({ // Additive filters (default off → current behavior). `availableOnly` implies configured. configuredOnly: boolParam, availableOnly: boolParam, + // Opt-in usage reporting: costs one aggregate query, so it is never implicit. + withUsage: boolParam, + // Rejected rather than silently coerced: a typo must not quietly return a + // different window than the caller asked for. + usageRange: z.enum(["1h", "24h", "7d", "30d"]).optional(), }); export async function OPTIONS() { @@ -35,6 +40,8 @@ export async function GET(request: NextRequest) { limit: url.searchParams.get("limit") || undefined, configuredOnly: url.searchParams.get("configuredOnly") || undefined, availableOnly: url.searchParams.get("availableOnly") || undefined, + withUsage: url.searchParams.get("withUsage") || undefined, + usageRange: url.searchParams.get("usageRange") || undefined, }); if (!parsed.success) { @@ -44,10 +51,12 @@ export async function GET(request: NextRequest) { ); } - const { category, limit, configuredOnly, availableOnly } = parsed.data; + const { category, limit, configuredOnly, availableOnly, withUsage, usageRange } = parsed.data; const rankings = await computeFreeProviderRankings(category, limit, { configuredOnly, availableOnly, + withUsage, + usageRange, }); return NextResponse.json({ rankings }, { headers: CORS_HEADERS }); diff --git a/src/lib/db/callLogStats.ts b/src/lib/db/callLogStats.ts index 7df9908bd1..f3845f31f1 100644 --- a/src/lib/db/callLogStats.ts +++ b/src/lib/db/callLogStats.ts @@ -25,6 +25,15 @@ export interface ProviderMetricRow { lastErrorStatus: number | null; } +/** One provider's traffic over a bounded window. See `getProviderUsageSince`. */ +export interface ProviderUsageRow { + provider: string; + requests: number; + successes: number; + avgLatencyMs: number | null; + lastRequestAt: string | null; +} + export interface SearchProviderStatRow { provider: string; requests: number; @@ -107,6 +116,43 @@ export function getProviderMetrics(): ProviderMetricRow[] { .all() as ProviderMetricRow[]; } +// --------------------------------------------------------------------------- +// /api/free-provider-rankings — windowed usage aggregate +// --------------------------------------------------------------------------- + +/** + * Per-provider usage over a time window: how much traffic a provider actually + * served, and how much of it succeeded. + * + * Deliberately NOT `getProviderMetrics()` with a `since` parameter: that query + * carries two correlated subqueries (`lastStatus`, `lastErrorStatus`) which a + * ranking never displays, and they dominate its cost — `call_logs` is indexed + * on `timestamp` alone, so each correlated pass rescans the whole window per + * provider. Here a single bounded `GROUP BY` uses `idx_cl_timestamp` and stops + * there. The rules are shared with its neighbour, not the query: same success + * definition, same `#10714` guard against providers whose connections are gone. + */ +export function getProviderUsageSince(since: string): ProviderUsageRow[] { + const db = getDbInstance(); + return db + .prepare( + `SELECT + c.provider, + COUNT(*) as requests, + SUM(CASE WHEN c.status >= 200 AND c.status < 400 THEN 1 ELSE 0 END) as successes, + ROUND(AVG(c.duration)) as avgLatencyMs, + MAX(c.timestamp) as lastRequestAt + FROM call_logs c + WHERE c.provider IS NOT NULL AND c.provider != '-' + AND c.timestamp >= @since + AND EXISTS ( + SELECT 1 FROM provider_connections pc WHERE pc.provider = c.provider + ) + GROUP BY c.provider` + ) + .all({ since }) as ProviderUsageRow[]; +} + // --------------------------------------------------------------------------- // /api/search/stats — search provider aggregates + recent entries // --------------------------------------------------------------------------- diff --git a/src/lib/freeProviderRankings.ts b/src/lib/freeProviderRankings.ts index a0fa1ecd07..37c99095c6 100644 --- a/src/lib/freeProviderRankings.ts +++ b/src/lib/freeProviderRankings.ts @@ -12,9 +12,14 @@ import { NOAUTH_PROVIDERS, OAUTH_PROVIDERS, APIKEY_PROVIDERS } from "@/shared/co import { REGISTRY } from "@omniroute/open-sse/config/providerRegistry"; import { listModelIntelligence } from "./db/modelIntelligence"; import { getProviderConnections } from "./db/providers"; +import { getProviderUsageSince, type ProviderUsageRow } from "./db/callLogStats"; import { getCustomModels } from "./db/models"; // Type-only: reuse the health vocabulary instead of forking it. -import type { ProviderHealthState } from "./monitoring/providerHealthMatrix"; +import { RANGE_MS } from "./monitoring/providerHealthMatrix"; +import type { + ProviderHealthState, + ProviderHealthMatrixRange, +} from "./monitoring/providerHealthMatrix"; import type { ProviderAuthType } from "./freeProviderRankingsAuthType"; // Re-exported for backward-compat / same-module ergonomics (#6915) — the @@ -248,8 +253,31 @@ export interface ProviderReliability { }>; /** Provider aggregate; absent entirely for providers with no loaded connection. */ state: ProviderHealthState; + /** + * What the provider actually served over a window, from `call_logs`. Present + * only when the caller asks for it (`withUsage`). Complements `state`, which + * describes the connection right now and cannot see a provider that answers + * every call with an error. + */ + usage?: ProviderUsage; } +export interface ProviderUsage { + requests: number; + successes: number; + /** `null` below `MIN_USAGE_REQUESTS` — too small a sample to state a rate. */ + successRate: number | null; + avgLatencyMs: number | null; + lastRequestAt: string | null; + windowHours: number; +} + +/** + * Below this many requests in the window, no rate is reported: 1 failure out of + * 2 calls is not "50% broken", and a provider nobody called is not "0% healthy". + */ +const MIN_USAGE_REQUESTS = 5; + /** * Options controlling the additive "configured" / "available" filters. * Both default off (undefined/false) → output identical to current behavior. @@ -259,6 +287,14 @@ export interface FreeProviderRankingFilterOptions { configuredOnly?: boolean; /** Keep only providers that have ≥1 non-exhausted, non-rate-limited connection (implies configured). */ availableOnly?: boolean; + /** + * Also report what each provider actually served (`reliability.usage`). + * Off by default: it costs one aggregate query over `call_logs`, which a + * caller that only needs the ranking should not pay. + */ + withUsage?: boolean; + /** Window for `withUsage`. Defaults to `24h`, the health matrix's own default. */ + usageRange?: ProviderHealthMatrixRange; } /** Group connection states by provider id (shared by filter and reliability attach). */ @@ -381,6 +417,37 @@ export function attachProviderReliability( }); } +/** + * Pure enrichment: attach `usage` to the `reliability` of every ranking that has + * a row in the windowed aggregate. Rankings without `reliability` (no connection + * loaded) are returned unchanged, never mutated. + */ +export function attachProviderUsage( + rankings: FreeProviderRanking[], + usageRows: ProviderUsageRow[], + windowHours: number +): FreeProviderRanking[] { + const byProvider = new Map(usageRows.map((row) => [row.provider, row])); + return rankings.map((ranking) => { + const row = byProvider.get(ranking.id); + if (!row || !ranking.reliability) return ranking; + return { + ...ranking, + reliability: { + ...ranking.reliability, + usage: { + requests: row.requests, + successes: row.successes, + successRate: row.requests >= MIN_USAGE_REQUESTS ? row.successes / row.requests : null, + avgLatencyMs: row.avgLatencyMs ?? null, + lastRequestAt: row.lastRequestAt ?? null, + windowHours, + }, + }, + }; + }); +} + /** * Compute rankings for free providers based on ELO scores. * @@ -474,6 +541,20 @@ export async function computeFreeProviderRankings( // `availableOnly` already drops providers with no healthy connection, so under // it `state` is never `down`; `down` needs `configuredOnly` alone. filtered = attachProviderReliability(filtered, connections); + + // Third dimension, opt-in: what the provider actually served. `state` above + // reads the connection as it stands now and cannot see a provider that + // answers every call with an error — only the call log can. + if (opts.withUsage) { + const range = opts.usageRange ?? "24h"; + const windowMs = RANGE_MS[range]; + const since = new Date(Date.now() - windowMs).toISOString(); + filtered = attachProviderUsage( + filtered, + getProviderUsageSince(since), + windowMs / (60 * 60 * 1000) + ); + } } return filtered.slice(0, limit); diff --git a/src/lib/monitoring/providerHealthMatrix.ts b/src/lib/monitoring/providerHealthMatrix.ts index b1ff287bae..f740a7570c 100644 --- a/src/lib/monitoring/providerHealthMatrix.ts +++ b/src/lib/monitoring/providerHealthMatrix.ts @@ -121,7 +121,8 @@ interface CallLogTargetStats { lastErrorStatus: number | null; } -const RANGE_MS: Record = { +/** Exported so other surfaces reporting over a window use the same scale. */ +export const RANGE_MS: Record = { "1h": 60 * 60 * 1000, "24h": 24 * 60 * 60 * 1000, "7d": 7 * 24 * 60 * 60 * 1000, diff --git a/tests/unit/db-call-log-stats-3500.test.ts b/tests/unit/db-call-log-stats-3500.test.ts index aa6d585546..5216132d8b 100644 --- a/tests/unit/db-call-log-stats-3500.test.ts +++ b/tests/unit/db-call-log-stats-3500.test.ts @@ -271,3 +271,97 @@ test("#3500 getSearchProviderCounts — ordered by cnt desc", () => { assert.ok(bing.cnt > rare.cnt, "bing cnt > rare_provider cnt"); } }); + +// --------------------------------------------------------------------------- +// getProviderUsageSince — windowed usage aggregate for the rankings API +// --------------------------------------------------------------------------- + +const USAGE_CUTOFF = "2025-07-01T00:00:00.000Z"; +const IN_WINDOW = "2025-07-02T12:00:00.000Z"; +const OUT_OF_WINDOW = "2025-06-01T12:00:00.000Z"; + +function seedConnection(provider: string) { + const now = new Date().toISOString(); + core + .getDbInstance() + .prepare( + `INSERT INTO provider_connections (id, provider, created_at, updated_at) VALUES (?, ?, ?, ?)` + ) + .run(`conn-usage-${provider}`, provider, now, now); +} + +test("getProviderUsageSince — only counts rows inside the window", () => { + seedConnection("usage-window"); + insertCallLog({ provider: "usage-window", status: 200, timestamp: IN_WINDOW }); + insertCallLog({ provider: "usage-window", status: 200, timestamp: IN_WINDOW }); + insertCallLog({ provider: "usage-window", status: 200, timestamp: OUT_OF_WINDOW }); + insertCallLog({ provider: "usage-window", status: 500, timestamp: OUT_OF_WINDOW }); + + const row = mod + .getProviderUsageSince(USAGE_CUTOFF) + .find((r) => r.provider === "usage-window"); + assert.ok(row, "provider must be present"); + assert.equal(row.requests, 2, "rows before the cutoff must not be counted"); + assert.equal(row.successes, 2); +}); + +test("getProviderUsageSince — 2xx/3xx count as success, 4xx/5xx do not", () => { + seedConnection("usage-status"); + for (const status of [200, 204, 301, 399]) { + insertCallLog({ provider: "usage-status", status, timestamp: IN_WINDOW }); + } + for (const status of [400, 429, 500, 503]) { + insertCallLog({ provider: "usage-status", status, timestamp: IN_WINDOW }); + } + + const row = mod + .getProviderUsageSince(USAGE_CUTOFF) + .find((r) => r.provider === "usage-status"); + assert.ok(row); + assert.equal(row.requests, 8); + assert.equal(row.successes, 4, "same success rule as getProviderMetrics"); +}); + +test("getProviderUsageSince — a provider with no live connection is excluded (#10714)", () => { + // No seedConnection() on purpose: rows exist in call_logs but the provider was deleted. + insertCallLog({ provider: "usage-ghost", status: 200, timestamp: IN_WINDOW }); + + const rows = mod.getProviderUsageSince(USAGE_CUTOFF); + assert.equal( + rows.find((r) => r.provider === "usage-ghost"), + undefined, + "a deleted provider must not resurface from retained logs" + ); +}); + +test("getProviderUsageSince — latency and lastRequestAt are bounded by the window too", () => { + seedConnection("usage-latency"); + insertCallLog({ + provider: "usage-latency", + status: 200, + duration: 100, + timestamp: IN_WINDOW, + }); + insertCallLog({ + provider: "usage-latency", + status: 200, + duration: 900, + timestamp: OUT_OF_WINDOW, + }); + + const row = mod + .getProviderUsageSince(USAGE_CUTOFF) + .find((r) => r.provider === "usage-latency"); + assert.ok(row); + assert.equal(row.avgLatencyMs, 100, "the out-of-window 900ms row must not weigh in"); + assert.equal(row.lastRequestAt, IN_WINDOW); +}); + +test("getProviderUsageSince — providers '-' and NULL are excluded", () => { + seedConnection("-"); + insertCallLog({ provider: "-", status: 200, timestamp: IN_WINDOW }); + + const rows = mod.getProviderUsageSince(USAGE_CUTOFF); + assert.equal(rows.find((r) => r.provider === "-"), undefined); + assert.equal(rows.find((r) => r.provider === null), undefined); +}); diff --git a/tests/unit/free-provider-rankings-usage-route.test.ts b/tests/unit/free-provider-rankings-usage-route.test.ts new file mode 100644 index 0000000000..e493ddca07 --- /dev/null +++ b/tests/unit/free-provider-rankings-usage-route.test.ts @@ -0,0 +1,87 @@ +/** + * Contract of the opt-in usage parameters on GET /api/free-provider-rankings. + * + * Two guarantees are worth a test rather than a reading of the code: + * - an unknown `usageRange` is rejected, never coerced to a default window + * (a typo must not silently answer for a different period); + * - without `withUsage`, the aggregate query over `call_logs` is not issued — + * asserted on a spy, so the opt-in cannot rot into an always-on cost. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import type { NextRequest } from "next/server"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omni-rankings-usage-route-")); +const ORIGINAL_DATA_DIR = process.env.DATA_DIR; +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../src/lib/db/core.ts"); +const route = await import("../../src/app/api/free-provider-rankings/route.ts"); + +function get(query: string): NextRequest { + return new Request(`http://localhost/api/free-provider-rankings${query}`) as NextRequest; +} + +test.after(() => { + core.resetDbInstance(); + if (ORIGINAL_DATA_DIR === undefined) delete process.env.DATA_DIR; + else process.env.DATA_DIR = ORIGINAL_DATA_DIR; + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +test("route: an unknown usageRange is rejected with 400, not coerced", async () => { + const res = await route.GET(get("?configuredOnly=1&withUsage=1&usageRange=42h")); + assert.equal(res.status, 400); + const body = (await res.json()) as { details?: Record }; + assert.ok(body.details?.usageRange, "the offending parameter must be named"); +}); + +test("route: every documented window is accepted", async () => { + for (const range of ["1h", "24h", "7d", "30d"]) { + const res = await route.GET(get(`?configuredOnly=1&withUsage=1&usageRange=${range}`)); + assert.equal(res.status, 200, `${range} must be accepted`); + } +}); + +/** + * Counts statements touching `call_logs`. Instrumenting the DB handle rather + * than the module export is deliberate: ESM namespaces are sealed (redefining + * an export throws), and the invariant worth protecting is "no query hits + * call_logs", not "this particular function was not called". + */ +function countCallLogQueries(): { stop: () => number } { + const db = core.getDbInstance() as { prepare: (sql: string) => unknown }; + const original = db.prepare.bind(db); + let hits = 0; + db.prepare = (sql: string) => { + if (/from\s+call_logs/i.test(sql)) hits += 1; + return original(sql); + }; + return { + stop: () => { + db.prepare = original; + return hits; + }, + }; +} + +test("route: without withUsage, call_logs is never queried", async () => { + const spy = countCallLogQueries(); + const res = await route.GET(get("?configuredOnly=1")); + const hits = spy.stop(); + + assert.equal(res.status, 200); + assert.equal(hits, 0, "the default path must not pay for the usage aggregate"); +}); + +test("route: with withUsage, call_logs is queried exactly once", async () => { + const spy = countCallLogQueries(); + const res = await route.GET(get("?configuredOnly=1&withUsage=1")); + const hits = spy.stop(); + + assert.equal(res.status, 200); + assert.equal(hits, 1, "one aggregate, never one query per provider"); +}); diff --git a/tests/unit/freeProviderRankings-filters.test.ts b/tests/unit/freeProviderRankings-filters.test.ts index 8ea815d21b..317191d175 100644 --- a/tests/unit/freeProviderRankings-filters.test.ts +++ b/tests/unit/freeProviderRankings-filters.test.ts @@ -12,6 +12,7 @@ import { isProviderUsable, filterFreeProviderRankings, attachProviderReliability, + attachProviderUsage, type ConnectionState, type FreeProviderRanking, } from "../../src/lib/freeProviderRankings.ts"; @@ -266,3 +267,99 @@ test("attachProviderReliability: input rankings are never mutated (pure function assert.notEqual(out[0], rankings[0], "returns new objects"); assert.equal(JSON.stringify(rankings), before, "input untouched"); }); + +// ──────────────── attachProviderUsage ──────────────── + +const WINDOW_HOURS = 24; + +function usage(provider: string, requests: number, successes: number) { + return { + provider, + requests, + successes, + avgLatencyMs: 120, + lastRequestAt: "2025-07-02T12:00:00.000Z", + }; +} + +/** A ranking already carrying #10909's reliability, which `usage` extends. */ +function rankingWithReliability(id: string): FreeProviderRanking { + return { + ...ranking(id), + reliability: { + connections: [{ testStatus: "active", rateLimitedUntil: null, state: "healthy" }], + state: "healthy", + }, + }; +} + +test("attachProviderUsage: fills usage from the windowed aggregate", () => { + const out = attachProviderUsage( + [rankingWithReliability("alpha")], + [usage("alpha", 100, 90)], + WINDOW_HOURS + ); + assert.deepEqual(out[0].reliability?.usage, { + requests: 100, + successes: 90, + successRate: 0.9, + avgLatencyMs: 120, + lastRequestAt: "2025-07-02T12:00:00.000Z", + windowHours: 24, + }); +}); + +test("attachProviderUsage: zero requests -> successRate null, never 0", () => { + const out = attachProviderUsage( + [rankingWithReliability("alpha")], + [usage("alpha", 0, 0)], + WINDOW_HOURS + ); + // A provider nobody called has no success *rate*; reporting 0 would read as + // "always fails" on a brand new provider. + assert.equal(out[0].reliability?.usage?.successRate, null); + assert.equal(out[0].reliability?.usage?.requests, 0); +}); + +test("attachProviderUsage: below MIN_REQUESTS -> successRate null, requests still exposed", () => { + const out = attachProviderUsage( + [rankingWithReliability("alpha")], + [usage("alpha", 2, 1)], + WINDOW_HOURS + ); + // 1 failure out of 2 is not "50% broken" — it is too small a sample to say. + assert.equal(out[0].reliability?.usage?.successRate, null); + assert.equal(out[0].reliability?.usage?.requests, 2); + assert.equal(out[0].reliability?.usage?.successes, 1); +}); + +test("attachProviderUsage: above the sample floor, all failing -> successRate 0 (not null)", () => { + const out = attachProviderUsage( + [rankingWithReliability("alpha")], + [usage("alpha", 50, 0)], + WINDOW_HOURS + ); + // This is the very case the field exists for: null here would hide the outage. + assert.equal(out[0].reliability?.usage?.successRate, 0); +}); + +test("attachProviderUsage: a provider with no usage row gets no usage field", () => { + const out = attachProviderUsage([rankingWithReliability("alpha")], [], WINDOW_HOURS); + assert.ok(out[0].reliability, "reliability itself is preserved"); + assert.equal(out[0].reliability?.usage, undefined); +}); + +test("attachProviderUsage: a ranking without reliability is left untouched", () => { + const bare = ranking("beta"); + const out = attachProviderUsage([bare], [usage("beta", 100, 90)], WINDOW_HOURS); + assert.deepEqual(out[0], bare, "no connection loaded => nothing to extend"); +}); + +test("attachProviderUsage: inputs are never mutated", () => { + const rankings = [rankingWithReliability("alpha")]; + const before = JSON.stringify(rankings); + const out = attachProviderUsage(rankings, [usage("alpha", 100, 90)], WINDOW_HOURS); + assert.notEqual(out[0], rankings[0]); + assert.notEqual(out[0].reliability, rankings[0].reliability); + assert.equal(JSON.stringify(rankings), before); +});