From c8dee8c5f7a6005f90e5b69b3293d23327ce6813 Mon Sep 17 00:00:00 2001 From: PizzaV <103120356+pizzav-xyz@users.noreply.github.com> Date: Sat, 13 Jun 2026 23:50:54 +0200 Subject: [PATCH] feat: free provider rankings page by Arena AI ELO scores (#3799) Adds a dashboard page + /api/free-provider-rankings route ranking free providers by model ELO/intelligence scores. Pure computation over the existing provider registry + model-intelligence data (no external fetch). Sidebar entry included. --- .../dashboard/free-provider-rankings/page.tsx | 249 ++++++++++++++++++ src/app/api/free-provider-rankings/route.ts | 40 +++ src/i18n/messages/en.json | 24 ++ src/lib/freeProviderRankings.ts | 239 +++++++++++++++++ src/shared/constants/sidebarVisibility.ts | 8 + tests/unit/freeProviderRankings.test.ts | 143 ++++++++++ 6 files changed, 703 insertions(+) create mode 100644 src/app/(dashboard)/dashboard/free-provider-rankings/page.tsx create mode 100644 src/app/api/free-provider-rankings/route.ts create mode 100644 src/lib/freeProviderRankings.ts create mode 100644 tests/unit/freeProviderRankings.test.ts diff --git a/src/app/(dashboard)/dashboard/free-provider-rankings/page.tsx b/src/app/(dashboard)/dashboard/free-provider-rankings/page.tsx new file mode 100644 index 0000000000..a112f299d5 --- /dev/null +++ b/src/app/(dashboard)/dashboard/free-provider-rankings/page.tsx @@ -0,0 +1,249 @@ +"use client"; + +import { useState, useEffect, useCallback } from "react"; +import { useTranslations } from "next-intl"; +import { Card } from "@/shared/components"; + +interface ProviderModelScore { + modelId: string; + modelName: string; + score: number; + eloRaw: number | null; + confidence: string | null; + category: string; +} + +interface FreeProviderRanking { + id: string; + name: string; + icon: string; + color: string; + textIcon?: string; + category: "noauth" | "oauth" | "apikey"; + topModel: ProviderModelScore | null; + averageScore: number; + modelCount: number; +} + +/** + * Convert a normalized task-fit score (0.4–0.98) to a human-readable label. + * The score represents relative ranking quality, not a percentage. + */ +function scoreLabel(score: number): string { + if (score >= 0.9) return "Elite"; + if (score >= 0.8) return "Excellent"; + if (score >= 0.7) return "Very Good"; + if (score >= 0.6) return "Good"; + if (score >= 0.5) return "Average"; + return "Below Average"; +} + +function scoreColor(score: number): string { + if (score >= 0.85) return "text-green-400"; + if (score >= 0.7) return "text-emerald-400"; + if (score >= 0.55) return "text-yellow-400"; + return "text-orange-400"; +} + +const CATEGORY_OPTIONS = [ + { value: "", labelKey: "allCategories" }, + { value: "default", labelKey: "categoryDefault" }, + { value: "coding", labelKey: "categoryCoding" }, + { value: "review", labelKey: "categoryReview" }, + { value: "documentation", labelKey: "categoryDocumentation" }, + { value: "debugging", labelKey: "categoryDebugging" }, +]; + +export default function FreeProviderRankingsPage() { + const t = useTranslations("freeProviderRankingsPage"); + const [rankings, setRankings] = useState([]); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(""); + const [filter, setFilter] = useState(""); + + const fetchRankings = useCallback( + async (category?: string) => { + setLoading(true); + setError(""); + try { + const url = category + ? `/api/free-provider-rankings?category=${category}` + : "/api/free-provider-rankings"; + const res = await fetch(url); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + const data = await res.json(); + setRankings(data.rankings || []); + } catch (err) { + setError(err instanceof Error ? err.message : t("errorLoading")); + } finally { + setLoading(false); + } + }, + [t] + ); + + useEffect(() => { + fetchRankings(filter || undefined); + }, [filter, fetchRankings]); + + return ( +
+ {/* Header */} +
+
+

{t("title")}

+

{t("subtitle")}

+
+
+ + {/* Filters */} +
+ {CATEGORY_OPTIONS.map((opt) => ( + + ))} +
+ + {error &&
{error}
} + + {loading ? ( +
+
{t("loading")}
+
+ ) : ( + <> + {/* Top 3 Podium */} + {rankings.length >= 3 && ( +
+ {rankings.slice(0, 3).map((provider, idx) => ( + +
+
+
{idx === 0 ? "🥇" : idx === 1 ? "🥈" : "🥉"}
+
+
+
+ {provider.textIcon || provider.name.charAt(0)} +
+

{provider.name}

+
+ {provider.topModel && ( +

+ {t("bestModel")}: {provider.topModel.modelName} +

+ )} + {provider.topModel && ( +

+ {scoreLabel(provider.topModel.score)} +

+ )} +
+
{idx + 1}
+
+ + ))} +
+ )} + + {/* Full List */} + {rankings.length > 0 && ( + +
+ + + + + + + + + + + + + + {rankings.map((provider, idx) => ( + + + + + + + + + + ))} + +
{t("colRank")}{t("colProvider")}{t("colTopModel")}{t("colScore")}{t("colAvgScore")}{t("colModels")}{t("colType")}
{idx + 1} +
+
+ {provider.textIcon || provider.name.charAt(0)} +
+ {provider.name} +
+ 		+ {provider.topModel?.modelName || "—"} + + {provider.topModel ? ( + + {scoreLabel(provider.topModel.score)} + + ) : ( + "—" + )} + + {scoreLabel(provider.averageScore)} + {provider.modelCount} + + {provider.category.toUpperCase()} + +
+
+ + )} + + {rankings.length === 0 && !error && ( + +
{t("emptyState")}
+ + )} + + )} +
+ ); +} diff --git a/src/app/api/free-provider-rankings/route.ts b/src/app/api/free-provider-rankings/route.ts new file mode 100644 index 0000000000..73b21dde8a --- /dev/null +++ b/src/app/api/free-provider-rankings/route.ts @@ -0,0 +1,40 @@ +import { NextRequest, NextResponse } from "next/server"; +import { z } from "zod"; +import { CORS_HEADERS, handleCorsOptions } from "@/shared/utils/cors"; +import { computeFreeProviderRankings } from "@/lib/freeProviderRankings"; + +const QuerySchema = z.object({ + category: z.string().min(1).max(50).optional(), + limit: z + .string() + .optional() + .transform((val) => { + if (!val) return 50; + const n = Number(val); + return Number.isFinite(n) && n >= 1 ? Math.min(Math.round(n), 100) : 50; + }), +}); + +export async function OPTIONS() { + return handleCorsOptions(); +} + +export async function GET(request: NextRequest) { + const url = new URL(request.url); + const parsed = QuerySchema.safeParse({ + category: url.searchParams.get("category") || undefined, + limit: url.searchParams.get("limit") || undefined, + }); + + if (!parsed.success) { + return NextResponse.json( + { error: "Invalid query parameters", details: parsed.error.flatten().fieldErrors }, + { status: 400, headers: CORS_HEADERS } + ); + } + + const { category, limit } = parsed.data; + const rankings = computeFreeProviderRankings(category, limit); + + return NextResponse.json({ rankings }, { headers: CORS_HEADERS }); +} diff --git a/src/i18n/messages/en.json b/src/i18n/messages/en.json index 3e77d4eb57..aa75994899 100644 --- a/src/i18n/messages/en.json +++ b/src/i18n/messages/en.json @@ -969,6 +969,9 @@ "analyticsCompression": "Compression", "costsBudget": "Budget", "costsFreeTiers": "Free-Tier Budget", + "costsFreeTiersSubtitle": "Track aggregated free tier quotas across 50+ providers", + "freeProviderRankings": "Free Provider Rankings", + "freeProviderRankingsSubtitle": "Best free providers ranked by model ELO scores", "costsQuotaShare": "Quota Sharing", "costsPricing": "Pricing", "logsProxy": "Proxy Logs", @@ -8704,5 +8707,26 @@ "regenerateRunning": "Regenerating skills…", "regenerateSuccess": "Skills regenerated successfully.", "regenerateError": "Failed to regenerate skills." + }, + "freeProviderRankingsPage": { + "title": "Free Provider Rankings", + "subtitle": "Best free providers ranked by model ELO scores from Arena AI leaderboards", + "loading": "Loading rankings...", + "errorLoading": "Failed to load rankings", + "emptyState": "No rankings available. Ensure Arena ELO sync is enabled (ARENA_ELO_SYNC_ENABLED=true).", + "bestModel": "Best", + "allCategories": "All Categories", + "categoryDefault": "Default", + "categoryCoding": "Coding", + "categoryReview": "Review", + "categoryDocumentation": "Documentation", + "categoryDebugging": "Debugging", + "colRank": "Rank", + "colProvider": "Provider", + "colTopModel": "Top Model", + "colScore": "Score", + "colAvgScore": "Avg Score", + "colModels": "Models", + "colType": "Type" } } diff --git a/src/lib/freeProviderRankings.ts b/src/lib/freeProviderRankings.ts new file mode 100644 index 0000000000..c717d5c474 --- /dev/null +++ b/src/lib/freeProviderRankings.ts @@ -0,0 +1,239 @@ +/** + * freeProviderRankings.ts — Compute rankings for free providers based on model ELO scores. + * + * Joins free providers (no-auth, OAuth, API key) with their models from the registry + * and their intelligence scores from the `model_intelligence` DB table. + * + * Uses flexible matching to bridge naming gaps between registry model IDs + * and Arena-normalized model names (e.g., "kimi-k2.6" vs "kimi-k2"). + */ + +import { NOAUTH_PROVIDERS, OAUTH_PROVIDERS, APIKEY_PROVIDERS } from "@/shared/constants/providers"; +import { REGISTRY } from "@omniroute/open-sse/config/providerRegistry"; +import { listModelIntelligence } from "./db/modelIntelligence"; + +export interface ProviderModelScore { + modelId: string; + modelName: string; + score: number; + eloRaw: number | null; + confidence: string | null; + category: string; +} + +export interface FreeProviderRanking { + id: string; + name: string; + icon: string; + color: string; + textIcon?: string; + category: "noauth" | "oauth" | "apikey"; + topModel: ProviderModelScore | null; + averageScore: number; + modelCount: number; +} + +/** + * Get all free providers from all categories. + */ +function getFreeProviders() { + const providers: Array<{ + id: string; + name: string; + icon: string; + color: string; + textIcon?: string; + category: "noauth" | "oauth" | "apikey"; + }> = []; + + // No-auth providers are always free + for (const [id, p] of Object.entries(NOAUTH_PROVIDERS)) { + providers.push({ + id, + name: p.name, + icon: p.icon, + color: p.color, + textIcon: p.textIcon, + category: "noauth", + }); + } + + // OAuth providers with free tier + for (const [id, p] of Object.entries(OAUTH_PROVIDERS)) { + if ("hasFree" in p && p.hasFree) { + providers.push({ + id, + name: p.name, + icon: p.icon, + color: p.color, + textIcon: "textIcon" in p ? (p as any).textIcon : undefined, + category: "oauth", + }); + } + } + + // API key providers with free tier + for (const [id, p] of Object.entries(APIKEY_PROVIDERS)) { + if ("hasFree" in p && p.hasFree) { + providers.push({ + id, + name: p.name, + icon: p.icon, + color: p.color, + textIcon: "textIcon" in p ? (p as any).textIcon : undefined, + category: "apikey", + }); + } + } + + return providers; +} + +/** + * Get models for a provider from the registry. + */ +function getProviderModels(providerId: string) { + const entry = REGISTRY[providerId]; + return entry?.models ?? []; +} + +/** + * Strip trailing version suffixes from a model ID for fuzzy matching. + * E.g., "kimi-k2.6" → "kimi-k2", "gpt-5.5" → "gpt-5" + */ +export function stripVersionSuffix(id: string): string { + return id.replace(/\.\d+(\.\d+)*$/, ""); +} + +/** + * Find the best matching intelligence entry for a registry model ID. + * + * Strategy (in order): + * 1. Exact match on normalized model ID + * 2. Exact match on model ID with version suffix stripped + * 3. Prefix match (intelligence entry model is a prefix of registry ID) + * + * @param modelId - The registry model ID (e.g., "kimi-k2.6") + * @param intelMap - Map of normalized model names → intelligence entries + * @returns The best matching intelligence entry, or null + */ +export function findMatchingIntelligence( + modelId: string, + intelMap: Map< + string, + Array<{ score: number; eloRaw: number | null; confidence: string | null; category: string }> + > +): { score: number; eloRaw: number | null; confidence: string | null; category: string } | null { + const normalizedId = modelId.toLowerCase(); + + // Strategy 1: Exact match + const exactMatches = intelMap.get(normalizedId); + if (exactMatches && exactMatches.length > 0) { + return exactMatches.reduce((prev, curr) => (curr.score > prev.score ? curr : prev)); + } + + // Strategy 2: Strip version suffix and match + const stripped = stripVersionSuffix(normalizedId); + if (stripped !== normalizedId) { + const strippedMatches = intelMap.get(stripped); + if (strippedMatches && strippedMatches.length > 0) { + return strippedMatches.reduce((prev, curr) => (curr.score > prev.score ? curr : prev)); + } + } + + // Strategy 3: Prefix match (intelligence entry model is a prefix of registry ID) + let bestPrefixMatch: { + score: number; + eloRaw: number | null; + confidence: string | null; + category: string; + } | null = null; + for (const [modelName, entries] of intelMap) { + if (normalizedId.startsWith(modelName + "-") || normalizedId.startsWith(modelName + ".")) { + const best = entries.reduce((prev, curr) => (curr.score > prev.score ? curr : prev)); + if (!bestPrefixMatch || best.score > bestPrefixMatch.score) { + bestPrefixMatch = best; + } + } + } + + return bestPrefixMatch; +} + +/** + * Compute rankings for free providers based on ELO scores. + * + * @param category - Optional filter for task category (e.g., "coding", "default") + * @param limit - Maximum number of providers to return + */ +export function computeFreeProviderRankings( + category?: string, + limit: number = 50 +): FreeProviderRanking[] { + const freeProviders = getFreeProviders(); + const intelligenceEntries = listModelIntelligence({ + source: "arena_elo", + category: category || undefined, + }); + + // Create a map for fast lookup: model name → intelligence entries + const intelMap = new Map(); + for (const entry of intelligenceEntries) { + const modelKey = entry.model.toLowerCase(); + if (!intelMap.has(modelKey)) { + intelMap.set(modelKey, []); + } + intelMap.get(modelKey)!.push(entry); + } + + const rankings: FreeProviderRanking[] = []; + + for (const provider of freeProviders) { + const models = getProviderModels(provider.id); + if (models.length === 0) continue; + + const modelScores: ProviderModelScore[] = []; + + for (const model of models) { + const match = findMatchingIntelligence(model.id, intelMap); + + if (match) { + modelScores.push({ + modelId: model.id, + modelName: model.name, + score: match.score, + eloRaw: match.eloRaw, + confidence: match.confidence, + category: match.category, + }); + } + } + + if (modelScores.length === 0) continue; + + // Sort models by score descending + modelScores.sort((a, b) => b.score - a.score); + + const topModel = modelScores[0]; + const averageScore = modelScores.reduce((sum, m) => sum + m.score, 0) / modelScores.length; + + rankings.push({ + ...provider, + topModel, + averageScore, + modelCount: modelScores.length, + }); + } + + // Sort providers by top model score descending, then by average score + rankings.sort((a, b) => { + if (a.topModel && b.topModel) { + return b.topModel.score - a.topModel.score; + } + if (a.topModel) return -1; + if (b.topModel) return 1; + return b.averageScore - a.averageScore; + }); + + return rankings.slice(0, limit); +} diff --git a/src/shared/constants/sidebarVisibility.ts b/src/shared/constants/sidebarVisibility.ts index 1e22507272..11dc8eeec7 100644 --- a/src/shared/constants/sidebarVisibility.ts +++ b/src/shared/constants/sidebarVisibility.ts @@ -49,6 +49,7 @@ export const HIDEABLE_SIDEBAR_ITEM_IDS = [ "costs-budget", "costs-free-tiers", "costs-quota-share", + "free-provider-rankings", // Monitoring > Audit "audit", "audit-mcp", @@ -477,6 +478,13 @@ const COSTS_ITEMS: readonly SidebarItemDefinition[] = [ subtitleKey: "costsFreeTiersSubtitle", icon: "request_quote", }, + { + id: "free-provider-rankings", + href: "/dashboard/free-provider-rankings", + i18nKey: "freeProviderRankings", + subtitleKey: "freeProviderRankingsSubtitle", + icon: "leaderboard", + }, ]; const AUDIT_GROUP: SidebarItemGroup = { diff --git a/tests/unit/freeProviderRankings.test.ts b/tests/unit/freeProviderRankings.test.ts new file mode 100644 index 0000000000..2a688a5f71 --- /dev/null +++ b/tests/unit/freeProviderRankings.test.ts @@ -0,0 +1,143 @@ +/** + * Unit tests for src/lib/freeProviderRankings.ts + * + * Tests exported pure functions (stripVersionSuffix, findMatchingIntelligence) + * that don't require DB or module mocking. + */ + +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; + +const { stripVersionSuffix, findMatchingIntelligence } = + await import("../../src/lib/freeProviderRankings.ts"); + +type IntelEntry = { + score: number; + eloRaw: number | null; + confidence: string | null; + category: string; +}; + +function e( + score: number, + category = "default", + eloRaw: number | null = null, + confidence: string | null = "high" +): IntelEntry { + return { score, eloRaw, confidence, category }; +} + +function buildMap(entries: Record): Map { + const map = new Map(); + for (const [key, vals] of Object.entries(entries)) { + map.set(key, vals); + } + return map; +} + +describe("stripVersionSuffix", () => { + it("strips trailing .N from kimi-k2.6", () => { + assert.equal(stripVersionSuffix("kimi-k2.6"), "kimi-k2"); + }); + + it("strips trailing .N from gpt-5.5", () => { + assert.equal(stripVersionSuffix("gpt-5.5"), "gpt-5"); + }); + + it("strips multi-segment .N.N from model-1.2.3", () => { + assert.equal(stripVersionSuffix("model-1.2.3"), "model-1"); + }); + + it("does not strip dash-separated names like claude-sonnet-4-5", () => { + assert.equal(stripVersionSuffix("claude-sonnet-4-5"), "claude-sonnet-4-5"); + }); + + it("returns unchanged for short names", () => { + assert.equal(stripVersionSuffix("gpt"), "gpt"); + }); +}); + +describe("findMatchingIntelligence", () => { + it("exact match returns highest score entry", () => { + const map = buildMap({ "gpt-5": [e(0.8), e(0.95)] }); + const result = findMatchingIntelligence("gpt-5", map); + assert.ok(result); + assert.equal(result.score, 0.95); + }); + + it("exact match is case-insensitive", () => { + const map = buildMap({ "gpt-5": [e(0.9)] }); + const result = findMatchingIntelligence("GPT-5", map); + assert.ok(result); + assert.equal(result.score, 0.9); + }); + + it("version suffix fuzzy: kimi-k2.6 matches kimi-k2", () => { + const map = buildMap({ "kimi-k2": [e(0.88)] }); + const result = findMatchingIntelligence("kimi-k2.6", map); + assert.ok(result); + assert.equal(result.score, 0.88); + }); + + it("version suffix fuzzy: gpt-5.5 matches gpt-5", () => { + const map = buildMap({ "gpt-5": [e(0.92)] }); + const result = findMatchingIntelligence("gpt-5.5", map); + assert.ok(result); + assert.equal(result.score, 0.92); + }); + + it("prefix match: claude-sonnet-4-5 matches claude-sonnet-4", () => { + const map = buildMap({ "claude-sonnet-4": [e(0.85)] }); + const result = findMatchingIntelligence("claude-sonnet-4-5", map); + assert.ok(result); + assert.equal(result.score, 0.85); + }); + + it("returns null when no match", () => { + const map = buildMap({ "gpt-5": [e(0.9)] }); + assert.equal(findMatchingIntelligence("unknown-model", map), null); + }); + + it("returns null for empty map", () => { + assert.equal(findMatchingIntelligence("gpt-5", new Map()), null); + }); + + it("prefers exact match over prefix match", () => { + const map = buildMap({ + "claude-sonnet-4-5": [e(0.95)], + "claude-sonnet-4": [e(0.7)], + }); + const result = findMatchingIntelligence("claude-sonnet-4-5", map); + assert.ok(result); + assert.equal(result.score, 0.95); + }); + + it("version match wins over prefix match", () => { + const map = buildMap({ + "kimi-k2": [e(0.9)], + kimi: [e(0.5)], + }); + const result = findMatchingIntelligence("kimi-k2.6", map); + assert.ok(result); + assert.equal(result.score, 0.9); + }); + + it("preserves category, eloRaw, confidence from matched entry", () => { + const map = buildMap({ "gpt-5": [e(0.9, "coding", 1400, "high")] }); + const result = findMatchingIntelligence("gpt-5", map); + assert.ok(result); + assert.equal(result.category, "coding"); + assert.equal(result.eloRaw, 1400); + assert.equal(result.confidence, "high"); + }); + + it("prefix match picks highest-scoring candidate", () => { + const map = buildMap({ + gpt: [e(0.6)], + "gpt-4": [e(0.8)], + }); + const result = findMatchingIntelligence("gpt-4.1", map); + assert.ok(result); + assert.equal(result.score, 0.8); + }); +});