feat: free provider rankings page by Arena AI ELO scores (#3799)

Adds a dashboard page + /api/free-provider-rankings route ranking free providers by model ELO/intelligence scores. Pure computation over the existing provider registry + model-intelligence data (no external fetch). Sidebar entry included.
This commit is contained in:
PizzaV
2026-06-13 23:50:54 +02:00
committed by GitHub
parent 76a07cf7a5
commit c8dee8c5f7
6 changed files with 703 additions and 0 deletions

View File

@@ -0,0 +1,249 @@
"use client";
import { useState, useEffect, useCallback } from "react";
import { useTranslations } from "next-intl";
import { Card } from "@/shared/components";
interface ProviderModelScore {
modelId: string;
modelName: string;
score: number;
eloRaw: number | null;
confidence: string | null;
category: string;
}
interface FreeProviderRanking {
id: string;
name: string;
icon: string;
color: string;
textIcon?: string;
category: "noauth" | "oauth" | "apikey";
topModel: ProviderModelScore | null;
averageScore: number;
modelCount: number;
}
/**
* Convert a normalized task-fit score (0.40.98) to a human-readable label.
* The score represents relative ranking quality, not a percentage.
*/
function scoreLabel(score: number): string {
if (score >= 0.9) return "Elite";
if (score >= 0.8) return "Excellent";
if (score >= 0.7) return "Very Good";
if (score >= 0.6) return "Good";
if (score >= 0.5) return "Average";
return "Below Average";
}
function scoreColor(score: number): string {
if (score >= 0.85) return "text-green-400";
if (score >= 0.7) return "text-emerald-400";
if (score >= 0.55) return "text-yellow-400";
return "text-orange-400";
}
const CATEGORY_OPTIONS = [
{ value: "", labelKey: "allCategories" },
{ value: "default", labelKey: "categoryDefault" },
{ value: "coding", labelKey: "categoryCoding" },
{ value: "review", labelKey: "categoryReview" },
{ value: "documentation", labelKey: "categoryDocumentation" },
{ value: "debugging", labelKey: "categoryDebugging" },
];
export default function FreeProviderRankingsPage() {
const t = useTranslations("freeProviderRankingsPage");
const [rankings, setRankings] = useState<FreeProviderRanking[]>([]);
const [loading, setLoading] = useState(true);
const [error, setError] = useState("");
const [filter, setFilter] = useState<string>("");
const fetchRankings = useCallback(
async (category?: string) => {
setLoading(true);
setError("");
try {
const url = category
? `/api/free-provider-rankings?category=${category}`
: "/api/free-provider-rankings";
const res = await fetch(url);
if (!res.ok) throw new Error(`HTTP ${res.status}`);
const data = await res.json();
setRankings(data.rankings || []);
} catch (err) {
setError(err instanceof Error ? err.message : t("errorLoading"));
} finally {
setLoading(false);
}
},
[t]
);
useEffect(() => {
fetchRankings(filter || undefined);
}, [filter, fetchRankings]);
return (
<div className="flex flex-col gap-6">
{/* Header */}
<div className="flex items-center justify-between">
<div>
<h1 className="text-2xl font-bold">{t("title")}</h1>
<p className="text-sm text-text-muted mt-1">{t("subtitle")}</p>
</div>
</div>
{/* Filters */}
<div className="flex items-center gap-2 flex-wrap">
{CATEGORY_OPTIONS.map((opt) => (
<button
key={opt.value}
onClick={() => setFilter(opt.value)}
className={`px-4 py-2 text-sm font-medium rounded-lg border transition-colors ${
filter === opt.value
? "bg-violet-500 border-violet-500 text-white"
: "border-border text-text-muted hover:text-text-main hover:border-violet-500/50"
}`}
>
{t(opt.labelKey)}
</button>
))}
</div>
{error && <div className="p-3 rounded-lg bg-red-500/10 text-red-400 text-sm">{error}</div>}
{loading ? (
<div className="flex items-center justify-center min-h-[200px]">
<div className="text-text-muted">{t("loading")}</div>
</div>
) : (
<>
{/* Top 3 Podium */}
{rankings.length >= 3 && (
<div className="grid grid-cols-1 md:grid-cols-3 gap-4">
{rankings.slice(0, 3).map((provider, idx) => (
<Card key={provider.id} className="relative overflow-hidden">
<div
className={`absolute top-0 left-0 right-0 h-1 ${
idx === 0
? "bg-gradient-to-r from-amber-400 to-yellow-600"
: idx === 1
? "bg-gradient-to-r from-gray-300 to-gray-500"
: "bg-gradient-to-r from-amber-600 to-orange-800"
}`}
/>
<div className="flex items-center gap-4">
<div className="text-4xl">{idx === 0 ? "🥇" : idx === 1 ? "🥈" : "🥉"}</div>
<div className="flex-1 min-w-0">
<div className="flex items-center gap-2">
<div
className="w-6 h-6 rounded flex items-center justify-center text-[10px] font-bold text-white"
style={{ backgroundColor: provider.color }}
>
{provider.textIcon || provider.name.charAt(0)}
</div>
<p className="font-semibold truncate">{provider.name}</p>
</div>
{provider.topModel && (
<p className="text-sm text-text-muted mt-1 truncate">
{t("bestModel")}: {provider.topModel.modelName}
</p>
)}
{provider.topModel && (
<p
className={`text-lg font-bold mt-1 ${scoreColor(provider.topModel.score)}`}
>
{scoreLabel(provider.topModel.score)}
</p>
)}
</div>
<div className="text-5xl font-black text-text-muted/20">{idx + 1}</div>
</div>
</Card>
))}
</div>
)}
{/* Full List */}
{rankings.length > 0 && (
<Card>
<div className="overflow-x-auto">
<table className="w-full">
<thead>
<tr className="text-left text-sm text-text-muted border-b border-border">
<th className="pb-3 font-medium w-16">{t("colRank")}</th>
<th className="pb-3 font-medium">{t("colProvider")}</th>
<th className="pb-3 font-medium">{t("colTopModel")}</th>
<th className="pb-3 font-medium text-right">{t("colScore")}</th>
<th className="pb-3 font-medium text-right">{t("colAvgScore")}</th>
<th className="pb-3 font-medium text-right">{t("colModels")}</th>
<th className="pb-3 font-medium text-right">{t("colType")}</th>
</tr>
</thead>
<tbody>
{rankings.map((provider, idx) => (
<tr key={provider.id} className="border-b border-border/50 last:border-b-0">
<td className="py-3 text-text-muted font-mono">{idx + 1}</td>
<td className="py-3">
<div className="flex items-center gap-2">
<div
className="w-6 h-6 rounded flex items-center justify-center text-[10px] font-bold text-white"
style={{ backgroundColor: provider.color }}
>
{provider.textIcon || provider.name.charAt(0)}
</div>
<span className="font-medium">{provider.name}</span>
</div>
</td>
<td className="py-3 text-text-muted truncate max-w-[200px]">
{provider.topModel?.modelName || "—"}
</td>
<td className="py-3 text-right">
{provider.topModel ? (
<span
className={`font-mono font-medium ${scoreColor(provider.topModel.score)}`}
>
{scoreLabel(provider.topModel.score)}
</span>
) : (
"—"
)}
</td>
<td className="py-3 text-right font-mono text-text-muted">
{scoreLabel(provider.averageScore)}
</td>
<td className="py-3 text-right text-text-muted">{provider.modelCount}</td>
<td className="py-3 text-right">
<span
className={`text-xs px-2 py-1 rounded ${
provider.category === "noauth"
? "bg-green-500/10 text-green-500"
: provider.category === "oauth"
? "bg-blue-500/10 text-blue-500"
: "bg-purple-500/10 text-purple-500"
}`}
>
{provider.category.toUpperCase()}
</span>
</td>
</tr>
))}
</tbody>
</table>
</div>
</Card>
)}
{rankings.length === 0 && !error && (
<Card>
<div className="text-center py-12 text-text-muted">{t("emptyState")}</div>
</Card>
)}
</>
)}
</div>
);
}

View File

@@ -0,0 +1,40 @@
import { NextRequest, NextResponse } from "next/server";
import { z } from "zod";
import { CORS_HEADERS, handleCorsOptions } from "@/shared/utils/cors";
import { computeFreeProviderRankings } from "@/lib/freeProviderRankings";
const QuerySchema = z.object({
category: z.string().min(1).max(50).optional(),
limit: z
.string()
.optional()
.transform((val) => {
if (!val) return 50;
const n = Number(val);
return Number.isFinite(n) && n >= 1 ? Math.min(Math.round(n), 100) : 50;
}),
});
export async function OPTIONS() {
return handleCorsOptions();
}
export async function GET(request: NextRequest) {
const url = new URL(request.url);
const parsed = QuerySchema.safeParse({
category: url.searchParams.get("category") || undefined,
limit: url.searchParams.get("limit") || undefined,
});
if (!parsed.success) {
return NextResponse.json(
{ error: "Invalid query parameters", details: parsed.error.flatten().fieldErrors },
{ status: 400, headers: CORS_HEADERS }
);
}
const { category, limit } = parsed.data;
const rankings = computeFreeProviderRankings(category, limit);
return NextResponse.json({ rankings }, { headers: CORS_HEADERS });
}

View File

@@ -969,6 +969,9 @@
"analyticsCompression": "Compression",
"costsBudget": "Budget",
"costsFreeTiers": "Free-Tier Budget",
"costsFreeTiersSubtitle": "Track aggregated free tier quotas across 50+ providers",
"freeProviderRankings": "Free Provider Rankings",
"freeProviderRankingsSubtitle": "Best free providers ranked by model ELO scores",
"costsQuotaShare": "Quota Sharing",
"costsPricing": "Pricing",
"logsProxy": "Proxy Logs",
@@ -8704,5 +8707,26 @@
"regenerateRunning": "Regenerating skills…",
"regenerateSuccess": "Skills regenerated successfully.",
"regenerateError": "Failed to regenerate skills."
},
"freeProviderRankingsPage": {
"title": "Free Provider Rankings",
"subtitle": "Best free providers ranked by model ELO scores from Arena AI leaderboards",
"loading": "Loading rankings...",
"errorLoading": "Failed to load rankings",
"emptyState": "No rankings available. Ensure Arena ELO sync is enabled (ARENA_ELO_SYNC_ENABLED=true).",
"bestModel": "Best",
"allCategories": "All Categories",
"categoryDefault": "Default",
"categoryCoding": "Coding",
"categoryReview": "Review",
"categoryDocumentation": "Documentation",
"categoryDebugging": "Debugging",
"colRank": "Rank",
"colProvider": "Provider",
"colTopModel": "Top Model",
"colScore": "Score",
"colAvgScore": "Avg Score",
"colModels": "Models",
"colType": "Type"
}
}

View File

@@ -0,0 +1,239 @@
/**
* freeProviderRankings.ts — Compute rankings for free providers based on model ELO scores.
*
* Joins free providers (no-auth, OAuth, API key) with their models from the registry
* and their intelligence scores from the `model_intelligence` DB table.
*
* Uses flexible matching to bridge naming gaps between registry model IDs
* and Arena-normalized model names (e.g., "kimi-k2.6" vs "kimi-k2").
*/
import { NOAUTH_PROVIDERS, OAUTH_PROVIDERS, APIKEY_PROVIDERS } from "@/shared/constants/providers";
import { REGISTRY } from "@omniroute/open-sse/config/providerRegistry";
import { listModelIntelligence } from "./db/modelIntelligence";
export interface ProviderModelScore {
modelId: string;
modelName: string;
score: number;
eloRaw: number | null;
confidence: string | null;
category: string;
}
export interface FreeProviderRanking {
id: string;
name: string;
icon: string;
color: string;
textIcon?: string;
category: "noauth" | "oauth" | "apikey";
topModel: ProviderModelScore | null;
averageScore: number;
modelCount: number;
}
/**
* Get all free providers from all categories.
*/
function getFreeProviders() {
const providers: Array<{
id: string;
name: string;
icon: string;
color: string;
textIcon?: string;
category: "noauth" | "oauth" | "apikey";
}> = [];
// No-auth providers are always free
for (const [id, p] of Object.entries(NOAUTH_PROVIDERS)) {
providers.push({
id,
name: p.name,
icon: p.icon,
color: p.color,
textIcon: p.textIcon,
category: "noauth",
});
}
// OAuth providers with free tier
for (const [id, p] of Object.entries(OAUTH_PROVIDERS)) {
if ("hasFree" in p && p.hasFree) {
providers.push({
id,
name: p.name,
icon: p.icon,
color: p.color,
textIcon: "textIcon" in p ? (p as any).textIcon : undefined,
category: "oauth",
});
}
}
// API key providers with free tier
for (const [id, p] of Object.entries(APIKEY_PROVIDERS)) {
if ("hasFree" in p && p.hasFree) {
providers.push({
id,
name: p.name,
icon: p.icon,
color: p.color,
textIcon: "textIcon" in p ? (p as any).textIcon : undefined,
category: "apikey",
});
}
}
return providers;
}
/**
* Get models for a provider from the registry.
*/
function getProviderModels(providerId: string) {
const entry = REGISTRY[providerId];
return entry?.models ?? [];
}
/**
* Strip trailing version suffixes from a model ID for fuzzy matching.
* E.g., "kimi-k2.6" → "kimi-k2", "gpt-5.5" → "gpt-5"
*/
export function stripVersionSuffix(id: string): string {
return id.replace(/\.\d+(\.\d+)*$/, "");
}
/**
* Find the best matching intelligence entry for a registry model ID.
*
* Strategy (in order):
* 1. Exact match on normalized model ID
* 2. Exact match on model ID with version suffix stripped
* 3. Prefix match (intelligence entry model is a prefix of registry ID)
*
* @param modelId - The registry model ID (e.g., "kimi-k2.6")
* @param intelMap - Map of normalized model names → intelligence entries
* @returns The best matching intelligence entry, or null
*/
export function findMatchingIntelligence(
modelId: string,
intelMap: Map<
string,
Array<{ score: number; eloRaw: number | null; confidence: string | null; category: string }>
>
): { score: number; eloRaw: number | null; confidence: string | null; category: string } | null {
const normalizedId = modelId.toLowerCase();
// Strategy 1: Exact match
const exactMatches = intelMap.get(normalizedId);
if (exactMatches && exactMatches.length > 0) {
return exactMatches.reduce((prev, curr) => (curr.score > prev.score ? curr : prev));
}
// Strategy 2: Strip version suffix and match
const stripped = stripVersionSuffix(normalizedId);
if (stripped !== normalizedId) {
const strippedMatches = intelMap.get(stripped);
if (strippedMatches && strippedMatches.length > 0) {
return strippedMatches.reduce((prev, curr) => (curr.score > prev.score ? curr : prev));
}
}
// Strategy 3: Prefix match (intelligence entry model is a prefix of registry ID)
let bestPrefixMatch: {
score: number;
eloRaw: number | null;
confidence: string | null;
category: string;
} | null = null;
for (const [modelName, entries] of intelMap) {
if (normalizedId.startsWith(modelName + "-") || normalizedId.startsWith(modelName + ".")) {
const best = entries.reduce((prev, curr) => (curr.score > prev.score ? curr : prev));
if (!bestPrefixMatch || best.score > bestPrefixMatch.score) {
bestPrefixMatch = best;
}
}
}
return bestPrefixMatch;
}
/**
* Compute rankings for free providers based on ELO scores.
*
* @param category - Optional filter for task category (e.g., "coding", "default")
* @param limit - Maximum number of providers to return
*/
export function computeFreeProviderRankings(
category?: string,
limit: number = 50
): FreeProviderRanking[] {
const freeProviders = getFreeProviders();
const intelligenceEntries = listModelIntelligence({
source: "arena_elo",
category: category || undefined,
});
// Create a map for fast lookup: model name → intelligence entries
const intelMap = new Map<string, typeof intelligenceEntries>();
for (const entry of intelligenceEntries) {
const modelKey = entry.model.toLowerCase();
if (!intelMap.has(modelKey)) {
intelMap.set(modelKey, []);
}
intelMap.get(modelKey)!.push(entry);
}
const rankings: FreeProviderRanking[] = [];
for (const provider of freeProviders) {
const models = getProviderModels(provider.id);
if (models.length === 0) continue;
const modelScores: ProviderModelScore[] = [];
for (const model of models) {
const match = findMatchingIntelligence(model.id, intelMap);
if (match) {
modelScores.push({
modelId: model.id,
modelName: model.name,
score: match.score,
eloRaw: match.eloRaw,
confidence: match.confidence,
category: match.category,
});
}
}
if (modelScores.length === 0) continue;
// Sort models by score descending
modelScores.sort((a, b) => b.score - a.score);
const topModel = modelScores[0];
const averageScore = modelScores.reduce((sum, m) => sum + m.score, 0) / modelScores.length;
rankings.push({
...provider,
topModel,
averageScore,
modelCount: modelScores.length,
});
}
// Sort providers by top model score descending, then by average score
rankings.sort((a, b) => {
if (a.topModel && b.topModel) {
return b.topModel.score - a.topModel.score;
}
if (a.topModel) return -1;
if (b.topModel) return 1;
return b.averageScore - a.averageScore;
});
return rankings.slice(0, limit);
}

View File

@@ -49,6 +49,7 @@ export const HIDEABLE_SIDEBAR_ITEM_IDS = [
"costs-budget",
"costs-free-tiers",
"costs-quota-share",
"free-provider-rankings",
// Monitoring > Audit
"audit",
"audit-mcp",
@@ -477,6 +478,13 @@ const COSTS_ITEMS: readonly SidebarItemDefinition[] = [
subtitleKey: "costsFreeTiersSubtitle",
icon: "request_quote",
},
{
id: "free-provider-rankings",
href: "/dashboard/free-provider-rankings",
i18nKey: "freeProviderRankings",
subtitleKey: "freeProviderRankingsSubtitle",
icon: "leaderboard",
},
];
const AUDIT_GROUP: SidebarItemGroup = {

View File

@@ -0,0 +1,143 @@
/**
* Unit tests for src/lib/freeProviderRankings.ts
*
* Tests exported pure functions (stripVersionSuffix, findMatchingIntelligence)
* that don't require DB or module mocking.
*/
import { describe, it } from "node:test";
import assert from "node:assert/strict";
const { stripVersionSuffix, findMatchingIntelligence } =
await import("../../src/lib/freeProviderRankings.ts");
type IntelEntry = {
score: number;
eloRaw: number | null;
confidence: string | null;
category: string;
};
function e(
score: number,
category = "default",
eloRaw: number | null = null,
confidence: string | null = "high"
): IntelEntry {
return { score, eloRaw, confidence, category };
}
function buildMap(entries: Record<string, IntelEntry[]>): Map<string, IntelEntry[]> {
const map = new Map<string, IntelEntry[]>();
for (const [key, vals] of Object.entries(entries)) {
map.set(key, vals);
}
return map;
}
describe("stripVersionSuffix", () => {
it("strips trailing .N from kimi-k2.6", () => {
assert.equal(stripVersionSuffix("kimi-k2.6"), "kimi-k2");
});
it("strips trailing .N from gpt-5.5", () => {
assert.equal(stripVersionSuffix("gpt-5.5"), "gpt-5");
});
it("strips multi-segment .N.N from model-1.2.3", () => {
assert.equal(stripVersionSuffix("model-1.2.3"), "model-1");
});
it("does not strip dash-separated names like claude-sonnet-4-5", () => {
assert.equal(stripVersionSuffix("claude-sonnet-4-5"), "claude-sonnet-4-5");
});
it("returns unchanged for short names", () => {
assert.equal(stripVersionSuffix("gpt"), "gpt");
});
});
describe("findMatchingIntelligence", () => {
it("exact match returns highest score entry", () => {
const map = buildMap({ "gpt-5": [e(0.8), e(0.95)] });
const result = findMatchingIntelligence("gpt-5", map);
assert.ok(result);
assert.equal(result.score, 0.95);
});
it("exact match is case-insensitive", () => {
const map = buildMap({ "gpt-5": [e(0.9)] });
const result = findMatchingIntelligence("GPT-5", map);
assert.ok(result);
assert.equal(result.score, 0.9);
});
it("version suffix fuzzy: kimi-k2.6 matches kimi-k2", () => {
const map = buildMap({ "kimi-k2": [e(0.88)] });
const result = findMatchingIntelligence("kimi-k2.6", map);
assert.ok(result);
assert.equal(result.score, 0.88);
});
it("version suffix fuzzy: gpt-5.5 matches gpt-5", () => {
const map = buildMap({ "gpt-5": [e(0.92)] });
const result = findMatchingIntelligence("gpt-5.5", map);
assert.ok(result);
assert.equal(result.score, 0.92);
});
it("prefix match: claude-sonnet-4-5 matches claude-sonnet-4", () => {
const map = buildMap({ "claude-sonnet-4": [e(0.85)] });
const result = findMatchingIntelligence("claude-sonnet-4-5", map);
assert.ok(result);
assert.equal(result.score, 0.85);
});
it("returns null when no match", () => {
const map = buildMap({ "gpt-5": [e(0.9)] });
assert.equal(findMatchingIntelligence("unknown-model", map), null);
});
it("returns null for empty map", () => {
assert.equal(findMatchingIntelligence("gpt-5", new Map()), null);
});
it("prefers exact match over prefix match", () => {
const map = buildMap({
"claude-sonnet-4-5": [e(0.95)],
"claude-sonnet-4": [e(0.7)],
});
const result = findMatchingIntelligence("claude-sonnet-4-5", map);
assert.ok(result);
assert.equal(result.score, 0.95);
});
it("version match wins over prefix match", () => {
const map = buildMap({
"kimi-k2": [e(0.9)],
kimi: [e(0.5)],
});
const result = findMatchingIntelligence("kimi-k2.6", map);
assert.ok(result);
assert.equal(result.score, 0.9);
});
it("preserves category, eloRaw, confidence from matched entry", () => {
const map = buildMap({ "gpt-5": [e(0.9, "coding", 1400, "high")] });
const result = findMatchingIntelligence("gpt-5", map);
assert.ok(result);
assert.equal(result.category, "coding");
assert.equal(result.eloRaw, 1400);
assert.equal(result.confidence, "high");
});
it("prefix match picks highest-scoring candidate", () => {
const map = buildMap({
gpt: [e(0.6)],
"gpt-4": [e(0.8)],
});
const result = findMatchingIntelligence("gpt-4.1", map);
assert.ok(result);
assert.equal(result.score, 0.8);
});
});