mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-07-26 09:52:11 +03:00
feat: free provider rankings page by Arena AI ELO scores (#3799)
Adds a dashboard page + /api/free-provider-rankings route ranking free providers by model ELO/intelligence scores. Pure computation over the existing provider registry + model-intelligence data (no external fetch). Sidebar entry included.
This commit is contained in:
249
src/app/(dashboard)/dashboard/free-provider-rankings/page.tsx
Normal file
249
src/app/(dashboard)/dashboard/free-provider-rankings/page.tsx
Normal file
@@ -0,0 +1,249 @@
|
||||
"use client";
|
||||
|
||||
import { useState, useEffect, useCallback } from "react";
|
||||
import { useTranslations } from "next-intl";
|
||||
import { Card } from "@/shared/components";
|
||||
|
||||
interface ProviderModelScore {
|
||||
modelId: string;
|
||||
modelName: string;
|
||||
score: number;
|
||||
eloRaw: number | null;
|
||||
confidence: string | null;
|
||||
category: string;
|
||||
}
|
||||
|
||||
interface FreeProviderRanking {
|
||||
id: string;
|
||||
name: string;
|
||||
icon: string;
|
||||
color: string;
|
||||
textIcon?: string;
|
||||
category: "noauth" | "oauth" | "apikey";
|
||||
topModel: ProviderModelScore | null;
|
||||
averageScore: number;
|
||||
modelCount: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a normalized task-fit score (0.4–0.98) to a human-readable label.
|
||||
* The score represents relative ranking quality, not a percentage.
|
||||
*/
|
||||
function scoreLabel(score: number): string {
|
||||
if (score >= 0.9) return "Elite";
|
||||
if (score >= 0.8) return "Excellent";
|
||||
if (score >= 0.7) return "Very Good";
|
||||
if (score >= 0.6) return "Good";
|
||||
if (score >= 0.5) return "Average";
|
||||
return "Below Average";
|
||||
}
|
||||
|
||||
function scoreColor(score: number): string {
|
||||
if (score >= 0.85) return "text-green-400";
|
||||
if (score >= 0.7) return "text-emerald-400";
|
||||
if (score >= 0.55) return "text-yellow-400";
|
||||
return "text-orange-400";
|
||||
}
|
||||
|
||||
const CATEGORY_OPTIONS = [
|
||||
{ value: "", labelKey: "allCategories" },
|
||||
{ value: "default", labelKey: "categoryDefault" },
|
||||
{ value: "coding", labelKey: "categoryCoding" },
|
||||
{ value: "review", labelKey: "categoryReview" },
|
||||
{ value: "documentation", labelKey: "categoryDocumentation" },
|
||||
{ value: "debugging", labelKey: "categoryDebugging" },
|
||||
];
|
||||
|
||||
export default function FreeProviderRankingsPage() {
|
||||
const t = useTranslations("freeProviderRankingsPage");
|
||||
const [rankings, setRankings] = useState<FreeProviderRanking[]>([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState("");
|
||||
const [filter, setFilter] = useState<string>("");
|
||||
|
||||
const fetchRankings = useCallback(
|
||||
async (category?: string) => {
|
||||
setLoading(true);
|
||||
setError("");
|
||||
try {
|
||||
const url = category
|
||||
? `/api/free-provider-rankings?category=${category}`
|
||||
: "/api/free-provider-rankings";
|
||||
const res = await fetch(url);
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
const data = await res.json();
|
||||
setRankings(data.rankings || []);
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : t("errorLoading"));
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
},
|
||||
[t]
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
fetchRankings(filter || undefined);
|
||||
}, [filter, fetchRankings]);
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-6">
|
||||
{/* Header */}
|
||||
<div className="flex items-center justify-between">
|
||||
<div>
|
||||
<h1 className="text-2xl font-bold">{t("title")}</h1>
|
||||
<p className="text-sm text-text-muted mt-1">{t("subtitle")}</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Filters */}
|
||||
<div className="flex items-center gap-2 flex-wrap">
|
||||
{CATEGORY_OPTIONS.map((opt) => (
|
||||
<button
|
||||
key={opt.value}
|
||||
onClick={() => setFilter(opt.value)}
|
||||
className={`px-4 py-2 text-sm font-medium rounded-lg border transition-colors ${
|
||||
filter === opt.value
|
||||
? "bg-violet-500 border-violet-500 text-white"
|
||||
: "border-border text-text-muted hover:text-text-main hover:border-violet-500/50"
|
||||
}`}
|
||||
>
|
||||
{t(opt.labelKey)}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
|
||||
{error && <div className="p-3 rounded-lg bg-red-500/10 text-red-400 text-sm">{error}</div>}
|
||||
|
||||
{loading ? (
|
||||
<div className="flex items-center justify-center min-h-[200px]">
|
||||
<div className="text-text-muted">{t("loading")}</div>
|
||||
</div>
|
||||
) : (
|
||||
<>
|
||||
{/* Top 3 Podium */}
|
||||
{rankings.length >= 3 && (
|
||||
<div className="grid grid-cols-1 md:grid-cols-3 gap-4">
|
||||
{rankings.slice(0, 3).map((provider, idx) => (
|
||||
<Card key={provider.id} className="relative overflow-hidden">
|
||||
<div
|
||||
className={`absolute top-0 left-0 right-0 h-1 ${
|
||||
idx === 0
|
||||
? "bg-gradient-to-r from-amber-400 to-yellow-600"
|
||||
: idx === 1
|
||||
? "bg-gradient-to-r from-gray-300 to-gray-500"
|
||||
: "bg-gradient-to-r from-amber-600 to-orange-800"
|
||||
}`}
|
||||
/>
|
||||
<div className="flex items-center gap-4">
|
||||
<div className="text-4xl">{idx === 0 ? "🥇" : idx === 1 ? "🥈" : "🥉"}</div>
|
||||
<div className="flex-1 min-w-0">
|
||||
<div className="flex items-center gap-2">
|
||||
<div
|
||||
className="w-6 h-6 rounded flex items-center justify-center text-[10px] font-bold text-white"
|
||||
style={{ backgroundColor: provider.color }}
|
||||
>
|
||||
{provider.textIcon || provider.name.charAt(0)}
|
||||
</div>
|
||||
<p className="font-semibold truncate">{provider.name}</p>
|
||||
</div>
|
||||
{provider.topModel && (
|
||||
<p className="text-sm text-text-muted mt-1 truncate">
|
||||
{t("bestModel")}: {provider.topModel.modelName}
|
||||
</p>
|
||||
)}
|
||||
{provider.topModel && (
|
||||
<p
|
||||
className={`text-lg font-bold mt-1 ${scoreColor(provider.topModel.score)}`}
|
||||
>
|
||||
{scoreLabel(provider.topModel.score)}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
<div className="text-5xl font-black text-text-muted/20">{idx + 1}</div>
|
||||
</div>
|
||||
</Card>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Full List */}
|
||||
{rankings.length > 0 && (
|
||||
<Card>
|
||||
<div className="overflow-x-auto">
|
||||
<table className="w-full">
|
||||
<thead>
|
||||
<tr className="text-left text-sm text-text-muted border-b border-border">
|
||||
<th className="pb-3 font-medium w-16">{t("colRank")}</th>
|
||||
<th className="pb-3 font-medium">{t("colProvider")}</th>
|
||||
<th className="pb-3 font-medium">{t("colTopModel")}</th>
|
||||
<th className="pb-3 font-medium text-right">{t("colScore")}</th>
|
||||
<th className="pb-3 font-medium text-right">{t("colAvgScore")}</th>
|
||||
<th className="pb-3 font-medium text-right">{t("colModels")}</th>
|
||||
<th className="pb-3 font-medium text-right">{t("colType")}</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{rankings.map((provider, idx) => (
|
||||
<tr key={provider.id} className="border-b border-border/50 last:border-b-0">
|
||||
<td className="py-3 text-text-muted font-mono">{idx + 1}</td>
|
||||
<td className="py-3">
|
||||
<div className="flex items-center gap-2">
|
||||
<div
|
||||
className="w-6 h-6 rounded flex items-center justify-center text-[10px] font-bold text-white"
|
||||
style={{ backgroundColor: provider.color }}
|
||||
>
|
||||
{provider.textIcon || provider.name.charAt(0)}
|
||||
</div>
|
||||
<span className="font-medium">{provider.name}</span>
|
||||
</div>
|
||||
</td>
|
||||
<td className="py-3 text-text-muted truncate max-w-[200px]">
|
||||
{provider.topModel?.modelName || "—"}
|
||||
</td>
|
||||
<td className="py-3 text-right">
|
||||
{provider.topModel ? (
|
||||
<span
|
||||
className={`font-mono font-medium ${scoreColor(provider.topModel.score)}`}
|
||||
>
|
||||
{scoreLabel(provider.topModel.score)}
|
||||
</span>
|
||||
) : (
|
||||
"—"
|
||||
)}
|
||||
</td>
|
||||
<td className="py-3 text-right font-mono text-text-muted">
|
||||
{scoreLabel(provider.averageScore)}
|
||||
</td>
|
||||
<td className="py-3 text-right text-text-muted">{provider.modelCount}</td>
|
||||
<td className="py-3 text-right">
|
||||
<span
|
||||
className={`text-xs px-2 py-1 rounded ${
|
||||
provider.category === "noauth"
|
||||
? "bg-green-500/10 text-green-500"
|
||||
: provider.category === "oauth"
|
||||
? "bg-blue-500/10 text-blue-500"
|
||||
: "bg-purple-500/10 text-purple-500"
|
||||
}`}
|
||||
>
|
||||
{provider.category.toUpperCase()}
|
||||
</span>
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</Card>
|
||||
)}
|
||||
|
||||
{rankings.length === 0 && !error && (
|
||||
<Card>
|
||||
<div className="text-center py-12 text-text-muted">{t("emptyState")}</div>
|
||||
</Card>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
40
src/app/api/free-provider-rankings/route.ts
Normal file
40
src/app/api/free-provider-rankings/route.ts
Normal file
@@ -0,0 +1,40 @@
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { z } from "zod";
|
||||
import { CORS_HEADERS, handleCorsOptions } from "@/shared/utils/cors";
|
||||
import { computeFreeProviderRankings } from "@/lib/freeProviderRankings";
|
||||
|
||||
const QuerySchema = z.object({
|
||||
category: z.string().min(1).max(50).optional(),
|
||||
limit: z
|
||||
.string()
|
||||
.optional()
|
||||
.transform((val) => {
|
||||
if (!val) return 50;
|
||||
const n = Number(val);
|
||||
return Number.isFinite(n) && n >= 1 ? Math.min(Math.round(n), 100) : 50;
|
||||
}),
|
||||
});
|
||||
|
||||
export async function OPTIONS() {
|
||||
return handleCorsOptions();
|
||||
}
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
const url = new URL(request.url);
|
||||
const parsed = QuerySchema.safeParse({
|
||||
category: url.searchParams.get("category") || undefined,
|
||||
limit: url.searchParams.get("limit") || undefined,
|
||||
});
|
||||
|
||||
if (!parsed.success) {
|
||||
return NextResponse.json(
|
||||
{ error: "Invalid query parameters", details: parsed.error.flatten().fieldErrors },
|
||||
{ status: 400, headers: CORS_HEADERS }
|
||||
);
|
||||
}
|
||||
|
||||
const { category, limit } = parsed.data;
|
||||
const rankings = computeFreeProviderRankings(category, limit);
|
||||
|
||||
return NextResponse.json({ rankings }, { headers: CORS_HEADERS });
|
||||
}
|
||||
@@ -969,6 +969,9 @@
|
||||
"analyticsCompression": "Compression",
|
||||
"costsBudget": "Budget",
|
||||
"costsFreeTiers": "Free-Tier Budget",
|
||||
"costsFreeTiersSubtitle": "Track aggregated free tier quotas across 50+ providers",
|
||||
"freeProviderRankings": "Free Provider Rankings",
|
||||
"freeProviderRankingsSubtitle": "Best free providers ranked by model ELO scores",
|
||||
"costsQuotaShare": "Quota Sharing",
|
||||
"costsPricing": "Pricing",
|
||||
"logsProxy": "Proxy Logs",
|
||||
@@ -8704,5 +8707,26 @@
|
||||
"regenerateRunning": "Regenerating skills…",
|
||||
"regenerateSuccess": "Skills regenerated successfully.",
|
||||
"regenerateError": "Failed to regenerate skills."
|
||||
},
|
||||
"freeProviderRankingsPage": {
|
||||
"title": "Free Provider Rankings",
|
||||
"subtitle": "Best free providers ranked by model ELO scores from Arena AI leaderboards",
|
||||
"loading": "Loading rankings...",
|
||||
"errorLoading": "Failed to load rankings",
|
||||
"emptyState": "No rankings available. Ensure Arena ELO sync is enabled (ARENA_ELO_SYNC_ENABLED=true).",
|
||||
"bestModel": "Best",
|
||||
"allCategories": "All Categories",
|
||||
"categoryDefault": "Default",
|
||||
"categoryCoding": "Coding",
|
||||
"categoryReview": "Review",
|
||||
"categoryDocumentation": "Documentation",
|
||||
"categoryDebugging": "Debugging",
|
||||
"colRank": "Rank",
|
||||
"colProvider": "Provider",
|
||||
"colTopModel": "Top Model",
|
||||
"colScore": "Score",
|
||||
"colAvgScore": "Avg Score",
|
||||
"colModels": "Models",
|
||||
"colType": "Type"
|
||||
}
|
||||
}
|
||||
|
||||
239
src/lib/freeProviderRankings.ts
Normal file
239
src/lib/freeProviderRankings.ts
Normal file
@@ -0,0 +1,239 @@
|
||||
/**
|
||||
* freeProviderRankings.ts — Compute rankings for free providers based on model ELO scores.
|
||||
*
|
||||
* Joins free providers (no-auth, OAuth, API key) with their models from the registry
|
||||
* and their intelligence scores from the `model_intelligence` DB table.
|
||||
*
|
||||
* Uses flexible matching to bridge naming gaps between registry model IDs
|
||||
* and Arena-normalized model names (e.g., "kimi-k2.6" vs "kimi-k2").
|
||||
*/
|
||||
|
||||
import { NOAUTH_PROVIDERS, OAUTH_PROVIDERS, APIKEY_PROVIDERS } from "@/shared/constants/providers";
|
||||
import { REGISTRY } from "@omniroute/open-sse/config/providerRegistry";
|
||||
import { listModelIntelligence } from "./db/modelIntelligence";
|
||||
|
||||
export interface ProviderModelScore {
|
||||
modelId: string;
|
||||
modelName: string;
|
||||
score: number;
|
||||
eloRaw: number | null;
|
||||
confidence: string | null;
|
||||
category: string;
|
||||
}
|
||||
|
||||
export interface FreeProviderRanking {
|
||||
id: string;
|
||||
name: string;
|
||||
icon: string;
|
||||
color: string;
|
||||
textIcon?: string;
|
||||
category: "noauth" | "oauth" | "apikey";
|
||||
topModel: ProviderModelScore | null;
|
||||
averageScore: number;
|
||||
modelCount: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all free providers from all categories.
|
||||
*/
|
||||
function getFreeProviders() {
|
||||
const providers: Array<{
|
||||
id: string;
|
||||
name: string;
|
||||
icon: string;
|
||||
color: string;
|
||||
textIcon?: string;
|
||||
category: "noauth" | "oauth" | "apikey";
|
||||
}> = [];
|
||||
|
||||
// No-auth providers are always free
|
||||
for (const [id, p] of Object.entries(NOAUTH_PROVIDERS)) {
|
||||
providers.push({
|
||||
id,
|
||||
name: p.name,
|
||||
icon: p.icon,
|
||||
color: p.color,
|
||||
textIcon: p.textIcon,
|
||||
category: "noauth",
|
||||
});
|
||||
}
|
||||
|
||||
// OAuth providers with free tier
|
||||
for (const [id, p] of Object.entries(OAUTH_PROVIDERS)) {
|
||||
if ("hasFree" in p && p.hasFree) {
|
||||
providers.push({
|
||||
id,
|
||||
name: p.name,
|
||||
icon: p.icon,
|
||||
color: p.color,
|
||||
textIcon: "textIcon" in p ? (p as any).textIcon : undefined,
|
||||
category: "oauth",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// API key providers with free tier
|
||||
for (const [id, p] of Object.entries(APIKEY_PROVIDERS)) {
|
||||
if ("hasFree" in p && p.hasFree) {
|
||||
providers.push({
|
||||
id,
|
||||
name: p.name,
|
||||
icon: p.icon,
|
||||
color: p.color,
|
||||
textIcon: "textIcon" in p ? (p as any).textIcon : undefined,
|
||||
category: "apikey",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return providers;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get models for a provider from the registry.
|
||||
*/
|
||||
function getProviderModels(providerId: string) {
|
||||
const entry = REGISTRY[providerId];
|
||||
return entry?.models ?? [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip trailing version suffixes from a model ID for fuzzy matching.
|
||||
* E.g., "kimi-k2.6" → "kimi-k2", "gpt-5.5" → "gpt-5"
|
||||
*/
|
||||
export function stripVersionSuffix(id: string): string {
|
||||
return id.replace(/\.\d+(\.\d+)*$/, "");
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the best matching intelligence entry for a registry model ID.
|
||||
*
|
||||
* Strategy (in order):
|
||||
* 1. Exact match on normalized model ID
|
||||
* 2. Exact match on model ID with version suffix stripped
|
||||
* 3. Prefix match (intelligence entry model is a prefix of registry ID)
|
||||
*
|
||||
* @param modelId - The registry model ID (e.g., "kimi-k2.6")
|
||||
* @param intelMap - Map of normalized model names → intelligence entries
|
||||
* @returns The best matching intelligence entry, or null
|
||||
*/
|
||||
export function findMatchingIntelligence(
|
||||
modelId: string,
|
||||
intelMap: Map<
|
||||
string,
|
||||
Array<{ score: number; eloRaw: number | null; confidence: string | null; category: string }>
|
||||
>
|
||||
): { score: number; eloRaw: number | null; confidence: string | null; category: string } | null {
|
||||
const normalizedId = modelId.toLowerCase();
|
||||
|
||||
// Strategy 1: Exact match
|
||||
const exactMatches = intelMap.get(normalizedId);
|
||||
if (exactMatches && exactMatches.length > 0) {
|
||||
return exactMatches.reduce((prev, curr) => (curr.score > prev.score ? curr : prev));
|
||||
}
|
||||
|
||||
// Strategy 2: Strip version suffix and match
|
||||
const stripped = stripVersionSuffix(normalizedId);
|
||||
if (stripped !== normalizedId) {
|
||||
const strippedMatches = intelMap.get(stripped);
|
||||
if (strippedMatches && strippedMatches.length > 0) {
|
||||
return strippedMatches.reduce((prev, curr) => (curr.score > prev.score ? curr : prev));
|
||||
}
|
||||
}
|
||||
|
||||
// Strategy 3: Prefix match (intelligence entry model is a prefix of registry ID)
|
||||
let bestPrefixMatch: {
|
||||
score: number;
|
||||
eloRaw: number | null;
|
||||
confidence: string | null;
|
||||
category: string;
|
||||
} | null = null;
|
||||
for (const [modelName, entries] of intelMap) {
|
||||
if (normalizedId.startsWith(modelName + "-") || normalizedId.startsWith(modelName + ".")) {
|
||||
const best = entries.reduce((prev, curr) => (curr.score > prev.score ? curr : prev));
|
||||
if (!bestPrefixMatch || best.score > bestPrefixMatch.score) {
|
||||
bestPrefixMatch = best;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return bestPrefixMatch;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute rankings for free providers based on ELO scores.
|
||||
*
|
||||
* @param category - Optional filter for task category (e.g., "coding", "default")
|
||||
* @param limit - Maximum number of providers to return
|
||||
*/
|
||||
export function computeFreeProviderRankings(
|
||||
category?: string,
|
||||
limit: number = 50
|
||||
): FreeProviderRanking[] {
|
||||
const freeProviders = getFreeProviders();
|
||||
const intelligenceEntries = listModelIntelligence({
|
||||
source: "arena_elo",
|
||||
category: category || undefined,
|
||||
});
|
||||
|
||||
// Create a map for fast lookup: model name → intelligence entries
|
||||
const intelMap = new Map<string, typeof intelligenceEntries>();
|
||||
for (const entry of intelligenceEntries) {
|
||||
const modelKey = entry.model.toLowerCase();
|
||||
if (!intelMap.has(modelKey)) {
|
||||
intelMap.set(modelKey, []);
|
||||
}
|
||||
intelMap.get(modelKey)!.push(entry);
|
||||
}
|
||||
|
||||
const rankings: FreeProviderRanking[] = [];
|
||||
|
||||
for (const provider of freeProviders) {
|
||||
const models = getProviderModels(provider.id);
|
||||
if (models.length === 0) continue;
|
||||
|
||||
const modelScores: ProviderModelScore[] = [];
|
||||
|
||||
for (const model of models) {
|
||||
const match = findMatchingIntelligence(model.id, intelMap);
|
||||
|
||||
if (match) {
|
||||
modelScores.push({
|
||||
modelId: model.id,
|
||||
modelName: model.name,
|
||||
score: match.score,
|
||||
eloRaw: match.eloRaw,
|
||||
confidence: match.confidence,
|
||||
category: match.category,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (modelScores.length === 0) continue;
|
||||
|
||||
// Sort models by score descending
|
||||
modelScores.sort((a, b) => b.score - a.score);
|
||||
|
||||
const topModel = modelScores[0];
|
||||
const averageScore = modelScores.reduce((sum, m) => sum + m.score, 0) / modelScores.length;
|
||||
|
||||
rankings.push({
|
||||
...provider,
|
||||
topModel,
|
||||
averageScore,
|
||||
modelCount: modelScores.length,
|
||||
});
|
||||
}
|
||||
|
||||
// Sort providers by top model score descending, then by average score
|
||||
rankings.sort((a, b) => {
|
||||
if (a.topModel && b.topModel) {
|
||||
return b.topModel.score - a.topModel.score;
|
||||
}
|
||||
if (a.topModel) return -1;
|
||||
if (b.topModel) return 1;
|
||||
return b.averageScore - a.averageScore;
|
||||
});
|
||||
|
||||
return rankings.slice(0, limit);
|
||||
}
|
||||
@@ -49,6 +49,7 @@ export const HIDEABLE_SIDEBAR_ITEM_IDS = [
|
||||
"costs-budget",
|
||||
"costs-free-tiers",
|
||||
"costs-quota-share",
|
||||
"free-provider-rankings",
|
||||
// Monitoring > Audit
|
||||
"audit",
|
||||
"audit-mcp",
|
||||
@@ -477,6 +478,13 @@ const COSTS_ITEMS: readonly SidebarItemDefinition[] = [
|
||||
subtitleKey: "costsFreeTiersSubtitle",
|
||||
icon: "request_quote",
|
||||
},
|
||||
{
|
||||
id: "free-provider-rankings",
|
||||
href: "/dashboard/free-provider-rankings",
|
||||
i18nKey: "freeProviderRankings",
|
||||
subtitleKey: "freeProviderRankingsSubtitle",
|
||||
icon: "leaderboard",
|
||||
},
|
||||
];
|
||||
|
||||
const AUDIT_GROUP: SidebarItemGroup = {
|
||||
|
||||
143
tests/unit/freeProviderRankings.test.ts
Normal file
143
tests/unit/freeProviderRankings.test.ts
Normal file
@@ -0,0 +1,143 @@
|
||||
/**
|
||||
* Unit tests for src/lib/freeProviderRankings.ts
|
||||
*
|
||||
* Tests exported pure functions (stripVersionSuffix, findMatchingIntelligence)
|
||||
* that don't require DB or module mocking.
|
||||
*/
|
||||
|
||||
import { describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
const { stripVersionSuffix, findMatchingIntelligence } =
|
||||
await import("../../src/lib/freeProviderRankings.ts");
|
||||
|
||||
type IntelEntry = {
|
||||
score: number;
|
||||
eloRaw: number | null;
|
||||
confidence: string | null;
|
||||
category: string;
|
||||
};
|
||||
|
||||
function e(
|
||||
score: number,
|
||||
category = "default",
|
||||
eloRaw: number | null = null,
|
||||
confidence: string | null = "high"
|
||||
): IntelEntry {
|
||||
return { score, eloRaw, confidence, category };
|
||||
}
|
||||
|
||||
function buildMap(entries: Record<string, IntelEntry[]>): Map<string, IntelEntry[]> {
|
||||
const map = new Map<string, IntelEntry[]>();
|
||||
for (const [key, vals] of Object.entries(entries)) {
|
||||
map.set(key, vals);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
describe("stripVersionSuffix", () => {
|
||||
it("strips trailing .N from kimi-k2.6", () => {
|
||||
assert.equal(stripVersionSuffix("kimi-k2.6"), "kimi-k2");
|
||||
});
|
||||
|
||||
it("strips trailing .N from gpt-5.5", () => {
|
||||
assert.equal(stripVersionSuffix("gpt-5.5"), "gpt-5");
|
||||
});
|
||||
|
||||
it("strips multi-segment .N.N from model-1.2.3", () => {
|
||||
assert.equal(stripVersionSuffix("model-1.2.3"), "model-1");
|
||||
});
|
||||
|
||||
it("does not strip dash-separated names like claude-sonnet-4-5", () => {
|
||||
assert.equal(stripVersionSuffix("claude-sonnet-4-5"), "claude-sonnet-4-5");
|
||||
});
|
||||
|
||||
it("returns unchanged for short names", () => {
|
||||
assert.equal(stripVersionSuffix("gpt"), "gpt");
|
||||
});
|
||||
});
|
||||
|
||||
describe("findMatchingIntelligence", () => {
|
||||
it("exact match returns highest score entry", () => {
|
||||
const map = buildMap({ "gpt-5": [e(0.8), e(0.95)] });
|
||||
const result = findMatchingIntelligence("gpt-5", map);
|
||||
assert.ok(result);
|
||||
assert.equal(result.score, 0.95);
|
||||
});
|
||||
|
||||
it("exact match is case-insensitive", () => {
|
||||
const map = buildMap({ "gpt-5": [e(0.9)] });
|
||||
const result = findMatchingIntelligence("GPT-5", map);
|
||||
assert.ok(result);
|
||||
assert.equal(result.score, 0.9);
|
||||
});
|
||||
|
||||
it("version suffix fuzzy: kimi-k2.6 matches kimi-k2", () => {
|
||||
const map = buildMap({ "kimi-k2": [e(0.88)] });
|
||||
const result = findMatchingIntelligence("kimi-k2.6", map);
|
||||
assert.ok(result);
|
||||
assert.equal(result.score, 0.88);
|
||||
});
|
||||
|
||||
it("version suffix fuzzy: gpt-5.5 matches gpt-5", () => {
|
||||
const map = buildMap({ "gpt-5": [e(0.92)] });
|
||||
const result = findMatchingIntelligence("gpt-5.5", map);
|
||||
assert.ok(result);
|
||||
assert.equal(result.score, 0.92);
|
||||
});
|
||||
|
||||
it("prefix match: claude-sonnet-4-5 matches claude-sonnet-4", () => {
|
||||
const map = buildMap({ "claude-sonnet-4": [e(0.85)] });
|
||||
const result = findMatchingIntelligence("claude-sonnet-4-5", map);
|
||||
assert.ok(result);
|
||||
assert.equal(result.score, 0.85);
|
||||
});
|
||||
|
||||
it("returns null when no match", () => {
|
||||
const map = buildMap({ "gpt-5": [e(0.9)] });
|
||||
assert.equal(findMatchingIntelligence("unknown-model", map), null);
|
||||
});
|
||||
|
||||
it("returns null for empty map", () => {
|
||||
assert.equal(findMatchingIntelligence("gpt-5", new Map()), null);
|
||||
});
|
||||
|
||||
it("prefers exact match over prefix match", () => {
|
||||
const map = buildMap({
|
||||
"claude-sonnet-4-5": [e(0.95)],
|
||||
"claude-sonnet-4": [e(0.7)],
|
||||
});
|
||||
const result = findMatchingIntelligence("claude-sonnet-4-5", map);
|
||||
assert.ok(result);
|
||||
assert.equal(result.score, 0.95);
|
||||
});
|
||||
|
||||
it("version match wins over prefix match", () => {
|
||||
const map = buildMap({
|
||||
"kimi-k2": [e(0.9)],
|
||||
kimi: [e(0.5)],
|
||||
});
|
||||
const result = findMatchingIntelligence("kimi-k2.6", map);
|
||||
assert.ok(result);
|
||||
assert.equal(result.score, 0.9);
|
||||
});
|
||||
|
||||
it("preserves category, eloRaw, confidence from matched entry", () => {
|
||||
const map = buildMap({ "gpt-5": [e(0.9, "coding", 1400, "high")] });
|
||||
const result = findMatchingIntelligence("gpt-5", map);
|
||||
assert.ok(result);
|
||||
assert.equal(result.category, "coding");
|
||||
assert.equal(result.eloRaw, 1400);
|
||||
assert.equal(result.confidence, "high");
|
||||
});
|
||||
|
||||
it("prefix match picks highest-scoring candidate", () => {
|
||||
const map = buildMap({
|
||||
gpt: [e(0.6)],
|
||||
"gpt-4": [e(0.8)],
|
||||
});
|
||||
const result = findMatchingIntelligence("gpt-4.1", map);
|
||||
assert.ok(result);
|
||||
assert.equal(result.score, 0.8);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user