From c1986ef4b9bc3962fe630dc93144efeb70997305 Mon Sep 17 00:00:00 2001 From: zabrodschiipavel-sketch Date: Thu, 6 Aug 2026 03:46:19 +0300 Subject: [PATCH] feat(providers): enrich dashboard providers list with OpenRouter data (#9324) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Validated in local merge-train (devbox-vm-06-dev002) @ combined-tip (FAST gates green: static + changed tests + vitest — only pre-existing audit.test.ts flake). Evidence: /home/diegosouzapw/dev/proxys/OmniRoute/.claude/worktrees/merge-train-20260805-213228-suite.log --- .../providers/components/ProviderCard.tsx | 36 +- .../openRouterProviderStatsContext.tsx | 32 ++ .../(dashboard)/dashboard/providers/page.tsx | 9 +- .../dashboard/providers/providerPageUtils.ts | 37 +- .../api/providers/openrouter-stats/route.ts | 51 +++ src/instrumentation-node.ts | 13 + src/lib/catalog/openrouterProviderStats.ts | 342 ++++++++++++++++++ tests/unit/openrouter-provider-stats.test.ts | 203 +++++++++++ 8 files changed, 715 insertions(+), 8 deletions(-) create mode 100644 src/app/(dashboard)/dashboard/providers/context/openRouterProviderStatsContext.tsx create mode 100644 src/app/api/providers/openrouter-stats/route.ts create mode 100644 src/lib/catalog/openrouterProviderStats.ts create mode 100644 tests/unit/openrouter-provider-stats.test.ts diff --git a/src/app/(dashboard)/dashboard/providers/components/ProviderCard.tsx b/src/app/(dashboard)/dashboard/providers/components/ProviderCard.tsx index a1efc70ec6..4faea53e21 100644 --- a/src/app/(dashboard)/dashboard/providers/components/ProviderCard.tsx +++ b/src/app/(dashboard)/dashboard/providers/components/ProviderCard.tsx @@ -17,6 +17,7 @@ import { import { CategoryDot } from "./CategoryDot"; import { isCheaperInferenceProviderId, isKimiPartnerProviderId } from "../featuredProviders"; +import { useOpenRouterProviderStat } from "../context/openRouterProviderStatsContext"; interface ProviderStats { total?: number; @@ -228,6 +229,7 @@ const ProviderCard = forwardRef(function const isKimiPartner = isKimiPartnerProviderId(provider.id || providerId); const isCheaperInferencePartner = isCheaperInferenceProviderId(provider.id || providerId); const isSponsorPartner = isKimiPartner || isCheaperInferencePartner; + const openRouterStat = useOpenRouterProviderStat(provider.id || providerId); const codexServiceTierLabel = stats.codexServiceTier === "flex" ? providerText(t, "codexTierFlexLabel", "Flex") @@ -287,6 +289,36 @@ const ProviderCard = forwardRef(function ) : null; + const openRouterTooltipBits: string[] = []; + if (openRouterStat?.headquarters) openRouterTooltipBits.push(`HQ: ${openRouterStat.headquarters}`); + if (openRouterStat?.dataPolicy?.training === false) { + openRouterTooltipBits.push(providerText(t, "openRouterNoTraining", "Does not train on prompts")); + } + if (openRouterStat?.dataPolicy?.retainsPrompts === false) { + openRouterTooltipBits.push(providerText(t, "openRouterNoRetention", "Does not retain prompts")); + } + const openRouterTooltip = openRouterStat + ? providerText(t, "openRouterPopularityTooltip", "OpenRouter usage rank #{rank}", { + rank: openRouterStat.popularityRank, + }) + (openRouterTooltipBits.length ? ` — ${openRouterTooltipBits.join(" · ")}` : "") + : ""; + + // OpenRouter popularity badge — data refreshed daily from OpenRouter's + // provider directory + usage rankings (see src/lib/catalog/openrouterProviderStats.ts). + // Absent entirely for providers OpenRouter doesn't track; never affects routing. + const openRouterPopularityChip = openRouterStat ? ( + + trending_up + {providerText(t, "openRouterPopularityBadge", "OR #{rank}", { + rank: openRouterStat.popularityRank, + })} + + ) : null; + const dotLabels: Record = { free: tc("free"), "no-auth": t("noAuthLabel"), @@ -417,10 +449,12 @@ const ProviderCard = forwardRef(function isCompatible || isCcCompatible || isAnthropicCompatible || - isSponsorPartner) && ( + isSponsorPartner || + Boolean(openRouterStat)) && (
{kimiOfficialSupporterChip} {cheaperInferenceSupporterChip} + {openRouterPopularityChip} {provider.serviceKinds?.map((k) => ( = new Map(); +const Context = + createContext>(EMPTY_STATS_MAP); + +export function OpenRouterProviderStatsProvider({ + entries, + children, +}: { + entries: OpenRouterProviderStatsEntry[]; + children: ReactNode; +}) { + const bySlug = useMemo(() => new Map(entries.map((entry) => [entry.slug, entry])), [entries]); + return {children}; +} + +export function useOpenRouterProviderStat( + providerId: string | undefined +): OpenRouterProviderStatsEntry | undefined { + const bySlug = useContext(Context); + return providerId ? bySlug.get(providerId) : undefined; +} diff --git a/src/app/(dashboard)/dashboard/providers/page.tsx b/src/app/(dashboard)/dashboard/providers/page.tsx index 896e095757..c5f537a61e 100644 --- a/src/app/(dashboard)/dashboard/providers/page.tsx +++ b/src/app/(dashboard)/dashboard/providers/page.tsx @@ -34,7 +34,8 @@ import { upsertProviderNodeById, loadProviderPageData, } from "./providerPageUtils"; -import type { ProviderEntry } from "./providerPageUtils"; +import type { ProviderEntry, OpenRouterProviderStatsEntry } from "./providerPageUtils"; +import { OpenRouterProviderStatsProvider } from "./context/openRouterProviderStatsContext"; import { shouldSyncProviderDisplayMode, writeProviderDisplayModePreference, @@ -200,6 +201,9 @@ export default function ProvidersPage() { const [modelSearchQuery, setModelSearchQuery] = useState(""); const liveModelsByProviderId = useSyncedModelsByProvider(); const [showFreeOnly, setShowFreeOnly] = useState(false); + const [openRouterProviderStats, setOpenRouterProviderStats] = useState< + OpenRouterProviderStatsEntry[] + >([]); const [activeCategory, setActiveCategory] = useState(null); // #4240: media-category (serviceKind) filter — composes with activeCategory, // search and configured-only. null = no serviceKind filter. @@ -255,6 +259,7 @@ export default function ProvidersPage() { if (data.expirations) setExpirations(data.expirations); if (data.blockedProviders) setBlockedProviders(data.blockedProviders); setCodexGlobalServiceMode(getCodexGlobalServiceMode(data.settings)); + setOpenRouterProviderStats(data.openRouterProviderStats); } catch (error) { console.log("Error fetching data:", error); } finally { @@ -812,6 +817,7 @@ export default function ProvidersPage() { shouldShowFirstProviderHint(connections.length, searchQuery) && !showAllProviders; return ( +
{showFirstProviderHint && ( @@ -1814,6 +1820,7 @@ export default function ProvidersPage() {
)}
+ ); } diff --git a/src/app/(dashboard)/dashboard/providers/providerPageUtils.ts b/src/app/(dashboard)/dashboard/providers/providerPageUtils.ts index 7551903aa8..b48c434e60 100644 --- a/src/app/(dashboard)/dashboard/providers/providerPageUtils.ts +++ b/src/app/(dashboard)/dashboard/providers/providerPageUtils.ts @@ -550,6 +550,28 @@ export interface ProviderPageData { expirations: any | null; blockedProviders: string[] | null; settings: any | null; + /** OpenRouter-sourced popularity/identity enrichment, keyed by provider slug. Empty if the sync hasn't run yet or the fetch failed. */ + openRouterProviderStats: OpenRouterProviderStatsEntry[]; +} + +/** Mirrors ProviderPopularityEntry from src/lib/catalog/openrouterProviderStats.ts (kept local to avoid a server-only import from a client component). */ +export interface OpenRouterProviderStatsEntry { + slug: string; + displayName: string; + headquarters?: string; + statusPageUrl?: string | null; + byokEnabled?: boolean; + dataPolicy?: { + training?: boolean; + retainsPrompts?: boolean; + termsOfServiceURL?: string; + privacyPolicyURL?: string; + }; + iconUrl?: string; + modelCount: number; + totalTokens: number; + totalRequests: number; + popularityRank: number; } // Bound each first-paint request so a single stalled connection cannot freeze @@ -587,12 +609,14 @@ export async function loadProviderPageData( } }; - const [connectionsData, nodesData, expirationsData, settingsData] = await Promise.all([ - safeJson("/api/providers"), - safeJson("/api/provider-nodes"), - safeJson("/api/providers/expiration"), - safeJson("/api/settings", { cache: "no-store" }), - ]); + const [connectionsData, nodesData, expirationsData, settingsData, openRouterStatsData] = + await Promise.all([ + safeJson("/api/providers"), + safeJson("/api/provider-nodes"), + safeJson("/api/providers/expiration"), + safeJson("/api/settings", { cache: "no-store" }), + safeJson("/api/providers/openrouter-stats"), + ]); return { connections: Array.isArray(connectionsData?.connections) ? connectionsData.connections : [], @@ -603,5 +627,6 @@ export async function loadProviderPageData( ? settingsData.blockedProviders : null, settings: settingsData ?? null, + openRouterProviderStats: Array.isArray(openRouterStatsData?.data) ? openRouterStatsData.data : [], }; } diff --git a/src/app/api/providers/openrouter-stats/route.ts b/src/app/api/providers/openrouter-stats/route.ts new file mode 100644 index 0000000000..55efbcc1ff --- /dev/null +++ b/src/app/api/providers/openrouter-stats/route.ts @@ -0,0 +1,51 @@ +/** + * GET /api/providers/openrouter-stats + * Returns OpenRouter-sourced provider enrichment (popularity rank, HQ, data + * policy, ToS/privacy links) with persistent cache — see openrouterProviderStats.ts. + * + * Query params: + * ?refresh=true — Force-refresh, ignores TTL + */ + +import { NextRequest, NextResponse } from "next/server"; +import { isAuthenticated } from "@/shared/utils/apiAuth"; +import { + getOpenRouterProviderStats, + refreshOpenRouterProviderStats, +} from "@/lib/catalog/openrouterProviderStats"; + +export async function GET(req: NextRequest) { + if (!(await isAuthenticated(req))) { + return NextResponse.json( + { error: { message: "Authentication required", type: "invalid_request_error" } }, + { status: 401 } + ); + } + + const forceRefresh = req.nextUrl.searchParams.get("refresh") === "true"; + + if (forceRefresh) { + const result = await refreshOpenRouterProviderStats(); + return NextResponse.json({ + object: "list", + data: result.data, + meta: { + source: result.ok ? "fresh" : "error", + count: result.data.length, + error: result.error ?? undefined, + }, + }); + } + + const result = await getOpenRouterProviderStats(); + return NextResponse.json({ + object: "list", + data: result.data, + meta: { + source: result.fromCache ? (result.stale ? "stale-cache" : "cache") : "fresh", + cachedAt: result.cachedAt ?? undefined, + stale: result.stale, + count: result.data.length, + }, + }); +} diff --git a/src/instrumentation-node.ts b/src/instrumentation-node.ts index 822265d5cc..7cab7b0387 100755 --- a/src/instrumentation-node.ts +++ b/src/instrumentation-node.ts @@ -552,6 +552,19 @@ export async function registerNodejs(): Promise { console.warn("[STARTUP] Pricing sync failed to start (non-fatal):", msg); }), + // OpenRouter provider stats sync: provider directory + popularity enrichment + // for the dashboard Providers page. On by default; opt out with + // OPENROUTER_PROVIDER_STATS_ENABLED=false. Non-blocking, never fatal. + import("@/lib/catalog/openrouterProviderStats") + .then((m) => { + const started = m.initOpenRouterProviderStatsSync(); + if (started) console.log("[STARTUP] OpenRouter provider stats sync initialized"); + }) + .catch((err: unknown) => { + const msg = err instanceof Error ? err.message : String(err); + console.warn("[STARTUP] OpenRouter provider stats sync failed to start (non-fatal):", msg); + }), + // models.dev capability sync: opt-in via Settings > AI (self-gated by // settings.modelsDevSyncEnabled inside initModelsDevSync). Non-blocking, never fatal. import("@/lib/modelsDevSync") diff --git a/src/lib/catalog/openrouterProviderStats.ts b/src/lib/catalog/openrouterProviderStats.ts new file mode 100644 index 0000000000..4363fc2ca2 --- /dev/null +++ b/src/lib/catalog/openrouterProviderStats.ts @@ -0,0 +1,342 @@ +/** + * openrouterProviderStats.ts + * + * Enriches OmniRoute's provider directory with data scraped from OpenRouter's + * *internal* frontend API (openrouter.ai/api/frontend/v1/*) — undocumented and + * unversioned, unlike the public /api/v1/models consumed by openrouterCatalog.ts. + * Three bulk endpoints, one request each, refreshed once/day — never per-model: + * + * - all-providers → provider directory (HQ, ToS/privacy, data policy, status page) + * - catalog/models → model→provider endpoint mapping (for attribution) + * - rankings/models → usage volume (tokens/requests) per model+variant + * + * catalog/models rows are joined against rankings/models by `permaslug+variant` + * and the resulting usage is summed per `provider_slug` — that sum is the + * popularity signal. All three responses can drift or disappear without notice + * (nothing here is a documented contract), so every row is parsed defensively + * with zod `.safeParse()` and skipped on failure rather than failing the batch, + * and the whole refresh falls back to the last good cache on any fetch error + * (same stale-if-error shape as openrouterCatalog.ts / arenaEloSync.ts). + */ + +import fs from "fs"; +import path from "path"; +import { z } from "zod"; + +const ALL_PROVIDERS_URL = "https://openrouter.ai/api/frontend/v1/all-providers"; +const CATALOG_MODELS_URL = "https://openrouter.ai/api/frontend/v1/catalog/models"; +const RANKINGS_URL = "https://openrouter.ai/api/frontend/v1/rankings/models?view=week"; +const DEFAULT_TTL_MS = 24 * 60 * 60 * 1000; // 24 hours +const FETCH_TIMEOUT_MS = 15_000; + +function getTTL(): number { + const env = process.env.OPENROUTER_PROVIDER_STATS_TTL_MS; + const parsed = env ? parseInt(env, 10) : NaN; + return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_TTL_MS; +} + +function getCacheFilePath(): string { + const dataDir = process.env.DATA_DIR || path.join(process.cwd(), "data"); + const cacheDir = path.join(dataDir, "cache"); + if (!fs.existsSync(cacheDir)) { + fs.mkdirSync(cacheDir, { recursive: true }); + } + return path.join(cacheDir, "openrouter-provider-stats.json"); +} + +// ─── Response shapes (defensive — undocumented API) ───────────────────────── + +const DataPolicySchema = z + .object({ + training: z.boolean().optional(), + retainsPrompts: z.boolean().optional(), + termsOfServiceURL: z.string().optional(), + privacyPolicyURL: z.string().optional(), + }) + .partial() + .optional(); + +const ProviderDirectoryRowSchema = z.object({ + slug: z.string().min(1), + displayName: z.string().min(1).optional(), + name: z.string().min(1).optional(), + headquarters: z.string().optional(), + statusPageUrl: z.string().nullable().optional(), + byokEnabled: z.boolean().optional(), + dataPolicy: DataPolicySchema, + icon: z.object({ url: z.string().optional() }).partial().optional(), +}); + +const CatalogEndpointRowSchema = z.object({ + permaslug: z.string().min(1), + endpoint: z + .object({ + variant: z.string().optional(), + provider_slug: z.string().min(1), + }) + .passthrough(), +}); + +const RankingRowSchema = z.object({ + model_permaslug: z.string().min(1), + variant: z.string().optional(), + total_prompt_tokens: z.number().nonnegative().optional(), + total_completion_tokens: z.number().nonnegative().optional(), + count: z.number().nonnegative().optional(), +}); + +type ProviderDirectoryRow = z.infer; +type CatalogEndpointRow = z.infer; +type RankingRow = z.infer; + +export interface ProviderPopularityEntry { + slug: string; + displayName: string; + headquarters?: string; + statusPageUrl?: string | null; + byokEnabled?: boolean; + dataPolicy?: { + training?: boolean; + retainsPrompts?: boolean; + termsOfServiceURL?: string; + privacyPolicyURL?: string; + }; + iconUrl?: string; + modelCount: number; + totalTokens: number; + totalRequests: number; + popularityRank: number; +} + +interface CacheFile { + fetchedAt: string; + data: ProviderPopularityEntry[]; +} + +/** Parse an array response body with a zod row schema, skipping rows that fail validation. */ +function parseRows(raw: unknown, schema: T): z.infer[] { + if (!Array.isArray(raw)) return []; + const out: z.infer[] = []; + for (const row of raw) { + const result = schema.safeParse(row); + if (result.success) out.push(result.data); + } + return out; +} + +async function fetchJson(url: string): Promise { + const res = await fetch(url, { + headers: { + "User-Agent": "OmniRoute/2.0", + Accept: "application/json", + }, + signal: AbortSignal.timeout(FETCH_TIMEOUT_MS), + }); + if (!res.ok) { + throw new Error(`${url} returned ${res.status}: ${res.statusText}`); + } + return res.json(); +} + +/** + * Join catalog endpoint rows against ranking rows (by permaslug+variant), sum + * usage per provider_slug, then merge in directory metadata and rank by + * total token volume descending. Pure — no I/O — kept separate for testing. + */ +export function computeProviderPopularity( + directoryRows: ProviderDirectoryRow[], + catalogRows: CatalogEndpointRow[], + rankingRows: RankingRow[] +): ProviderPopularityEntry[] { + const usageByKey = new Map(); + for (const row of rankingRows) { + const key = `${row.model_permaslug}::${row.variant ?? "standard"}`; + const tokens = (row.total_prompt_tokens ?? 0) + (row.total_completion_tokens ?? 0); + const existing = usageByKey.get(key); + if (existing) { + existing.tokens += tokens; + existing.requests += row.count ?? 0; + } else { + usageByKey.set(key, { tokens, requests: row.count ?? 0 }); + } + } + + const bySlug = new Map(); + for (const row of catalogRows) { + const key = `${row.permaslug}::${row.endpoint.variant ?? "standard"}`; + const usage = usageByKey.get(key); + const slug = row.endpoint.provider_slug; + const agg = bySlug.get(slug) ?? { modelCount: 0, totalTokens: 0, totalRequests: 0 }; + agg.modelCount += 1; + if (usage) { + agg.totalTokens += usage.tokens; + agg.totalRequests += usage.requests; + } + bySlug.set(slug, agg); + } + + const directoryBySlug = new Map(directoryRows.map((row) => [row.slug, row])); + + const entries: ProviderPopularityEntry[] = Array.from(bySlug.entries()).map(([slug, agg]) => { + const directory = directoryBySlug.get(slug); + return { + slug, + displayName: directory?.displayName || directory?.name || slug, + headquarters: directory?.headquarters, + statusPageUrl: directory?.statusPageUrl ?? undefined, + byokEnabled: directory?.byokEnabled, + dataPolicy: directory?.dataPolicy, + iconUrl: directory?.icon?.url, + modelCount: agg.modelCount, + totalTokens: agg.totalTokens, + totalRequests: agg.totalRequests, + popularityRank: 0, // assigned below + }; + }); + + entries.sort((a, b) => b.totalTokens - a.totalTokens); + entries.forEach((entry, index) => { + entry.popularityRank = index + 1; + }); + + return entries; +} + +async function fetchFromAPI(): Promise { + const [directoryRaw, catalogRaw, rankingRaw] = await Promise.all([ + fetchJson(ALL_PROVIDERS_URL), + fetchJson(CATALOG_MODELS_URL), + fetchJson(RANKINGS_URL), + ]); + + const directoryRows = parseRows( + (directoryRaw as { data?: unknown })?.data, + ProviderDirectoryRowSchema + ); + const catalogRows = parseRows((catalogRaw as { data?: unknown })?.data, CatalogEndpointRowSchema); + const rankingRows = parseRows((rankingRaw as { data?: unknown })?.data, RankingRowSchema); + + return computeProviderPopularity(directoryRows, catalogRows, rankingRows); +} + +function readCache(): CacheFile | null { + const filePath = getCacheFilePath(); + try { + if (!fs.existsSync(filePath)) return null; + return JSON.parse(fs.readFileSync(filePath, "utf8")) as CacheFile; + } catch { + return null; + } +} + +function writeCache(data: ProviderPopularityEntry[]): void { + const filePath = getCacheFilePath(); + const cache: CacheFile = { fetchedAt: new Date().toISOString(), data }; + try { + fs.writeFileSync(filePath, JSON.stringify(cache, null, 2), "utf8"); + } catch (err) { + console.warn("[OpenRouterProviderStats] Failed to write cache:", err); + } +} + +/** Get provider popularity/enrichment stats, honoring the on-disk TTL cache. */ +export async function getOpenRouterProviderStats(): Promise<{ + data: ProviderPopularityEntry[]; + stale: boolean; + cachedAt: string | null; + fromCache: boolean; +}> { + const ttl = getTTL(); + const cache = readCache(); + const now = Date.now(); + + if (cache && cache.fetchedAt) { + const age = now - new Date(cache.fetchedAt).getTime(); + if (age < ttl) { + return { data: cache.data, stale: false, cachedAt: cache.fetchedAt, fromCache: true }; + } + } + + try { + const data = await fetchFromAPI(); + writeCache(data); + return { data, stale: false, cachedAt: null, fromCache: false }; + } catch (err) { + console.warn("[OpenRouterProviderStats] Fetch failed, using stale cache:", err); + if (cache) { + return { data: cache.data, stale: true, cachedAt: cache.fetchedAt, fromCache: true }; + } + return { data: [], stale: true, cachedAt: null, fromCache: false }; + } +} + +/** Force-refresh, ignoring TTL. Used by admin endpoints and manual refresh actions. */ +export async function refreshOpenRouterProviderStats(): Promise<{ + data: ProviderPopularityEntry[]; + ok: boolean; + error?: string; +}> { + try { + const data = await fetchFromAPI(); + writeCache(data); + return { data, ok: true }; + } catch (err) { + const error = err instanceof Error ? err.message : String(err); + return { data: [], ok: false, error }; + } +} + +// ─── Periodic sync (mirrors arenaEloSync.ts's startPeriodicSync shape) ────── + +let syncTimer: ReturnType | null = null; + +function getEffectiveOpenRouterProviderStatsEnabled(): boolean { + return process.env.OPENROUTER_PROVIDER_STATS_ENABLED !== "false"; +} + +function startPeriodicSync(intervalMs?: number): void { + if (syncTimer) return; // Already running + + const interval = intervalMs ?? getTTL(); + console.log(`[OpenRouterProviderStats] Starting periodic sync every ${interval / 1000}s`); + + refreshOpenRouterProviderStats() + .then((result) => { + if (result.ok) { + console.log(`[OpenRouterProviderStats] Initial sync complete: ${result.data.length} providers`); + } else { + console.warn(`[OpenRouterProviderStats] Initial sync failed: ${result.error}`); + } + }) + .catch((err) => { + console.warn( + "[OpenRouterProviderStats] Initial sync error:", + err instanceof Error ? err.message : err + ); + }); + + syncTimer = setInterval(() => { + refreshOpenRouterProviderStats().catch((err) => { + console.warn( + "[OpenRouterProviderStats] Periodic sync error:", + err instanceof Error ? err.message : err + ); + }); + }, interval); + syncTimer.unref(); +} + +/** + * Boot entry point — call once from server-init.ts. + * On by default; opt out via OPENROUTER_PROVIDER_STATS_ENABLED=false. + */ +export function initOpenRouterProviderStatsSync(): boolean { + if (!getEffectiveOpenRouterProviderStatsEnabled()) { + console.log( + "[OpenRouterProviderStats] Disabled via OPENROUTER_PROVIDER_STATS_ENABLED=false." + ); + return false; + } + startPeriodicSync(); + return true; +} diff --git a/tests/unit/openrouter-provider-stats.test.ts b/tests/unit/openrouter-provider-stats.test.ts new file mode 100644 index 0000000000..28667bec9f --- /dev/null +++ b/tests/unit/openrouter-provider-stats.test.ts @@ -0,0 +1,203 @@ +/** + * Unit tests for src/lib/catalog/openrouterProviderStats.ts + * + * Uses Node.js native test runner. All external fetch calls are mocked — + * no network access, no database (this module is file-cache only). + */ + +import { describe, it, beforeEach, afterEach } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { + computeProviderPopularity, + getOpenRouterProviderStats, + refreshOpenRouterProviderStats, +} from "../../src/lib/catalog/openrouterProviderStats.ts"; + +const originalFetch = globalThis.fetch; +const originalDataDir = process.env.DATA_DIR; +const originalTtl = process.env.OPENROUTER_PROVIDER_STATS_TTL_MS; + +function mockFetch(impl: (url: string) => Promise): void { + globalThis.fetch = impl as typeof fetch; +} + +function restoreFetch(): void { + globalThis.fetch = originalFetch; +} + +function jsonResponse(data: unknown, status = 200): Response { + return new Response(JSON.stringify(data), { + status, + headers: { "Content-Type": "application/json" }, + }); +} + +function useTempDataDir(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-or-provider-stats-test-")); + process.env.DATA_DIR = dir; + return dir; +} + +describe("computeProviderPopularity (pure join/aggregation)", () => { + it("sums usage across models for the same provider and ranks by total tokens desc", () => { + const directory = [ + { + slug: "deepinfra", + displayName: "DeepInfra", + headquarters: "US", + dataPolicy: { training: false, retainsPrompts: false }, + }, + { slug: "cerebras", displayName: "Cerebras", headquarters: "US" }, + ]; + const catalog = [ + { permaslug: "deepseek/deepseek-v4-flash", endpoint: { variant: "standard", provider_slug: "deepinfra" } }, + { permaslug: "qwen/qwen3-max", endpoint: { variant: "standard", provider_slug: "deepinfra" } }, + { permaslug: "deepseek/deepseek-v4-flash", endpoint: { variant: "standard", provider_slug: "cerebras" } }, + ]; + const rankings = [ + { model_permaslug: "deepseek/deepseek-v4-flash", variant: "standard", total_prompt_tokens: 100, total_completion_tokens: 20, count: 5 }, + { model_permaslug: "qwen/qwen3-max", variant: "standard", total_prompt_tokens: 900, total_completion_tokens: 100, count: 50 }, + ]; + + const result = computeProviderPopularity(directory, catalog, rankings); + + assert.equal(result.length, 2); + // deepinfra: (100+20) + (900+100) = 1120 tokens across 2 models; cerebras: 120 tokens, 1 model + assert.equal(result[0].slug, "deepinfra"); + assert.equal(result[0].totalTokens, 1120); + assert.equal(result[0].totalRequests, 55); + assert.equal(result[0].modelCount, 2); + assert.equal(result[0].popularityRank, 1); + assert.equal(result[0].displayName, "DeepInfra"); + assert.equal(result[0].dataPolicy?.training, false); + + assert.equal(result[1].slug, "cerebras"); + assert.equal(result[1].totalTokens, 120); + assert.equal(result[1].popularityRank, 2); + }); + + it("counts a model with no matching ranking row (zero usage) without crashing", () => { + const catalog = [ + { permaslug: "unknown/model", endpoint: { variant: "standard", provider_slug: "novita" } }, + ]; + const result = computeProviderPopularity([], catalog, []); + + assert.equal(result.length, 1); + assert.equal(result[0].slug, "novita"); + assert.equal(result[0].modelCount, 1); + assert.equal(result[0].totalTokens, 0); + assert.equal(result[0].totalRequests, 0); + }); + + it("falls back to the provider slug as displayName when no directory entry matches", () => { + const catalog = [ + { permaslug: "m/1", endpoint: { variant: "standard", provider_slug: "some-new-provider" } }, + ]; + const result = computeProviderPopularity([], catalog, []); + + assert.equal(result[0].displayName, "some-new-provider"); + assert.equal(result[0].headquarters, undefined); + }); + + it("returns an empty array for empty inputs", () => { + assert.deepEqual(computeProviderPopularity([], [], []), []); + }); +}); + +describe("getOpenRouterProviderStats / refreshOpenRouterProviderStats (cache + TTL + stale-if-error)", () => { + let tempDir: string; + + beforeEach(() => { + tempDir = useTempDataDir(); + process.env.OPENROUTER_PROVIDER_STATS_TTL_MS = String(24 * 60 * 60 * 1000); + }); + + afterEach(() => { + restoreFetch(); + fs.rmSync(tempDir, { recursive: true, force: true }); + if (originalDataDir === undefined) delete process.env.DATA_DIR; + else process.env.DATA_DIR = originalDataDir; + if (originalTtl === undefined) delete process.env.OPENROUTER_PROVIDER_STATS_TTL_MS; + else process.env.OPENROUTER_PROVIDER_STATS_TTL_MS = originalTtl; + }); + + it("fetches fresh data on first call and writes it to the on-disk cache", async () => { + mockFetch(async (url: string) => { + if (url.includes("all-providers")) { + return jsonResponse({ data: [{ slug: "deepinfra", displayName: "DeepInfra" }] }); + } + if (url.includes("catalog/models")) { + return jsonResponse({ + data: [{ permaslug: "m/1", endpoint: { variant: "standard", provider_slug: "deepinfra" } }], + }); + } + if (url.includes("rankings/models")) { + return jsonResponse({ + data: [{ model_permaslug: "m/1", variant: "standard", total_prompt_tokens: 10, total_completion_tokens: 5, count: 1 }], + }); + } + throw new Error(`Unexpected URL: ${url}`); + }); + + const result = await getOpenRouterProviderStats(); + + assert.equal(result.fromCache, false); + assert.equal(result.stale, false); + assert.equal(result.data.length, 1); + assert.equal(result.data[0].slug, "deepinfra"); + assert.equal(result.data[0].totalTokens, 15); + + const cachePath = path.join(tempDir, "cache", "openrouter-provider-stats.json"); + assert.equal(fs.existsSync(cachePath), true); + }); + + it("serves from cache within the TTL without calling fetch again", async () => { + let fetchCalls = 0; + mockFetch(async (url: string) => { + fetchCalls += 1; + if (url.includes("all-providers")) return jsonResponse({ data: [] }); + if (url.includes("catalog/models")) return jsonResponse({ data: [] }); + return jsonResponse({ data: [] }); + }); + + await getOpenRouterProviderStats(); + const callsAfterFirst = fetchCalls; + const second = await getOpenRouterProviderStats(); + + assert.equal(fetchCalls, callsAfterFirst); // no new fetch calls + assert.equal(second.fromCache, true); + assert.equal(second.stale, false); + }); + + it("falls back to stale cache when the refresh fetch fails", async () => { + mockFetch(async (url: string) => { + if (url.includes("all-providers")) return jsonResponse({ data: [{ slug: "cerebras", displayName: "Cerebras" }] }); + if (url.includes("catalog/models")) return jsonResponse({ data: [] }); + return jsonResponse({ data: [] }); + }); + await getOpenRouterProviderStats(); + + mockFetch(async () => { + throw new Error("network down"); + }); + const result = await refreshOpenRouterProviderStats(); + + assert.equal(result.ok, false); + assert.equal(typeof result.error, "string"); + }); + + it("returns empty data (not a throw) when there is no cache and the fetch fails", async () => { + mockFetch(async () => { + throw new Error("network down"); + }); + + const result = await getOpenRouterProviderStats(); + + assert.equal(result.data.length, 0); + assert.equal(result.stale, true); + }); +});