mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-06 07:12:12 +03:00
feat(providers): enrich dashboard providers list with OpenRouter data (#9324)
Validated in local merge-train (devbox-vm-06-dev002) @ combined-tip (FAST gates green: static + changed tests + vitest — only pre-existing audit.test.ts flake). Evidence: /home/diegosouzapw/dev/proxys/OmniRoute/.claude/worktrees/merge-train-20260805-213228-suite.log
This commit is contained in:
committed by
GitHub
parent
93711ec619
commit
c1986ef4b9
@@ -17,6 +17,7 @@ import {
|
||||
|
||||
import { CategoryDot } from "./CategoryDot";
|
||||
import { isCheaperInferenceProviderId, isKimiPartnerProviderId } from "../featuredProviders";
|
||||
import { useOpenRouterProviderStat } from "../context/openRouterProviderStatsContext";
|
||||
|
||||
interface ProviderStats {
|
||||
total?: number;
|
||||
@@ -228,6 +229,7 @@ const ProviderCard = forwardRef<ProviderCardHandle, ProviderCardProps>(function
|
||||
const isKimiPartner = isKimiPartnerProviderId(provider.id || providerId);
|
||||
const isCheaperInferencePartner = isCheaperInferenceProviderId(provider.id || providerId);
|
||||
const isSponsorPartner = isKimiPartner || isCheaperInferencePartner;
|
||||
const openRouterStat = useOpenRouterProviderStat(provider.id || providerId);
|
||||
const codexServiceTierLabel =
|
||||
stats.codexServiceTier === "flex"
|
||||
? providerText(t, "codexTierFlexLabel", "Flex")
|
||||
@@ -287,6 +289,36 @@ const ProviderCard = forwardRef<ProviderCardHandle, ProviderCardProps>(function
|
||||
</span>
|
||||
) : null;
|
||||
|
||||
const openRouterTooltipBits: string[] = [];
|
||||
if (openRouterStat?.headquarters) openRouterTooltipBits.push(`HQ: ${openRouterStat.headquarters}`);
|
||||
if (openRouterStat?.dataPolicy?.training === false) {
|
||||
openRouterTooltipBits.push(providerText(t, "openRouterNoTraining", "Does not train on prompts"));
|
||||
}
|
||||
if (openRouterStat?.dataPolicy?.retainsPrompts === false) {
|
||||
openRouterTooltipBits.push(providerText(t, "openRouterNoRetention", "Does not retain prompts"));
|
||||
}
|
||||
const openRouterTooltip = openRouterStat
|
||||
? providerText(t, "openRouterPopularityTooltip", "OpenRouter usage rank #{rank}", {
|
||||
rank: openRouterStat.popularityRank,
|
||||
}) + (openRouterTooltipBits.length ? ` — ${openRouterTooltipBits.join(" · ")}` : "")
|
||||
: "";
|
||||
|
||||
// OpenRouter popularity badge — data refreshed daily from OpenRouter's
|
||||
// provider directory + usage rankings (see src/lib/catalog/openrouterProviderStats.ts).
|
||||
// Absent entirely for providers OpenRouter doesn't track; never affects routing.
|
||||
const openRouterPopularityChip = openRouterStat ? (
|
||||
<span
|
||||
key="openrouter-popularity"
|
||||
className="inline-flex items-center gap-0.5 rounded-full border border-border bg-bg-subtle px-1.5 py-0 text-[9px] font-semibold leading-none text-text-muted"
|
||||
title={openRouterTooltip}
|
||||
>
|
||||
<span className="material-symbols-outlined text-[10px] leading-none">trending_up</span>
|
||||
{providerText(t, "openRouterPopularityBadge", "OR #{rank}", {
|
||||
rank: openRouterStat.popularityRank,
|
||||
})}
|
||||
</span>
|
||||
) : null;
|
||||
|
||||
const dotLabels: Record<string, string> = {
|
||||
free: tc("free"),
|
||||
"no-auth": t("noAuthLabel"),
|
||||
@@ -417,10 +449,12 @@ const ProviderCard = forwardRef<ProviderCardHandle, ProviderCardProps>(function
|
||||
isCompatible ||
|
||||
isCcCompatible ||
|
||||
isAnthropicCompatible ||
|
||||
isSponsorPartner) && (
|
||||
isSponsorPartner ||
|
||||
Boolean(openRouterStat)) && (
|
||||
<div className="flex flex-wrap items-center gap-1">
|
||||
{kimiOfficialSupporterChip}
|
||||
{cheaperInferenceSupporterChip}
|
||||
{openRouterPopularityChip}
|
||||
{provider.serviceKinds?.map((k) => (
|
||||
<span
|
||||
key={k}
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
"use client";
|
||||
|
||||
import { createContext, useContext, useMemo, type ReactNode } from "react";
|
||||
import type { OpenRouterProviderStatsEntry } from "../providerPageUtils";
|
||||
|
||||
/**
|
||||
* Makes OpenRouter provider popularity/identity enrichment available to every
|
||||
* ProviderCard without prop-drilling it through the ~15 render sites in
|
||||
* page.tsx (one per auth-type section). Looked up by provider slug — cards
|
||||
* for providers OpenRouter doesn't know about simply get `undefined`.
|
||||
*/
|
||||
const EMPTY_STATS_MAP: ReadonlyMap<string, OpenRouterProviderStatsEntry> = new Map();
|
||||
const Context =
|
||||
createContext<ReadonlyMap<string, OpenRouterProviderStatsEntry>>(EMPTY_STATS_MAP);
|
||||
|
||||
export function OpenRouterProviderStatsProvider({
|
||||
entries,
|
||||
children,
|
||||
}: {
|
||||
entries: OpenRouterProviderStatsEntry[];
|
||||
children: ReactNode;
|
||||
}) {
|
||||
const bySlug = useMemo(() => new Map(entries.map((entry) => [entry.slug, entry])), [entries]);
|
||||
return <Context.Provider value={bySlug}>{children}</Context.Provider>;
|
||||
}
|
||||
|
||||
export function useOpenRouterProviderStat(
|
||||
providerId: string | undefined
|
||||
): OpenRouterProviderStatsEntry | undefined {
|
||||
const bySlug = useContext(Context);
|
||||
return providerId ? bySlug.get(providerId) : undefined;
|
||||
}
|
||||
@@ -34,7 +34,8 @@ import {
|
||||
upsertProviderNodeById,
|
||||
loadProviderPageData,
|
||||
} from "./providerPageUtils";
|
||||
import type { ProviderEntry } from "./providerPageUtils";
|
||||
import type { ProviderEntry, OpenRouterProviderStatsEntry } from "./providerPageUtils";
|
||||
import { OpenRouterProviderStatsProvider } from "./context/openRouterProviderStatsContext";
|
||||
import {
|
||||
shouldSyncProviderDisplayMode,
|
||||
writeProviderDisplayModePreference,
|
||||
@@ -200,6 +201,9 @@ export default function ProvidersPage() {
|
||||
const [modelSearchQuery, setModelSearchQuery] = useState("");
|
||||
const liveModelsByProviderId = useSyncedModelsByProvider();
|
||||
const [showFreeOnly, setShowFreeOnly] = useState(false);
|
||||
const [openRouterProviderStats, setOpenRouterProviderStats] = useState<
|
||||
OpenRouterProviderStatsEntry[]
|
||||
>([]);
|
||||
const [activeCategory, setActiveCategory] = useState<string | null>(null);
|
||||
// #4240: media-category (serviceKind) filter — composes with activeCategory,
|
||||
// search and configured-only. null = no serviceKind filter.
|
||||
@@ -255,6 +259,7 @@ export default function ProvidersPage() {
|
||||
if (data.expirations) setExpirations(data.expirations);
|
||||
if (data.blockedProviders) setBlockedProviders(data.blockedProviders);
|
||||
setCodexGlobalServiceMode(getCodexGlobalServiceMode(data.settings));
|
||||
setOpenRouterProviderStats(data.openRouterProviderStats);
|
||||
} catch (error) {
|
||||
console.log("Error fetching data:", error);
|
||||
} finally {
|
||||
@@ -812,6 +817,7 @@ export default function ProvidersPage() {
|
||||
shouldShowFirstProviderHint(connections.length, searchQuery) && !showAllProviders;
|
||||
|
||||
return (
|
||||
<OpenRouterProviderStatsProvider entries={openRouterProviderStats}>
|
||||
<div className="flex flex-col gap-6">
|
||||
{showFirstProviderHint && (
|
||||
<Card padding="lg">
|
||||
@@ -1814,6 +1820,7 @@ export default function ProvidersPage() {
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</OpenRouterProviderStatsProvider>
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -550,6 +550,28 @@ export interface ProviderPageData {
|
||||
expirations: any | null;
|
||||
blockedProviders: string[] | null;
|
||||
settings: any | null;
|
||||
/** OpenRouter-sourced popularity/identity enrichment, keyed by provider slug. Empty if the sync hasn't run yet or the fetch failed. */
|
||||
openRouterProviderStats: OpenRouterProviderStatsEntry[];
|
||||
}
|
||||
|
||||
/** Mirrors ProviderPopularityEntry from src/lib/catalog/openrouterProviderStats.ts (kept local to avoid a server-only import from a client component). */
|
||||
export interface OpenRouterProviderStatsEntry {
|
||||
slug: string;
|
||||
displayName: string;
|
||||
headquarters?: string;
|
||||
statusPageUrl?: string | null;
|
||||
byokEnabled?: boolean;
|
||||
dataPolicy?: {
|
||||
training?: boolean;
|
||||
retainsPrompts?: boolean;
|
||||
termsOfServiceURL?: string;
|
||||
privacyPolicyURL?: string;
|
||||
};
|
||||
iconUrl?: string;
|
||||
modelCount: number;
|
||||
totalTokens: number;
|
||||
totalRequests: number;
|
||||
popularityRank: number;
|
||||
}
|
||||
|
||||
// Bound each first-paint request so a single stalled connection cannot freeze
|
||||
@@ -587,12 +609,14 @@ export async function loadProviderPageData(
|
||||
}
|
||||
};
|
||||
|
||||
const [connectionsData, nodesData, expirationsData, settingsData] = await Promise.all([
|
||||
safeJson("/api/providers"),
|
||||
safeJson("/api/provider-nodes"),
|
||||
safeJson("/api/providers/expiration"),
|
||||
safeJson("/api/settings", { cache: "no-store" }),
|
||||
]);
|
||||
const [connectionsData, nodesData, expirationsData, settingsData, openRouterStatsData] =
|
||||
await Promise.all([
|
||||
safeJson("/api/providers"),
|
||||
safeJson("/api/provider-nodes"),
|
||||
safeJson("/api/providers/expiration"),
|
||||
safeJson("/api/settings", { cache: "no-store" }),
|
||||
safeJson("/api/providers/openrouter-stats"),
|
||||
]);
|
||||
|
||||
return {
|
||||
connections: Array.isArray(connectionsData?.connections) ? connectionsData.connections : [],
|
||||
@@ -603,5 +627,6 @@ export async function loadProviderPageData(
|
||||
? settingsData.blockedProviders
|
||||
: null,
|
||||
settings: settingsData ?? null,
|
||||
openRouterProviderStats: Array.isArray(openRouterStatsData?.data) ? openRouterStatsData.data : [],
|
||||
};
|
||||
}
|
||||
|
||||
51
src/app/api/providers/openrouter-stats/route.ts
Normal file
51
src/app/api/providers/openrouter-stats/route.ts
Normal file
@@ -0,0 +1,51 @@
|
||||
/**
|
||||
* GET /api/providers/openrouter-stats
|
||||
* Returns OpenRouter-sourced provider enrichment (popularity rank, HQ, data
|
||||
* policy, ToS/privacy links) with persistent cache — see openrouterProviderStats.ts.
|
||||
*
|
||||
* Query params:
|
||||
* ?refresh=true — Force-refresh, ignores TTL
|
||||
*/
|
||||
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { isAuthenticated } from "@/shared/utils/apiAuth";
|
||||
import {
|
||||
getOpenRouterProviderStats,
|
||||
refreshOpenRouterProviderStats,
|
||||
} from "@/lib/catalog/openrouterProviderStats";
|
||||
|
||||
export async function GET(req: NextRequest) {
|
||||
if (!(await isAuthenticated(req))) {
|
||||
return NextResponse.json(
|
||||
{ error: { message: "Authentication required", type: "invalid_request_error" } },
|
||||
{ status: 401 }
|
||||
);
|
||||
}
|
||||
|
||||
const forceRefresh = req.nextUrl.searchParams.get("refresh") === "true";
|
||||
|
||||
if (forceRefresh) {
|
||||
const result = await refreshOpenRouterProviderStats();
|
||||
return NextResponse.json({
|
||||
object: "list",
|
||||
data: result.data,
|
||||
meta: {
|
||||
source: result.ok ? "fresh" : "error",
|
||||
count: result.data.length,
|
||||
error: result.error ?? undefined,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const result = await getOpenRouterProviderStats();
|
||||
return NextResponse.json({
|
||||
object: "list",
|
||||
data: result.data,
|
||||
meta: {
|
||||
source: result.fromCache ? (result.stale ? "stale-cache" : "cache") : "fresh",
|
||||
cachedAt: result.cachedAt ?? undefined,
|
||||
stale: result.stale,
|
||||
count: result.data.length,
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -552,6 +552,19 @@ export async function registerNodejs(): Promise<void> {
|
||||
console.warn("[STARTUP] Pricing sync failed to start (non-fatal):", msg);
|
||||
}),
|
||||
|
||||
// OpenRouter provider stats sync: provider directory + popularity enrichment
|
||||
// for the dashboard Providers page. On by default; opt out with
|
||||
// OPENROUTER_PROVIDER_STATS_ENABLED=false. Non-blocking, never fatal.
|
||||
import("@/lib/catalog/openrouterProviderStats")
|
||||
.then((m) => {
|
||||
const started = m.initOpenRouterProviderStatsSync();
|
||||
if (started) console.log("[STARTUP] OpenRouter provider stats sync initialized");
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
console.warn("[STARTUP] OpenRouter provider stats sync failed to start (non-fatal):", msg);
|
||||
}),
|
||||
|
||||
// models.dev capability sync: opt-in via Settings > AI (self-gated by
|
||||
// settings.modelsDevSyncEnabled inside initModelsDevSync). Non-blocking, never fatal.
|
||||
import("@/lib/modelsDevSync")
|
||||
|
||||
342
src/lib/catalog/openrouterProviderStats.ts
Normal file
342
src/lib/catalog/openrouterProviderStats.ts
Normal file
@@ -0,0 +1,342 @@
|
||||
/**
|
||||
* openrouterProviderStats.ts
|
||||
*
|
||||
* Enriches OmniRoute's provider directory with data scraped from OpenRouter's
|
||||
* *internal* frontend API (openrouter.ai/api/frontend/v1/*) — undocumented and
|
||||
* unversioned, unlike the public /api/v1/models consumed by openrouterCatalog.ts.
|
||||
* Three bulk endpoints, one request each, refreshed once/day — never per-model:
|
||||
*
|
||||
* - all-providers → provider directory (HQ, ToS/privacy, data policy, status page)
|
||||
* - catalog/models → model→provider endpoint mapping (for attribution)
|
||||
* - rankings/models → usage volume (tokens/requests) per model+variant
|
||||
*
|
||||
* catalog/models rows are joined against rankings/models by `permaslug+variant`
|
||||
* and the resulting usage is summed per `provider_slug` — that sum is the
|
||||
* popularity signal. All three responses can drift or disappear without notice
|
||||
* (nothing here is a documented contract), so every row is parsed defensively
|
||||
* with zod `.safeParse()` and skipped on failure rather than failing the batch,
|
||||
* and the whole refresh falls back to the last good cache on any fetch error
|
||||
* (same stale-if-error shape as openrouterCatalog.ts / arenaEloSync.ts).
|
||||
*/
|
||||
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { z } from "zod";
|
||||
|
||||
const ALL_PROVIDERS_URL = "https://openrouter.ai/api/frontend/v1/all-providers";
|
||||
const CATALOG_MODELS_URL = "https://openrouter.ai/api/frontend/v1/catalog/models";
|
||||
const RANKINGS_URL = "https://openrouter.ai/api/frontend/v1/rankings/models?view=week";
|
||||
const DEFAULT_TTL_MS = 24 * 60 * 60 * 1000; // 24 hours
|
||||
const FETCH_TIMEOUT_MS = 15_000;
|
||||
|
||||
function getTTL(): number {
|
||||
const env = process.env.OPENROUTER_PROVIDER_STATS_TTL_MS;
|
||||
const parsed = env ? parseInt(env, 10) : NaN;
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_TTL_MS;
|
||||
}
|
||||
|
||||
function getCacheFilePath(): string {
|
||||
const dataDir = process.env.DATA_DIR || path.join(process.cwd(), "data");
|
||||
const cacheDir = path.join(dataDir, "cache");
|
||||
if (!fs.existsSync(cacheDir)) {
|
||||
fs.mkdirSync(cacheDir, { recursive: true });
|
||||
}
|
||||
return path.join(cacheDir, "openrouter-provider-stats.json");
|
||||
}
|
||||
|
||||
// ─── Response shapes (defensive — undocumented API) ─────────────────────────
|
||||
|
||||
const DataPolicySchema = z
|
||||
.object({
|
||||
training: z.boolean().optional(),
|
||||
retainsPrompts: z.boolean().optional(),
|
||||
termsOfServiceURL: z.string().optional(),
|
||||
privacyPolicyURL: z.string().optional(),
|
||||
})
|
||||
.partial()
|
||||
.optional();
|
||||
|
||||
const ProviderDirectoryRowSchema = z.object({
|
||||
slug: z.string().min(1),
|
||||
displayName: z.string().min(1).optional(),
|
||||
name: z.string().min(1).optional(),
|
||||
headquarters: z.string().optional(),
|
||||
statusPageUrl: z.string().nullable().optional(),
|
||||
byokEnabled: z.boolean().optional(),
|
||||
dataPolicy: DataPolicySchema,
|
||||
icon: z.object({ url: z.string().optional() }).partial().optional(),
|
||||
});
|
||||
|
||||
const CatalogEndpointRowSchema = z.object({
|
||||
permaslug: z.string().min(1),
|
||||
endpoint: z
|
||||
.object({
|
||||
variant: z.string().optional(),
|
||||
provider_slug: z.string().min(1),
|
||||
})
|
||||
.passthrough(),
|
||||
});
|
||||
|
||||
const RankingRowSchema = z.object({
|
||||
model_permaslug: z.string().min(1),
|
||||
variant: z.string().optional(),
|
||||
total_prompt_tokens: z.number().nonnegative().optional(),
|
||||
total_completion_tokens: z.number().nonnegative().optional(),
|
||||
count: z.number().nonnegative().optional(),
|
||||
});
|
||||
|
||||
type ProviderDirectoryRow = z.infer<typeof ProviderDirectoryRowSchema>;
|
||||
type CatalogEndpointRow = z.infer<typeof CatalogEndpointRowSchema>;
|
||||
type RankingRow = z.infer<typeof RankingRowSchema>;
|
||||
|
||||
export interface ProviderPopularityEntry {
|
||||
slug: string;
|
||||
displayName: string;
|
||||
headquarters?: string;
|
||||
statusPageUrl?: string | null;
|
||||
byokEnabled?: boolean;
|
||||
dataPolicy?: {
|
||||
training?: boolean;
|
||||
retainsPrompts?: boolean;
|
||||
termsOfServiceURL?: string;
|
||||
privacyPolicyURL?: string;
|
||||
};
|
||||
iconUrl?: string;
|
||||
modelCount: number;
|
||||
totalTokens: number;
|
||||
totalRequests: number;
|
||||
popularityRank: number;
|
||||
}
|
||||
|
||||
interface CacheFile {
|
||||
fetchedAt: string;
|
||||
data: ProviderPopularityEntry[];
|
||||
}
|
||||
|
||||
/** Parse an array response body with a zod row schema, skipping rows that fail validation. */
|
||||
function parseRows<T extends z.ZodTypeAny>(raw: unknown, schema: T): z.infer<T>[] {
|
||||
if (!Array.isArray(raw)) return [];
|
||||
const out: z.infer<T>[] = [];
|
||||
for (const row of raw) {
|
||||
const result = schema.safeParse(row);
|
||||
if (result.success) out.push(result.data);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async function fetchJson(url: string): Promise<unknown> {
|
||||
const res = await fetch(url, {
|
||||
headers: {
|
||||
"User-Agent": "OmniRoute/2.0",
|
||||
Accept: "application/json",
|
||||
},
|
||||
signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
|
||||
});
|
||||
if (!res.ok) {
|
||||
throw new Error(`${url} returned ${res.status}: ${res.statusText}`);
|
||||
}
|
||||
return res.json();
|
||||
}
|
||||
|
||||
/**
|
||||
* Join catalog endpoint rows against ranking rows (by permaslug+variant), sum
|
||||
* usage per provider_slug, then merge in directory metadata and rank by
|
||||
* total token volume descending. Pure — no I/O — kept separate for testing.
|
||||
*/
|
||||
export function computeProviderPopularity(
|
||||
directoryRows: ProviderDirectoryRow[],
|
||||
catalogRows: CatalogEndpointRow[],
|
||||
rankingRows: RankingRow[]
|
||||
): ProviderPopularityEntry[] {
|
||||
const usageByKey = new Map<string, { tokens: number; requests: number }>();
|
||||
for (const row of rankingRows) {
|
||||
const key = `${row.model_permaslug}::${row.variant ?? "standard"}`;
|
||||
const tokens = (row.total_prompt_tokens ?? 0) + (row.total_completion_tokens ?? 0);
|
||||
const existing = usageByKey.get(key);
|
||||
if (existing) {
|
||||
existing.tokens += tokens;
|
||||
existing.requests += row.count ?? 0;
|
||||
} else {
|
||||
usageByKey.set(key, { tokens, requests: row.count ?? 0 });
|
||||
}
|
||||
}
|
||||
|
||||
const bySlug = new Map<string, { modelCount: number; totalTokens: number; totalRequests: number }>();
|
||||
for (const row of catalogRows) {
|
||||
const key = `${row.permaslug}::${row.endpoint.variant ?? "standard"}`;
|
||||
const usage = usageByKey.get(key);
|
||||
const slug = row.endpoint.provider_slug;
|
||||
const agg = bySlug.get(slug) ?? { modelCount: 0, totalTokens: 0, totalRequests: 0 };
|
||||
agg.modelCount += 1;
|
||||
if (usage) {
|
||||
agg.totalTokens += usage.tokens;
|
||||
agg.totalRequests += usage.requests;
|
||||
}
|
||||
bySlug.set(slug, agg);
|
||||
}
|
||||
|
||||
const directoryBySlug = new Map(directoryRows.map((row) => [row.slug, row]));
|
||||
|
||||
const entries: ProviderPopularityEntry[] = Array.from(bySlug.entries()).map(([slug, agg]) => {
|
||||
const directory = directoryBySlug.get(slug);
|
||||
return {
|
||||
slug,
|
||||
displayName: directory?.displayName || directory?.name || slug,
|
||||
headquarters: directory?.headquarters,
|
||||
statusPageUrl: directory?.statusPageUrl ?? undefined,
|
||||
byokEnabled: directory?.byokEnabled,
|
||||
dataPolicy: directory?.dataPolicy,
|
||||
iconUrl: directory?.icon?.url,
|
||||
modelCount: agg.modelCount,
|
||||
totalTokens: agg.totalTokens,
|
||||
totalRequests: agg.totalRequests,
|
||||
popularityRank: 0, // assigned below
|
||||
};
|
||||
});
|
||||
|
||||
entries.sort((a, b) => b.totalTokens - a.totalTokens);
|
||||
entries.forEach((entry, index) => {
|
||||
entry.popularityRank = index + 1;
|
||||
});
|
||||
|
||||
return entries;
|
||||
}
|
||||
|
||||
async function fetchFromAPI(): Promise<ProviderPopularityEntry[]> {
|
||||
const [directoryRaw, catalogRaw, rankingRaw] = await Promise.all([
|
||||
fetchJson(ALL_PROVIDERS_URL),
|
||||
fetchJson(CATALOG_MODELS_URL),
|
||||
fetchJson(RANKINGS_URL),
|
||||
]);
|
||||
|
||||
const directoryRows = parseRows(
|
||||
(directoryRaw as { data?: unknown })?.data,
|
||||
ProviderDirectoryRowSchema
|
||||
);
|
||||
const catalogRows = parseRows((catalogRaw as { data?: unknown })?.data, CatalogEndpointRowSchema);
|
||||
const rankingRows = parseRows((rankingRaw as { data?: unknown })?.data, RankingRowSchema);
|
||||
|
||||
return computeProviderPopularity(directoryRows, catalogRows, rankingRows);
|
||||
}
|
||||
|
||||
function readCache(): CacheFile | null {
|
||||
const filePath = getCacheFilePath();
|
||||
try {
|
||||
if (!fs.existsSync(filePath)) return null;
|
||||
return JSON.parse(fs.readFileSync(filePath, "utf8")) as CacheFile;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function writeCache(data: ProviderPopularityEntry[]): void {
|
||||
const filePath = getCacheFilePath();
|
||||
const cache: CacheFile = { fetchedAt: new Date().toISOString(), data };
|
||||
try {
|
||||
fs.writeFileSync(filePath, JSON.stringify(cache, null, 2), "utf8");
|
||||
} catch (err) {
|
||||
console.warn("[OpenRouterProviderStats] Failed to write cache:", err);
|
||||
}
|
||||
}
|
||||
|
||||
/** Get provider popularity/enrichment stats, honoring the on-disk TTL cache. */
|
||||
export async function getOpenRouterProviderStats(): Promise<{
|
||||
data: ProviderPopularityEntry[];
|
||||
stale: boolean;
|
||||
cachedAt: string | null;
|
||||
fromCache: boolean;
|
||||
}> {
|
||||
const ttl = getTTL();
|
||||
const cache = readCache();
|
||||
const now = Date.now();
|
||||
|
||||
if (cache && cache.fetchedAt) {
|
||||
const age = now - new Date(cache.fetchedAt).getTime();
|
||||
if (age < ttl) {
|
||||
return { data: cache.data, stale: false, cachedAt: cache.fetchedAt, fromCache: true };
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const data = await fetchFromAPI();
|
||||
writeCache(data);
|
||||
return { data, stale: false, cachedAt: null, fromCache: false };
|
||||
} catch (err) {
|
||||
console.warn("[OpenRouterProviderStats] Fetch failed, using stale cache:", err);
|
||||
if (cache) {
|
||||
return { data: cache.data, stale: true, cachedAt: cache.fetchedAt, fromCache: true };
|
||||
}
|
||||
return { data: [], stale: true, cachedAt: null, fromCache: false };
|
||||
}
|
||||
}
|
||||
|
||||
/** Force-refresh, ignoring TTL. Used by admin endpoints and manual refresh actions. */
|
||||
export async function refreshOpenRouterProviderStats(): Promise<{
|
||||
data: ProviderPopularityEntry[];
|
||||
ok: boolean;
|
||||
error?: string;
|
||||
}> {
|
||||
try {
|
||||
const data = await fetchFromAPI();
|
||||
writeCache(data);
|
||||
return { data, ok: true };
|
||||
} catch (err) {
|
||||
const error = err instanceof Error ? err.message : String(err);
|
||||
return { data: [], ok: false, error };
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Periodic sync (mirrors arenaEloSync.ts's startPeriodicSync shape) ──────
|
||||
|
||||
let syncTimer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
function getEffectiveOpenRouterProviderStatsEnabled(): boolean {
|
||||
return process.env.OPENROUTER_PROVIDER_STATS_ENABLED !== "false";
|
||||
}
|
||||
|
||||
function startPeriodicSync(intervalMs?: number): void {
|
||||
if (syncTimer) return; // Already running
|
||||
|
||||
const interval = intervalMs ?? getTTL();
|
||||
console.log(`[OpenRouterProviderStats] Starting periodic sync every ${interval / 1000}s`);
|
||||
|
||||
refreshOpenRouterProviderStats()
|
||||
.then((result) => {
|
||||
if (result.ok) {
|
||||
console.log(`[OpenRouterProviderStats] Initial sync complete: ${result.data.length} providers`);
|
||||
} else {
|
||||
console.warn(`[OpenRouterProviderStats] Initial sync failed: ${result.error}`);
|
||||
}
|
||||
})
|
||||
.catch((err) => {
|
||||
console.warn(
|
||||
"[OpenRouterProviderStats] Initial sync error:",
|
||||
err instanceof Error ? err.message : err
|
||||
);
|
||||
});
|
||||
|
||||
syncTimer = setInterval(() => {
|
||||
refreshOpenRouterProviderStats().catch((err) => {
|
||||
console.warn(
|
||||
"[OpenRouterProviderStats] Periodic sync error:",
|
||||
err instanceof Error ? err.message : err
|
||||
);
|
||||
});
|
||||
}, interval);
|
||||
syncTimer.unref();
|
||||
}
|
||||
|
||||
/**
|
||||
* Boot entry point — call once from server-init.ts.
|
||||
* On by default; opt out via OPENROUTER_PROVIDER_STATS_ENABLED=false.
|
||||
*/
|
||||
export function initOpenRouterProviderStatsSync(): boolean {
|
||||
if (!getEffectiveOpenRouterProviderStatsEnabled()) {
|
||||
console.log(
|
||||
"[OpenRouterProviderStats] Disabled via OPENROUTER_PROVIDER_STATS_ENABLED=false."
|
||||
);
|
||||
return false;
|
||||
}
|
||||
startPeriodicSync();
|
||||
return true;
|
||||
}
|
||||
203
tests/unit/openrouter-provider-stats.test.ts
Normal file
203
tests/unit/openrouter-provider-stats.test.ts
Normal file
@@ -0,0 +1,203 @@
|
||||
/**
|
||||
* Unit tests for src/lib/catalog/openrouterProviderStats.ts
|
||||
*
|
||||
* Uses Node.js native test runner. All external fetch calls are mocked —
|
||||
* no network access, no database (this module is file-cache only).
|
||||
*/
|
||||
|
||||
import { describe, it, beforeEach, afterEach } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
import {
|
||||
computeProviderPopularity,
|
||||
getOpenRouterProviderStats,
|
||||
refreshOpenRouterProviderStats,
|
||||
} from "../../src/lib/catalog/openrouterProviderStats.ts";
|
||||
|
||||
const originalFetch = globalThis.fetch;
|
||||
const originalDataDir = process.env.DATA_DIR;
|
||||
const originalTtl = process.env.OPENROUTER_PROVIDER_STATS_TTL_MS;
|
||||
|
||||
function mockFetch(impl: (url: string) => Promise<Response>): void {
|
||||
globalThis.fetch = impl as typeof fetch;
|
||||
}
|
||||
|
||||
function restoreFetch(): void {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
|
||||
function jsonResponse(data: unknown, status = 200): Response {
|
||||
return new Response(JSON.stringify(data), {
|
||||
status,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
}
|
||||
|
||||
function useTempDataDir(): string {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-or-provider-stats-test-"));
|
||||
process.env.DATA_DIR = dir;
|
||||
return dir;
|
||||
}
|
||||
|
||||
describe("computeProviderPopularity (pure join/aggregation)", () => {
|
||||
it("sums usage across models for the same provider and ranks by total tokens desc", () => {
|
||||
const directory = [
|
||||
{
|
||||
slug: "deepinfra",
|
||||
displayName: "DeepInfra",
|
||||
headquarters: "US",
|
||||
dataPolicy: { training: false, retainsPrompts: false },
|
||||
},
|
||||
{ slug: "cerebras", displayName: "Cerebras", headquarters: "US" },
|
||||
];
|
||||
const catalog = [
|
||||
{ permaslug: "deepseek/deepseek-v4-flash", endpoint: { variant: "standard", provider_slug: "deepinfra" } },
|
||||
{ permaslug: "qwen/qwen3-max", endpoint: { variant: "standard", provider_slug: "deepinfra" } },
|
||||
{ permaslug: "deepseek/deepseek-v4-flash", endpoint: { variant: "standard", provider_slug: "cerebras" } },
|
||||
];
|
||||
const rankings = [
|
||||
{ model_permaslug: "deepseek/deepseek-v4-flash", variant: "standard", total_prompt_tokens: 100, total_completion_tokens: 20, count: 5 },
|
||||
{ model_permaslug: "qwen/qwen3-max", variant: "standard", total_prompt_tokens: 900, total_completion_tokens: 100, count: 50 },
|
||||
];
|
||||
|
||||
const result = computeProviderPopularity(directory, catalog, rankings);
|
||||
|
||||
assert.equal(result.length, 2);
|
||||
// deepinfra: (100+20) + (900+100) = 1120 tokens across 2 models; cerebras: 120 tokens, 1 model
|
||||
assert.equal(result[0].slug, "deepinfra");
|
||||
assert.equal(result[0].totalTokens, 1120);
|
||||
assert.equal(result[0].totalRequests, 55);
|
||||
assert.equal(result[0].modelCount, 2);
|
||||
assert.equal(result[0].popularityRank, 1);
|
||||
assert.equal(result[0].displayName, "DeepInfra");
|
||||
assert.equal(result[0].dataPolicy?.training, false);
|
||||
|
||||
assert.equal(result[1].slug, "cerebras");
|
||||
assert.equal(result[1].totalTokens, 120);
|
||||
assert.equal(result[1].popularityRank, 2);
|
||||
});
|
||||
|
||||
it("counts a model with no matching ranking row (zero usage) without crashing", () => {
|
||||
const catalog = [
|
||||
{ permaslug: "unknown/model", endpoint: { variant: "standard", provider_slug: "novita" } },
|
||||
];
|
||||
const result = computeProviderPopularity([], catalog, []);
|
||||
|
||||
assert.equal(result.length, 1);
|
||||
assert.equal(result[0].slug, "novita");
|
||||
assert.equal(result[0].modelCount, 1);
|
||||
assert.equal(result[0].totalTokens, 0);
|
||||
assert.equal(result[0].totalRequests, 0);
|
||||
});
|
||||
|
||||
it("falls back to the provider slug as displayName when no directory entry matches", () => {
|
||||
const catalog = [
|
||||
{ permaslug: "m/1", endpoint: { variant: "standard", provider_slug: "some-new-provider" } },
|
||||
];
|
||||
const result = computeProviderPopularity([], catalog, []);
|
||||
|
||||
assert.equal(result[0].displayName, "some-new-provider");
|
||||
assert.equal(result[0].headquarters, undefined);
|
||||
});
|
||||
|
||||
it("returns an empty array for empty inputs", () => {
|
||||
assert.deepEqual(computeProviderPopularity([], [], []), []);
|
||||
});
|
||||
});
|
||||
|
||||
describe("getOpenRouterProviderStats / refreshOpenRouterProviderStats (cache + TTL + stale-if-error)", () => {
|
||||
let tempDir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
tempDir = useTempDataDir();
|
||||
process.env.OPENROUTER_PROVIDER_STATS_TTL_MS = String(24 * 60 * 60 * 1000);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
restoreFetch();
|
||||
fs.rmSync(tempDir, { recursive: true, force: true });
|
||||
if (originalDataDir === undefined) delete process.env.DATA_DIR;
|
||||
else process.env.DATA_DIR = originalDataDir;
|
||||
if (originalTtl === undefined) delete process.env.OPENROUTER_PROVIDER_STATS_TTL_MS;
|
||||
else process.env.OPENROUTER_PROVIDER_STATS_TTL_MS = originalTtl;
|
||||
});
|
||||
|
||||
it("fetches fresh data on first call and writes it to the on-disk cache", async () => {
|
||||
mockFetch(async (url: string) => {
|
||||
if (url.includes("all-providers")) {
|
||||
return jsonResponse({ data: [{ slug: "deepinfra", displayName: "DeepInfra" }] });
|
||||
}
|
||||
if (url.includes("catalog/models")) {
|
||||
return jsonResponse({
|
||||
data: [{ permaslug: "m/1", endpoint: { variant: "standard", provider_slug: "deepinfra" } }],
|
||||
});
|
||||
}
|
||||
if (url.includes("rankings/models")) {
|
||||
return jsonResponse({
|
||||
data: [{ model_permaslug: "m/1", variant: "standard", total_prompt_tokens: 10, total_completion_tokens: 5, count: 1 }],
|
||||
});
|
||||
}
|
||||
throw new Error(`Unexpected URL: ${url}`);
|
||||
});
|
||||
|
||||
const result = await getOpenRouterProviderStats();
|
||||
|
||||
assert.equal(result.fromCache, false);
|
||||
assert.equal(result.stale, false);
|
||||
assert.equal(result.data.length, 1);
|
||||
assert.equal(result.data[0].slug, "deepinfra");
|
||||
assert.equal(result.data[0].totalTokens, 15);
|
||||
|
||||
const cachePath = path.join(tempDir, "cache", "openrouter-provider-stats.json");
|
||||
assert.equal(fs.existsSync(cachePath), true);
|
||||
});
|
||||
|
||||
it("serves from cache within the TTL without calling fetch again", async () => {
|
||||
let fetchCalls = 0;
|
||||
mockFetch(async (url: string) => {
|
||||
fetchCalls += 1;
|
||||
if (url.includes("all-providers")) return jsonResponse({ data: [] });
|
||||
if (url.includes("catalog/models")) return jsonResponse({ data: [] });
|
||||
return jsonResponse({ data: [] });
|
||||
});
|
||||
|
||||
await getOpenRouterProviderStats();
|
||||
const callsAfterFirst = fetchCalls;
|
||||
const second = await getOpenRouterProviderStats();
|
||||
|
||||
assert.equal(fetchCalls, callsAfterFirst); // no new fetch calls
|
||||
assert.equal(second.fromCache, true);
|
||||
assert.equal(second.stale, false);
|
||||
});
|
||||
|
||||
it("falls back to stale cache when the refresh fetch fails", async () => {
|
||||
mockFetch(async (url: string) => {
|
||||
if (url.includes("all-providers")) return jsonResponse({ data: [{ slug: "cerebras", displayName: "Cerebras" }] });
|
||||
if (url.includes("catalog/models")) return jsonResponse({ data: [] });
|
||||
return jsonResponse({ data: [] });
|
||||
});
|
||||
await getOpenRouterProviderStats();
|
||||
|
||||
mockFetch(async () => {
|
||||
throw new Error("network down");
|
||||
});
|
||||
const result = await refreshOpenRouterProviderStats();
|
||||
|
||||
assert.equal(result.ok, false);
|
||||
assert.equal(typeof result.error, "string");
|
||||
});
|
||||
|
||||
it("returns empty data (not a throw) when there is no cache and the fetch fails", async () => {
|
||||
mockFetch(async () => {
|
||||
throw new Error("network down");
|
||||
});
|
||||
|
||||
const result = await getOpenRouterProviderStats();
|
||||
|
||||
assert.equal(result.data.length, 0);
|
||||
assert.equal(result.stale, true);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user