Release v3.5.8 (#1092)

* chore: bump to v3.5.8-dev

* fix(combo): quarantine degraded provider-model pairs adaptively (#1090)

Integrated into release/v3.5.8

* fix(healthcheck): keep active accounts routable after refresh failure (#1085)

Integrated into release/v3.5.8

* chore: squash PR 1089

* Merge PR 1088: docs(deps): bump axios from 1.14.0 to 1.15.0

---------

Co-authored-by: diegosouzapw <diegosouzapw@users.noreply.github.com>
Co-authored-by: Wellington Fonseca <wf.tecnologia@hotmail.com>
Co-authored-by: xiaoge1688 <129356184+xiaoge1688@users.noreply.github.com>
This commit is contained in:
Diego Rodrigues de Sa e Souza
2026-04-09 18:18:45 -03:00
committed by GitHub
parent bba7479688
commit 336d889034
12 changed files with 401 additions and 157 deletions

21
package-lock.json generated
View File

@@ -1,12 +1,12 @@
{
"name": "omniroute",
"version": "3.5.7",
"version": "3.5.8",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "omniroute",
"version": "3.5.7",
"version": "3.5.8",
"hasInstallScript": true,
"license": "MIT",
"workspaces": [
@@ -17,6 +17,7 @@
"@modelcontextprotocol/sdk": "^1.27.1",
"@monaco-editor/react": "^4.7.0",
"@swc/helpers": "0.5.21",
"axios": "^1.15.0",
"bcryptjs": "^3.0.3",
"better-sqlite3": "^12.6.2",
"bottleneck": "^2.19.5",
@@ -7782,7 +7783,6 @@
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
"integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
"dev": true,
"license": "MIT"
},
"node_modules/atomic-sleep": {
@@ -7831,10 +7831,9 @@
}
},
"node_modules/axios": {
"version": "1.14.0",
"resolved": "https://registry.npmjs.org/axios/-/axios-1.14.0.tgz",
"integrity": "sha512-3Y8yrqLSwjuzpXuZ0oIYZ/XGgLwUIBU3uLvbcpb0pidD9ctpShJd43KSlEEkVQg6DS0G9NKyzOvBfUtDKEyHvQ==",
"dev": true,
"version": "1.15.0",
"resolved": "https://registry.npmjs.org/axios/-/axios-1.15.0.tgz",
"integrity": "sha512-wWyJDlAatxk30ZJer+GeCWS209sA42X+N5jU2jy6oHTp7ufw8uzUTVFBX9+wTfAlhiJXGS0Bq7X6efruWjuK9Q==",
"license": "MIT",
"dependencies": {
"follow-redirects": "^1.15.11",
@@ -8575,7 +8574,6 @@
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
"integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
"dev": true,
"license": "MIT",
"dependencies": {
"delayed-stream": "~1.0.0"
@@ -9627,7 +9625,6 @@
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
"integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=0.4.0"
@@ -9959,7 +9956,6 @@
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz",
"integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==",
"dev": true,
"license": "MIT",
"dependencies": {
"es-errors": "^1.3.0",
@@ -11134,7 +11130,6 @@
"version": "4.0.5",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz",
"integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==",
"dev": true,
"license": "MIT",
"dependencies": {
"asynckit": "^0.4.0",
@@ -11151,7 +11146,6 @@
"version": "1.52.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
"integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">= 0.6"
@@ -11161,7 +11155,6 @@
"version": "2.1.35",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
"integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
"dev": true,
"license": "MIT",
"dependencies": {
"mime-db": "1.52.0"
@@ -11605,7 +11598,6 @@
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz",
"integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==",
"dev": true,
"license": "MIT",
"dependencies": {
"has-symbols": "^1.0.3"
@@ -16671,7 +16663,6 @@
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-2.1.0.tgz",
"integrity": "sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=10"

View File

@@ -1,6 +1,6 @@
{
"name": "omniroute",
"version": "3.5.7",
"version": "3.5.8",
"description": "Smart AI Router with auto fallback — route to FREE & cheap models, zero downtime. Works with Cursor, Cline, Claude Desktop, Codex, and any OpenAI-compatible tool.",
"type": "module",
"bin": {
@@ -93,6 +93,7 @@
"@modelcontextprotocol/sdk": "^1.27.1",
"@monaco-editor/react": "^4.7.0",
"@swc/helpers": "0.5.21",
"axios": "^1.15.0",
"bcryptjs": "^3.0.3",
"better-sqlite3": "^12.6.2",
"bottleneck": "^2.19.5",

View File

@@ -72,33 +72,43 @@ export default function DiversityScoreCard() {
riskLabel = "Moderate Distribution";
}
const providerEntries = Object.entries(data.providers || {})
.sort(([, a], [, b]) => b.share - a.share)
.slice(0, 4);
return (
<Card className="p-5 flex flex-col gap-5">
<div className="flex items-start justify-between gap-3">
<div>
<div className="flex items-center gap-2">
<span className="material-symbols-outlined text-[20px] text-primary">pie_chart</span>
<h3 className="font-semibold text-text-main">Provider Diversity</h3>
</div>
<p className="text-sm text-text-muted mt-1">
Provider concentration snapshot for the recent traffic window.
</p>
<Card className="p-5">
{/* Header */}
<div className="flex items-center justify-between gap-3 mb-4">
<div className="flex items-center gap-2">
<span className="material-symbols-outlined text-[20px] text-primary">pie_chart</span>
<h3 className="font-semibold text-text-main">Provider Diversity</h3>
<span className="text-xs text-text-muted hidden sm:inline">
Provider concentration snapshot for the recent traffic window.
</span>
</div>
<div className="flex items-center gap-3">
<span className="text-xs text-text-muted hidden md:inline">
Window: {data.windowSize} reqs · Last {Math.round(data.ttlMs / 60000)} mins
</span>
<span
className={`shrink-0 rounded-full px-2.5 py-1 text-xs font-medium ${
scorePercentage < 40
? "bg-red-500/10 text-red-500"
: scorePercentage < 70
? "bg-amber-500/10 text-amber-500"
: "bg-green-500/10 text-green-500"
}`}
>
Shannon entropy
</span>
</div>
<span
className={`shrink-0 rounded-full px-2.5 py-1 text-xs font-medium ${
scorePercentage < 40
? "bg-red-500/10 text-red-500"
: scorePercentage < 70
? "bg-amber-500/10 text-amber-500"
: "bg-green-500/10 text-green-500"
}`}
>
Shannon entropy
</span>
</div>
<div className="grid gap-4 sm:grid-cols-[112px_minmax(0,1fr)] sm:items-center">
<div className="relative mx-auto h-28 w-28">
{/* Horizontal layout: gauge + risk label + provider bars */}
<div className="flex flex-col sm:flex-row items-center gap-5">
{/* Gauge */}
<div className="relative shrink-0 h-20 w-20">
<svg className="h-full w-full -rotate-90" viewBox="0 0 36 36">
<path
className="text-border/70"
@@ -118,58 +128,54 @@ export default function DiversityScoreCard() {
/>
</svg>
<div className="absolute inset-0 flex flex-col items-center justify-center">
<span className={`text-2xl font-semibold tabular-nums ${riskColor}`}>
<span className={`text-lg font-semibold tabular-nums ${riskColor}`}>
{scorePercentage}%
</span>
<span className="text-[11px] uppercase tracking-[0.18em] text-text-muted">score</span>
<span className="text-[9px] uppercase tracking-[0.15em] text-text-muted">score</span>
</div>
</div>
<div className="space-y-4">
<div className="rounded-xl border border-border/30 bg-surface/20 px-4 py-3">
<div className={`text-sm font-medium ${riskColor}`}>{riskLabel}</div>
<div className="text-xs text-text-muted mt-1">
Higher values indicate traffic is spread across multiple providers instead of
clustering on one vendor.
</div>
{/* Risk label */}
<div className="shrink-0 text-center sm:text-left">
<div className={`text-sm font-medium ${riskColor}`}>{riskLabel}</div>
<div className="text-xs text-text-muted mt-0.5 max-w-[200px]">
Higher values mean traffic is spread across multiple providers.
</div>
</div>
{Object.keys(data.providers || {}).length === 0 ? (
<div className="rounded-xl border border-dashed border-border/50 px-4 py-5 text-sm text-text-muted">
{/* Divider */}
<div className="hidden sm:block w-px h-14 bg-border/30 shrink-0" />
{/* Provider bars */}
<div className="flex-1 min-w-0 w-full">
{providerEntries.length === 0 ? (
<div className="text-sm text-text-muted text-center py-2">
No recent usage data available.
</div>
) : (
<div className="space-y-3">
{Object.entries(data.providers)
.sort(([, a], [, b]) => b.share - a.share)
.slice(0, 4)
.map(([provider, stat]) => (
<div key={provider} className="space-y-1.5">
<div className="flex items-center justify-between gap-3 text-sm">
<span className="font-medium text-text-main capitalize">{provider}</span>
<span className="tabular-nums text-text-muted">
{Math.round(stat.share * 100)}%
</span>
</div>
<div className="h-2 rounded-full bg-surface/50 overflow-hidden">
<div
className={`h-full rounded-full ${gaugeColor}`}
style={{ width: `${Math.round(stat.share * 100)}%` }}
/>
</div>
<div className="grid grid-cols-1 sm:grid-cols-2 gap-x-6 gap-y-2">
{providerEntries.map(([provider, stat]) => (
<div key={provider} className="space-y-1">
<div className="flex items-center justify-between gap-3 text-sm">
<span className="font-medium text-text-main capitalize truncate">
{provider}
</span>
<span className="tabular-nums text-text-muted shrink-0">
{Math.round(stat.share * 100)}%
</span>
</div>
))}
<div className="h-1.5 rounded-full bg-surface/50 overflow-hidden">
<div
className={`h-full rounded-full ${gaugeColor}`}
style={{ width: `${Math.round(stat.share * 100)}%` }}
/>
</div>
</div>
))}
</div>
)}
</div>
</div>
<div className="grid grid-cols-2 gap-3 border-t border-border/30 pt-4 text-xs text-text-muted">
<div className="rounded-lg bg-surface/20 px-3 py-2">Window: {data.windowSize} reqs</div>
<div className="rounded-lg bg-surface/20 px-3 py-2 text-right">
Based on last {Math.round(data.ttlMs / 60000)} mins
</div>
</div>
</Card>
);
}

View File

@@ -45,12 +45,12 @@ export default function AnalyticsPage() {
/>
{activeTab === "overview" && (
<div className="grid gap-6 xl:grid-cols-[minmax(0,1fr)_340px] xl:items-start">
<>
<Suspense fallback={<CardSkeleton />}>
<UsageAnalytics />
</Suspense>
<DiversityScoreCard />
</div>
</>
)}
{activeTab === "evals" && <EvalsTab />}
{activeTab === "search" && <SearchAnalyticsTab />}

View File

@@ -21,6 +21,23 @@
/** @type {Map<string, UnavailableEntry>} */
const unavailable = new Map();
/** @type {Map<string, { failureCount: number, lastFailureAt: number }>} */
const failureState = new Map();
const FAILURE_WINDOW_MS = 30 * 60 * 1000;
const PROBLEMATIC_STATUS_COOLDOWNS = {
429: 5 * 60 * 1000,
408: 60 * 1000,
500: 2 * 60 * 1000,
502: 2 * 60 * 1000,
503: 2 * 60 * 1000,
504: 2 * 60 * 1000,
};
const MIN_PROBLEMATIC_COOLDOWN_MS = 60 * 1000;
const MAX_PROBLEMATIC_COOLDOWN_MS = 30 * 60 * 1000;
/**
* Build a composite key for provider+model.
* @param {string} provider
@@ -62,15 +79,62 @@ export function isModelAvailable(provider, model) {
*/
export function setModelUnavailable(provider, model, cooldownMs = 60000, reason) {
const key = makeKey(provider, model);
const now = Date.now();
const safeCooldownMs = Number.isFinite(cooldownMs) && cooldownMs > 0 ? cooldownMs : 60000;
const existing = unavailable.get(key);
const existingRemainingMs =
existing && Date.now() - existing.unavailableSince < existing.cooldownMs
? existing.cooldownMs - (Date.now() - existing.unavailableSince)
: 0;
const effectiveCooldownMs = Math.max(safeCooldownMs, existingRemainingMs);
unavailable.set(key, {
provider,
model,
unavailableSince: Date.now(),
cooldownMs,
unavailableSince: now,
cooldownMs: effectiveCooldownMs,
reason: reason || "unknown",
});
}
/**
* Marca provider/model como problemático com cooldown adaptativo.
* Mantém retrocompatibilidade: não altera o comportamento de setModelUnavailable,
* apenas oferece uma estratégia mais agressiva para falhas recorrentes.
*
* @param {string} provider
* @param {string} model
* @param {{ status?: number, baseCooldownMs?: number, reason?: string }} [options]
* @returns {{ cooldownMs: number, failureCount: number }}
*/
export function markModelAsProblematic(provider, model, options = {}) {
const key = makeKey(provider, model);
const now = Date.now();
const status = Number.isFinite(options.status) ? Number(options.status) : null;
const statusBaseCooldown =
status && Object.prototype.hasOwnProperty.call(PROBLEMATIC_STATUS_COOLDOWNS, status)
? PROBLEMATIC_STATUS_COOLDOWNS[status]
: 0;
const baseCooldownMs =
Number.isFinite(options.baseCooldownMs) && Number(options.baseCooldownMs) > 0
? Number(options.baseCooldownMs)
: 0;
const prev = failureState.get(key);
const withinFailureWindow = prev && now - prev.lastFailureAt <= FAILURE_WINDOW_MS;
const failureCount = withinFailureWindow ? prev.failureCount + 1 : 1;
failureState.set(key, { failureCount, lastFailureAt: now });
const cooldownBase = Math.max(baseCooldownMs, statusBaseCooldown, MIN_PROBLEMATIC_COOLDOWN_MS);
const cooldownMs = Math.min(
cooldownBase * Math.pow(2, Math.max(0, failureCount - 1)),
MAX_PROBLEMATIC_COOLDOWN_MS
);
setModelUnavailable(provider, model, cooldownMs, options.reason || "problematic_model");
return { cooldownMs, failureCount };
}
/**
* Clear unavailability for a model (e.g. after manual reset).
*
@@ -79,7 +143,9 @@ export function setModelUnavailable(provider, model, cooldownMs = 60000, reason)
* @returns {boolean} true if entry existed and was removed
*/
export function clearModelUnavailability(provider, model) {
return unavailable.delete(makeKey(provider, model));
const key = makeKey(provider, model);
failureState.delete(key);
return unavailable.delete(key);
}
/**
@@ -125,4 +191,5 @@ export function getUnavailableCount() {
*/
export function resetAllAvailability() {
unavailable.clear();
failureState.clear();
}

View File

@@ -10,7 +10,12 @@
* updates the DB, and logs the result.
*/
import { getProviderConnections, updateProviderConnection, getSettings, resolveProxyForConnection } from "@/lib/localDb";
import {
getProviderConnections,
updateProviderConnection,
getSettings,
resolveProxyForConnection,
} from "@/lib/localDb";
import {
getAccessToken,
supportsTokenRefresh,
@@ -25,6 +30,25 @@ const EXPIRED_RETRY_BACKOFF_MIN = 5; // backoff between expired retries (minutes
const LOG_PREFIX = "[HealthCheck]";
const TRUE_ENV_VALUES = new Set(["1", "true", "yes", "on"]);
export function buildRefreshFailureUpdate(conn: any, now: string) {
const wasExpired = conn.testStatus === "expired";
const retryCount = (conn.expiredRetryCount ?? 0) + (wasExpired ? 1 : 0);
return {
lastHealthCheckAt: now,
// A failed background refresh should not evict otherwise healthy accounts
// from request routing. Keep non-expired connections active and only persist
// the refresh error metadata for observability.
testStatus: wasExpired ? "expired" : "active",
lastError: "Health check: token refresh failed",
lastErrorAt: now,
lastErrorType: "token_refresh_failed",
lastErrorSource: "oauth",
errorCode: "refresh_failed",
...(wasExpired ? { expiredRetryCount: retryCount, expiredRetryAt: now } : {}),
};
}
function isEnvFlagEnabled(name: string): boolean {
const value = process.env[name];
if (!value) return false;
@@ -304,22 +328,13 @@ async function checkConnection(conn) {
await updateProviderConnection(conn.id, updateData);
log(`${LOG_PREFIX}${conn.provider}/${conn.name || conn.email || conn.id} refreshed`);
} else {
const wasExpired = conn.testStatus === "expired";
const retryCount = (conn.expiredRetryCount ?? 0) + (wasExpired ? 1 : 0);
await updateProviderConnection(conn.id, {
lastHealthCheckAt: now,
testStatus: wasExpired ? "expired" : "error",
lastError: "Health check: token refresh failed",
lastErrorAt: now,
lastErrorType: "token_refresh_failed",
lastErrorSource: "oauth",
errorCode: "refresh_failed",
...(wasExpired ? { expiredRetryCount: retryCount, expiredRetryAt: now } : {}),
});
const updateData = buildRefreshFailureUpdate(conn, now);
await updateProviderConnection(conn.id, updateData);
logWarn(
`${LOG_PREFIX}${conn.provider}/${conn.name || conn.email || conn.id} refresh failed` +
(wasExpired ? ` (expired retry ${retryCount}/${EXPIRED_RETRY_MAX})` : "")
(conn.testStatus === "expired"
? ` (${updateData.expiredRetryCount}/${EXPIRED_RETRY_MAX} expired retries used)`
: "")
);
}
}

View File

@@ -6,6 +6,7 @@ import { CardSkeleton } from "./Loading";
import { fmtCompact as fmt, fmtFull, fmtCost } from "@/shared/utils/formatting";
import {
StatCard,
CompactStatGrid,
ActivityHeatmap,
DailyTrendChart,
AccountDonut,
@@ -109,7 +110,7 @@ export default function UsageAnalytics() {
const ioRatio = s.completionTokens > 0 ? (s.promptTokens / s.completionTokens).toFixed(1) : "—";
return (
<div className="flex flex-col gap-5">
<div className="flex flex-col gap-5 min-w-0">
{/* Header + Time Range */}
<div className="flex items-center justify-between">
<h2 className="text-xl font-semibold flex items-center gap-2">
@@ -133,8 +134,8 @@ export default function UsageAnalytics() {
</div>
</div>
{/* Summary Cards — Row 1: Core metrics */}
<div className="grid grid-cols-2 md:grid-cols-8 gap-3">
{/* Primary KPI Cards */}
<div className="grid grid-cols-2 md:grid-cols-4 gap-3">
<StatCard
icon="generating_tokens"
label="Total Tokens"
@@ -159,56 +160,71 @@ export default function UsageAnalytics() {
value={fmtCost(s.totalCost)}
color="text-amber-500"
/>
<StatCard icon="group" label="Accounts" value={s.uniqueAccounts || 0} />
<StatCard icon="vpn_key" label="API Keys" value={s.uniqueApiKeys || 0} />
<StatCard icon="model_training" label="Models" value={s.uniqueModels || 0} />
<StatCard
icon="swap_horiz"
label="Fallback Rate"
value={`${Number(s.fallbackRatePct || 0).toFixed(1)}%`}
subValue={`${fmtFull(s.fallbackCount || 0)} fallbacks`}
color="text-amber-500"
/>
</div>
{/* Summary Cards — Row 2: Derived insights */}
<div className="grid grid-cols-2 md:grid-cols-8 gap-3">
<StatCard
icon="speed"
label="Avg Tokens/Req"
value={fmt(avgTokensPerReq)}
color="text-cyan-500"
/>
<StatCard
icon="request_quote"
label="Cost/Request"
value={fmtCost(costPerReq)}
color="text-orange-500"
/>
<StatCard
icon="compare_arrows"
label="I/O Ratio"
value={`${ioRatio}x`}
color="text-violet-500"
/>
<StatCard icon="star" label="Top Model" value={topModel} color="text-pink-500" />
<StatCard icon="cloud" label="Top Provider" value={topProvider} color="text-teal-500" />
<StatCard icon="today" label="Busiest Day" value={busiestDay} color="text-rose-500" />
<StatCard icon="dns" label="Providers" value={providerCount} color="text-indigo-500" />
<StatCard
icon="network_node"
label="Diversity Score"
value={`${providerDiversity.toFixed(1)}%`}
color="text-sky-500"
/>
</div>
{/* Secondary Metrics — compact grid with sections */}
<CompactStatGrid
sections={[
{
title: "Infrastructure",
items: [
{ icon: "group", label: "Accounts", value: s.uniqueAccounts || 0 },
{ icon: "dns", label: "Providers", value: providerCount, color: "text-indigo-500" },
{ icon: "vpn_key", label: "API Keys", value: s.uniqueApiKeys || 0 },
{ icon: "model_training", label: "Models", value: s.uniqueModels || 0 },
],
},
{
title: "Performance",
items: [
{
icon: "speed",
label: "Avg Tokens/Req",
value: fmt(avgTokensPerReq),
color: "text-cyan-500",
},
{
icon: "request_quote",
label: "Cost/Req",
value: fmtCost(costPerReq),
color: "text-orange-500",
},
{
icon: "compare_arrows",
label: "I/O Ratio",
value: `${ioRatio}x`,
color: "text-violet-500",
},
{
icon: "swap_horiz",
label: "Fallback Rate",
value: `${Number(s.fallbackRatePct || 0).toFixed(1)}%`,
color: "text-amber-500",
},
],
},
{
title: "Highlights",
wideValues: true,
items: [
{ icon: "star", label: "Top Model", value: topModel, color: "text-pink-500" },
{ icon: "cloud", label: "Top Provider", value: topProvider, color: "text-teal-500" },
{ icon: "today", label: "Busiest Day", value: busiestDay, color: "text-rose-500" },
{
icon: "network_node",
label: "Diversity",
value: `${providerDiversity.toFixed(1)}%`,
color: "text-sky-500",
},
],
},
]}
/>
{/* Activity Heatmap + Weekly Widgets */}
<div
style={{ display: "grid", gridTemplateColumns: "2fr 1fr", gap: 16, alignItems: "stretch" }}
>
<div className="grid grid-cols-1 lg:grid-cols-[2fr_1fr] gap-4 items-stretch">
<ActivityHeatmap activityMap={analytics?.activityMap} />
<div style={{ display: "flex", flexDirection: "column", gap: 16 }}>
<div className="flex flex-col gap-4">
<MostActiveDay7d activityMap={analytics?.activityMap} />
<WeeklySquares7d activityMap={analytics?.activityMap} />
</div>

View File

@@ -84,7 +84,7 @@ export function SortIndicator({ active, sortOrder }: { active: boolean; sortOrde
);
}
// ── StatCard ───────────────────────────────────────────────────────────────
// ── StatCard (primary KPI) ─────────────────────────────────────────────────
export function StatCard({
icon,
@@ -100,13 +100,73 @@ export function StatCard({
color?: string;
}) {
return (
<Card className="px-4 py-3 flex flex-col gap-1">
<div className="flex items-center gap-2 text-text-muted text-xs uppercase font-semibold tracking-wider">
<span className="material-symbols-outlined text-[16px]">{icon}</span>
{label}
<Card className="px-4 py-3 flex flex-col gap-1 min-w-0">
<div className="flex items-center gap-1.5 text-text-muted text-[11px] uppercase font-semibold tracking-wide min-w-0">
<span className="material-symbols-outlined text-[14px] shrink-0">{icon}</span>
<span className="truncate">{label}</span>
</div>
<span className={`text-2xl font-bold ${color} truncate`} title={String(value)}>
{value}
</span>
{subValue && <span className="text-xs text-text-muted truncate">{subValue}</span>}
</Card>
);
}
// ── CompactStatGrid (secondary metrics in a single card, grouped) ─────────
export type CompactStatSection = {
title: string;
items: Array<{ icon: string; label: string; value: any; color?: string }>;
/** On mobile use 1 column instead of 2 — useful when values can be long (model names, etc.) */
wideValues?: boolean;
};
export function CompactStatGrid({ sections }: { sections: CompactStatSection[] }) {
return (
<Card className="px-5 py-4">
<div className="flex flex-col gap-3">
{sections.map((section, si) => (
<div key={si}>
{si > 0 && (
<div className="border-t border-black/[0.06] dark:border-white/[0.06] mb-3" />
)}
<div className="text-[10px] uppercase font-semibold tracking-widest text-text-muted/50 mb-2">
{section.title}
</div>
<div
className={
section.wideValues
? "grid grid-cols-1 sm:grid-cols-2 md:grid-cols-4 gap-x-5 gap-y-2"
: "grid grid-cols-2 md:grid-cols-4 gap-x-5 gap-y-2"
}
>
{section.items.map((stat, i) => (
<div key={i} className="flex items-center justify-between gap-2 min-w-0 py-0.5">
<div
className={`flex items-center gap-1.5 ${section.wideValues ? "shrink-0" : "min-w-0"}`}
>
<span className="material-symbols-outlined text-[14px] text-text-muted shrink-0">
{stat.icon}
</span>
<span
className={`text-[11px] uppercase font-semibold tracking-wide text-text-muted ${section.wideValues ? "whitespace-nowrap" : "truncate"}`}
>
{stat.label}
</span>
</div>
<span
className={`text-sm font-bold text-right ${section.wideValues ? "truncate min-w-0" : "shrink-0"} ${stat.color || "text-text-main"}`}
title={String(stat.value)}
>
{stat.value}
</span>
</div>
))}
</div>
</div>
))}
</div>
<span className={`text-2xl font-bold ${color}`}>{value}</span>
{subValue && <span className="text-xs text-text-muted">{subValue}</span>}
</Card>
);
}
@@ -189,7 +249,7 @@ export function ActivityHeatmap({ activityMap }) {
}
return (
<Card className="p-4 h-full">
<Card className="p-4 h-full min-w-0 overflow-hidden">
<div className="flex items-center justify-between mb-3">
<h3 className="text-sm font-semibold text-text-muted uppercase tracking-wider">Activity</h3>
<span className="text-xs text-text-muted">

View File

@@ -11,6 +11,7 @@
export {
SortIndicator,
StatCard,
CompactStatGrid,
ActivityHeatmap,
DailyTrendChart,
AccountDonut,

View File

@@ -35,7 +35,7 @@ import {
import { getCircuitBreaker } from "../../shared/utils/circuitBreaker";
import {
isModelAvailable,
setModelUnavailable,
markModelAsProblematic,
clearModelUnavailability,
} from "../../domain/modelAvailability";
import { markAccountExhaustedFrom429 } from "../../domain/quotaCache";
@@ -468,6 +468,7 @@ async function handleSingleModelChat(
let excludeConnectionId = null;
let lastError = null;
let lastStatus = null;
let lastCooldownMs = 0;
while (true) {
const credentials = await getProviderCredentials(
@@ -484,11 +485,15 @@ async function handleSingleModelChat(
);
if (!credentials || credentials.allRateLimited) {
if (lastStatus === 429 || lastStatus === 503) {
setModelUnavailable(provider, model, 60000, `HTTP ${lastStatus}`);
if ([408, 429, 500, 502, 503, 504].includes(Number(lastStatus))) {
const quarantine = markModelAsProblematic(provider, model, {
status: Number(lastStatus),
baseCooldownMs: lastCooldownMs,
reason: `HTTP ${lastStatus}`,
});
log.info(
"AVAILABILITY",
`${provider}/${model} marked unavailable — all accounts exhausted (HTTP ${lastStatus})`
`${provider}/${model} marked unavailable — all accounts exhausted (HTTP ${lastStatus}, cooldown ${Math.ceil(quarantine.cooldownMs / 1000)}s, failureCount ${quarantine.failureCount})`
);
}
return handleNoCredentials(
@@ -654,7 +659,7 @@ async function handleSingleModelChat(
}
// 7. Fallback to next account
const { shouldFallback } = await markAccountUnavailable(
const { shouldFallback, cooldownMs } = await markAccountUnavailable(
credentials.connectionId,
result.status,
result.error,
@@ -663,6 +668,9 @@ async function handleSingleModelChat(
);
if (shouldFallback) {
if (Number.isFinite(cooldownMs) && cooldownMs > 0) {
lastCooldownMs = cooldownMs;
}
log.warn("AUTH", `Account ${accountId}... unavailable (${result.status}), trying fallback`);
excludeConnectionId = credentials.connectionId;
lastError = result.error;

View File

@@ -13,6 +13,7 @@ import assert from "node:assert/strict";
import {
isModelAvailable,
setModelUnavailable,
markModelAsProblematic,
clearModelUnavailability,
getAvailabilityReport,
getUnavailableCount,
@@ -58,6 +59,41 @@ describe("modelAvailability", () => {
while (Date.now() - start < 5) {} // spin wait
assert.equal(isModelAvailable("anthropic", "claude-sonnet-4-20250514"), true);
});
it("should apply adaptive quarantine for repeated problematic failures", () => {
const first = markModelAsProblematic("nvidia", "z-ai/glm4.7", {
status: 502,
baseCooldownMs: 3000,
reason: "HTTP 502",
});
const second = markModelAsProblematic("nvidia", "z-ai/glm4.7", {
status: 502,
baseCooldownMs: 3000,
reason: "HTTP 502",
});
assert.equal(first.failureCount, 1);
assert.equal(first.cooldownMs, 120000);
assert.equal(second.failureCount, 2);
assert.equal(second.cooldownMs, 240000);
const report = getAvailabilityReport();
const entry = report.find((r) => r.provider === "nvidia" && r.model === "z-ai/glm4.7");
assert.ok(entry, "nvidia model should be quarantined");
assert.ok(entry.remainingMs >= 200000, "cooldown should be preserved/escalated");
});
it("should reset failure history after model recovery", () => {
clearModelUnavailability("nvidia", "z-ai/glm4.7");
const afterReset = markModelAsProblematic("nvidia", "z-ai/glm4.7", {
status: 502,
baseCooldownMs: 3000,
reason: "HTTP 502",
});
assert.equal(afterReset.failureCount, 1);
assert.equal(afterReset.cooldownMs, 120000);
});
});
// ──────────────── T-19: Cost Rules ────────────────

View File

@@ -0,0 +1,43 @@
import test from "node:test";
import assert from "node:assert/strict";
process.env.NODE_ENV = "test";
const tokenHealthCheck = await import("../../src/lib/tokenHealthCheck.ts");
test("buildRefreshFailureUpdate keeps active connections routable after refresh failure", () => {
const now = "2026-04-09T04:40:00.000Z";
const update = tokenHealthCheck.buildRefreshFailureUpdate(
{
testStatus: "active",
expiredRetryCount: 2,
},
now
);
assert.equal(update.testStatus, "active");
assert.equal(update.lastError, "Health check: token refresh failed");
assert.equal(update.lastErrorType, "token_refresh_failed");
assert.equal(update.lastErrorSource, "oauth");
assert.equal(update.errorCode, "refresh_failed");
assert.equal(update.lastHealthCheckAt, now);
assert.equal("expiredRetryCount" in update, false);
assert.equal("expiredRetryAt" in update, false);
});
test("buildRefreshFailureUpdate preserves expired retry tracking", () => {
const now = "2026-04-09T04:41:00.000Z";
const update = tokenHealthCheck.buildRefreshFailureUpdate(
{
testStatus: "expired",
expiredRetryCount: 2,
},
now
);
assert.equal(update.testStatus, "expired");
assert.equal(update.expiredRetryCount, 3);
assert.equal(update.expiredRetryAt, now);
});