mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-07-26 09:52:11 +03:00
Release v3.5.8 (#1092)
* chore: bump to v3.5.8-dev * fix(combo): quarantine degraded provider-model pairs adaptively (#1090) Integrated into release/v3.5.8 * fix(healthcheck): keep active accounts routable after refresh failure (#1085) Integrated into release/v3.5.8 * chore: squash PR 1089 * Merge PR 1088: docs(deps): bump axios from 1.14.0 to 1.15.0 --------- Co-authored-by: diegosouzapw <diegosouzapw@users.noreply.github.com> Co-authored-by: Wellington Fonseca <wf.tecnologia@hotmail.com> Co-authored-by: xiaoge1688 <129356184+xiaoge1688@users.noreply.github.com>
This commit is contained in:
committed by
GitHub
parent
bba7479688
commit
336d889034
21
package-lock.json
generated
21
package-lock.json
generated
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "omniroute",
|
||||
"version": "3.5.7",
|
||||
"version": "3.5.8",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "omniroute",
|
||||
"version": "3.5.7",
|
||||
"version": "3.5.8",
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"workspaces": [
|
||||
@@ -17,6 +17,7 @@
|
||||
"@modelcontextprotocol/sdk": "^1.27.1",
|
||||
"@monaco-editor/react": "^4.7.0",
|
||||
"@swc/helpers": "0.5.21",
|
||||
"axios": "^1.15.0",
|
||||
"bcryptjs": "^3.0.3",
|
||||
"better-sqlite3": "^12.6.2",
|
||||
"bottleneck": "^2.19.5",
|
||||
@@ -7782,7 +7783,6 @@
|
||||
"version": "0.4.0",
|
||||
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
|
||||
"integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/atomic-sleep": {
|
||||
@@ -7831,10 +7831,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/axios": {
|
||||
"version": "1.14.0",
|
||||
"resolved": "https://registry.npmjs.org/axios/-/axios-1.14.0.tgz",
|
||||
"integrity": "sha512-3Y8yrqLSwjuzpXuZ0oIYZ/XGgLwUIBU3uLvbcpb0pidD9ctpShJd43KSlEEkVQg6DS0G9NKyzOvBfUtDKEyHvQ==",
|
||||
"dev": true,
|
||||
"version": "1.15.0",
|
||||
"resolved": "https://registry.npmjs.org/axios/-/axios-1.15.0.tgz",
|
||||
"integrity": "sha512-wWyJDlAatxk30ZJer+GeCWS209sA42X+N5jU2jy6oHTp7ufw8uzUTVFBX9+wTfAlhiJXGS0Bq7X6efruWjuK9Q==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"follow-redirects": "^1.15.11",
|
||||
@@ -8575,7 +8574,6 @@
|
||||
"version": "1.0.8",
|
||||
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
|
||||
"integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"delayed-stream": "~1.0.0"
|
||||
@@ -9627,7 +9625,6 @@
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
|
||||
"integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.4.0"
|
||||
@@ -9959,7 +9956,6 @@
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz",
|
||||
"integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"es-errors": "^1.3.0",
|
||||
@@ -11134,7 +11130,6 @@
|
||||
"version": "4.0.5",
|
||||
"resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz",
|
||||
"integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"asynckit": "^0.4.0",
|
||||
@@ -11151,7 +11146,6 @@
|
||||
"version": "1.52.0",
|
||||
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
|
||||
"integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 0.6"
|
||||
@@ -11161,7 +11155,6 @@
|
||||
"version": "2.1.35",
|
||||
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
|
||||
"integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"mime-db": "1.52.0"
|
||||
@@ -11605,7 +11598,6 @@
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz",
|
||||
"integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"has-symbols": "^1.0.3"
|
||||
@@ -16671,7 +16663,6 @@
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-2.1.0.tgz",
|
||||
"integrity": "sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "omniroute",
|
||||
"version": "3.5.7",
|
||||
"version": "3.5.8",
|
||||
"description": "Smart AI Router with auto fallback — route to FREE & cheap models, zero downtime. Works with Cursor, Cline, Claude Desktop, Codex, and any OpenAI-compatible tool.",
|
||||
"type": "module",
|
||||
"bin": {
|
||||
@@ -93,6 +93,7 @@
|
||||
"@modelcontextprotocol/sdk": "^1.27.1",
|
||||
"@monaco-editor/react": "^4.7.0",
|
||||
"@swc/helpers": "0.5.21",
|
||||
"axios": "^1.15.0",
|
||||
"bcryptjs": "^3.0.3",
|
||||
"better-sqlite3": "^12.6.2",
|
||||
"bottleneck": "^2.19.5",
|
||||
|
||||
@@ -72,33 +72,43 @@ export default function DiversityScoreCard() {
|
||||
riskLabel = "Moderate Distribution";
|
||||
}
|
||||
|
||||
const providerEntries = Object.entries(data.providers || {})
|
||||
.sort(([, a], [, b]) => b.share - a.share)
|
||||
.slice(0, 4);
|
||||
|
||||
return (
|
||||
<Card className="p-5 flex flex-col gap-5">
|
||||
<div className="flex items-start justify-between gap-3">
|
||||
<div>
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="material-symbols-outlined text-[20px] text-primary">pie_chart</span>
|
||||
<h3 className="font-semibold text-text-main">Provider Diversity</h3>
|
||||
</div>
|
||||
<p className="text-sm text-text-muted mt-1">
|
||||
Provider concentration snapshot for the recent traffic window.
|
||||
</p>
|
||||
<Card className="p-5">
|
||||
{/* Header */}
|
||||
<div className="flex items-center justify-between gap-3 mb-4">
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="material-symbols-outlined text-[20px] text-primary">pie_chart</span>
|
||||
<h3 className="font-semibold text-text-main">Provider Diversity</h3>
|
||||
<span className="text-xs text-text-muted hidden sm:inline">
|
||||
— Provider concentration snapshot for the recent traffic window.
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-3">
|
||||
<span className="text-xs text-text-muted hidden md:inline">
|
||||
Window: {data.windowSize} reqs · Last {Math.round(data.ttlMs / 60000)} mins
|
||||
</span>
|
||||
<span
|
||||
className={`shrink-0 rounded-full px-2.5 py-1 text-xs font-medium ${
|
||||
scorePercentage < 40
|
||||
? "bg-red-500/10 text-red-500"
|
||||
: scorePercentage < 70
|
||||
? "bg-amber-500/10 text-amber-500"
|
||||
: "bg-green-500/10 text-green-500"
|
||||
}`}
|
||||
>
|
||||
Shannon entropy
|
||||
</span>
|
||||
</div>
|
||||
<span
|
||||
className={`shrink-0 rounded-full px-2.5 py-1 text-xs font-medium ${
|
||||
scorePercentage < 40
|
||||
? "bg-red-500/10 text-red-500"
|
||||
: scorePercentage < 70
|
||||
? "bg-amber-500/10 text-amber-500"
|
||||
: "bg-green-500/10 text-green-500"
|
||||
}`}
|
||||
>
|
||||
Shannon entropy
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div className="grid gap-4 sm:grid-cols-[112px_minmax(0,1fr)] sm:items-center">
|
||||
<div className="relative mx-auto h-28 w-28">
|
||||
{/* Horizontal layout: gauge + risk label + provider bars */}
|
||||
<div className="flex flex-col sm:flex-row items-center gap-5">
|
||||
{/* Gauge */}
|
||||
<div className="relative shrink-0 h-20 w-20">
|
||||
<svg className="h-full w-full -rotate-90" viewBox="0 0 36 36">
|
||||
<path
|
||||
className="text-border/70"
|
||||
@@ -118,58 +128,54 @@ export default function DiversityScoreCard() {
|
||||
/>
|
||||
</svg>
|
||||
<div className="absolute inset-0 flex flex-col items-center justify-center">
|
||||
<span className={`text-2xl font-semibold tabular-nums ${riskColor}`}>
|
||||
<span className={`text-lg font-semibold tabular-nums ${riskColor}`}>
|
||||
{scorePercentage}%
|
||||
</span>
|
||||
<span className="text-[11px] uppercase tracking-[0.18em] text-text-muted">score</span>
|
||||
<span className="text-[9px] uppercase tracking-[0.15em] text-text-muted">score</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="space-y-4">
|
||||
<div className="rounded-xl border border-border/30 bg-surface/20 px-4 py-3">
|
||||
<div className={`text-sm font-medium ${riskColor}`}>{riskLabel}</div>
|
||||
<div className="text-xs text-text-muted mt-1">
|
||||
Higher values indicate traffic is spread across multiple providers instead of
|
||||
clustering on one vendor.
|
||||
</div>
|
||||
{/* Risk label */}
|
||||
<div className="shrink-0 text-center sm:text-left">
|
||||
<div className={`text-sm font-medium ${riskColor}`}>{riskLabel}</div>
|
||||
<div className="text-xs text-text-muted mt-0.5 max-w-[200px]">
|
||||
Higher values mean traffic is spread across multiple providers.
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{Object.keys(data.providers || {}).length === 0 ? (
|
||||
<div className="rounded-xl border border-dashed border-border/50 px-4 py-5 text-sm text-text-muted">
|
||||
{/* Divider */}
|
||||
<div className="hidden sm:block w-px h-14 bg-border/30 shrink-0" />
|
||||
|
||||
{/* Provider bars */}
|
||||
<div className="flex-1 min-w-0 w-full">
|
||||
{providerEntries.length === 0 ? (
|
||||
<div className="text-sm text-text-muted text-center py-2">
|
||||
No recent usage data available.
|
||||
</div>
|
||||
) : (
|
||||
<div className="space-y-3">
|
||||
{Object.entries(data.providers)
|
||||
.sort(([, a], [, b]) => b.share - a.share)
|
||||
.slice(0, 4)
|
||||
.map(([provider, stat]) => (
|
||||
<div key={provider} className="space-y-1.5">
|
||||
<div className="flex items-center justify-between gap-3 text-sm">
|
||||
<span className="font-medium text-text-main capitalize">{provider}</span>
|
||||
<span className="tabular-nums text-text-muted">
|
||||
{Math.round(stat.share * 100)}%
|
||||
</span>
|
||||
</div>
|
||||
<div className="h-2 rounded-full bg-surface/50 overflow-hidden">
|
||||
<div
|
||||
className={`h-full rounded-full ${gaugeColor}`}
|
||||
style={{ width: `${Math.round(stat.share * 100)}%` }}
|
||||
/>
|
||||
</div>
|
||||
<div className="grid grid-cols-1 sm:grid-cols-2 gap-x-6 gap-y-2">
|
||||
{providerEntries.map(([provider, stat]) => (
|
||||
<div key={provider} className="space-y-1">
|
||||
<div className="flex items-center justify-between gap-3 text-sm">
|
||||
<span className="font-medium text-text-main capitalize truncate">
|
||||
{provider}
|
||||
</span>
|
||||
<span className="tabular-nums text-text-muted shrink-0">
|
||||
{Math.round(stat.share * 100)}%
|
||||
</span>
|
||||
</div>
|
||||
))}
|
||||
<div className="h-1.5 rounded-full bg-surface/50 overflow-hidden">
|
||||
<div
|
||||
className={`h-full rounded-full ${gaugeColor}`}
|
||||
style={{ width: `${Math.round(stat.share * 100)}%` }}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-2 gap-3 border-t border-border/30 pt-4 text-xs text-text-muted">
|
||||
<div className="rounded-lg bg-surface/20 px-3 py-2">Window: {data.windowSize} reqs</div>
|
||||
<div className="rounded-lg bg-surface/20 px-3 py-2 text-right">
|
||||
Based on last {Math.round(data.ttlMs / 60000)} mins
|
||||
</div>
|
||||
</div>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -45,12 +45,12 @@ export default function AnalyticsPage() {
|
||||
/>
|
||||
|
||||
{activeTab === "overview" && (
|
||||
<div className="grid gap-6 xl:grid-cols-[minmax(0,1fr)_340px] xl:items-start">
|
||||
<>
|
||||
<Suspense fallback={<CardSkeleton />}>
|
||||
<UsageAnalytics />
|
||||
</Suspense>
|
||||
<DiversityScoreCard />
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
{activeTab === "evals" && <EvalsTab />}
|
||||
{activeTab === "search" && <SearchAnalyticsTab />}
|
||||
|
||||
@@ -21,6 +21,23 @@
|
||||
/** @type {Map<string, UnavailableEntry>} */
|
||||
const unavailable = new Map();
|
||||
|
||||
/** @type {Map<string, { failureCount: number, lastFailureAt: number }>} */
|
||||
const failureState = new Map();
|
||||
|
||||
const FAILURE_WINDOW_MS = 30 * 60 * 1000;
|
||||
|
||||
const PROBLEMATIC_STATUS_COOLDOWNS = {
|
||||
429: 5 * 60 * 1000,
|
||||
408: 60 * 1000,
|
||||
500: 2 * 60 * 1000,
|
||||
502: 2 * 60 * 1000,
|
||||
503: 2 * 60 * 1000,
|
||||
504: 2 * 60 * 1000,
|
||||
};
|
||||
|
||||
const MIN_PROBLEMATIC_COOLDOWN_MS = 60 * 1000;
|
||||
const MAX_PROBLEMATIC_COOLDOWN_MS = 30 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* Build a composite key for provider+model.
|
||||
* @param {string} provider
|
||||
@@ -62,15 +79,62 @@ export function isModelAvailable(provider, model) {
|
||||
*/
|
||||
export function setModelUnavailable(provider, model, cooldownMs = 60000, reason) {
|
||||
const key = makeKey(provider, model);
|
||||
const now = Date.now();
|
||||
const safeCooldownMs = Number.isFinite(cooldownMs) && cooldownMs > 0 ? cooldownMs : 60000;
|
||||
const existing = unavailable.get(key);
|
||||
const existingRemainingMs =
|
||||
existing && Date.now() - existing.unavailableSince < existing.cooldownMs
|
||||
? existing.cooldownMs - (Date.now() - existing.unavailableSince)
|
||||
: 0;
|
||||
const effectiveCooldownMs = Math.max(safeCooldownMs, existingRemainingMs);
|
||||
|
||||
unavailable.set(key, {
|
||||
provider,
|
||||
model,
|
||||
unavailableSince: Date.now(),
|
||||
cooldownMs,
|
||||
unavailableSince: now,
|
||||
cooldownMs: effectiveCooldownMs,
|
||||
reason: reason || "unknown",
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Marca provider/model como problemático com cooldown adaptativo.
|
||||
* Mantém retrocompatibilidade: não altera o comportamento de setModelUnavailable,
|
||||
* apenas oferece uma estratégia mais agressiva para falhas recorrentes.
|
||||
*
|
||||
* @param {string} provider
|
||||
* @param {string} model
|
||||
* @param {{ status?: number, baseCooldownMs?: number, reason?: string }} [options]
|
||||
* @returns {{ cooldownMs: number, failureCount: number }}
|
||||
*/
|
||||
export function markModelAsProblematic(provider, model, options = {}) {
|
||||
const key = makeKey(provider, model);
|
||||
const now = Date.now();
|
||||
const status = Number.isFinite(options.status) ? Number(options.status) : null;
|
||||
const statusBaseCooldown =
|
||||
status && Object.prototype.hasOwnProperty.call(PROBLEMATIC_STATUS_COOLDOWNS, status)
|
||||
? PROBLEMATIC_STATUS_COOLDOWNS[status]
|
||||
: 0;
|
||||
const baseCooldownMs =
|
||||
Number.isFinite(options.baseCooldownMs) && Number(options.baseCooldownMs) > 0
|
||||
? Number(options.baseCooldownMs)
|
||||
: 0;
|
||||
|
||||
const prev = failureState.get(key);
|
||||
const withinFailureWindow = prev && now - prev.lastFailureAt <= FAILURE_WINDOW_MS;
|
||||
const failureCount = withinFailureWindow ? prev.failureCount + 1 : 1;
|
||||
failureState.set(key, { failureCount, lastFailureAt: now });
|
||||
|
||||
const cooldownBase = Math.max(baseCooldownMs, statusBaseCooldown, MIN_PROBLEMATIC_COOLDOWN_MS);
|
||||
const cooldownMs = Math.min(
|
||||
cooldownBase * Math.pow(2, Math.max(0, failureCount - 1)),
|
||||
MAX_PROBLEMATIC_COOLDOWN_MS
|
||||
);
|
||||
|
||||
setModelUnavailable(provider, model, cooldownMs, options.reason || "problematic_model");
|
||||
return { cooldownMs, failureCount };
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear unavailability for a model (e.g. after manual reset).
|
||||
*
|
||||
@@ -79,7 +143,9 @@ export function setModelUnavailable(provider, model, cooldownMs = 60000, reason)
|
||||
* @returns {boolean} true if entry existed and was removed
|
||||
*/
|
||||
export function clearModelUnavailability(provider, model) {
|
||||
return unavailable.delete(makeKey(provider, model));
|
||||
const key = makeKey(provider, model);
|
||||
failureState.delete(key);
|
||||
return unavailable.delete(key);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -125,4 +191,5 @@ export function getUnavailableCount() {
|
||||
*/
|
||||
export function resetAllAvailability() {
|
||||
unavailable.clear();
|
||||
failureState.clear();
|
||||
}
|
||||
|
||||
@@ -10,7 +10,12 @@
|
||||
* updates the DB, and logs the result.
|
||||
*/
|
||||
|
||||
import { getProviderConnections, updateProviderConnection, getSettings, resolveProxyForConnection } from "@/lib/localDb";
|
||||
import {
|
||||
getProviderConnections,
|
||||
updateProviderConnection,
|
||||
getSettings,
|
||||
resolveProxyForConnection,
|
||||
} from "@/lib/localDb";
|
||||
import {
|
||||
getAccessToken,
|
||||
supportsTokenRefresh,
|
||||
@@ -25,6 +30,25 @@ const EXPIRED_RETRY_BACKOFF_MIN = 5; // backoff between expired retries (minutes
|
||||
const LOG_PREFIX = "[HealthCheck]";
|
||||
const TRUE_ENV_VALUES = new Set(["1", "true", "yes", "on"]);
|
||||
|
||||
export function buildRefreshFailureUpdate(conn: any, now: string) {
|
||||
const wasExpired = conn.testStatus === "expired";
|
||||
const retryCount = (conn.expiredRetryCount ?? 0) + (wasExpired ? 1 : 0);
|
||||
|
||||
return {
|
||||
lastHealthCheckAt: now,
|
||||
// A failed background refresh should not evict otherwise healthy accounts
|
||||
// from request routing. Keep non-expired connections active and only persist
|
||||
// the refresh error metadata for observability.
|
||||
testStatus: wasExpired ? "expired" : "active",
|
||||
lastError: "Health check: token refresh failed",
|
||||
lastErrorAt: now,
|
||||
lastErrorType: "token_refresh_failed",
|
||||
lastErrorSource: "oauth",
|
||||
errorCode: "refresh_failed",
|
||||
...(wasExpired ? { expiredRetryCount: retryCount, expiredRetryAt: now } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function isEnvFlagEnabled(name: string): boolean {
|
||||
const value = process.env[name];
|
||||
if (!value) return false;
|
||||
@@ -304,22 +328,13 @@ async function checkConnection(conn) {
|
||||
await updateProviderConnection(conn.id, updateData);
|
||||
log(`${LOG_PREFIX} ✓ ${conn.provider}/${conn.name || conn.email || conn.id} refreshed`);
|
||||
} else {
|
||||
const wasExpired = conn.testStatus === "expired";
|
||||
const retryCount = (conn.expiredRetryCount ?? 0) + (wasExpired ? 1 : 0);
|
||||
|
||||
await updateProviderConnection(conn.id, {
|
||||
lastHealthCheckAt: now,
|
||||
testStatus: wasExpired ? "expired" : "error",
|
||||
lastError: "Health check: token refresh failed",
|
||||
lastErrorAt: now,
|
||||
lastErrorType: "token_refresh_failed",
|
||||
lastErrorSource: "oauth",
|
||||
errorCode: "refresh_failed",
|
||||
...(wasExpired ? { expiredRetryCount: retryCount, expiredRetryAt: now } : {}),
|
||||
});
|
||||
const updateData = buildRefreshFailureUpdate(conn, now);
|
||||
await updateProviderConnection(conn.id, updateData);
|
||||
logWarn(
|
||||
`${LOG_PREFIX} ✗ ${conn.provider}/${conn.name || conn.email || conn.id} refresh failed` +
|
||||
(wasExpired ? ` (expired retry ${retryCount}/${EXPIRED_RETRY_MAX})` : "")
|
||||
(conn.testStatus === "expired"
|
||||
? ` (${updateData.expiredRetryCount}/${EXPIRED_RETRY_MAX} expired retries used)`
|
||||
: "")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import { CardSkeleton } from "./Loading";
|
||||
import { fmtCompact as fmt, fmtFull, fmtCost } from "@/shared/utils/formatting";
|
||||
import {
|
||||
StatCard,
|
||||
CompactStatGrid,
|
||||
ActivityHeatmap,
|
||||
DailyTrendChart,
|
||||
AccountDonut,
|
||||
@@ -109,7 +110,7 @@ export default function UsageAnalytics() {
|
||||
const ioRatio = s.completionTokens > 0 ? (s.promptTokens / s.completionTokens).toFixed(1) : "—";
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-5">
|
||||
<div className="flex flex-col gap-5 min-w-0">
|
||||
{/* Header + Time Range */}
|
||||
<div className="flex items-center justify-between">
|
||||
<h2 className="text-xl font-semibold flex items-center gap-2">
|
||||
@@ -133,8 +134,8 @@ export default function UsageAnalytics() {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Summary Cards — Row 1: Core metrics */}
|
||||
<div className="grid grid-cols-2 md:grid-cols-8 gap-3">
|
||||
{/* Primary KPI Cards */}
|
||||
<div className="grid grid-cols-2 md:grid-cols-4 gap-3">
|
||||
<StatCard
|
||||
icon="generating_tokens"
|
||||
label="Total Tokens"
|
||||
@@ -159,56 +160,71 @@ export default function UsageAnalytics() {
|
||||
value={fmtCost(s.totalCost)}
|
||||
color="text-amber-500"
|
||||
/>
|
||||
<StatCard icon="group" label="Accounts" value={s.uniqueAccounts || 0} />
|
||||
<StatCard icon="vpn_key" label="API Keys" value={s.uniqueApiKeys || 0} />
|
||||
<StatCard icon="model_training" label="Models" value={s.uniqueModels || 0} />
|
||||
<StatCard
|
||||
icon="swap_horiz"
|
||||
label="Fallback Rate"
|
||||
value={`${Number(s.fallbackRatePct || 0).toFixed(1)}%`}
|
||||
subValue={`${fmtFull(s.fallbackCount || 0)} fallbacks`}
|
||||
color="text-amber-500"
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Summary Cards — Row 2: Derived insights */}
|
||||
<div className="grid grid-cols-2 md:grid-cols-8 gap-3">
|
||||
<StatCard
|
||||
icon="speed"
|
||||
label="Avg Tokens/Req"
|
||||
value={fmt(avgTokensPerReq)}
|
||||
color="text-cyan-500"
|
||||
/>
|
||||
<StatCard
|
||||
icon="request_quote"
|
||||
label="Cost/Request"
|
||||
value={fmtCost(costPerReq)}
|
||||
color="text-orange-500"
|
||||
/>
|
||||
<StatCard
|
||||
icon="compare_arrows"
|
||||
label="I/O Ratio"
|
||||
value={`${ioRatio}x`}
|
||||
color="text-violet-500"
|
||||
/>
|
||||
<StatCard icon="star" label="Top Model" value={topModel} color="text-pink-500" />
|
||||
<StatCard icon="cloud" label="Top Provider" value={topProvider} color="text-teal-500" />
|
||||
<StatCard icon="today" label="Busiest Day" value={busiestDay} color="text-rose-500" />
|
||||
<StatCard icon="dns" label="Providers" value={providerCount} color="text-indigo-500" />
|
||||
<StatCard
|
||||
icon="network_node"
|
||||
label="Diversity Score"
|
||||
value={`${providerDiversity.toFixed(1)}%`}
|
||||
color="text-sky-500"
|
||||
/>
|
||||
</div>
|
||||
{/* Secondary Metrics — compact grid with sections */}
|
||||
<CompactStatGrid
|
||||
sections={[
|
||||
{
|
||||
title: "Infrastructure",
|
||||
items: [
|
||||
{ icon: "group", label: "Accounts", value: s.uniqueAccounts || 0 },
|
||||
{ icon: "dns", label: "Providers", value: providerCount, color: "text-indigo-500" },
|
||||
{ icon: "vpn_key", label: "API Keys", value: s.uniqueApiKeys || 0 },
|
||||
{ icon: "model_training", label: "Models", value: s.uniqueModels || 0 },
|
||||
],
|
||||
},
|
||||
{
|
||||
title: "Performance",
|
||||
items: [
|
||||
{
|
||||
icon: "speed",
|
||||
label: "Avg Tokens/Req",
|
||||
value: fmt(avgTokensPerReq),
|
||||
color: "text-cyan-500",
|
||||
},
|
||||
{
|
||||
icon: "request_quote",
|
||||
label: "Cost/Req",
|
||||
value: fmtCost(costPerReq),
|
||||
color: "text-orange-500",
|
||||
},
|
||||
{
|
||||
icon: "compare_arrows",
|
||||
label: "I/O Ratio",
|
||||
value: `${ioRatio}x`,
|
||||
color: "text-violet-500",
|
||||
},
|
||||
{
|
||||
icon: "swap_horiz",
|
||||
label: "Fallback Rate",
|
||||
value: `${Number(s.fallbackRatePct || 0).toFixed(1)}%`,
|
||||
color: "text-amber-500",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
title: "Highlights",
|
||||
wideValues: true,
|
||||
items: [
|
||||
{ icon: "star", label: "Top Model", value: topModel, color: "text-pink-500" },
|
||||
{ icon: "cloud", label: "Top Provider", value: topProvider, color: "text-teal-500" },
|
||||
{ icon: "today", label: "Busiest Day", value: busiestDay, color: "text-rose-500" },
|
||||
{
|
||||
icon: "network_node",
|
||||
label: "Diversity",
|
||||
value: `${providerDiversity.toFixed(1)}%`,
|
||||
color: "text-sky-500",
|
||||
},
|
||||
],
|
||||
},
|
||||
]}
|
||||
/>
|
||||
|
||||
{/* Activity Heatmap + Weekly Widgets */}
|
||||
<div
|
||||
style={{ display: "grid", gridTemplateColumns: "2fr 1fr", gap: 16, alignItems: "stretch" }}
|
||||
>
|
||||
<div className="grid grid-cols-1 lg:grid-cols-[2fr_1fr] gap-4 items-stretch">
|
||||
<ActivityHeatmap activityMap={analytics?.activityMap} />
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 16 }}>
|
||||
<div className="flex flex-col gap-4">
|
||||
<MostActiveDay7d activityMap={analytics?.activityMap} />
|
||||
<WeeklySquares7d activityMap={analytics?.activityMap} />
|
||||
</div>
|
||||
|
||||
@@ -84,7 +84,7 @@ export function SortIndicator({ active, sortOrder }: { active: boolean; sortOrde
|
||||
);
|
||||
}
|
||||
|
||||
// ── StatCard ───────────────────────────────────────────────────────────────
|
||||
// ── StatCard (primary KPI) ─────────────────────────────────────────────────
|
||||
|
||||
export function StatCard({
|
||||
icon,
|
||||
@@ -100,13 +100,73 @@ export function StatCard({
|
||||
color?: string;
|
||||
}) {
|
||||
return (
|
||||
<Card className="px-4 py-3 flex flex-col gap-1">
|
||||
<div className="flex items-center gap-2 text-text-muted text-xs uppercase font-semibold tracking-wider">
|
||||
<span className="material-symbols-outlined text-[16px]">{icon}</span>
|
||||
{label}
|
||||
<Card className="px-4 py-3 flex flex-col gap-1 min-w-0">
|
||||
<div className="flex items-center gap-1.5 text-text-muted text-[11px] uppercase font-semibold tracking-wide min-w-0">
|
||||
<span className="material-symbols-outlined text-[14px] shrink-0">{icon}</span>
|
||||
<span className="truncate">{label}</span>
|
||||
</div>
|
||||
<span className={`text-2xl font-bold ${color} truncate`} title={String(value)}>
|
||||
{value}
|
||||
</span>
|
||||
{subValue && <span className="text-xs text-text-muted truncate">{subValue}</span>}
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
|
||||
// ── CompactStatGrid (secondary metrics in a single card, grouped) ─────────
|
||||
|
||||
export type CompactStatSection = {
|
||||
title: string;
|
||||
items: Array<{ icon: string; label: string; value: any; color?: string }>;
|
||||
/** On mobile use 1 column instead of 2 — useful when values can be long (model names, etc.) */
|
||||
wideValues?: boolean;
|
||||
};
|
||||
|
||||
export function CompactStatGrid({ sections }: { sections: CompactStatSection[] }) {
|
||||
return (
|
||||
<Card className="px-5 py-4">
|
||||
<div className="flex flex-col gap-3">
|
||||
{sections.map((section, si) => (
|
||||
<div key={si}>
|
||||
{si > 0 && (
|
||||
<div className="border-t border-black/[0.06] dark:border-white/[0.06] mb-3" />
|
||||
)}
|
||||
<div className="text-[10px] uppercase font-semibold tracking-widest text-text-muted/50 mb-2">
|
||||
{section.title}
|
||||
</div>
|
||||
<div
|
||||
className={
|
||||
section.wideValues
|
||||
? "grid grid-cols-1 sm:grid-cols-2 md:grid-cols-4 gap-x-5 gap-y-2"
|
||||
: "grid grid-cols-2 md:grid-cols-4 gap-x-5 gap-y-2"
|
||||
}
|
||||
>
|
||||
{section.items.map((stat, i) => (
|
||||
<div key={i} className="flex items-center justify-between gap-2 min-w-0 py-0.5">
|
||||
<div
|
||||
className={`flex items-center gap-1.5 ${section.wideValues ? "shrink-0" : "min-w-0"}`}
|
||||
>
|
||||
<span className="material-symbols-outlined text-[14px] text-text-muted shrink-0">
|
||||
{stat.icon}
|
||||
</span>
|
||||
<span
|
||||
className={`text-[11px] uppercase font-semibold tracking-wide text-text-muted ${section.wideValues ? "whitespace-nowrap" : "truncate"}`}
|
||||
>
|
||||
{stat.label}
|
||||
</span>
|
||||
</div>
|
||||
<span
|
||||
className={`text-sm font-bold text-right ${section.wideValues ? "truncate min-w-0" : "shrink-0"} ${stat.color || "text-text-main"}`}
|
||||
title={String(stat.value)}
|
||||
>
|
||||
{stat.value}
|
||||
</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
<span className={`text-2xl font-bold ${color}`}>{value}</span>
|
||||
{subValue && <span className="text-xs text-text-muted">{subValue}</span>}
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
@@ -189,7 +249,7 @@ export function ActivityHeatmap({ activityMap }) {
|
||||
}
|
||||
|
||||
return (
|
||||
<Card className="p-4 h-full">
|
||||
<Card className="p-4 h-full min-w-0 overflow-hidden">
|
||||
<div className="flex items-center justify-between mb-3">
|
||||
<h3 className="text-sm font-semibold text-text-muted uppercase tracking-wider">Activity</h3>
|
||||
<span className="text-xs text-text-muted">
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
export {
|
||||
SortIndicator,
|
||||
StatCard,
|
||||
CompactStatGrid,
|
||||
ActivityHeatmap,
|
||||
DailyTrendChart,
|
||||
AccountDonut,
|
||||
|
||||
@@ -35,7 +35,7 @@ import {
|
||||
import { getCircuitBreaker } from "../../shared/utils/circuitBreaker";
|
||||
import {
|
||||
isModelAvailable,
|
||||
setModelUnavailable,
|
||||
markModelAsProblematic,
|
||||
clearModelUnavailability,
|
||||
} from "../../domain/modelAvailability";
|
||||
import { markAccountExhaustedFrom429 } from "../../domain/quotaCache";
|
||||
@@ -468,6 +468,7 @@ async function handleSingleModelChat(
|
||||
let excludeConnectionId = null;
|
||||
let lastError = null;
|
||||
let lastStatus = null;
|
||||
let lastCooldownMs = 0;
|
||||
|
||||
while (true) {
|
||||
const credentials = await getProviderCredentials(
|
||||
@@ -484,11 +485,15 @@ async function handleSingleModelChat(
|
||||
);
|
||||
|
||||
if (!credentials || credentials.allRateLimited) {
|
||||
if (lastStatus === 429 || lastStatus === 503) {
|
||||
setModelUnavailable(provider, model, 60000, `HTTP ${lastStatus}`);
|
||||
if ([408, 429, 500, 502, 503, 504].includes(Number(lastStatus))) {
|
||||
const quarantine = markModelAsProblematic(provider, model, {
|
||||
status: Number(lastStatus),
|
||||
baseCooldownMs: lastCooldownMs,
|
||||
reason: `HTTP ${lastStatus}`,
|
||||
});
|
||||
log.info(
|
||||
"AVAILABILITY",
|
||||
`${provider}/${model} marked unavailable — all accounts exhausted (HTTP ${lastStatus})`
|
||||
`${provider}/${model} marked unavailable — all accounts exhausted (HTTP ${lastStatus}, cooldown ${Math.ceil(quarantine.cooldownMs / 1000)}s, failureCount ${quarantine.failureCount})`
|
||||
);
|
||||
}
|
||||
return handleNoCredentials(
|
||||
@@ -654,7 +659,7 @@ async function handleSingleModelChat(
|
||||
}
|
||||
|
||||
// 7. Fallback to next account
|
||||
const { shouldFallback } = await markAccountUnavailable(
|
||||
const { shouldFallback, cooldownMs } = await markAccountUnavailable(
|
||||
credentials.connectionId,
|
||||
result.status,
|
||||
result.error,
|
||||
@@ -663,6 +668,9 @@ async function handleSingleModelChat(
|
||||
);
|
||||
|
||||
if (shouldFallback) {
|
||||
if (Number.isFinite(cooldownMs) && cooldownMs > 0) {
|
||||
lastCooldownMs = cooldownMs;
|
||||
}
|
||||
log.warn("AUTH", `Account ${accountId}... unavailable (${result.status}), trying fallback`);
|
||||
excludeConnectionId = credentials.connectionId;
|
||||
lastError = result.error;
|
||||
|
||||
@@ -13,6 +13,7 @@ import assert from "node:assert/strict";
|
||||
import {
|
||||
isModelAvailable,
|
||||
setModelUnavailable,
|
||||
markModelAsProblematic,
|
||||
clearModelUnavailability,
|
||||
getAvailabilityReport,
|
||||
getUnavailableCount,
|
||||
@@ -58,6 +59,41 @@ describe("modelAvailability", () => {
|
||||
while (Date.now() - start < 5) {} // spin wait
|
||||
assert.equal(isModelAvailable("anthropic", "claude-sonnet-4-20250514"), true);
|
||||
});
|
||||
|
||||
it("should apply adaptive quarantine for repeated problematic failures", () => {
|
||||
const first = markModelAsProblematic("nvidia", "z-ai/glm4.7", {
|
||||
status: 502,
|
||||
baseCooldownMs: 3000,
|
||||
reason: "HTTP 502",
|
||||
});
|
||||
const second = markModelAsProblematic("nvidia", "z-ai/glm4.7", {
|
||||
status: 502,
|
||||
baseCooldownMs: 3000,
|
||||
reason: "HTTP 502",
|
||||
});
|
||||
|
||||
assert.equal(first.failureCount, 1);
|
||||
assert.equal(first.cooldownMs, 120000);
|
||||
assert.equal(second.failureCount, 2);
|
||||
assert.equal(second.cooldownMs, 240000);
|
||||
|
||||
const report = getAvailabilityReport();
|
||||
const entry = report.find((r) => r.provider === "nvidia" && r.model === "z-ai/glm4.7");
|
||||
assert.ok(entry, "nvidia model should be quarantined");
|
||||
assert.ok(entry.remainingMs >= 200000, "cooldown should be preserved/escalated");
|
||||
});
|
||||
|
||||
it("should reset failure history after model recovery", () => {
|
||||
clearModelUnavailability("nvidia", "z-ai/glm4.7");
|
||||
const afterReset = markModelAsProblematic("nvidia", "z-ai/glm4.7", {
|
||||
status: 502,
|
||||
baseCooldownMs: 3000,
|
||||
reason: "HTTP 502",
|
||||
});
|
||||
|
||||
assert.equal(afterReset.failureCount, 1);
|
||||
assert.equal(afterReset.cooldownMs, 120000);
|
||||
});
|
||||
});
|
||||
|
||||
// ──────────────── T-19: Cost Rules ────────────────
|
||||
|
||||
43
tests/unit/token-health-check.test.mjs
Normal file
43
tests/unit/token-health-check.test.mjs
Normal file
@@ -0,0 +1,43 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
process.env.NODE_ENV = "test";
|
||||
|
||||
const tokenHealthCheck = await import("../../src/lib/tokenHealthCheck.ts");
|
||||
|
||||
test("buildRefreshFailureUpdate keeps active connections routable after refresh failure", () => {
|
||||
const now = "2026-04-09T04:40:00.000Z";
|
||||
|
||||
const update = tokenHealthCheck.buildRefreshFailureUpdate(
|
||||
{
|
||||
testStatus: "active",
|
||||
expiredRetryCount: 2,
|
||||
},
|
||||
now
|
||||
);
|
||||
|
||||
assert.equal(update.testStatus, "active");
|
||||
assert.equal(update.lastError, "Health check: token refresh failed");
|
||||
assert.equal(update.lastErrorType, "token_refresh_failed");
|
||||
assert.equal(update.lastErrorSource, "oauth");
|
||||
assert.equal(update.errorCode, "refresh_failed");
|
||||
assert.equal(update.lastHealthCheckAt, now);
|
||||
assert.equal("expiredRetryCount" in update, false);
|
||||
assert.equal("expiredRetryAt" in update, false);
|
||||
});
|
||||
|
||||
test("buildRefreshFailureUpdate preserves expired retry tracking", () => {
|
||||
const now = "2026-04-09T04:41:00.000Z";
|
||||
|
||||
const update = tokenHealthCheck.buildRefreshFailureUpdate(
|
||||
{
|
||||
testStatus: "expired",
|
||||
expiredRetryCount: 2,
|
||||
},
|
||||
now
|
||||
);
|
||||
|
||||
assert.equal(update.testStatus, "expired");
|
||||
assert.equal(update.expiredRetryCount, 3);
|
||||
assert.equal(update.expiredRetryAt, now);
|
||||
});
|
||||
Reference in New Issue
Block a user