diff --git a/package-lock.json b/package-lock.json index 86659d40a2..942427e486 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "omniroute", - "version": "3.5.7", + "version": "3.5.8", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "omniroute", - "version": "3.5.7", + "version": "3.5.8", "hasInstallScript": true, "license": "MIT", "workspaces": [ @@ -17,6 +17,7 @@ "@modelcontextprotocol/sdk": "^1.27.1", "@monaco-editor/react": "^4.7.0", "@swc/helpers": "0.5.21", + "axios": "^1.15.0", "bcryptjs": "^3.0.3", "better-sqlite3": "^12.6.2", "bottleneck": "^2.19.5", @@ -7782,7 +7783,6 @@ "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "dev": true, "license": "MIT" }, "node_modules/atomic-sleep": { @@ -7831,10 +7831,9 @@ } }, "node_modules/axios": { - "version": "1.14.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.14.0.tgz", - "integrity": "sha512-3Y8yrqLSwjuzpXuZ0oIYZ/XGgLwUIBU3uLvbcpb0pidD9ctpShJd43KSlEEkVQg6DS0G9NKyzOvBfUtDKEyHvQ==", - "dev": true, + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.15.0.tgz", + "integrity": "sha512-wWyJDlAatxk30ZJer+GeCWS209sA42X+N5jU2jy6oHTp7ufw8uzUTVFBX9+wTfAlhiJXGS0Bq7X6efruWjuK9Q==", "license": "MIT", "dependencies": { "follow-redirects": "^1.15.11", @@ -8575,7 +8574,6 @@ "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "dev": true, "license": "MIT", "dependencies": { "delayed-stream": "~1.0.0" @@ -9627,7 +9625,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.4.0" @@ -9959,7 +9956,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", - "dev": true, "license": "MIT", "dependencies": { "es-errors": "^1.3.0", @@ -11134,7 +11130,6 @@ "version": "4.0.5", "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", - "dev": true, "license": "MIT", "dependencies": { "asynckit": "^0.4.0", @@ -11151,7 +11146,6 @@ "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.6" @@ -11161,7 +11155,6 @@ "version": "2.1.35", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "dev": true, "license": "MIT", "dependencies": { "mime-db": "1.52.0" @@ -11605,7 +11598,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", - "dev": true, "license": "MIT", "dependencies": { "has-symbols": "^1.0.3" @@ -16671,7 +16663,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-2.1.0.tgz", "integrity": "sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==", - "dev": true, "license": "MIT", "engines": { "node": ">=10" diff --git a/package.json b/package.json index c103af4c89..abf1113300 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "omniroute", - "version": "3.5.7", + "version": "3.5.8", "description": "Smart AI Router with auto fallback — route to FREE & cheap models, zero downtime. Works with Cursor, Cline, Claude Desktop, Codex, and any OpenAI-compatible tool.", "type": "module", "bin": { @@ -93,6 +93,7 @@ "@modelcontextprotocol/sdk": "^1.27.1", "@monaco-editor/react": "^4.7.0", "@swc/helpers": "0.5.21", + "axios": "^1.15.0", "bcryptjs": "^3.0.3", "better-sqlite3": "^12.6.2", "bottleneck": "^2.19.5", diff --git a/src/app/(dashboard)/dashboard/analytics/components/DiversityScoreCard.tsx b/src/app/(dashboard)/dashboard/analytics/components/DiversityScoreCard.tsx index ffddf3fab2..3d21b80b41 100644 --- a/src/app/(dashboard)/dashboard/analytics/components/DiversityScoreCard.tsx +++ b/src/app/(dashboard)/dashboard/analytics/components/DiversityScoreCard.tsx @@ -72,33 +72,43 @@ export default function DiversityScoreCard() { riskLabel = "Moderate Distribution"; } + const providerEntries = Object.entries(data.providers || {}) + .sort(([, a], [, b]) => b.share - a.share) + .slice(0, 4); + return ( - -
-
-
- pie_chart -

Provider Diversity

-
-

- Provider concentration snapshot for the recent traffic window. -

+ + {/* Header */} +
+
+ pie_chart +

Provider Diversity

+ + — Provider concentration snapshot for the recent traffic window. + +
+
+ + Window: {data.windowSize} reqs · Last {Math.round(data.ttlMs / 60000)} mins + + + Shannon entropy +
- - Shannon entropy -
-
-
+ {/* Horizontal layout: gauge + risk label + provider bars */} +
+ {/* Gauge */} +
- + {scorePercentage}% - score + score
-
-
-
{riskLabel}
-
- Higher values indicate traffic is spread across multiple providers instead of - clustering on one vendor. -
+ {/* Risk label */} +
+
{riskLabel}
+
+ Higher values mean traffic is spread across multiple providers.
+
- {Object.keys(data.providers || {}).length === 0 ? ( -
+ {/* Divider */} +
+ + {/* Provider bars */} +
+ {providerEntries.length === 0 ? ( +
No recent usage data available.
) : ( -
- {Object.entries(data.providers) - .sort(([, a], [, b]) => b.share - a.share) - .slice(0, 4) - .map(([provider, stat]) => ( -
-
- {provider} - - {Math.round(stat.share * 100)}% - -
-
-
-
+
+ {providerEntries.map(([provider, stat]) => ( +
+
+ + {provider} + + + {Math.round(stat.share * 100)}% +
- ))} +
+
+
+
+ ))}
)}
- -
-
Window: {data.windowSize} reqs
-
- Based on last {Math.round(data.ttlMs / 60000)} mins -
-
); } diff --git a/src/app/(dashboard)/dashboard/analytics/page.tsx b/src/app/(dashboard)/dashboard/analytics/page.tsx index 2f7a2006ad..ba2022064f 100644 --- a/src/app/(dashboard)/dashboard/analytics/page.tsx +++ b/src/app/(dashboard)/dashboard/analytics/page.tsx @@ -45,12 +45,12 @@ export default function AnalyticsPage() { /> {activeTab === "overview" && ( -
+ <> }> -
+ )} {activeTab === "evals" && } {activeTab === "search" && } diff --git a/src/domain/modelAvailability.ts b/src/domain/modelAvailability.ts index e9a266e4a3..d4cfedd2f6 100644 --- a/src/domain/modelAvailability.ts +++ b/src/domain/modelAvailability.ts @@ -21,6 +21,23 @@ /** @type {Map} */ const unavailable = new Map(); +/** @type {Map} */ +const failureState = new Map(); + +const FAILURE_WINDOW_MS = 30 * 60 * 1000; + +const PROBLEMATIC_STATUS_COOLDOWNS = { + 429: 5 * 60 * 1000, + 408: 60 * 1000, + 500: 2 * 60 * 1000, + 502: 2 * 60 * 1000, + 503: 2 * 60 * 1000, + 504: 2 * 60 * 1000, +}; + +const MIN_PROBLEMATIC_COOLDOWN_MS = 60 * 1000; +const MAX_PROBLEMATIC_COOLDOWN_MS = 30 * 60 * 1000; + /** * Build a composite key for provider+model. * @param {string} provider @@ -62,15 +79,62 @@ export function isModelAvailable(provider, model) { */ export function setModelUnavailable(provider, model, cooldownMs = 60000, reason) { const key = makeKey(provider, model); + const now = Date.now(); + const safeCooldownMs = Number.isFinite(cooldownMs) && cooldownMs > 0 ? cooldownMs : 60000; + const existing = unavailable.get(key); + const existingRemainingMs = + existing && Date.now() - existing.unavailableSince < existing.cooldownMs + ? existing.cooldownMs - (Date.now() - existing.unavailableSince) + : 0; + const effectiveCooldownMs = Math.max(safeCooldownMs, existingRemainingMs); + unavailable.set(key, { provider, model, - unavailableSince: Date.now(), - cooldownMs, + unavailableSince: now, + cooldownMs: effectiveCooldownMs, reason: reason || "unknown", }); } +/** + * Marca provider/model como problemático com cooldown adaptativo. + * Mantém retrocompatibilidade: não altera o comportamento de setModelUnavailable, + * apenas oferece uma estratégia mais agressiva para falhas recorrentes. + * + * @param {string} provider + * @param {string} model + * @param {{ status?: number, baseCooldownMs?: number, reason?: string }} [options] + * @returns {{ cooldownMs: number, failureCount: number }} + */ +export function markModelAsProblematic(provider, model, options = {}) { + const key = makeKey(provider, model); + const now = Date.now(); + const status = Number.isFinite(options.status) ? Number(options.status) : null; + const statusBaseCooldown = + status && Object.prototype.hasOwnProperty.call(PROBLEMATIC_STATUS_COOLDOWNS, status) + ? PROBLEMATIC_STATUS_COOLDOWNS[status] + : 0; + const baseCooldownMs = + Number.isFinite(options.baseCooldownMs) && Number(options.baseCooldownMs) > 0 + ? Number(options.baseCooldownMs) + : 0; + + const prev = failureState.get(key); + const withinFailureWindow = prev && now - prev.lastFailureAt <= FAILURE_WINDOW_MS; + const failureCount = withinFailureWindow ? prev.failureCount + 1 : 1; + failureState.set(key, { failureCount, lastFailureAt: now }); + + const cooldownBase = Math.max(baseCooldownMs, statusBaseCooldown, MIN_PROBLEMATIC_COOLDOWN_MS); + const cooldownMs = Math.min( + cooldownBase * Math.pow(2, Math.max(0, failureCount - 1)), + MAX_PROBLEMATIC_COOLDOWN_MS + ); + + setModelUnavailable(provider, model, cooldownMs, options.reason || "problematic_model"); + return { cooldownMs, failureCount }; +} + /** * Clear unavailability for a model (e.g. after manual reset). * @@ -79,7 +143,9 @@ export function setModelUnavailable(provider, model, cooldownMs = 60000, reason) * @returns {boolean} true if entry existed and was removed */ export function clearModelUnavailability(provider, model) { - return unavailable.delete(makeKey(provider, model)); + const key = makeKey(provider, model); + failureState.delete(key); + return unavailable.delete(key); } /** @@ -125,4 +191,5 @@ export function getUnavailableCount() { */ export function resetAllAvailability() { unavailable.clear(); + failureState.clear(); } diff --git a/src/lib/tokenHealthCheck.ts b/src/lib/tokenHealthCheck.ts index f265182205..6557728b18 100644 --- a/src/lib/tokenHealthCheck.ts +++ b/src/lib/tokenHealthCheck.ts @@ -10,7 +10,12 @@ * updates the DB, and logs the result. */ -import { getProviderConnections, updateProviderConnection, getSettings, resolveProxyForConnection } from "@/lib/localDb"; +import { + getProviderConnections, + updateProviderConnection, + getSettings, + resolveProxyForConnection, +} from "@/lib/localDb"; import { getAccessToken, supportsTokenRefresh, @@ -25,6 +30,25 @@ const EXPIRED_RETRY_BACKOFF_MIN = 5; // backoff between expired retries (minutes const LOG_PREFIX = "[HealthCheck]"; const TRUE_ENV_VALUES = new Set(["1", "true", "yes", "on"]); +export function buildRefreshFailureUpdate(conn: any, now: string) { + const wasExpired = conn.testStatus === "expired"; + const retryCount = (conn.expiredRetryCount ?? 0) + (wasExpired ? 1 : 0); + + return { + lastHealthCheckAt: now, + // A failed background refresh should not evict otherwise healthy accounts + // from request routing. Keep non-expired connections active and only persist + // the refresh error metadata for observability. + testStatus: wasExpired ? "expired" : "active", + lastError: "Health check: token refresh failed", + lastErrorAt: now, + lastErrorType: "token_refresh_failed", + lastErrorSource: "oauth", + errorCode: "refresh_failed", + ...(wasExpired ? { expiredRetryCount: retryCount, expiredRetryAt: now } : {}), + }; +} + function isEnvFlagEnabled(name: string): boolean { const value = process.env[name]; if (!value) return false; @@ -304,22 +328,13 @@ async function checkConnection(conn) { await updateProviderConnection(conn.id, updateData); log(`${LOG_PREFIX} ✓ ${conn.provider}/${conn.name || conn.email || conn.id} refreshed`); } else { - const wasExpired = conn.testStatus === "expired"; - const retryCount = (conn.expiredRetryCount ?? 0) + (wasExpired ? 1 : 0); - - await updateProviderConnection(conn.id, { - lastHealthCheckAt: now, - testStatus: wasExpired ? "expired" : "error", - lastError: "Health check: token refresh failed", - lastErrorAt: now, - lastErrorType: "token_refresh_failed", - lastErrorSource: "oauth", - errorCode: "refresh_failed", - ...(wasExpired ? { expiredRetryCount: retryCount, expiredRetryAt: now } : {}), - }); + const updateData = buildRefreshFailureUpdate(conn, now); + await updateProviderConnection(conn.id, updateData); logWarn( `${LOG_PREFIX} ✗ ${conn.provider}/${conn.name || conn.email || conn.id} refresh failed` + - (wasExpired ? ` (expired retry ${retryCount}/${EXPIRED_RETRY_MAX})` : "") + (conn.testStatus === "expired" + ? ` (${updateData.expiredRetryCount}/${EXPIRED_RETRY_MAX} expired retries used)` + : "") ); } } diff --git a/src/shared/components/UsageAnalytics.tsx b/src/shared/components/UsageAnalytics.tsx index 96f0103745..0d699358a8 100644 --- a/src/shared/components/UsageAnalytics.tsx +++ b/src/shared/components/UsageAnalytics.tsx @@ -6,6 +6,7 @@ import { CardSkeleton } from "./Loading"; import { fmtCompact as fmt, fmtFull, fmtCost } from "@/shared/utils/formatting"; import { StatCard, + CompactStatGrid, ActivityHeatmap, DailyTrendChart, AccountDonut, @@ -109,7 +110,7 @@ export default function UsageAnalytics() { const ioRatio = s.completionTokens > 0 ? (s.promptTokens / s.completionTokens).toFixed(1) : "—"; return ( -
+
{/* Header + Time Range */}

@@ -133,8 +134,8 @@ export default function UsageAnalytics() {

- {/* Summary Cards — Row 1: Core metrics */} -
+ {/* Primary KPI Cards */} +
- - - -
- {/* Summary Cards — Row 2: Derived insights */} -
- - - - - - - - -
+ {/* Secondary Metrics — compact grid with sections */} + {/* Activity Heatmap + Weekly Widgets */} -
+
-
+
diff --git a/src/shared/components/analytics/charts.tsx b/src/shared/components/analytics/charts.tsx index bb8aa09484..f1968951d2 100644 --- a/src/shared/components/analytics/charts.tsx +++ b/src/shared/components/analytics/charts.tsx @@ -84,7 +84,7 @@ export function SortIndicator({ active, sortOrder }: { active: boolean; sortOrde ); } -// ── StatCard ─────────────────────────────────────────────────────────────── +// ── StatCard (primary KPI) ───────────────────────────────────────────────── export function StatCard({ icon, @@ -100,13 +100,73 @@ export function StatCard({ color?: string; }) { return ( - -
- {icon} - {label} + +
+ {icon} + {label} +
+ + {value} + + {subValue && {subValue}} + + ); +} + +// ── CompactStatGrid (secondary metrics in a single card, grouped) ───────── + +export type CompactStatSection = { + title: string; + items: Array<{ icon: string; label: string; value: any; color?: string }>; + /** On mobile use 1 column instead of 2 — useful when values can be long (model names, etc.) */ + wideValues?: boolean; +}; + +export function CompactStatGrid({ sections }: { sections: CompactStatSection[] }) { + return ( + +
+ {sections.map((section, si) => ( +
+ {si > 0 && ( +
+ )} +
+ {section.title} +
+
+ {section.items.map((stat, i) => ( +
+
+ + {stat.icon} + + + {stat.label} + +
+ + {stat.value} + +
+ ))} +
+
+ ))}
- {value} - {subValue && {subValue}} ); } @@ -189,7 +249,7 @@ export function ActivityHeatmap({ activityMap }) { } return ( - +

Activity

diff --git a/src/shared/components/analytics/index.tsx b/src/shared/components/analytics/index.tsx index d08a5d778b..ba28edafef 100644 --- a/src/shared/components/analytics/index.tsx +++ b/src/shared/components/analytics/index.tsx @@ -11,6 +11,7 @@ export { SortIndicator, StatCard, + CompactStatGrid, ActivityHeatmap, DailyTrendChart, AccountDonut, diff --git a/src/sse/handlers/chat.ts b/src/sse/handlers/chat.ts index 9fa80b8c9d..42171c063d 100644 --- a/src/sse/handlers/chat.ts +++ b/src/sse/handlers/chat.ts @@ -35,7 +35,7 @@ import { import { getCircuitBreaker } from "../../shared/utils/circuitBreaker"; import { isModelAvailable, - setModelUnavailable, + markModelAsProblematic, clearModelUnavailability, } from "../../domain/modelAvailability"; import { markAccountExhaustedFrom429 } from "../../domain/quotaCache"; @@ -468,6 +468,7 @@ async function handleSingleModelChat( let excludeConnectionId = null; let lastError = null; let lastStatus = null; + let lastCooldownMs = 0; while (true) { const credentials = await getProviderCredentials( @@ -484,11 +485,15 @@ async function handleSingleModelChat( ); if (!credentials || credentials.allRateLimited) { - if (lastStatus === 429 || lastStatus === 503) { - setModelUnavailable(provider, model, 60000, `HTTP ${lastStatus}`); + if ([408, 429, 500, 502, 503, 504].includes(Number(lastStatus))) { + const quarantine = markModelAsProblematic(provider, model, { + status: Number(lastStatus), + baseCooldownMs: lastCooldownMs, + reason: `HTTP ${lastStatus}`, + }); log.info( "AVAILABILITY", - `${provider}/${model} marked unavailable — all accounts exhausted (HTTP ${lastStatus})` + `${provider}/${model} marked unavailable — all accounts exhausted (HTTP ${lastStatus}, cooldown ${Math.ceil(quarantine.cooldownMs / 1000)}s, failureCount ${quarantine.failureCount})` ); } return handleNoCredentials( @@ -654,7 +659,7 @@ async function handleSingleModelChat( } // 7. Fallback to next account - const { shouldFallback } = await markAccountUnavailable( + const { shouldFallback, cooldownMs } = await markAccountUnavailable( credentials.connectionId, result.status, result.error, @@ -663,6 +668,9 @@ async function handleSingleModelChat( ); if (shouldFallback) { + if (Number.isFinite(cooldownMs) && cooldownMs > 0) { + lastCooldownMs = cooldownMs; + } log.warn("AUTH", `Account ${accountId}... unavailable (${result.status}), trying fallback`); excludeConnectionId = credentials.connectionId; lastError = result.error; diff --git a/tests/unit/batch-a-domain.test.mjs b/tests/unit/batch-a-domain.test.mjs index eb951f48ef..92e43a1edd 100644 --- a/tests/unit/batch-a-domain.test.mjs +++ b/tests/unit/batch-a-domain.test.mjs @@ -13,6 +13,7 @@ import assert from "node:assert/strict"; import { isModelAvailable, setModelUnavailable, + markModelAsProblematic, clearModelUnavailability, getAvailabilityReport, getUnavailableCount, @@ -58,6 +59,41 @@ describe("modelAvailability", () => { while (Date.now() - start < 5) {} // spin wait assert.equal(isModelAvailable("anthropic", "claude-sonnet-4-20250514"), true); }); + + it("should apply adaptive quarantine for repeated problematic failures", () => { + const first = markModelAsProblematic("nvidia", "z-ai/glm4.7", { + status: 502, + baseCooldownMs: 3000, + reason: "HTTP 502", + }); + const second = markModelAsProblematic("nvidia", "z-ai/glm4.7", { + status: 502, + baseCooldownMs: 3000, + reason: "HTTP 502", + }); + + assert.equal(first.failureCount, 1); + assert.equal(first.cooldownMs, 120000); + assert.equal(second.failureCount, 2); + assert.equal(second.cooldownMs, 240000); + + const report = getAvailabilityReport(); + const entry = report.find((r) => r.provider === "nvidia" && r.model === "z-ai/glm4.7"); + assert.ok(entry, "nvidia model should be quarantined"); + assert.ok(entry.remainingMs >= 200000, "cooldown should be preserved/escalated"); + }); + + it("should reset failure history after model recovery", () => { + clearModelUnavailability("nvidia", "z-ai/glm4.7"); + const afterReset = markModelAsProblematic("nvidia", "z-ai/glm4.7", { + status: 502, + baseCooldownMs: 3000, + reason: "HTTP 502", + }); + + assert.equal(afterReset.failureCount, 1); + assert.equal(afterReset.cooldownMs, 120000); + }); }); // ──────────────── T-19: Cost Rules ──────────────── diff --git a/tests/unit/token-health-check.test.mjs b/tests/unit/token-health-check.test.mjs new file mode 100644 index 0000000000..6645ac767c --- /dev/null +++ b/tests/unit/token-health-check.test.mjs @@ -0,0 +1,43 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +process.env.NODE_ENV = "test"; + +const tokenHealthCheck = await import("../../src/lib/tokenHealthCheck.ts"); + +test("buildRefreshFailureUpdate keeps active connections routable after refresh failure", () => { + const now = "2026-04-09T04:40:00.000Z"; + + const update = tokenHealthCheck.buildRefreshFailureUpdate( + { + testStatus: "active", + expiredRetryCount: 2, + }, + now + ); + + assert.equal(update.testStatus, "active"); + assert.equal(update.lastError, "Health check: token refresh failed"); + assert.equal(update.lastErrorType, "token_refresh_failed"); + assert.equal(update.lastErrorSource, "oauth"); + assert.equal(update.errorCode, "refresh_failed"); + assert.equal(update.lastHealthCheckAt, now); + assert.equal("expiredRetryCount" in update, false); + assert.equal("expiredRetryAt" in update, false); +}); + +test("buildRefreshFailureUpdate preserves expired retry tracking", () => { + const now = "2026-04-09T04:41:00.000Z"; + + const update = tokenHealthCheck.buildRefreshFailureUpdate( + { + testStatus: "expired", + expiredRetryCount: 2, + }, + now + ); + + assert.equal(update.testStatus, "expired"); + assert.equal(update.expiredRetryCount, 3); + assert.equal(update.expiredRetryAt, now); +});