From b19e9772bc31b3db823eb63c12ee352e58828af6 Mon Sep 17 00:00:00 2001 From: tkgo11 <71800574+tkgo11@users.noreply.github.com> Date: Sun, 16 Aug 2026 12:15:46 +0900 Subject: [PATCH] fix(monitoring): canonicalize provider aliases in health matrix (#10370) * fix(monitoring): canonicalize provider aliases in health matrix * fix(monitoring): canonicalize aliases in health autopilot --------- Co-authored-by: tkgo11 <7.1800574e+07+tkgo11@users.noreply.github.com> --- src/lib/monitoring/providerHealthAutopilot.ts | 69 ++++++++++++------- src/lib/monitoring/providerHealthMatrix.ts | 44 ++++++++---- tests/unit/provider-health-matrix.test.ts | 63 +++++++++++++++++ .../serial/provider-health-autopilot.test.ts | 56 ++++++++++++++- 4 files changed, 190 insertions(+), 42 deletions(-) diff --git a/src/lib/monitoring/providerHealthAutopilot.ts b/src/lib/monitoring/providerHealthAutopilot.ts index 8fc54e2d86..e29844da36 100644 --- a/src/lib/monitoring/providerHealthAutopilot.ts +++ b/src/lib/monitoring/providerHealthAutopilot.ts @@ -1,11 +1,9 @@ import { createHash } from "crypto"; -import { - getProviderConnections, - updateProviderConnection, -} from "@/lib/db/providers"; +import { getProviderConnections, updateProviderConnection } from "@/lib/db/providers"; import { getCachedProviderConnectionById } from "@/lib/localDb"; import { clearProviderFailure, clearModelLock } from "@omniroute/open-sse/services/accountFallback"; +import { resolveProviderAlias } from "@omniroute/open-sse/services/model"; type JsonRecord = Record; @@ -113,6 +111,11 @@ function toString(value: unknown): string | null { return typeof value === "string" && value.trim() ? value.trim() : null; } +function canonicalProviderId(value: unknown): string | null { + const provider = toString(value); + return provider ? (resolveProviderAlias(provider) ?? provider) : null; +} + function toNumber(value: unknown): number | null { if (typeof value === "number" && Number.isFinite(value)) return value; if (typeof value === "string" && value.trim()) { @@ -253,7 +256,7 @@ export async function buildProviderHealthAutopilotReport( const checkedAt = new Date(now).toISOString(); const includeHealthy = options.includeHealthy === true; const includeActions = options.includeActions !== false; - const providerFilter = toString(options.provider); + const providerFilter = canonicalProviderId(options.provider); const [{ getAllCircuitBreakerStatuses }, { getAllModelLockouts }, quotaMonitor] = await Promise.all([ @@ -262,40 +265,45 @@ export async function buildProviderHealthAutopilotReport( import("@omniroute/open-sse/services/quotaMonitor.ts").catch(() => null), ]); - const connections = (await getProviderConnections( - providerFilter ? { provider: providerFilter } : {} - )) as JsonRecord[]; + // Connections generally use canonical ids, while breakers, lockouts, and quota + // snapshots can retain the alias used at dispatch time. Normalize the aggregation + // key, but preserve each raw source id for actions that must mutate runtime state. + const connections = ((await getProviderConnections({})) as JsonRecord[]).filter((connection) => { + const provider = canonicalProviderId(connection.provider); + return provider && (!providerFilter || provider === providerFilter); + }); const breakers = getAllCircuitBreakerStatuses().filter((breaker) => { const name = toString((breaker as JsonRecord).name); - if (!name || name.startsWith("test-") || name.startsWith("test_")) return false; - return !providerFilter || name === providerFilter; + const provider = canonicalProviderId(name); + if (!name || !provider || name.startsWith("test-") || name.startsWith("test_")) return false; + return !providerFilter || provider === providerFilter; }); const lockouts = (getAllModelLockouts() as JsonRecord[]).filter((lockout) => { - const provider = providerFromLockout(lockout); + const provider = canonicalProviderId(providerFromLockout(lockout)); return provider && (!providerFilter || provider === providerFilter); }); const quotaSnapshots = quotaMonitor?.getQuotaMonitorSnapshots ? (quotaMonitor.getQuotaMonitorSnapshots() as JsonRecord[]).filter((snapshot) => { - const provider = toString(snapshot.provider); + const provider = canonicalProviderId(snapshot.provider); return provider && (!providerFilter || provider === providerFilter); }) : []; const providerIds = new Set(); for (const connection of connections) { - const provider = toString(connection.provider); + const provider = canonicalProviderId(connection.provider); if (provider) providerIds.add(provider); } for (const breaker of breakers) { - const provider = toString((breaker as JsonRecord).name); + const provider = canonicalProviderId((breaker as JsonRecord).name); if (provider) providerIds.add(provider); } for (const lockout of lockouts) { - const provider = providerFromLockout(lockout); + const provider = canonicalProviderId(providerFromLockout(lockout)); if (provider) providerIds.add(provider); } for (const snapshot of quotaSnapshots) { - const provider = toString(snapshot.provider); + const provider = canonicalProviderId(snapshot.provider); if (provider) providerIds.add(provider); } if (providerFilter) providerIds.add(providerFilter); @@ -303,19 +311,21 @@ export async function buildProviderHealthAutopilotReport( const providers: ProviderAutopilotProvider[] = []; for (const provider of [...providerIds].sort()) { const providerConnections = connections.filter( - (connection) => connection.provider === provider + (connection) => canonicalProviderId(connection.provider) === provider ); - const breaker = breakers.find((entry) => (entry as JsonRecord).name === provider) as - | JsonRecord - | undefined; + const breaker = breakers.find( + (entry) => canonicalProviderId((entry as JsonRecord).name) === provider + ) as JsonRecord | undefined; const providerLockouts = lockouts.filter( - (lockout) => providerFromLockout(lockout) === provider + (lockout) => canonicalProviderId(providerFromLockout(lockout)) === provider + ); + const providerQuota = quotaSnapshots.filter( + (snapshot) => canonicalProviderId(snapshot.provider) === provider ); - const providerQuota = quotaSnapshots.filter((snapshot) => snapshot.provider === provider); const issues: ProviderAutopilotIssue[] = []; if (breaker && OPEN_BREAKER_STATES.has(String(breaker.state))) { - const target = { provider }; + const target = { provider: toString(breaker.name) ?? provider }; const evidence = { state: breaker.state, failureCount: toNumber(breaker.failureCount) ?? 0, @@ -342,7 +352,7 @@ export async function buildProviderHealthAutopilotReport( for (const connection of providerConnections) { const connectionId = toString(connection.id); if (!connectionId) continue; - const target = { provider, connectionId }; + const target = { provider: toString(connection.provider) ?? provider, connectionId }; const label = sanitizeConnectionLabel(connection); const cooldownUntil = parseTimeMs(connection.rateLimitedUntil); const terminal = isTerminalConnection(connection); @@ -450,7 +460,11 @@ export async function buildProviderHealthAutopilotReport( if (!connectionId || !model) continue; const connection = providerConnections.find((entry) => entry.id === connectionId); const terminalConnection = connection ? isTerminalConnection(connection) : false; - const target = { provider, connectionId, model }; + const target = { + provider: providerFromLockout(lockout) ?? provider, + connectionId, + model, + }; const evidence = { reason: lockout.reason ?? null, remainingMs: toNumber(lockout.remainingMs) ?? 0, @@ -483,7 +497,10 @@ export async function buildProviderHealthAutopilotReport( if (!status || !["warning", "exhausted", "error"].includes(status)) continue; const connectionId = toString(snapshot.accountId) ?? undefined; const sessionId = toString(snapshot.sessionId) ?? undefined; - const target = { provider, ...(connectionId ? { connectionId } : {}) }; + const target = { + provider: toString(snapshot.provider) ?? provider, + ...(connectionId ? { connectionId } : {}), + }; issues.push({ id: issueId("quota_monitor_warning", { ...target, diff --git a/src/lib/monitoring/providerHealthMatrix.ts b/src/lib/monitoring/providerHealthMatrix.ts index dcc99a3830..b1ff287bae 100644 --- a/src/lib/monitoring/providerHealthMatrix.ts +++ b/src/lib/monitoring/providerHealthMatrix.ts @@ -3,6 +3,7 @@ import { getProviderConnections } from "@/lib/db/providers"; import { getDbInstance } from "@/lib/db/core"; import { getAllCircuitBreakerStatuses } from "@/shared/utils/circuitBreaker"; import { getAllModelLockouts } from "@omniroute/open-sse/services/accountFallback"; +import { resolveProviderAlias } from "@omniroute/open-sse/services/model"; import { getWebSessionPoolHealth } from "@omniroute/open-sse/services/webSessionPoolHealth"; type JsonRecord = Record; @@ -133,6 +134,11 @@ function toString(value: unknown): string | null { return typeof value === "string" && value.trim() ? value.trim() : null; } +function canonicalProviderId(value: unknown): string | null { + const provider = toString(value); + return provider ? (resolveProviderAlias(provider) ?? provider) : null; +} + function toNumber(value: unknown): number { if (typeof value === "number" && Number.isFinite(value)) return value; if (typeof value === "string" && value.trim()) { @@ -345,40 +351,46 @@ export async function buildProviderHealthMatrix( const checkedAt = new Date(now).toISOString(); const range = normalizeRange(options.range); const cutoff = new Date(now - RANGE_MS[range]).toISOString(); - const providerFilter = toString(options.provider); + const providerFilter = canonicalProviderId(options.provider); const includeHealthy = options.includeHealthy !== false; - const [connections, breakers, lockouts, stats] = await Promise.all([ - getProviderConnections(providerFilter ? { provider: providerFilter } : {}), + // Connections use canonical ids while circuit breakers, lockouts, and historical + // call logs can retain the alias used at dispatch time. Normalize all sources here + // so a provider has one health row with every related signal attached. + const [connections, breakers, lockouts, rawStats] = await Promise.all([ + getProviderConnections({}), getAllCircuitBreakerStatuses(), getAllModelLockouts(), - Promise.resolve(queryCallLogTargetStats(cutoff, providerFilter)), + Promise.resolve(queryCallLogTargetStats(cutoff, null)), ]); const connectionRows = (connections as JsonRecord[]).filter((connection) => { - const provider = toString(connection.provider); + const provider = canonicalProviderId(connection.provider); return provider && (!providerFilter || provider === providerFilter); }); const breakerRows = (breakers as JsonRecord[]).filter((breaker) => { - const provider = toString(breaker.name); + const provider = canonicalProviderId(breaker.name); return provider && (!providerFilter || provider === providerFilter); }); const lockoutRows = (lockouts as JsonRecord[]).filter((lockout) => { - const provider = toString(lockout.provider); + const provider = canonicalProviderId(lockout.provider); return provider && (!providerFilter || provider === providerFilter); }); + const stats = rawStats + .map((row) => ({ ...row, provider: canonicalProviderId(row.provider) ?? row.provider })) + .filter((row) => !providerFilter || row.provider === providerFilter); const providerIds = new Set(); for (const connection of connectionRows) { - const provider = toString(connection.provider); + const provider = canonicalProviderId(connection.provider); if (provider) providerIds.add(provider); } for (const breaker of breakerRows) { - const provider = toString(breaker.name); + const provider = canonicalProviderId(breaker.name); if (provider) providerIds.add(provider); } for (const lockout of lockoutRows) { - const provider = toString(lockout.provider); + const provider = canonicalProviderId(lockout.provider); if (provider) providerIds.add(provider); } for (const row of stats) providerIds.add(row.provider); @@ -407,7 +419,7 @@ export async function buildProviderHealthMatrix( const lockoutsByTarget = new Map(); const lockoutCountByProvider = new Map(); for (const lockout of lockoutRows) { - const provider = toString(lockout.provider); + const provider = canonicalProviderId(lockout.provider); const connectionId = toString(lockout.connectionId); const model = toString(lockout.model); if (!provider || !model) continue; @@ -418,10 +430,12 @@ export async function buildProviderHealthMatrix( const providers: ProviderHealthMatrixProvider[] = []; for (const provider of [...providerIds].sort()) { const providerConnections = connectionRows.filter( - (connection) => toString(connection.provider) === provider + (connection) => canonicalProviderId(connection.provider) === provider ); const providerStats = stats.filter((row) => row.provider === provider); - const providerBreaker = breakerRows.find((breaker) => toString(breaker.name) === provider); + const providerBreaker = breakerRows.find( + (breaker) => canonicalProviderId(breaker.name) === provider + ); const circuitBreaker = providerBreaker ? { state: toString(providerBreaker.state) || "CLOSED", @@ -443,7 +457,7 @@ export async function buildProviderHealthMatrix( if (!accountRows.has(key)) accountRows.set(key, null); } for (const lockout of lockoutRows) { - if (toString(lockout.provider) !== provider) continue; + if (canonicalProviderId(lockout.provider) !== provider) continue; const key = accountKey(provider, toString(lockout.connectionId)); if (!accountRows.has(key)) accountRows.set(key, null); } @@ -466,7 +480,7 @@ export async function buildProviderHealthMatrix( modelIds.add(stat.model); } for (const lockout of lockoutRows) { - if (toString(lockout.provider) !== provider) continue; + if (canonicalProviderId(lockout.provider) !== provider) continue; if ((toString(lockout.connectionId) ?? "") !== (connectionId ?? "")) continue; const model = toString(lockout.model); if (model) modelIds.add(model); diff --git a/tests/unit/provider-health-matrix.test.ts b/tests/unit/provider-health-matrix.test.ts index 7ed151cd28..7e4edd12ab 100644 --- a/tests/unit/provider-health-matrix.test.ts +++ b/tests/unit/provider-health-matrix.test.ts @@ -22,6 +22,8 @@ const route = await import("../../src/app/api/providers/health-matrix/route.ts") const accountFallback = await import("@omniroute/open-sse/services/accountFallback"); const PROVIDER = "matrix-test-provider"; +const ALIAS_PROVIDER = "nous"; +const CANONICAL_ALIAS_PROVIDER = "nous-research"; async function resetStorage() { core.resetDbInstance(); @@ -33,6 +35,13 @@ async function resetStorage() { } } accountFallback.clearProviderFailure(PROVIDER); + accountFallback.clearProviderFailure(ALIAS_PROVIDER); + accountFallback.clearProviderFailure(CANONICAL_ALIAS_PROVIDER); + for (const lockout of accountFallback.getAllModelLockouts()) { + if (lockout.provider === ALIAS_PROVIDER || lockout.provider === CANONICAL_ALIAS_PROVIDER) { + accountFallback.clearModelLock(lockout.provider, lockout.connectionId, lockout.model); + } + } } async function enableManagementAuth() { @@ -139,6 +148,60 @@ test("provider health matrix combines connections, synced models, logs and locko assert.equal(locked.lockoutReason, "quota_exhausted"); }); +test("provider health matrix collapses alias-keyed signals into one canonical provider", async () => { + const connection = (await providersDb.createProviderConnection({ + id: "matrix-nous-connection", + provider: CANONICAL_ALIAS_PROVIDER, + authType: "apikey", + name: "nous-key", + apiKey: "test-key", + isActive: true, + })) as Record; + const connectionId = String(connection.id); + + accountFallback.lockModel( + ALIAS_PROVIDER, + connectionId, + "nous-locked-model", + "quota_exhausted", + 60_000, + {} + ); + accountFallback.recordProviderFailure(ALIAS_PROVIDER, undefined, undefined, { + failureThreshold: 1, + resetTimeoutMs: 60_000, + }); + + const report = await matrix.buildProviderHealthMatrix({ includeHealthy: true, range: "24h" }); + const canonicalRows = report.providers.filter( + (provider) => provider.provider === CANONICAL_ALIAS_PROVIDER + ); + + assert.equal(canonicalRows.length, 1, "the canonical provider must have exactly one health row"); + assert.equal( + report.providers.some((provider) => provider.provider === ALIAS_PROVIDER), + false, + "the alias must not create a duplicate provider row" + ); + + const provider = canonicalRows[0]; + assert.equal(provider.connections.total, 1); + assert.equal(provider.circuitBreaker?.state, "OPEN"); + assert.equal(provider.modelLockoutCount, 1); + assert.equal(provider.accounts[0]?.models[0]?.model, "nous-locked-model"); + assert.equal(provider.accounts[0]?.models[0]?.isLockedOut, true); + + const filteredByAlias = await matrix.buildProviderHealthMatrix({ + provider: ALIAS_PROVIDER, + includeHealthy: true, + range: "24h", + }); + assert.equal(filteredByAlias.providers.length, 1); + assert.equal(filteredByAlias.providers[0]?.provider, CANONICAL_ALIAS_PROVIDER); + assert.equal(filteredByAlias.providers[0]?.connections.total, 1); + assert.equal(filteredByAlias.providers[0]?.circuitBreaker?.state, "OPEN"); +}); + test("provider health matrix treats recovered models as degraded instead of error", async () => { const connection = (await providersDb.createProviderConnection({ id: "matrix-recovered-connection", diff --git a/tests/unit/serial/provider-health-autopilot.test.ts b/tests/unit/serial/provider-health-autopilot.test.ts index c7ea6c4fc5..67e0b22909 100644 --- a/tests/unit/serial/provider-health-autopilot.test.ts +++ b/tests/unit/serial/provider-health-autopilot.test.ts @@ -17,7 +17,8 @@ const core = await import("../../../src/lib/db/core.ts"); const settingsDb = await import("../../../src/lib/db/settings.ts"); const providersDb = await import("../../../src/lib/db/providers.ts"); const autopilot = await import("../../../src/lib/monitoring/providerHealthAutopilot.ts"); -const actionsRoute = await import("../../../src/app/api/providers/health-autopilot/actions/route.ts"); +const actionsRoute = + await import("../../../src/app/api/providers/health-autopilot/actions/route.ts"); const reportRoute = await import("../../../src/app/api/providers/health-autopilot/route.ts"); const routeGuard = await import("../../../src/server/authz/routeGuard.ts"); const authzPipeline = await import("../../../src/server/authz/pipeline.ts"); @@ -113,6 +114,59 @@ test("provider health autopilot reports actionable cooldown and model lockout is } }); +test("provider health autopilot canonicalizes alias-keyed signals while preserving raw breaker actions", async () => { + const canonicalProvider = "nous-research"; + const aliasProvider = "nous"; + const connection = await createCooldownConnection(canonicalProvider); + for (let failure = 0; failure < 20; failure += 1) { + accountFallback.recordProviderFailure(aliasProvider); + } + accountFallback.lockModel( + aliasProvider, + String(connection.id), + "alias-locked-model", + "quota", + 60_000, + {} + ); + + try { + const report = await autopilot.buildProviderHealthAutopilotReport({ + provider: aliasProvider, + includeHealthy: true, + }); + assert.equal(report.providers.length, 1); + const provider = report.providers[0]; + assert.equal(provider.provider, canonicalProvider); + assert.equal(provider.signals.connections.total, 1); + assert.equal(provider.signals.modelLockouts, 1); + + const clearBreaker = findAction(report, "clear_provider_breaker"); + assert.ok(clearBreaker); + assert.equal(clearBreaker.target.provider, aliasProvider); + + const applied = await autopilot.executeProviderHealthAutopilotAction({ + type: clearBreaker.type, + target: clearBreaker.target, + preconditionsHash: clearBreaker.preconditionsHash, + confirm: true, + }); + assert.equal(applied.status, 200); + + const afterReset = await autopilot.buildProviderHealthAutopilotReport({ + provider: aliasProvider, + includeHealthy: true, + }); + assert.equal( + afterReset.providers[0].issues.some((issue) => issue.kind === "provider_circuit_open"), + false + ); + } finally { + accountFallback.clearModelLock(aliasProvider, String(connection.id), "alias-locked-model"); + accountFallback.clearProviderFailure(aliasProvider); + } +}); + test("provider health autopilot action clears cooldown with stale-state protection", async () => { await enableManagementAuth(); const connection = await createCooldownConnection();