fix(monitoring): canonicalize provider aliases in health matrix (#10370)

* fix(monitoring): canonicalize provider aliases in health matrix * fix(monitoring): canonicalize aliases in health autopilot --------- Co-authored-by: tkgo11 <7.1800574e+07+tkgo11@users.noreply.github.com>
2026-08-16 12:12:23 +03:00 · 2026-08-16 12:15:46 +09:00
parent 684ea70fb3
commit b19e9772bc
4 changed files with 190 additions and 42 deletions
--- a/src/lib/monitoring/providerHealthAutopilot.ts
+++ b/src/lib/monitoring/providerHealthAutopilot.ts
@@ -1,11 +1,9 @@
 import { createHash } from "crypto";

-import {
-  getProviderConnections,
-  updateProviderConnection,
-} from "@/lib/db/providers";
+import { getProviderConnections, updateProviderConnection } from "@/lib/db/providers";
 import { getCachedProviderConnectionById } from "@/lib/localDb";
 import { clearProviderFailure, clearModelLock } from "@omniroute/open-sse/services/accountFallback";
+import { resolveProviderAlias } from "@omniroute/open-sse/services/model";

 type JsonRecord = Record<string, unknown>;

@@ -113,6 +111,11 @@ function toString(value: unknown): string | null {
  return typeof value === "string" && value.trim() ? value.trim() : null;
 }

+function canonicalProviderId(value: unknown): string | null {
+  const provider = toString(value);
+  return provider ? (resolveProviderAlias(provider) ?? provider) : null;
+}
+
 function toNumber(value: unknown): number | null {
  if (typeof value === "number" && Number.isFinite(value)) return value;
  if (typeof value === "string" && value.trim()) {
@@ -253,7 +256,7 @@ export async function buildProviderHealthAutopilotReport(
  const checkedAt = new Date(now).toISOString();
  const includeHealthy = options.includeHealthy === true;
  const includeActions = options.includeActions !== false;
-  const providerFilter = toString(options.provider);
+  const providerFilter = canonicalProviderId(options.provider);

  const [{ getAllCircuitBreakerStatuses }, { getAllModelLockouts }, quotaMonitor] =
    await Promise.all([
@@ -262,40 +265,45 @@ export async function buildProviderHealthAutopilotReport(
      import("@omniroute/open-sse/services/quotaMonitor.ts").catch(() => null),
    ]);

-  const connections = (await getProviderConnections(
-    providerFilter ? { provider: providerFilter } : {}
-  )) as JsonRecord[];
+  // Connections generally use canonical ids, while breakers, lockouts, and quota
+  // snapshots can retain the alias used at dispatch time. Normalize the aggregation
+  // key, but preserve each raw source id for actions that must mutate runtime state.
+  const connections = ((await getProviderConnections({})) as JsonRecord[]).filter((connection) => {
+    const provider = canonicalProviderId(connection.provider);
+    return provider && (!providerFilter || provider === providerFilter);
+  });
  const breakers = getAllCircuitBreakerStatuses().filter((breaker) => {
    const name = toString((breaker as JsonRecord).name);
-    if (!name || name.startsWith("test-") || name.startsWith("test_")) return false;
-    return !providerFilter || name === providerFilter;
+    const provider = canonicalProviderId(name);
+    if (!name || !provider || name.startsWith("test-") || name.startsWith("test_")) return false;
+    return !providerFilter || provider === providerFilter;
  });
  const lockouts = (getAllModelLockouts() as JsonRecord[]).filter((lockout) => {
-    const provider = providerFromLockout(lockout);
+    const provider = canonicalProviderId(providerFromLockout(lockout));
    return provider && (!providerFilter || provider === providerFilter);
  });
  const quotaSnapshots = quotaMonitor?.getQuotaMonitorSnapshots
    ? (quotaMonitor.getQuotaMonitorSnapshots() as JsonRecord[]).filter((snapshot) => {
-        const provider = toString(snapshot.provider);
+        const provider = canonicalProviderId(snapshot.provider);
        return provider && (!providerFilter || provider === providerFilter);
      })
    : [];

  const providerIds = new Set<string>();
  for (const connection of connections) {
-    const provider = toString(connection.provider);
+    const provider = canonicalProviderId(connection.provider);
    if (provider) providerIds.add(provider);
  }
  for (const breaker of breakers) {
-    const provider = toString((breaker as JsonRecord).name);
+    const provider = canonicalProviderId((breaker as JsonRecord).name);
    if (provider) providerIds.add(provider);
  }
  for (const lockout of lockouts) {
-    const provider = providerFromLockout(lockout);
+    const provider = canonicalProviderId(providerFromLockout(lockout));
    if (provider) providerIds.add(provider);
  }
  for (const snapshot of quotaSnapshots) {
-    const provider = toString(snapshot.provider);
+    const provider = canonicalProviderId(snapshot.provider);
    if (provider) providerIds.add(provider);
  }
  if (providerFilter) providerIds.add(providerFilter);
@@ -303,19 +311,21 @@ export async function buildProviderHealthAutopilotReport(
  const providers: ProviderAutopilotProvider[] = [];
  for (const provider of [...providerIds].sort()) {
    const providerConnections = connections.filter(
-      (connection) => connection.provider === provider
+      (connection) => canonicalProviderId(connection.provider) === provider
    );
-    const breaker = breakers.find((entry) => (entry as JsonRecord).name === provider) as
-      | JsonRecord
-      | undefined;
+    const breaker = breakers.find(
+      (entry) => canonicalProviderId((entry as JsonRecord).name) === provider
+    ) as JsonRecord | undefined;
    const providerLockouts = lockouts.filter(
-      (lockout) => providerFromLockout(lockout) === provider
+      (lockout) => canonicalProviderId(providerFromLockout(lockout)) === provider
+    );
+    const providerQuota = quotaSnapshots.filter(
+      (snapshot) => canonicalProviderId(snapshot.provider) === provider
    );
-    const providerQuota = quotaSnapshots.filter((snapshot) => snapshot.provider === provider);
    const issues: ProviderAutopilotIssue[] = [];

    if (breaker && OPEN_BREAKER_STATES.has(String(breaker.state))) {
-      const target = { provider };
+      const target = { provider: toString(breaker.name) ?? provider };
      const evidence = {
        state: breaker.state,
        failureCount: toNumber(breaker.failureCount) ?? 0,
@@ -342,7 +352,7 @@ export async function buildProviderHealthAutopilotReport(
    for (const connection of providerConnections) {
      const connectionId = toString(connection.id);
      if (!connectionId) continue;
-      const target = { provider, connectionId };
+      const target = { provider: toString(connection.provider) ?? provider, connectionId };
      const label = sanitizeConnectionLabel(connection);
      const cooldownUntil = parseTimeMs(connection.rateLimitedUntil);
      const terminal = isTerminalConnection(connection);
@@ -450,7 +460,11 @@ export async function buildProviderHealthAutopilotReport(
      if (!connectionId || !model) continue;
      const connection = providerConnections.find((entry) => entry.id === connectionId);
      const terminalConnection = connection ? isTerminalConnection(connection) : false;
-      const target = { provider, connectionId, model };
+      const target = {
+        provider: providerFromLockout(lockout) ?? provider,
+        connectionId,
+        model,
+      };
      const evidence = {
        reason: lockout.reason ?? null,
        remainingMs: toNumber(lockout.remainingMs) ?? 0,
@@ -483,7 +497,10 @@ export async function buildProviderHealthAutopilotReport(
      if (!status || !["warning", "exhausted", "error"].includes(status)) continue;
      const connectionId = toString(snapshot.accountId) ?? undefined;
      const sessionId = toString(snapshot.sessionId) ?? undefined;
-      const target = { provider, ...(connectionId ? { connectionId } : {}) };
+      const target = {
+        provider: toString(snapshot.provider) ?? provider,
+        ...(connectionId ? { connectionId } : {}),
+      };
      issues.push({
        id: issueId("quota_monitor_warning", {
          ...target,
--- a/src/lib/monitoring/providerHealthMatrix.ts
+++ b/src/lib/monitoring/providerHealthMatrix.ts
@@ -3,6 +3,7 @@ import { getProviderConnections } from "@/lib/db/providers";
 import { getDbInstance } from "@/lib/db/core";
 import { getAllCircuitBreakerStatuses } from "@/shared/utils/circuitBreaker";
 import { getAllModelLockouts } from "@omniroute/open-sse/services/accountFallback";
+import { resolveProviderAlias } from "@omniroute/open-sse/services/model";
 import { getWebSessionPoolHealth } from "@omniroute/open-sse/services/webSessionPoolHealth";

 type JsonRecord = Record<string, unknown>;
@@ -133,6 +134,11 @@ function toString(value: unknown): string | null {
  return typeof value === "string" && value.trim() ? value.trim() : null;
 }

+function canonicalProviderId(value: unknown): string | null {
+  const provider = toString(value);
+  return provider ? (resolveProviderAlias(provider) ?? provider) : null;
+}
+
 function toNumber(value: unknown): number {
  if (typeof value === "number" && Number.isFinite(value)) return value;
  if (typeof value === "string" && value.trim()) {
@@ -345,40 +351,46 @@ export async function buildProviderHealthMatrix(
  const checkedAt = new Date(now).toISOString();
  const range = normalizeRange(options.range);
  const cutoff = new Date(now - RANGE_MS[range]).toISOString();
-  const providerFilter = toString(options.provider);
+  const providerFilter = canonicalProviderId(options.provider);
  const includeHealthy = options.includeHealthy !== false;

-  const [connections, breakers, lockouts, stats] = await Promise.all([
-    getProviderConnections(providerFilter ? { provider: providerFilter } : {}),
+  // Connections use canonical ids while circuit breakers, lockouts, and historical
+  // call logs can retain the alias used at dispatch time. Normalize all sources here
+  // so a provider has one health row with every related signal attached.
+  const [connections, breakers, lockouts, rawStats] = await Promise.all([
+    getProviderConnections({}),
    getAllCircuitBreakerStatuses(),
    getAllModelLockouts(),
-    Promise.resolve(queryCallLogTargetStats(cutoff, providerFilter)),
+    Promise.resolve(queryCallLogTargetStats(cutoff, null)),
  ]);

  const connectionRows = (connections as JsonRecord[]).filter((connection) => {
-    const provider = toString(connection.provider);
+    const provider = canonicalProviderId(connection.provider);
    return provider && (!providerFilter || provider === providerFilter);
  });
  const breakerRows = (breakers as JsonRecord[]).filter((breaker) => {
-    const provider = toString(breaker.name);
+    const provider = canonicalProviderId(breaker.name);
    return provider && (!providerFilter || provider === providerFilter);
  });
  const lockoutRows = (lockouts as JsonRecord[]).filter((lockout) => {
-    const provider = toString(lockout.provider);
+    const provider = canonicalProviderId(lockout.provider);
    return provider && (!providerFilter || provider === providerFilter);
  });
+  const stats = rawStats
+    .map((row) => ({ ...row, provider: canonicalProviderId(row.provider) ?? row.provider }))
+    .filter((row) => !providerFilter || row.provider === providerFilter);

  const providerIds = new Set<string>();
  for (const connection of connectionRows) {
-    const provider = toString(connection.provider);
+    const provider = canonicalProviderId(connection.provider);
    if (provider) providerIds.add(provider);
  }
  for (const breaker of breakerRows) {
-    const provider = toString(breaker.name);
+    const provider = canonicalProviderId(breaker.name);
    if (provider) providerIds.add(provider);
  }
  for (const lockout of lockoutRows) {
-    const provider = toString(lockout.provider);
+    const provider = canonicalProviderId(lockout.provider);
    if (provider) providerIds.add(provider);
  }
  for (const row of stats) providerIds.add(row.provider);
@@ -407,7 +419,7 @@ export async function buildProviderHealthMatrix(
  const lockoutsByTarget = new Map<string, JsonRecord>();
  const lockoutCountByProvider = new Map<string, number>();
  for (const lockout of lockoutRows) {
-    const provider = toString(lockout.provider);
+    const provider = canonicalProviderId(lockout.provider);
    const connectionId = toString(lockout.connectionId);
    const model = toString(lockout.model);
    if (!provider || !model) continue;
@@ -418,10 +430,12 @@ export async function buildProviderHealthMatrix(
  const providers: ProviderHealthMatrixProvider[] = [];
  for (const provider of [...providerIds].sort()) {
    const providerConnections = connectionRows.filter(
-      (connection) => toString(connection.provider) === provider
+      (connection) => canonicalProviderId(connection.provider) === provider
    );
    const providerStats = stats.filter((row) => row.provider === provider);
-    const providerBreaker = breakerRows.find((breaker) => toString(breaker.name) === provider);
+    const providerBreaker = breakerRows.find(
+      (breaker) => canonicalProviderId(breaker.name) === provider
+    );
    const circuitBreaker = providerBreaker
      ? {
          state: toString(providerBreaker.state) || "CLOSED",
@@ -443,7 +457,7 @@ export async function buildProviderHealthMatrix(
      if (!accountRows.has(key)) accountRows.set(key, null);
    }
    for (const lockout of lockoutRows) {
-      if (toString(lockout.provider) !== provider) continue;
+      if (canonicalProviderId(lockout.provider) !== provider) continue;
      const key = accountKey(provider, toString(lockout.connectionId));
      if (!accountRows.has(key)) accountRows.set(key, null);
    }
@@ -466,7 +480,7 @@ export async function buildProviderHealthMatrix(
        modelIds.add(stat.model);
      }
      for (const lockout of lockoutRows) {
-        if (toString(lockout.provider) !== provider) continue;
+        if (canonicalProviderId(lockout.provider) !== provider) continue;
        if ((toString(lockout.connectionId) ?? "") !== (connectionId ?? "")) continue;
        const model = toString(lockout.model);
        if (model) modelIds.add(model);
--- a/tests/unit/provider-health-matrix.test.ts
+++ b/tests/unit/provider-health-matrix.test.ts
@@ -22,6 +22,8 @@ const route = await import("../../src/app/api/providers/health-matrix/route.ts")
 const accountFallback = await import("@omniroute/open-sse/services/accountFallback");

 const PROVIDER = "matrix-test-provider";
+const ALIAS_PROVIDER = "nous";
+const CANONICAL_ALIAS_PROVIDER = "nous-research";

 async function resetStorage() {
  core.resetDbInstance();
@@ -33,6 +35,13 @@ async function resetStorage() {
    }
  }
  accountFallback.clearProviderFailure(PROVIDER);
+  accountFallback.clearProviderFailure(ALIAS_PROVIDER);
+  accountFallback.clearProviderFailure(CANONICAL_ALIAS_PROVIDER);
+  for (const lockout of accountFallback.getAllModelLockouts()) {
+    if (lockout.provider === ALIAS_PROVIDER || lockout.provider === CANONICAL_ALIAS_PROVIDER) {
+      accountFallback.clearModelLock(lockout.provider, lockout.connectionId, lockout.model);
+    }
+  }
 }

 async function enableManagementAuth() {
@@ -139,6 +148,60 @@ test("provider health matrix combines connections, synced models, logs and locko
  assert.equal(locked.lockoutReason, "quota_exhausted");
 });

+test("provider health matrix collapses alias-keyed signals into one canonical provider", async () => {
+  const connection = (await providersDb.createProviderConnection({
+    id: "matrix-nous-connection",
+    provider: CANONICAL_ALIAS_PROVIDER,
+    authType: "apikey",
+    name: "nous-key",
+    apiKey: "test-key",
+    isActive: true,
+  })) as Record<string, unknown>;
+  const connectionId = String(connection.id);
+
+  accountFallback.lockModel(
+    ALIAS_PROVIDER,
+    connectionId,
+    "nous-locked-model",
+    "quota_exhausted",
+    60_000,
+    {}
+  );
+  accountFallback.recordProviderFailure(ALIAS_PROVIDER, undefined, undefined, {
+    failureThreshold: 1,
+    resetTimeoutMs: 60_000,
+  });
+
+  const report = await matrix.buildProviderHealthMatrix({ includeHealthy: true, range: "24h" });
+  const canonicalRows = report.providers.filter(
+    (provider) => provider.provider === CANONICAL_ALIAS_PROVIDER
+  );
+
+  assert.equal(canonicalRows.length, 1, "the canonical provider must have exactly one health row");
+  assert.equal(
+    report.providers.some((provider) => provider.provider === ALIAS_PROVIDER),
+    false,
+    "the alias must not create a duplicate provider row"
+  );
+
+  const provider = canonicalRows[0];
+  assert.equal(provider.connections.total, 1);
+  assert.equal(provider.circuitBreaker?.state, "OPEN");
+  assert.equal(provider.modelLockoutCount, 1);
+  assert.equal(provider.accounts[0]?.models[0]?.model, "nous-locked-model");
+  assert.equal(provider.accounts[0]?.models[0]?.isLockedOut, true);
+
+  const filteredByAlias = await matrix.buildProviderHealthMatrix({
+    provider: ALIAS_PROVIDER,
+    includeHealthy: true,
+    range: "24h",
+  });
+  assert.equal(filteredByAlias.providers.length, 1);
+  assert.equal(filteredByAlias.providers[0]?.provider, CANONICAL_ALIAS_PROVIDER);
+  assert.equal(filteredByAlias.providers[0]?.connections.total, 1);
+  assert.equal(filteredByAlias.providers[0]?.circuitBreaker?.state, "OPEN");
+});
+
 test("provider health matrix treats recovered models as degraded instead of error", async () => {
  const connection = (await providersDb.createProviderConnection({
    id: "matrix-recovered-connection",
--- a/tests/unit/serial/provider-health-autopilot.test.ts
+++ b/tests/unit/serial/provider-health-autopilot.test.ts
@@ -17,7 +17,8 @@ const core = await import("../../../src/lib/db/core.ts");
 const settingsDb = await import("../../../src/lib/db/settings.ts");
 const providersDb = await import("../../../src/lib/db/providers.ts");
 const autopilot = await import("../../../src/lib/monitoring/providerHealthAutopilot.ts");
-const actionsRoute = await import("../../../src/app/api/providers/health-autopilot/actions/route.ts");
+const actionsRoute =
+  await import("../../../src/app/api/providers/health-autopilot/actions/route.ts");
 const reportRoute = await import("../../../src/app/api/providers/health-autopilot/route.ts");
 const routeGuard = await import("../../../src/server/authz/routeGuard.ts");
 const authzPipeline = await import("../../../src/server/authz/pipeline.ts");
@@ -113,6 +114,59 @@ test("provider health autopilot reports actionable cooldown and model lockout is
  }
 });

+test("provider health autopilot canonicalizes alias-keyed signals while preserving raw breaker actions", async () => {
+  const canonicalProvider = "nous-research";
+  const aliasProvider = "nous";
+  const connection = await createCooldownConnection(canonicalProvider);
+  for (let failure = 0; failure < 20; failure += 1) {
+    accountFallback.recordProviderFailure(aliasProvider);
+  }
+  accountFallback.lockModel(
+    aliasProvider,
+    String(connection.id),
+    "alias-locked-model",
+    "quota",
+    60_000,
+    {}
+  );
+
+  try {
+    const report = await autopilot.buildProviderHealthAutopilotReport({
+      provider: aliasProvider,
+      includeHealthy: true,
+    });
+    assert.equal(report.providers.length, 1);
+    const provider = report.providers[0];
+    assert.equal(provider.provider, canonicalProvider);
+    assert.equal(provider.signals.connections.total, 1);
+    assert.equal(provider.signals.modelLockouts, 1);
+
+    const clearBreaker = findAction(report, "clear_provider_breaker");
+    assert.ok(clearBreaker);
+    assert.equal(clearBreaker.target.provider, aliasProvider);
+
+    const applied = await autopilot.executeProviderHealthAutopilotAction({
+      type: clearBreaker.type,
+      target: clearBreaker.target,
+      preconditionsHash: clearBreaker.preconditionsHash,
+      confirm: true,
+    });
+    assert.equal(applied.status, 200);
+
+    const afterReset = await autopilot.buildProviderHealthAutopilotReport({
+      provider: aliasProvider,
+      includeHealthy: true,
+    });
+    assert.equal(
+      afterReset.providers[0].issues.some((issue) => issue.kind === "provider_circuit_open"),
+      false
+    );
+  } finally {
+    accountFallback.clearModelLock(aliasProvider, String(connection.id), "alias-locked-model");
+    accountFallback.clearProviderFailure(aliasProvider);
+  }
+});
+
 test("provider health autopilot action clears cooldown with stale-state protection", async () => {
  await enableManagementAuth();
  const connection = await createCooldownConnection();