mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-09-18 21:02:50 +03:00
fix(monitoring): serve cached credentialHealth off the request path (#12533)
Validado em lote numa worktree combinada com os 10 PRs desta leva sobre o tip de `release/v3.8.51`: `typecheck:core` limpo, `check-file-size` OK e **241/242** nos 29 arquivos de teste que os PRs tocam. A única "falha" não é falha: `tests/unit/autoCombo/strict-zero-cost-filter.test.ts` é um teste em estilo Vitest que eu incluí por engano na invocação do runner nativo do Node — ele quebra no import (`@vitest/runner`), não numa asserção. Ao investigar, descobri que esse arquivo não roda em nenhum dos dois runners hoje (o glob do `test:unit` não lista `autoCombo` e o `include` do Vitest só pega `.tsx` nessa pasta); é um problema pré-existente do repositório, sem relação com esta leva, e vou registrá-lo separadamente. O #12636 conflitava apenas na lista de testes do `@omniroute/opencode-plugin/package.json`, de forma aditiva: o tip já tinha `models-fetcher.test.ts` (do #12607, irmão desta mesma leva) e o #12636 acrescenta `telemetry.test.ts`. Fiz a união dos dois lados (25 arquivos contra 24 de cada) em vez de escolher um, o que teria removido um arquivo da suíte do plugin em silêncio. Obrigado, @RaviTharuma.
This commit is contained in:
@@ -14,14 +14,23 @@ import { requireManagementAuth } from "@/lib/api/requireManagementAuth";
|
||||
* Returns system info, provider health (circuit breakers),
|
||||
* rate limit status, and database stats.
|
||||
*/
|
||||
// §8.2 optimization: short-TTL cache for the health payload. Health is a
|
||||
// frequently-polled endpoint and rebuilding it every request (DB reads +
|
||||
// status aggregation across 8 subsystems) is wasteful under rapid polling. 1s
|
||||
// stays near-real-time for monitoring; the cache is invalidated on DELETE
|
||||
// (circuit-breaker reset) so a manual reset is reflected immediately.
|
||||
// §8.2 / #12532: short-TTL cache with stale-while-revalidate. Health is a
|
||||
// frequently-polled endpoint; rebuilding it on the request path (DB reads +
|
||||
// status aggregation) shares the event loop with GET /healthz. After the first
|
||||
// fill, scrapes always receive the last payload immediately. An expired entry
|
||||
// is refreshed in the background — never by awaiting live credential probes.
|
||||
let healthPayloadCache: { payload: unknown; expiresAt: number } | null = null;
|
||||
let healthPayloadRefreshInFlight = false;
|
||||
let healthPayloadCacheGeneration = 0;
|
||||
const HEALTH_PAYLOAD_TTL_MS = 1000;
|
||||
|
||||
/** Test-only: drop the in-process health payload cache. */
|
||||
export function __test_resetMonitoringHealthPayloadCache(): void {
|
||||
healthPayloadCache = null;
|
||||
healthPayloadRefreshInFlight = false;
|
||||
healthPayloadCacheGeneration += 1;
|
||||
}
|
||||
|
||||
// GHSA-mvf8-qc78-5mxm: the full health payload fingerprints the host (version,
|
||||
// node version, pid, memory, provider config). An anonymous caller — the common
|
||||
// case on a keyless install, and what a liveness/load-balancer probe needs — gets
|
||||
@@ -34,15 +43,70 @@ function publicHealthView(payload: unknown): Record<string, unknown> {
|
||||
};
|
||||
}
|
||||
|
||||
function serveHealthPayload(fullView: boolean, payload: unknown) {
|
||||
return NextResponse.json(fullView ? payload : publicHealthView(payload));
|
||||
}
|
||||
|
||||
function scheduleHealthPayloadRefresh(): void {
|
||||
if (healthPayloadRefreshInFlight) return;
|
||||
healthPayloadRefreshInFlight = true;
|
||||
setImmediate(() => {
|
||||
rebuildHealthPayload()
|
||||
.catch((error) => {
|
||||
console.warn(
|
||||
"[API] GET /api/monitoring/health background refresh failed:",
|
||||
error instanceof Error ? error.message : error
|
||||
);
|
||||
})
|
||||
.finally(() => {
|
||||
healthPayloadRefreshInFlight = false;
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export async function GET(request: Request) {
|
||||
const fullView = (await requireManagementAuth(request, { alwaysRequireAuth: true })) === null;
|
||||
const cachedNow = Date.now();
|
||||
if (healthPayloadCache && cachedNow <= healthPayloadCache.expiresAt) {
|
||||
return NextResponse.json(
|
||||
fullView ? healthPayloadCache.payload : publicHealthView(healthPayloadCache.payload)
|
||||
);
|
||||
if (healthPayloadCache) {
|
||||
if (cachedNow > healthPayloadCache.expiresAt) {
|
||||
scheduleHealthPayloadRefresh();
|
||||
}
|
||||
return serveHealthPayload(fullView, healthPayloadCache.payload);
|
||||
}
|
||||
|
||||
try {
|
||||
const payload = await rebuildHealthPayload();
|
||||
return serveHealthPayload(fullView, payload);
|
||||
} catch (error) {
|
||||
console.error("[API] GET /api/monitoring/health error:", error);
|
||||
return NextResponse.json({
|
||||
status: "degraded",
|
||||
error: "Health check partially unavailable",
|
||||
timestamp: new Date().toISOString(),
|
||||
providerBreakers: [],
|
||||
providerHealth: {},
|
||||
rateLimitStatus: {},
|
||||
learnedLimits: {},
|
||||
lockouts: [],
|
||||
quotaMonitor: {
|
||||
active: 0,
|
||||
alerting: 0,
|
||||
exhausted: 0,
|
||||
errors: 0,
|
||||
statusCounts: { starting: 0, idle: 0, healthy: 0, warning: 0, exhausted: 0, error: 0 },
|
||||
byProvider: {},
|
||||
monitors: [],
|
||||
},
|
||||
sessions: { activeCount: 0, stickyBoundCount: 0, byApiKey: {}, top: [] },
|
||||
adaptiveAdmission: null,
|
||||
chatAdmission: null,
|
||||
dedup: { inflightRequests: 0 },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function rebuildHealthPayload(): Promise<unknown> {
|
||||
const generation = healthPayloadCacheGeneration;
|
||||
const readHealthValue = <T>(label: string, reader: () => T, fallback: T): T => {
|
||||
try {
|
||||
return reader();
|
||||
@@ -64,179 +128,150 @@ export async function GET(request: Request) {
|
||||
byProvider: {},
|
||||
};
|
||||
|
||||
try {
|
||||
const [
|
||||
circuitBreakerModule,
|
||||
rateLimitModule,
|
||||
accountFallbackModule,
|
||||
requestDedupModule,
|
||||
quotaMonitorModule,
|
||||
sessionManagerModule,
|
||||
credentialHealthModule,
|
||||
localHealthModule,
|
||||
adaptiveAdmissionModule,
|
||||
chatAdmissionModule,
|
||||
settingsResult,
|
||||
connectionsResult,
|
||||
] = await Promise.allSettled([
|
||||
import("@/shared/utils/circuitBreaker"),
|
||||
import("@omniroute/open-sse/services/rateLimitManager"),
|
||||
import("@omniroute/open-sse/services/accountFallback"),
|
||||
import("@omniroute/open-sse/services/requestDedup.ts"),
|
||||
import("@omniroute/open-sse/services/quotaMonitor.ts"),
|
||||
import("@omniroute/open-sse/services/sessionManager.ts"),
|
||||
import("@/lib/credentialHealth/cache"),
|
||||
import("@/lib/localHealthCheck"),
|
||||
import("@omniroute/open-sse/services/admission/runtime.ts"),
|
||||
import("@/shared/middleware/chatBodyAdmission"),
|
||||
getCachedSettings(),
|
||||
getProviderConnections(),
|
||||
]);
|
||||
const [
|
||||
circuitBreakerModule,
|
||||
rateLimitModule,
|
||||
accountFallbackModule,
|
||||
requestDedupModule,
|
||||
quotaMonitorModule,
|
||||
sessionManagerModule,
|
||||
credentialHealthModule,
|
||||
localHealthModule,
|
||||
adaptiveAdmissionModule,
|
||||
chatAdmissionModule,
|
||||
settingsResult,
|
||||
connectionsResult,
|
||||
] = await Promise.allSettled([
|
||||
import("@/shared/utils/circuitBreaker"),
|
||||
import("@omniroute/open-sse/services/rateLimitManager"),
|
||||
import("@omniroute/open-sse/services/accountFallback"),
|
||||
import("@omniroute/open-sse/services/requestDedup.ts"),
|
||||
import("@omniroute/open-sse/services/quotaMonitor.ts"),
|
||||
import("@omniroute/open-sse/services/sessionManager.ts"),
|
||||
import("@/lib/credentialHealth/cache"),
|
||||
import("@/lib/localHealthCheck"),
|
||||
import("@omniroute/open-sse/services/admission/runtime.ts"),
|
||||
import("@/shared/middleware/chatBodyAdmission"),
|
||||
getCachedSettings(),
|
||||
getProviderConnections(),
|
||||
]);
|
||||
|
||||
const circuitBreakers =
|
||||
circuitBreakerModule.status === "fulfilled"
|
||||
? readHealthValue(
|
||||
"circuit breakers",
|
||||
() => circuitBreakerModule.value.getAllCircuitBreakerStatuses(),
|
||||
[]
|
||||
)
|
||||
: [];
|
||||
const rateLimitStatus =
|
||||
rateLimitModule.status === "fulfilled"
|
||||
? readHealthValue("rate limits", () => rateLimitModule.value.getAllRateLimitStatus(), {})
|
||||
: {};
|
||||
const learnedLimits =
|
||||
rateLimitModule.status === "fulfilled"
|
||||
? readHealthValue("learned limits", () => rateLimitModule.value.getLearnedLimits(), {})
|
||||
: {};
|
||||
const lockouts =
|
||||
accountFallbackModule.status === "fulfilled"
|
||||
? readHealthValue(
|
||||
"model lockouts",
|
||||
() => accountFallbackModule.value.getAllModelLockouts(),
|
||||
[]
|
||||
)
|
||||
: [];
|
||||
const quotaMonitorSummary =
|
||||
quotaMonitorModule.status === "fulfilled"
|
||||
? readHealthValue(
|
||||
"quota monitor summary",
|
||||
() => quotaMonitorModule.value.getQuotaMonitorSummary(),
|
||||
fallbackQuotaMonitorSummary
|
||||
)
|
||||
: fallbackQuotaMonitorSummary;
|
||||
const quotaMonitorMonitors =
|
||||
quotaMonitorModule.status === "fulfilled"
|
||||
? readHealthValue(
|
||||
"quota monitor snapshots",
|
||||
() => quotaMonitorModule.value.getQuotaMonitorSnapshots(),
|
||||
[]
|
||||
)
|
||||
: [];
|
||||
const activeSessions =
|
||||
sessionManagerModule.status === "fulfilled"
|
||||
? readHealthValue(
|
||||
"active sessions",
|
||||
() => sessionManagerModule.value.getActiveSessions(),
|
||||
[]
|
||||
)
|
||||
: [];
|
||||
const activeSessionsByKey =
|
||||
sessionManagerModule.status === "fulfilled"
|
||||
? readHealthValue(
|
||||
"active sessions by key",
|
||||
() => sessionManagerModule.value.getAllActiveSessionCountsByKey(),
|
||||
{}
|
||||
)
|
||||
: {};
|
||||
const credentialHealth =
|
||||
credentialHealthModule.status === "fulfilled"
|
||||
? readHealthValue(
|
||||
"credential health",
|
||||
() => credentialHealthModule.value.getCredentialHealthSummary(),
|
||||
undefined
|
||||
)
|
||||
: undefined;
|
||||
const localProviders =
|
||||
localHealthModule.status === "fulfilled"
|
||||
? readHealthValue(
|
||||
"local providers",
|
||||
() => localHealthModule.value.getAllHealthStatuses(),
|
||||
{}
|
||||
)
|
||||
: {};
|
||||
const settings = settingsResult.status === "fulfilled" ? settingsResult.value : {};
|
||||
const connections = connectionsResult.status === "fulfilled" ? connectionsResult.value : [];
|
||||
const adaptiveAdmission =
|
||||
adaptiveAdmissionModule.status === "fulfilled"
|
||||
? readHealthValue(
|
||||
"adaptive admission",
|
||||
() => adaptiveAdmissionModule.value.getAdaptiveAdmissionRuntime().snapshot(),
|
||||
null
|
||||
)
|
||||
: null;
|
||||
// #11244: the STRUCTURAL admission gate (chatBodyAdmission.ts — bounded
|
||||
// heavyweight lease + shed counters), exposed next to but distinct from the
|
||||
// adaptive shadow-mode snapshot above. Additive key — nothing existing moves.
|
||||
const chatAdmission =
|
||||
chatAdmissionModule.status === "fulfilled"
|
||||
? readHealthValue(
|
||||
"chat admission",
|
||||
() => chatAdmissionModule.value.perConnectionAdmissionController.snapshot(),
|
||||
null
|
||||
)
|
||||
: null;
|
||||
const circuitBreakers =
|
||||
circuitBreakerModule.status === "fulfilled"
|
||||
? readHealthValue(
|
||||
"circuit breakers",
|
||||
() => circuitBreakerModule.value.getAllCircuitBreakerStatuses(),
|
||||
[]
|
||||
)
|
||||
: [];
|
||||
const rateLimitStatus =
|
||||
rateLimitModule.status === "fulfilled"
|
||||
? readHealthValue("rate limits", () => rateLimitModule.value.getAllRateLimitStatus(), {})
|
||||
: {};
|
||||
const learnedLimits =
|
||||
rateLimitModule.status === "fulfilled"
|
||||
? readHealthValue("learned limits", () => rateLimitModule.value.getLearnedLimits(), {})
|
||||
: {};
|
||||
const lockouts =
|
||||
accountFallbackModule.status === "fulfilled"
|
||||
? readHealthValue(
|
||||
"model lockouts",
|
||||
() => accountFallbackModule.value.getAllModelLockouts(),
|
||||
[]
|
||||
)
|
||||
: [];
|
||||
const quotaMonitorSummary =
|
||||
quotaMonitorModule.status === "fulfilled"
|
||||
? readHealthValue(
|
||||
"quota monitor summary",
|
||||
() => quotaMonitorModule.value.getQuotaMonitorSummary(),
|
||||
fallbackQuotaMonitorSummary
|
||||
)
|
||||
: fallbackQuotaMonitorSummary;
|
||||
const quotaMonitorMonitors =
|
||||
quotaMonitorModule.status === "fulfilled"
|
||||
? readHealthValue(
|
||||
"quota monitor snapshots",
|
||||
() => quotaMonitorModule.value.getQuotaMonitorSnapshots(),
|
||||
[]
|
||||
)
|
||||
: [];
|
||||
const activeSessions =
|
||||
sessionManagerModule.status === "fulfilled"
|
||||
? readHealthValue("active sessions", () => sessionManagerModule.value.getActiveSessions(), [])
|
||||
: [];
|
||||
const activeSessionsByKey =
|
||||
sessionManagerModule.status === "fulfilled"
|
||||
? readHealthValue(
|
||||
"active sessions by key",
|
||||
() => sessionManagerModule.value.getAllActiveSessionCountsByKey(),
|
||||
{}
|
||||
)
|
||||
: {};
|
||||
const credentialHealth =
|
||||
credentialHealthModule.status === "fulfilled"
|
||||
? readHealthValue(
|
||||
"credential health",
|
||||
() => credentialHealthModule.value.getCachedCredentialHealthSummary(),
|
||||
undefined
|
||||
)
|
||||
: undefined;
|
||||
const localProviders =
|
||||
localHealthModule.status === "fulfilled"
|
||||
? readHealthValue("local providers", () => localHealthModule.value.getAllHealthStatuses(), {})
|
||||
: {};
|
||||
const settings = settingsResult.status === "fulfilled" ? settingsResult.value : {};
|
||||
const connections = connectionsResult.status === "fulfilled" ? connectionsResult.value : [];
|
||||
const adaptiveAdmission =
|
||||
adaptiveAdmissionModule.status === "fulfilled"
|
||||
? readHealthValue(
|
||||
"adaptive admission",
|
||||
() => adaptiveAdmissionModule.value.getAdaptiveAdmissionRuntime().snapshot(),
|
||||
null
|
||||
)
|
||||
: null;
|
||||
// #11244: the STRUCTURAL admission gate (chatBodyAdmission.ts — bounded
|
||||
// heavyweight lease + shed counters), exposed next to but distinct from the
|
||||
// adaptive shadow-mode snapshot above. Additive key — nothing existing moves.
|
||||
const chatAdmission =
|
||||
chatAdmissionModule.status === "fulfilled"
|
||||
? readHealthValue(
|
||||
"chat admission",
|
||||
() => chatAdmissionModule.value.perConnectionAdmissionController.snapshot(),
|
||||
null
|
||||
)
|
||||
: null;
|
||||
|
||||
const payload = buildHealthPayload({
|
||||
appVersion: APP_CONFIG.version,
|
||||
// #10427: surface the artifact's git SHA so a deployment can be audited over HTTP
|
||||
// instead of SSH + grepping compiled chunks (the 2026-08-14 gateway outage).
|
||||
buildSha: readRunningBuildSha(),
|
||||
catalogCount: Object.keys(AI_PROVIDERS).length,
|
||||
settings,
|
||||
connections,
|
||||
circuitBreakers,
|
||||
rateLimitStatus,
|
||||
learnedLimits,
|
||||
lockouts,
|
||||
localProviders,
|
||||
inflightRequests:
|
||||
requestDedupModule.status === "fulfilled"
|
||||
? readHealthValue(
|
||||
"inflight requests",
|
||||
() => requestDedupModule.value.getInflightCount(),
|
||||
0
|
||||
)
|
||||
: 0,
|
||||
quotaMonitorSummary,
|
||||
quotaMonitorMonitors,
|
||||
activeSessions,
|
||||
activeSessionsByKey,
|
||||
credentialHealth,
|
||||
adaptiveAdmission,
|
||||
chatAdmission,
|
||||
});
|
||||
const payload = buildHealthPayload({
|
||||
appVersion: APP_CONFIG.version,
|
||||
// #10427: surface the artifact's git SHA so a deployment can be audited over HTTP
|
||||
// instead of SSH + grepping compiled chunks (the 2026-08-14 gateway outage).
|
||||
buildSha: readRunningBuildSha(),
|
||||
catalogCount: Object.keys(AI_PROVIDERS).length,
|
||||
settings,
|
||||
connections,
|
||||
circuitBreakers,
|
||||
rateLimitStatus,
|
||||
learnedLimits,
|
||||
lockouts,
|
||||
localProviders,
|
||||
inflightRequests:
|
||||
requestDedupModule.status === "fulfilled"
|
||||
? readHealthValue("inflight requests", () => requestDedupModule.value.getInflightCount(), 0)
|
||||
: 0,
|
||||
quotaMonitorSummary,
|
||||
quotaMonitorMonitors,
|
||||
activeSessions,
|
||||
activeSessionsByKey,
|
||||
credentialHealth,
|
||||
adaptiveAdmission,
|
||||
chatAdmission,
|
||||
});
|
||||
|
||||
if (generation === healthPayloadCacheGeneration) {
|
||||
healthPayloadCache = { payload, expiresAt: Date.now() + HEALTH_PAYLOAD_TTL_MS };
|
||||
return NextResponse.json(fullView ? payload : publicHealthView(payload));
|
||||
} catch (error) {
|
||||
console.error("[API] GET /api/monitoring/health error:", error);
|
||||
return NextResponse.json({
|
||||
status: "degraded",
|
||||
error: "Health check partially unavailable",
|
||||
timestamp: new Date().toISOString(),
|
||||
providerBreakers: [],
|
||||
providerHealth: {},
|
||||
rateLimitStatus: {},
|
||||
learnedLimits: {},
|
||||
lockouts: [],
|
||||
quotaMonitor: { ...fallbackQuotaMonitorSummary, monitors: [] },
|
||||
sessions: { activeCount: 0, stickyBoundCount: 0, byApiKey: {}, top: [] },
|
||||
adaptiveAdmission: null,
|
||||
chatAdmission: null,
|
||||
dedup: { inflightRequests: 0 },
|
||||
});
|
||||
}
|
||||
return payload;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -175,29 +175,80 @@ export function getAllCredentialHealth(): Record<string, CredentialHealthStatus>
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cache summary stats for health API.
|
||||
*/
|
||||
export function getCredentialHealthSummary(): {
|
||||
export interface CredentialHealthSummary {
|
||||
total: number;
|
||||
healthy: number;
|
||||
failed: number;
|
||||
unknown: number;
|
||||
stale: number;
|
||||
} {
|
||||
const all = getAllCredentialHealth();
|
||||
const entries = Object.values(all);
|
||||
const now = Date.now();
|
||||
}
|
||||
|
||||
return {
|
||||
total: entries.length,
|
||||
healthy: entries.filter((e) => e.status === "active").length,
|
||||
failed: entries.filter((e) => e.status === "error").length,
|
||||
unknown: entries.filter((e) => e.status === "unknown").length,
|
||||
stale: entries.filter((e) => now - e.lastTested.getTime() > STALE_THRESHOLD_MS).length,
|
||||
/**
|
||||
* Snapshot credential health for GET /api/monitoring/health.
|
||||
*
|
||||
* Never probes upstream and never expires entries on read. Expired / old
|
||||
* rows stay in the counts so a scrape can return immediately while the
|
||||
* background scheduler refreshes them (#12532).
|
||||
*/
|
||||
export function getCachedCredentialHealthSummary(): CredentialHealthSummary {
|
||||
const state = getCacheState();
|
||||
const now = Date.now();
|
||||
let total = 0;
|
||||
let healthy = 0;
|
||||
let failed = 0;
|
||||
let unknown = 0;
|
||||
let stale = 0;
|
||||
|
||||
for (const entry of state.cache.values()) {
|
||||
total += 1;
|
||||
if (entry.status.status === "active") healthy += 1;
|
||||
else if (entry.status.status === "error") failed += 1;
|
||||
else unknown += 1;
|
||||
if (now - entry.status.lastTested.getTime() > STALE_THRESHOLD_MS || now > entry.expiresAt) {
|
||||
stale += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return { total, healthy, failed, unknown, stale };
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cache summary stats for health API.
|
||||
* Monitoring scrapes must use the stale-safe snapshot (no live probes).
|
||||
*/
|
||||
export function getCredentialHealthSummary(): CredentialHealthSummary {
|
||||
return getCachedCredentialHealthSummary();
|
||||
}
|
||||
|
||||
/** Test-only: drop every cached credential-health row. */
|
||||
export function __test_resetCredentialHealthCache(): void {
|
||||
globalThis.__omnirouteCredentialCache = {
|
||||
initialized: false,
|
||||
cache: new Map(),
|
||||
};
|
||||
}
|
||||
|
||||
/** Test-only: insert a cache row, including expired / stale timestamps. */
|
||||
export function __test_putCredentialHealth(entry: {
|
||||
connectionId: string;
|
||||
provider: string;
|
||||
status: "active" | "error" | "unknown";
|
||||
lastTested: Date;
|
||||
expiresAt?: number;
|
||||
}): void {
|
||||
const state = getCacheState();
|
||||
state.cache.set(entry.connectionId, {
|
||||
status: {
|
||||
connectionId: entry.connectionId,
|
||||
provider: entry.provider,
|
||||
status: entry.status,
|
||||
lastTested: entry.lastTested,
|
||||
consecutiveFailures: 0,
|
||||
},
|
||||
expiresAt: entry.expiresAt ?? Date.now() + DEFAULT_TTL_MS,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark cache as initialized (called by scheduler on startup).
|
||||
*/
|
||||
|
||||
@@ -17,13 +17,12 @@
|
||||
* - Resets to default on success
|
||||
*/
|
||||
|
||||
import { setImmediate as yieldToEventLoop } from "node:timers/promises";
|
||||
|
||||
import { testSingleConnection } from "@/app/api/providers/[id]/test/route";
|
||||
import { getProviderConnections } from "@/lib/db/providers";
|
||||
import { getCachedSettings } from "@/lib/db/readCache";
|
||||
import {
|
||||
setCredentialHealth,
|
||||
initCredentialCache,
|
||||
} from "@/lib/credentialHealth/cache";
|
||||
import { setCredentialHealth, initCredentialCache } from "@/lib/credentialHealth/cache";
|
||||
import {
|
||||
isCredentialProbeInconclusive,
|
||||
resolveInconclusiveProbeRecheckDelayMs,
|
||||
@@ -386,6 +385,9 @@ export async function sweep(): Promise<void> {
|
||||
}
|
||||
|
||||
for (const batch of batches) {
|
||||
// Yield so GET /healthz and cached /api/monitoring/health can drain
|
||||
// while this background sweep talks to providers (#12532).
|
||||
await yieldToEventLoop();
|
||||
await Promise.allSettled(
|
||||
batch.map((conn) =>
|
||||
testConnection(conn.id, conn.provider, getConnIntervalMs(conn, globalIntervalMs))
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
/**
|
||||
* Integration test for the short-TTL cache on GET /api/monitoring/health.
|
||||
*
|
||||
* Health is a frequently-polled endpoint; rebuilding it every request (DB reads
|
||||
* + status aggregation across subsystems) is wasteful under rapid polling. The
|
||||
* route caches the payload for HEALTH_PAYLOAD_TTL_MS (1s) and invalidates it on
|
||||
* DELETE (circuit-breaker reset). We assert the behavior via the payload's
|
||||
* `timestamp` field, which is stamped at build time: identical timestamp ⇒ the
|
||||
* cached payload was served; a fresh timestamp ⇒ it was rebuilt.
|
||||
* Health is a frequently-polled endpoint; rebuilding it on the request path
|
||||
* (DB reads + status aggregation) starves GET /healthz (#12532). The route
|
||||
* caches the payload for HEALTH_PAYLOAD_TTL_MS (1s). After the first fill,
|
||||
* expired entries are served immediately (stale-while-revalidate) and
|
||||
* refreshed off the request path. DELETE (circuit-breaker reset) invalidates
|
||||
* the cache so the next GET rebuilds. We assert via `timestamp`.
|
||||
*/
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
@@ -20,7 +20,8 @@ process.env.REQUIRE_API_KEY = "false";
|
||||
process.env.JWT_SECRET = "test-health-cache-secret";
|
||||
|
||||
await import("../../src/lib/db/core.ts");
|
||||
const { GET, DELETE } = await import("../../src/app/api/monitoring/health/route.ts");
|
||||
const { GET, DELETE, __test_resetMonitoringHealthPayloadCache } =
|
||||
await import("../../src/app/api/monitoring/health/route.ts");
|
||||
|
||||
// GHSA-mvf8-qc78-5mxm: the detailed health payload (the one carrying `timestamp`)
|
||||
// is reserved for a management principal — GET now takes the Request and an
|
||||
@@ -52,19 +53,22 @@ async function healthTimestamp(): Promise<string> {
|
||||
}
|
||||
|
||||
test("GET within the TTL serves the cached payload (identical timestamp)", async () => {
|
||||
__test_resetMonitoringHealthPayloadCache();
|
||||
const t1 = await healthTimestamp();
|
||||
const t2 = await healthTimestamp();
|
||||
assert.equal(t2, t1, "a second GET within the TTL must return the cached payload");
|
||||
});
|
||||
|
||||
test("cache expires after the TTL — a fresh payload is built", async () => {
|
||||
test("expired cache is served immediately (stale-while-revalidate)", async () => {
|
||||
__test_resetMonitoringHealthPayloadCache();
|
||||
const t1 = await healthTimestamp();
|
||||
await new Promise((r) => setTimeout(r, 1100)); // TTL is 1000ms
|
||||
const t2 = await healthTimestamp();
|
||||
assert.notEqual(t2, t1, "after the 1s TTL the payload must be rebuilt");
|
||||
assert.equal(t2, t1, "after the 1s TTL the stale cached payload must be returned immediately");
|
||||
});
|
||||
|
||||
test("DELETE (circuit-breaker reset) invalidates the cache immediately", async () => {
|
||||
__test_resetMonitoringHealthPayloadCache();
|
||||
const t1 = await healthTimestamp(); // populate cache
|
||||
const delRes = await DELETE(authedRequest("DELETE"));
|
||||
assert.ok(delRes.status < 400, `DELETE should succeed, got ${delRes.status}`);
|
||||
|
||||
111
tests/unit/monitoring-health-cached-credential.test.ts
Normal file
111
tests/unit/monitoring-health-cached-credential.test.ts
Normal file
@@ -0,0 +1,111 @@
|
||||
/**
|
||||
* #12532 — GET /api/monitoring/health must serve cached credentialHealth
|
||||
* immediately and must not run live credential probes on the request path.
|
||||
*/
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-health-cred-cache-"));
|
||||
process.env.DATA_DIR = TEST_DATA_DIR;
|
||||
process.env.REQUIRE_API_KEY = "false";
|
||||
process.env.JWT_SECRET = "test-health-cred-cache-secret";
|
||||
|
||||
await import("../../src/lib/db/core.ts");
|
||||
|
||||
const {
|
||||
getCachedCredentialHealthSummary,
|
||||
getCredentialHealthSummary,
|
||||
__test_resetCredentialHealthCache,
|
||||
__test_putCredentialHealth,
|
||||
} = await import("../../src/lib/credentialHealth/cache.ts");
|
||||
|
||||
const { GET, __test_resetMonitoringHealthPayloadCache } =
|
||||
await import("../../src/app/api/monitoring/health/route.ts");
|
||||
|
||||
const { SignJWT } = await import("jose");
|
||||
const AUTH_TOKEN = await new SignJWT({ authenticated: true })
|
||||
.setProtectedHeader({ alg: "HS256" })
|
||||
.setExpirationTime("30d")
|
||||
.sign(new TextEncoder().encode(process.env.JWT_SECRET as string));
|
||||
|
||||
function authedRequest(): Request {
|
||||
return new Request("http://localhost/api/monitoring/health", {
|
||||
method: "GET",
|
||||
headers: { cookie: `auth_token=${AUTH_TOKEN}` },
|
||||
});
|
||||
}
|
||||
|
||||
const STALE_MS = 11 * 60 * 1000;
|
||||
|
||||
test("getCachedCredentialHealthSummary includes expired and stale rows without deleting them", () => {
|
||||
__test_resetCredentialHealthCache();
|
||||
const lastTested = new Date(Date.now() - STALE_MS);
|
||||
__test_putCredentialHealth({
|
||||
connectionId: "conn-stale",
|
||||
provider: "openai",
|
||||
status: "active",
|
||||
lastTested,
|
||||
expiresAt: Date.now() - 1000,
|
||||
});
|
||||
|
||||
const summary = getCachedCredentialHealthSummary();
|
||||
assert.deepEqual(summary, {
|
||||
total: 1,
|
||||
healthy: 1,
|
||||
failed: 0,
|
||||
unknown: 0,
|
||||
stale: 1,
|
||||
});
|
||||
assert.deepEqual(getCredentialHealthSummary(), summary);
|
||||
assert.deepEqual(getCachedCredentialHealthSummary(), summary);
|
||||
});
|
||||
|
||||
test("GET /api/monitoring/health returns the stale cached summary immediately", async () => {
|
||||
__test_resetCredentialHealthCache();
|
||||
__test_resetMonitoringHealthPayloadCache();
|
||||
const lastTested = new Date(Date.now() - STALE_MS);
|
||||
__test_putCredentialHealth({
|
||||
connectionId: "conn-stale-get",
|
||||
provider: "anthropic",
|
||||
status: "error",
|
||||
lastTested,
|
||||
expiresAt: Date.now() - 5000,
|
||||
});
|
||||
|
||||
const started = Date.now();
|
||||
const res = await GET(authedRequest());
|
||||
const elapsedMs = Date.now() - started;
|
||||
const body = (await res.json()) as {
|
||||
credentialHealth?: {
|
||||
total: number;
|
||||
healthy: number;
|
||||
failed: number;
|
||||
unknown: number;
|
||||
stale: number;
|
||||
};
|
||||
};
|
||||
|
||||
assert.equal(res.status, 200);
|
||||
assert.deepEqual(body.credentialHealth, {
|
||||
total: 1,
|
||||
healthy: 0,
|
||||
failed: 1,
|
||||
unknown: 0,
|
||||
stale: 1,
|
||||
});
|
||||
assert.ok(elapsedMs < 2000, `stale summary must return immediately, took ${elapsedMs}ms`);
|
||||
});
|
||||
|
||||
test("monitoring health route never imports live credential probes", () => {
|
||||
const source = fs.readFileSync(
|
||||
path.join(process.cwd(), "src/app/api/monitoring/health/route.ts"),
|
||||
"utf8"
|
||||
);
|
||||
assert.doesNotMatch(source, /testSingleConnection/);
|
||||
assert.doesNotMatch(source, /credentialHealth\/scheduler/);
|
||||
assert.doesNotMatch(source, /forceSweep/);
|
||||
assert.match(source, /getCachedCredentialHealthSummary/);
|
||||
});
|
||||
Reference in New Issue
Block a user