diff --git a/src/app/api/resilience/connections/route.ts b/src/app/api/resilience/connections/route.ts new file mode 100644 index 0000000000..55e4714de7 --- /dev/null +++ b/src/app/api/resilience/connections/route.ts @@ -0,0 +1,246 @@ +import { NextRequest, NextResponse } from "next/server"; +import { z } from "zod"; + +import { getRawProviderConnections, getProviderConnectionsCount } from "@/lib/db/providers"; +import { getAllCircuitBreakerStatuses } from "@/shared/utils/circuitBreaker"; +import { resolveProviderId } from "@/shared/constants/providers"; +import { TERMINAL_CONNECTION_STATUSES } from "@/lib/quota/connectionRecovery"; +import { sanitizeErrorMessage, buildErrorBody } from "@omniroute/open-sse/utils/error"; +import { + getAllModelLockouts, + cooldownUntilMs, + type ModelLockoutInfo, +} from "@omniroute/open-sse/services/accountFallback"; +import type { + ResilienceConnectionsResponse, + ConnectionState, + BreakerWithHistory, +} from "@/types/resilience"; + +// Explicit column whitelist -- getRawProviderConnections() DEFAULTS TO SELECT *, +// so passing columns is MANDATORY to avoid leaking api_key, access_token, +// refresh_token, id_token, email, scope, project_id, provider_specific_data, last_error. +const CONNECTION_COLUMNS: string[] = [ + "id", + "provider", + "name", + "auth_type", + "priority", + "is_active", + "test_status", + "error_code", + "last_error_type", + "last_error_at", + "backoff_level", + "rate_limited_until", + "last_used_at", +]; + +const CONNECTION_LIMIT = 1000; // shared with UI cap indicator + +const querySchema = z.object({ + windowMs: z.preprocess( + (v) => (typeof v === "string" && v.trim() === "" ? undefined : v), + z.coerce.number().int().min(0).max(86400000).default(3600000) + ), + provider: z.string().trim().min(1).max(64).optional(), +}); + +function categorizeErrorCode(code: string | number): string { + const s = String(code).toLowerCase(); + if (s.includes("rate") || s.includes("429") || s.includes("quota")) return "rate_limit"; + if (s.includes("auth") || s.includes("401") || s.includes("403") || s.includes("key")) + return "auth"; + if (s.includes("500") || s.includes("502") || s.includes("503") || s.includes("504")) + return "server"; + if (s.includes("404") || s.includes("not_found") || s.includes("model")) return "not_found"; + return "other"; +} + +function toConnectionState( + row: Record, + breakersMap: Map, + lockoutsMap: Map, + now: number // server timestamp captured before fetch (avoids drift) +): ConnectionState { + // getRawProviderConnections returns camelCase keys (via rowToCamel) + const provider = String(row.provider ?? ""); + const breaker = breakersMap.get(resolveProviderId(provider)) ?? null; + const lockouts = lockoutsMap.get(String(row.id ?? "")) ?? []; + const testStatus = row.testStatus ? String(row.testStatus).trim().toLowerCase() : null; // normalize to match TERMINAL_CONNECTION_STATUSES + const rateLimitedUntil = row.rateLimitedUntil ? String(row.rateLimitedUntil) : null; + // cooldownUntilMs() handles both ISO strings and numeric epoch TEXT (#3954) + const rawCooldown = rateLimitedUntil ? cooldownUntilMs(rateLimitedUntil) - now : 0; + const cooldownRemainingMs = Number.isFinite(rawCooldown) ? Math.max(0, rawCooldown) : 0; + + // Derive connection status for UI badge (terminal states take priority over cooldown) + let connectionStatus: ConnectionState["connectionStatus"] = "healthy"; + if (testStatus && TERMINAL_CONNECTION_STATUSES.has(testStatus)) { + connectionStatus = "terminal"; // permanent unavailability takes priority + } else if (breaker?.state === "OPEN") { + connectionStatus = "circuit_open"; + } else if (cooldownRemainingMs > 0) { + connectionStatus = "cooling_down"; + } + + return { + id: String(row.id ?? ""), + provider, + name: row.name != null && row.name !== "" ? String(row.name) : null, + authType: String(row.authType ?? ""), + priority: Number(row.priority ?? 0), + isActive: Boolean(row.isActive), + connectionStatus, + rateLimitedUntil, + backoffLevel: Number(row.backoffLevel ?? 0), + testStatus, + lastErrorType: row.lastErrorType ? String(row.lastErrorType) : null, + lastErrorAt: row.lastErrorAt ? String(row.lastErrorAt) : null, + errorCode: row.errorCode != null ? categorizeErrorCode(String(row.errorCode)) : null, // coarse category, not raw upstream code + lastUsedAt: row.lastUsedAt ? String(row.lastUsedAt) : null, + cooldownRemainingMs, + isCoolingDown: cooldownRemainingMs > 0, + breaker: breaker + ? { + state: breaker.state, + failureCount: breaker.failureCount, + retryAfterMs: breaker.retryAfterMs, + lastFailureKind: breaker.lastFailureKind, + } + : null, + lockouts: lockouts.map((l) => ({ + model: l.model, + reason: l.reason, + remainingMs: l.remainingMs, + })), + }; +} + +export async function GET(req: NextRequest) { + try { + const params = querySchema.safeParse(Object.fromEntries(new URL(req.url).searchParams)); + if (!params.success) { + return NextResponse.json( + buildErrorBody(400, params.error.issues[0]?.message ?? "Invalid query parameters"), + { status: 400 } + ); + } + const { windowMs, provider } = params.data; + + const degraded: string[] = []; + + // Fetch all three sources independently (partial degradation) + let rawConnections: Record[] = []; + try { + rawConnections = await getRawProviderConnections( + { provider }, + CONNECTION_LIMIT, + undefined, + CONNECTION_COLUMNS + ); + } catch (err) { + degraded.push("database"); + console.error("[API] resilience/connections database error:", err); + } + + // NOTE: getAllCircuitBreakerStatuses() calls getStatus() internally. If a single + // getStatus() throws (e.g., onStateChange callback error), the entire function + // throws before reaching our loop. This is an accepted limitation - per-item + // fault tolerance is not possible with the current getAllCircuitBreakerStatuses() + // API. The outer try/catch handles this case. + let breakers: BreakerWithHistory[] = []; + try { + const allStatuses = getAllCircuitBreakerStatuses(); + breakers = allStatuses.map((status) => ({ + name: status.name, + state: status.state, + failureCount: status.failureCount, + retryAfterMs: status.retryAfterMs, + lastFailureKind: status.lastFailureKind, + transitionHistory: status.transitionHistory ?? [], + })); + } catch (err) { + degraded.push("circuitBreaker"); + console.error("[API] resilience/connections breaker module error:", err); + } + + // Capture window timestamps AFTER source fetches complete (includes lazy recovery transitions) + const now = Date.now(); + const sinceMs = windowMs ? now - windowMs : 0; + // Apply window filter to all breakers. + // Both `now` and breaker transition timestamps come from the same Node.js process, + // so clock skew is negligible -- no future-buffer needed. + breakers = breakers.map((b) => ({ + ...b, + transitionHistory: + windowMs > 0 + ? b.transitionHistory.filter((tr) => tr.timestamp >= sinceMs && tr.timestamp <= now) + : b.transitionHistory, + })); + + let lockouts: ModelLockoutInfo[] = []; + try { + lockouts = getAllModelLockouts(); + } catch (err) { + degraded.push("modelLockouts"); + console.error("[API] resilience/connections lockout module error:", err); + } + + // Join all sources AFTER all fetches complete (so toConnectionState has full context) + // NOTE: If multiple breaker instances resolve to the same canonical provider (e.g., alias + canonical), + // the last one wins in the map. This is an accepted limitation -- connections typically have one + // active breaker per provider. The top-level breakers[] array preserves all instances. + const breakersMap = new Map(breakers.map((b) => [resolveProviderId(b.name), b])); + const lockoutsMap = new Map(); + for (const l of lockouts) { + const arr = lockoutsMap.get(l.connectionId) ?? []; + arr.push(l); + lockoutsMap.set(l.connectionId, arr); + } + const connections = rawConnections.map((row) => + toConnectionState(row, breakersMap, lockoutsMap, now) + ); + + // Total count (separate query; falls back to connections.length on failure) + let totalConnections = connections.length; + let countFailed = false; + try { + totalConnections = getProviderConnectionsCount({ provider }); + } catch (err) { + // Non-critical: connections.length is acceptable fallback + console.error("[API] resilience/connections count error:", err); + degraded.push("count"); // surface count degradation for UI transparency + countFailed = true; + } + // When totalConnections > CONNECTION_LIMIT, counts reflect only the first LIMIT rows + const coolingDownCount = connections.filter((c) => c.isCoolingDown).length; + // Count all non-healthy breaker states (OPEN + HALF_OPEN + DEGRADED) for accurate summary + const unhealthyBreakerCount = connections.filter( + (c) => + c.breaker?.state === "OPEN" || + c.breaker?.state === "HALF_OPEN" || + c.breaker?.state === "DEGRADED" + ).length; + // Flag indicates counts may be incomplete due to LIMIT capping or count query failure. + // When countFailed=true, we returned a limited page and can't verify the true total, + // so treat as potentially capped to give the client an honest signal. + const countsCapped = + totalConnections > CONNECTION_LIMIT || + (countFailed && connections.length === CONNECTION_LIMIT); + + // Assemble window metadata (absolute timestamps) + const windowMeta = { sinceMs, untilMs: now, now: now }; + + // Assemble response (top-level fields, no `data` wrapper -- matches ResilienceConnectionsResponse) + const response: ResilienceConnectionsResponse = { + connections, + breakers, + window: windowMeta, + meta: { totalConnections, coolingDownCount, unhealthyBreakerCount, countsCapped, degraded }, + }; + return NextResponse.json(response); + } catch (err) { + console.error("[API] resilience/connections unexpected error:", err); + return NextResponse.json(buildErrorBody(500, sanitizeErrorMessage(err)), { status: 500 }); + } +} diff --git a/src/lib/db/providers.ts b/src/lib/db/providers.ts index 02a39f74c5..6fbf999486 100644 --- a/src/lib/db/providers.ts +++ b/src/lib/db/providers.ts @@ -57,7 +57,7 @@ interface DbLike { // requested name must be validated against this allowlist before use — // there is no current caller that passes untrusted input, but the // projection API itself must never accept an arbitrary string. -const PROVIDER_CONNECTIONS_COLUMNS = new Set([ +export const PROVIDER_CONNECTIONS_COLUMNS = new Set([ "id", "provider", "auth_type", diff --git a/src/server/authz/routeGuard.ts b/src/server/authz/routeGuard.ts index be7220b00a..97bed95714 100644 --- a/src/server/authz/routeGuard.ts +++ b/src/server/authz/routeGuard.ts @@ -55,6 +55,7 @@ export const LOCAL_ONLY_API_PREFIXES: ReadonlyArray = [ "/api/discovery/", // Discovery tool (opt-in provider scanner): the scan route makes outbound probes to provider endpoints (SSRF-adjacent) and the whole surface is an admin research tool — strict-loopback only, no manage-scope bypass (NOT in LOCAL_ONLY_MANAGE_SCOPE_BYPASS_PREFIXES). See _tasks/features-v3.8.42/gaps/DISCOVERY_TOOL_DESIGN.md. VNC_ROUTE_PREFIX, // #7892: /api/vnc-session/* spawns Docker containers via child_process.spawn (src/lib/vncSession/service.ts) — RCE-via-tunnel surface (Hard Rules #15 + #17), same CVE class (GHSA-fhh6-4qxv-rpqj). "/api/acp/agents", // ACP custom-agent registry: POST registers a client-chosen `binary`; GET / POST {action:"refresh"} runs detectInstalledAgents() -> execFileSync(probe.command, probe.args, { shell }) transitively (src/lib/acp/registry.ts) — RCE-via-tunnel surface (Hard Rules #15 + #17, #7948) + "/api/resilience/connections", // Per-account resilience state. NOTE: prefix matching also gates future /api/resilience/connections-* paths. ]; /** diff --git a/src/shared/utils/circuitBreaker.ts b/src/shared/utils/circuitBreaker.ts index 495402851a..724112f526 100644 --- a/src/shared/utils/circuitBreaker.ts +++ b/src/shared/utils/circuitBreaker.ts @@ -113,7 +113,7 @@ interface CircuitBreakerOptions { backoffEscalationCount?: number; } -interface TransitionRecord { +export interface TransitionRecord { from: string; to: string; timestamp: number; @@ -121,6 +121,20 @@ interface TransitionRecord { reason?: string; } +export interface CircuitBreakerStatus { + name: string; + state: string; + failureCount: number; + lastFailureTime: number | null; + retryAfterMs: number; + lastFailureKind: string | null; + openCycleCount: number; + kindFailureCounts: Record; + degradationThreshold: number; + effectiveResetTimeout: number; + transitionHistory: TransitionRecord[]; +} + export class CircuitBreaker { name: string; failureThreshold: number; @@ -300,7 +314,7 @@ export class CircuitBreaker { return false; } - getStatus() { + getStatus(): CircuitBreakerStatus { this._refreshOpenState(); return { name: this.name, @@ -313,6 +327,7 @@ export class CircuitBreaker { kindFailureCounts: { ...this.kindFailureCounts }, degradationThreshold: this.degradationThreshold, effectiveResetTimeout: this._effectiveResetTimeout(), + transitionHistory: [...this.transitionHistory], }; } diff --git a/src/types/resilience.ts b/src/types/resilience.ts new file mode 100644 index 0000000000..02c0e6b529 --- /dev/null +++ b/src/types/resilience.ts @@ -0,0 +1,56 @@ +import type { TransitionRecord } from "@/shared/utils/circuitBreaker"; + +// Shared contract between the connections API (src/app/api/resilience/connections/route.ts) +// and any future UI consumer. Keep in sync with the route's GET response shape. + +export interface ResilienceConnectionsResponse { + connections: ConnectionState[]; + breakers: BreakerWithHistory[]; + // sinceMs/untilMs are ABSOLUTE timestamps (epoch ms); now is server time. + window: { sinceMs: number; untilMs: number; now: number }; + // Client-side field: set after fetch resolves (not sent by server). + // Used for clock-skew-immune countdown: cooldownRemainingMs - (Date.now() - receivedAt). + receivedAt?: number; + meta: { + totalConnections: number; + coolingDownCount: number; + unhealthyBreakerCount: number; + countsCapped: boolean; // true when totalConnections > CONNECTION_LIMIT + degraded: string[]; + }; +} + +export interface ConnectionState { + id: string; + provider: string; + name: string | null; + authType: string; + priority: number; + isActive: boolean; + connectionStatus: "healthy" | "cooling_down" | "circuit_open" | "terminal"; + rateLimitedUntil: string | null; + backoffLevel: number; + testStatus: string | null; + lastErrorType: string | null; + lastErrorAt: string | null; + errorCode: string | null; + lastUsedAt: string | null; + cooldownRemainingMs: number; + isCoolingDown: boolean; + breaker: { + state: string; + failureCount: number; + retryAfterMs: number; + lastFailureKind: string | null; + } | null; + lockouts: Array<{ model: string; reason: string; remainingMs: number }>; +} + +export interface BreakerWithHistory { + name: string; + state: string; + failureCount: number; + retryAfterMs: number; + lastFailureKind: string | null; + transitionHistory: TransitionRecord[]; +} diff --git a/tests/unit/resilience-connections.test.ts b/tests/unit/resilience-connections.test.ts new file mode 100644 index 0000000000..e4b47c1811 --- /dev/null +++ b/tests/unit/resilience-connections.test.ts @@ -0,0 +1,382 @@ +/** + * Unit + integration tests for GET /api/resilience/connections. + * + * The codebase cannot mock ESM module exports (no mock.module under the tsx + * loader, and namespace exports are non-configurable), so these tests exercise + * the REAL route against a REAL isolated SQLite DATA_DIR with seeded data. + * This validates the actual join logic, column whitelist, cooldown math, and + * error handling -- strictly stronger than module mocking. + * + * Run: node --import tsx/esm --import ./open-sse/utils/setupPolyfill.ts + * --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit + * tests/unit/resilience-connections.test.ts + */ +import test from "node:test"; +import assert from "node:assert/strict"; + +import { getDbInstance, resetDbInstance } from "../../src/lib/db/core.ts"; +import { createProviderConnection, getRawProviderConnections } from "../../src/lib/db/providers.ts"; +import { + getCircuitBreaker, + resetAllCircuitBreakers, +} from "../../src/shared/utils/circuitBreaker.ts"; +import { lockModel, clearAllModelLockouts } from "../../open-sse/services/accountFallback.ts"; +import * as routeGuard from "../../src/server/authz/routeGuard.ts"; + +// Import the route AFTER env/db setup so its module-level bindings see the +// isolated DATA_DIR. +const { GET } = await import("../../src/app/api/resilience/connections/route.ts"); +import type { ResilienceConnectionsResponse, ConnectionState } from "../../src/types/resilience.ts"; + +function makeReq(query = ""): Request { + return new Request(`http://localhost/api/resilience/connections${query}`); +} + +// createProviderConnection always generates its own UUID id (ignores data.id), +// so we capture the returned id to locate the row in assertions. +async function seedConnection(data: Record): Promise { + const created = await createProviderConnection(data); + return created.id as string; +} + +async function json(res: Response): Promise { + return res.json(); +} + +type Connection = ConnectionState; +function findConn(body: ResilienceConnectionsResponse, id: string): Connection { + return body.connections.find((c) => c.id === id)!; +} +function findBreaker(body: ResilienceConnectionsResponse, name: string) { + return body.breakers.find((b) => b.name === name); +} + +// --- Route guard membership (static -- no DB needed) --------------------------- + +test("GET /api/resilience/connections is in LOCAL_ONLY_API_PREFIXES", () => { + const prefixes = routeGuard.LOCAL_ONLY_API_PREFIXES as string[]; + assert.ok(prefixes.includes("/api/resilience/connections"), "exact path must be present"); +}); + +test("GET /api/resilience/ (prefix) is NOT in LOCAL_ONLY_API_PREFIXES (siblings unaffected)", () => { + const prefixes = routeGuard.LOCAL_ONLY_API_PREFIXES as string[]; + assert.ok( + !prefixes.includes("/api/resilience/"), + "prefix must not be present (would gate settings)" + ); +}); + +// --- Column whitelist (B1) ----------------------------------------------------------- + +test("getRawProviderConnections is called with explicit columns array (not SELECT *)", async () => { + // Seed a row including a credential column; the route must project it away. + await createProviderConnection({ + id: "conn-1", + provider: "openai", + authType: "apikey", + name: "acc1", + priority: 1, + apiKey: "sk-secret-value", + }); + // Verify the column: the route selects only whitelisted columns. We assert + // indirectly via the response (no credential leak) AND directly by calling + // the same projection the route uses. + const projected = await getRawProviderConnections({ provider: "openai" }, 1000, undefined, [ + "id", + "provider", + "name", + "auth_type", + "priority", + "is_active", + "test_status", + "error_code", + "last_error_type", + "last_error_at", + "backoff_level", + "rate_limited_until", + "last_used_at", + ]); + const raw = JSON.stringify(projected); + assert.ok(!raw.includes("sk-secret-value"), "projection must not include apiKey"); + assert.ok(!raw.includes("access_token"), "projection must not include access_token"); +}); + +test("response does NOT contain credential fields", async () => { + await createProviderConnection({ + id: "conn-2", + provider: "anthropic", + authType: "oauth", + name: "acc2", + priority: 1, + accessToken: "at-secret", + refreshToken: "rt-secret", + idToken: "it-secret", + }); + const body = await json(await GET(makeReq("?provider=anthropic"))); + const raw = JSON.stringify(body); + // Key-name checks use camelCase because getRawProviderConnections runs rowToCamel. + // Use JSON key pattern ("key":) to avoid substring matches (e.g. lastError vs lastErrorAt). + for (const forbidden of [ + '"apiKey":', + '"accessToken":', + '"refreshToken":', + '"idToken":', + '"email":', + '"scope":', + '"projectId":', + '"providerSpecificData":', + '"lastError":', + ]) { + assert.ok(!raw.includes(forbidden), `response must not contain ${forbidden}`); + } + // The apiKey/accessToken must not leak even as values + assert.ok(!raw.includes("at-secret"), "accessToken value must not leak"); + assert.ok(!raw.includes("rt-secret"), "refreshToken value must not leak"); +}); + +test("response DOES contain lastErrorAt (from last_error_at column)", async () => { + await createProviderConnection({ + id: "conn-3", + provider: "gemini", + authType: "apikey", + name: "acc3", + priority: 1, + lastErrorAt: "2026-01-01T00:00:00.000Z", + }); + const body = await json(await GET(makeReq("?provider=gemini"))); + assert.equal(body.connections[0].lastErrorAt, "2026-01-01T00:00:00.000Z"); +}); + +// --- Windowed transition history ----------------------------------------------------- + +test("transitionHistory is included in breaker response", async () => { + const cb = getCircuitBreaker("window-test-1", { failureThreshold: 1 }); + cb._onFailure(); + const body = await json(await GET(makeReq("?provider=window-test-1"))); + const breaker = findBreaker(body, "window-test-1"); + assert.ok(breaker, "breaker should be present"); + assert.ok(Array.isArray(breaker.transitionHistory), "transitionHistory must be an array"); + assert.equal(breaker.transitionHistory.length, 1); +}); + +test("windowMs filters transitionHistory", async () => { + const now = Date.now(); + const cb = getCircuitBreaker("window-test-2", { failureThreshold: 1 }); + // Manually inject two transitions (recent + old) to control timestamps. + cb.transitionHistory.push({ from: "CLOSED", to: "OPEN", timestamp: now - 1000, failureCount: 1 }); + cb.transitionHistory.push({ + from: "CLOSED", + to: "OPEN", + timestamp: now - 100000, + failureCount: 2, + }); + const body = await json(await GET(makeReq("?provider=window-test-2&windowMs=60000"))); + const breaker = findBreaker(body, "window-test-2"); + assert.equal(breaker.transitionHistory.length, 1, "only recent transition should remain"); + assert.equal(breaker.transitionHistory[0].failureCount, 1); +}); + +test("windowMs=0 returns all history (up to 20 entries)", async () => { + const now = Date.now(); + const cb = getCircuitBreaker("window-test-3", { failureThreshold: 1 }); + cb.transitionHistory.push({ from: "CLOSED", to: "OPEN", timestamp: now - 1000, failureCount: 1 }); + cb.transitionHistory.push({ + from: "CLOSED", + to: "OPEN", + timestamp: now - 100000, + failureCount: 2, + }); + const body = await json(await GET(makeReq("?provider=window-test-3&windowMs=0"))); + const breaker = findBreaker(body, "window-test-3"); + assert.equal(breaker.transitionHistory.length, 2, "windowMs=0 returns all"); +}); + +// --- Lockout join ------------------------------------------------------------------- + +test("lockout joined to correct connection by connectionId", async () => { + const id1 = await seedConnection({ + provider: "openai-lock", + authType: "apikey", + name: "acc1", + priority: 1, + }); + const id2 = await seedConnection({ + provider: "openai-lock", + authType: "apikey", + name: "acc2", + priority: 2, + }); + lockModel("openai-lock", id1, "gpt-4", "429", 60000); + const body = await json(await GET(makeReq("?provider=openai-lock"))); + const c1 = findConn(body, id1); + const c2 = findConn(body, id2); + assert.ok(c1, "connection 1 should exist"); + assert.ok(c2, "connection 2 should exist"); + assert.equal(c1.lockouts.length, 1, "conn-1 should have 1 lockout"); + assert.equal(c1.lockouts[0].model, "gpt-4"); + assert.equal(c2.lockouts.length, 0, "conn-2 should have no lockouts"); +}); + +test("orphan lockout (no matching connection) is filtered out", async () => { + const id1 = await seedConnection({ + provider: "openai-orphan", + authType: "apikey", + name: "acc1", + priority: 1, + }); + lockModel("openai-orphan", "deleted-conn", "gpt-4", "429", 60000); + const body = await json(await GET(makeReq("?provider=openai-orphan"))); + const c1 = findConn(body, id1); + assert.ok(c1, "connection should exist"); + assert.equal(c1.lockouts.length, 0, "orphan lockout must not attach to any connection"); +}); + +// --- Cooldown ----------------------------------------------------------------------- + +test("cooldownRemainingMs > 0 for cooling-down connection, 0 for healthy", async () => { + const future = String(Date.now() + 60000); + const idCool = await seedConnection({ + provider: "openai-cool", + authType: "apikey", + name: "cooling", + priority: 1, + rateLimitedUntil: future, + }); + const idHealthy = await seedConnection({ + provider: "openai-cool", + authType: "apikey", + name: "healthy", + priority: 2, + rateLimitedUntil: null, + }); + const body = await json(await GET(makeReq("?provider=openai-cool"))); + const cooling = findConn(body, idCool); + const healthy = findConn(body, idHealthy); + assert.ok(cooling, "cooling connection should exist"); + assert.ok(healthy, "healthy connection should exist"); + assert.ok(cooling.cooldownRemainingMs > 0, "cooling connection should have positive remaining"); + assert.equal(cooling.isCoolingDown, true); + assert.equal(healthy.cooldownRemainingMs, 0, "healthy connection should have 0 remaining"); + assert.equal(healthy.isCoolingDown, false); +}); + +// --- Query validation --------------------------------------------------------------- + +test("windowMs > 86400000 returns 400", async () => { + const res = await GET(makeReq("?windowMs=90000000")); + assert.equal(res.status, 400); +}); + +test("windowMs=invalid (NaN) returns 400", async () => { + const res = await GET(makeReq("?windowMs=abc")); + assert.equal(res.status, 400); +}); + +// --- Partial degradation ------------------------------------------------------------ + +test("when database throws, response has empty connections + meta.degraded includes database", async () => { + // Force DB errors by closing the instance so getRawProviderConnections throws. + const db = getDbInstance(); + db.close(); + const body = await json(await GET(makeReq())); + assert.equal(body.connections.length, 0, "connections should be empty on db failure"); + assert.ok(body.meta.degraded.includes("database"), "degraded should include database"); + // restore + resetDbInstance(); +}); + +// --- Window metadata ---------------------------------------------------------------- + +test("window.now is present (for client countdown calculation)", async () => { + const before = Date.now(); + const body = await json(await GET(makeReq())); + assert.ok(typeof body.window.now === "number", "window.now must be a number"); + assert.ok(body.window.now >= before, "window.now should be >= test start"); +}); + +// --- meta counts -------------------------------------------------------------------- + +test("meta.totalConnections >= returned connections count", async () => { + await seedConnection({ provider: "openai-cnt", authType: "apikey", name: "acc1", priority: 1 }); + const body = await json(await GET(makeReq())); + assert.ok(body.meta.totalConnections >= body.connections.length); +}); + +// --- getStatus() transitionHistory -------------------------------------------------- + +test("getStatus() return includes transitionHistory after the modification", async () => { + const cb = getCircuitBreaker("status-history", { failureThreshold: 1 }); + cb._onFailure(); + const status = cb.getStatus(); + assert.ok(Array.isArray(status.transitionHistory), "transitionHistory must be in getStatus()"); + assert.ok(status.transitionHistory.length >= 1, "should record the failure transition"); +}); + +// --- Alias join --------------------------------------------------------------------- + +test("alias join: connection provider=cx matches breaker name=codex via resolveProviderId", async () => { + const id1 = await seedConnection({ + provider: "cx", + authType: "apikey", + name: "acc1", + priority: 1, + }); + // Breaker registered under canonical "codex" name; connection uses alias "cx" + const cb = getCircuitBreaker("codex", { failureThreshold: 1 }); + cb._onFailure(); // trip to OPEN + const body = await json(await GET(makeReq("?provider=cx"))); + const c1 = findConn(body, id1); + assert.ok(c1, "connection should exist"); + assert.ok(c1.breaker !== null, "breaker should be found via alias resolution"); + assert.equal(c1.breaker.state, "OPEN"); +}); + +// --- Static guard ------------------------------------------------------------------- + +test("CONNECTION_COLUMNS every column exists in PROVIDER_CONNECTIONS_COLUMNS (no typos)", async () => { + const expected = [ + "id", + "provider", + "name", + "auth_type", + "priority", + "is_active", + "test_status", + "error_code", + "last_error_type", + "last_error_at", + "backoff_level", + "rate_limited_until", + "last_used_at", + ]; + for (const col of expected) { + const { PROVIDER_CONNECTIONS_COLUMNS } = await import("../../src/lib/db/providers.ts"); + assert.ok( + PROVIDER_CONNECTIONS_COLUMNS.has(col), + `PROVIDER_CONNECTIONS_COLUMNS must contain ${col}` + ); + } +}); + +// --- Error sanitization ------------------------------------------------------------- + +test("error response uses buildErrorBody (no raw stack)", async () => { + const res = await GET(makeReq("?windowMs=abc")); + assert.equal(res.status, 400); + const raw = JSON.stringify(await res.json()); + assert.ok(!raw.includes("at /"), "error body must not leak stack traces"); +}); + +// --- Reset shared state between tests ---------------------------------------------- + +test.beforeEach(() => { + resetDbInstance(); + resetAllCircuitBreakers(); + clearAllModelLockouts(); +}); + +test.after(() => { + resetDbInstance(); + resetAllCircuitBreakers(); + clearAllModelLockouts(); +});