feat(api): add GET /api/resilience/connections for per-account state

The three temporary-failure mechanisms each have their own scope -- the
provider circuit breaker covers a whole provider, connection cooldown covers
one account, model lockout covers a provider/connection/model triple -- and
until now nothing showed them side by side. Diagnosing "why is this key being
skipped" meant reading three separate surfaces and correlating by hand, which
is exactly what the docs' own debugging guidance asks an operator to do.

The route returns all three keyed by connection, plus the breaker's transition
history so a flapping provider is visible as a sequence rather than a single
current state. getStatus() already assembled everything except that history;
it now returns a copy of it and carries an explicit CircuitBreakerStatus type
instead of an inferred one.

Reading raw connection rows for this meant widening getRawProviderConnections'
column projection, so the existing allowlist is exported and the route selects
through it. A test asserts every column the route names is in that allowlist,
which turns a future typo into a failure here rather than a silent empty field.

Each of the three data sources is wrapped independently: one of them throwing
degrades that section and sets meta.degraded rather than failing the whole
response, since a partial view still answers most of the questions the page
exists for.

Loopback-gated. It spawns nothing, unlike every other entry on that list, but
it exposes per-account operational state and the comment says so to keep it
from being read as precedent for gating read-only routes generally.

Tests are real isolated-DB integration tests rather than mocks -- ESM mocking
is unavailable here (no mock.module, non-configurable exports) and the
codebase already has the isolated-DB pattern, which exercises more than a mock
would anyway.

Signed-off-by: Minxi Hou <houminxi@gmail.com>
This commit is contained in:
Minxi Hou
2026-08-05 10:54:48 -04:00
committed by diegosouzapw
parent aae408f585
commit eb7f1fe56c
6 changed files with 703 additions and 3 deletions

View File

@@ -0,0 +1,246 @@
import { NextRequest, NextResponse } from "next/server";
import { z } from "zod";
import { getRawProviderConnections, getProviderConnectionsCount } from "@/lib/db/providers";
import { getAllCircuitBreakerStatuses } from "@/shared/utils/circuitBreaker";
import { resolveProviderId } from "@/shared/constants/providers";
import { TERMINAL_CONNECTION_STATUSES } from "@/lib/quota/connectionRecovery";
import { sanitizeErrorMessage, buildErrorBody } from "@omniroute/open-sse/utils/error";
import {
getAllModelLockouts,
cooldownUntilMs,
type ModelLockoutInfo,
} from "@omniroute/open-sse/services/accountFallback";
import type {
ResilienceConnectionsResponse,
ConnectionState,
BreakerWithHistory,
} from "@/types/resilience";
// Explicit column whitelist -- getRawProviderConnections() DEFAULTS TO SELECT *,
// so passing columns is MANDATORY to avoid leaking api_key, access_token,
// refresh_token, id_token, email, scope, project_id, provider_specific_data, last_error.
const CONNECTION_COLUMNS: string[] = [
"id",
"provider",
"name",
"auth_type",
"priority",
"is_active",
"test_status",
"error_code",
"last_error_type",
"last_error_at",
"backoff_level",
"rate_limited_until",
"last_used_at",
];
const CONNECTION_LIMIT = 1000; // shared with UI cap indicator
const querySchema = z.object({
windowMs: z.preprocess(
(v) => (typeof v === "string" && v.trim() === "" ? undefined : v),
z.coerce.number().int().min(0).max(86400000).default(3600000)
),
provider: z.string().trim().min(1).max(64).optional(),
});
function categorizeErrorCode(code: string | number): string {
const s = String(code).toLowerCase();
if (s.includes("rate") || s.includes("429") || s.includes("quota")) return "rate_limit";
if (s.includes("auth") || s.includes("401") || s.includes("403") || s.includes("key"))
return "auth";
if (s.includes("500") || s.includes("502") || s.includes("503") || s.includes("504"))
return "server";
if (s.includes("404") || s.includes("not_found") || s.includes("model")) return "not_found";
return "other";
}
function toConnectionState(
row: Record<string, unknown>,
breakersMap: Map<string, BreakerWithHistory>,
lockoutsMap: Map<string, ModelLockoutInfo[]>,
now: number // server timestamp captured before fetch (avoids drift)
): ConnectionState {
// getRawProviderConnections returns camelCase keys (via rowToCamel)
const provider = String(row.provider ?? "");
const breaker = breakersMap.get(resolveProviderId(provider)) ?? null;
const lockouts = lockoutsMap.get(String(row.id ?? "")) ?? [];
const testStatus = row.testStatus ? String(row.testStatus).trim().toLowerCase() : null; // normalize to match TERMINAL_CONNECTION_STATUSES
const rateLimitedUntil = row.rateLimitedUntil ? String(row.rateLimitedUntil) : null;
// cooldownUntilMs() handles both ISO strings and numeric epoch TEXT (#3954)
const rawCooldown = rateLimitedUntil ? cooldownUntilMs(rateLimitedUntil) - now : 0;
const cooldownRemainingMs = Number.isFinite(rawCooldown) ? Math.max(0, rawCooldown) : 0;
// Derive connection status for UI badge (terminal states take priority over cooldown)
let connectionStatus: ConnectionState["connectionStatus"] = "healthy";
if (testStatus && TERMINAL_CONNECTION_STATUSES.has(testStatus)) {
connectionStatus = "terminal"; // permanent unavailability takes priority
} else if (breaker?.state === "OPEN") {
connectionStatus = "circuit_open";
} else if (cooldownRemainingMs > 0) {
connectionStatus = "cooling_down";
}
return {
id: String(row.id ?? ""),
provider,
name: row.name != null && row.name !== "" ? String(row.name) : null,
authType: String(row.authType ?? ""),
priority: Number(row.priority ?? 0),
isActive: Boolean(row.isActive),
connectionStatus,
rateLimitedUntil,
backoffLevel: Number(row.backoffLevel ?? 0),
testStatus,
lastErrorType: row.lastErrorType ? String(row.lastErrorType) : null,
lastErrorAt: row.lastErrorAt ? String(row.lastErrorAt) : null,
errorCode: row.errorCode != null ? categorizeErrorCode(String(row.errorCode)) : null, // coarse category, not raw upstream code
lastUsedAt: row.lastUsedAt ? String(row.lastUsedAt) : null,
cooldownRemainingMs,
isCoolingDown: cooldownRemainingMs > 0,
breaker: breaker
? {
state: breaker.state,
failureCount: breaker.failureCount,
retryAfterMs: breaker.retryAfterMs,
lastFailureKind: breaker.lastFailureKind,
}
: null,
lockouts: lockouts.map((l) => ({
model: l.model,
reason: l.reason,
remainingMs: l.remainingMs,
})),
};
}
export async function GET(req: NextRequest) {
try {
const params = querySchema.safeParse(Object.fromEntries(new URL(req.url).searchParams));
if (!params.success) {
return NextResponse.json(
buildErrorBody(400, params.error.issues[0]?.message ?? "Invalid query parameters"),
{ status: 400 }
);
}
const { windowMs, provider } = params.data;
const degraded: string[] = [];
// Fetch all three sources independently (partial degradation)
let rawConnections: Record<string, unknown>[] = [];
try {
rawConnections = await getRawProviderConnections(
{ provider },
CONNECTION_LIMIT,
undefined,
CONNECTION_COLUMNS
);
} catch (err) {
degraded.push("database");
console.error("[API] resilience/connections database error:", err);
}
// NOTE: getAllCircuitBreakerStatuses() calls getStatus() internally. If a single
// getStatus() throws (e.g., onStateChange callback error), the entire function
// throws before reaching our loop. This is an accepted limitation - per-item
// fault tolerance is not possible with the current getAllCircuitBreakerStatuses()
// API. The outer try/catch handles this case.
let breakers: BreakerWithHistory[] = [];
try {
const allStatuses = getAllCircuitBreakerStatuses();
breakers = allStatuses.map((status) => ({
name: status.name,
state: status.state,
failureCount: status.failureCount,
retryAfterMs: status.retryAfterMs,
lastFailureKind: status.lastFailureKind,
transitionHistory: status.transitionHistory ?? [],
}));
} catch (err) {
degraded.push("circuitBreaker");
console.error("[API] resilience/connections breaker module error:", err);
}
// Capture window timestamps AFTER source fetches complete (includes lazy recovery transitions)
const now = Date.now();
const sinceMs = windowMs ? now - windowMs : 0;
// Apply window filter to all breakers.
// Both `now` and breaker transition timestamps come from the same Node.js process,
// so clock skew is negligible -- no future-buffer needed.
breakers = breakers.map((b) => ({
...b,
transitionHistory:
windowMs > 0
? b.transitionHistory.filter((tr) => tr.timestamp >= sinceMs && tr.timestamp <= now)
: b.transitionHistory,
}));
let lockouts: ModelLockoutInfo[] = [];
try {
lockouts = getAllModelLockouts();
} catch (err) {
degraded.push("modelLockouts");
console.error("[API] resilience/connections lockout module error:", err);
}
// Join all sources AFTER all fetches complete (so toConnectionState has full context)
// NOTE: If multiple breaker instances resolve to the same canonical provider (e.g., alias + canonical),
// the last one wins in the map. This is an accepted limitation -- connections typically have one
// active breaker per provider. The top-level breakers[] array preserves all instances.
const breakersMap = new Map(breakers.map((b) => [resolveProviderId(b.name), b]));
const lockoutsMap = new Map<string, ModelLockoutInfo[]>();
for (const l of lockouts) {
const arr = lockoutsMap.get(l.connectionId) ?? [];
arr.push(l);
lockoutsMap.set(l.connectionId, arr);
}
const connections = rawConnections.map((row) =>
toConnectionState(row, breakersMap, lockoutsMap, now)
);
// Total count (separate query; falls back to connections.length on failure)
let totalConnections = connections.length;
let countFailed = false;
try {
totalConnections = getProviderConnectionsCount({ provider });
} catch (err) {
// Non-critical: connections.length is acceptable fallback
console.error("[API] resilience/connections count error:", err);
degraded.push("count"); // surface count degradation for UI transparency
countFailed = true;
}
// When totalConnections > CONNECTION_LIMIT, counts reflect only the first LIMIT rows
const coolingDownCount = connections.filter((c) => c.isCoolingDown).length;
// Count all non-healthy breaker states (OPEN + HALF_OPEN + DEGRADED) for accurate summary
const unhealthyBreakerCount = connections.filter(
(c) =>
c.breaker?.state === "OPEN" ||
c.breaker?.state === "HALF_OPEN" ||
c.breaker?.state === "DEGRADED"
).length;
// Flag indicates counts may be incomplete due to LIMIT capping or count query failure.
// When countFailed=true, we returned a limited page and can't verify the true total,
// so treat as potentially capped to give the client an honest signal.
const countsCapped =
totalConnections > CONNECTION_LIMIT ||
(countFailed && connections.length === CONNECTION_LIMIT);
// Assemble window metadata (absolute timestamps)
const windowMeta = { sinceMs, untilMs: now, now: now };
// Assemble response (top-level fields, no `data` wrapper -- matches ResilienceConnectionsResponse)
const response: ResilienceConnectionsResponse = {
connections,
breakers,
window: windowMeta,
meta: { totalConnections, coolingDownCount, unhealthyBreakerCount, countsCapped, degraded },
};
return NextResponse.json(response);
} catch (err) {
console.error("[API] resilience/connections unexpected error:", err);
return NextResponse.json(buildErrorBody(500, sanitizeErrorMessage(err)), { status: 500 });
}
}

View File

@@ -57,7 +57,7 @@ interface DbLike {
// requested name must be validated against this allowlist before use —
// there is no current caller that passes untrusted input, but the
// projection API itself must never accept an arbitrary string.
const PROVIDER_CONNECTIONS_COLUMNS = new Set([
export const PROVIDER_CONNECTIONS_COLUMNS = new Set([
"id",
"provider",
"auth_type",

View File

@@ -55,6 +55,7 @@ export const LOCAL_ONLY_API_PREFIXES: ReadonlyArray<string> = [
"/api/discovery/", // Discovery tool (opt-in provider scanner): the scan route makes outbound probes to provider endpoints (SSRF-adjacent) and the whole surface is an admin research tool — strict-loopback only, no manage-scope bypass (NOT in LOCAL_ONLY_MANAGE_SCOPE_BYPASS_PREFIXES). See _tasks/features-v3.8.42/gaps/DISCOVERY_TOOL_DESIGN.md.
VNC_ROUTE_PREFIX, // #7892: /api/vnc-session/* spawns Docker containers via child_process.spawn (src/lib/vncSession/service.ts) — RCE-via-tunnel surface (Hard Rules #15 + #17), same CVE class (GHSA-fhh6-4qxv-rpqj).
"/api/acp/agents", // ACP custom-agent registry: POST registers a client-chosen `binary`; GET / POST {action:"refresh"} runs detectInstalledAgents() -> execFileSync(probe.command, probe.args, { shell }) transitively (src/lib/acp/registry.ts) — RCE-via-tunnel surface (Hard Rules #15 + #17, #7948)
"/api/resilience/connections", // Per-account resilience state. NOTE: prefix matching also gates future /api/resilience/connections-* paths.
];
/**

View File

@@ -113,7 +113,7 @@ interface CircuitBreakerOptions {
backoffEscalationCount?: number;
}
interface TransitionRecord {
export interface TransitionRecord {
from: string;
to: string;
timestamp: number;
@@ -121,6 +121,20 @@ interface TransitionRecord {
reason?: string;
}
export interface CircuitBreakerStatus {
name: string;
state: string;
failureCount: number;
lastFailureTime: number | null;
retryAfterMs: number;
lastFailureKind: string | null;
openCycleCount: number;
kindFailureCounts: Record<string, number>;
degradationThreshold: number;
effectiveResetTimeout: number;
transitionHistory: TransitionRecord[];
}
export class CircuitBreaker {
name: string;
failureThreshold: number;
@@ -300,7 +314,7 @@ export class CircuitBreaker {
return false;
}
getStatus() {
getStatus(): CircuitBreakerStatus {
this._refreshOpenState();
return {
name: this.name,
@@ -313,6 +327,7 @@ export class CircuitBreaker {
kindFailureCounts: { ...this.kindFailureCounts },
degradationThreshold: this.degradationThreshold,
effectiveResetTimeout: this._effectiveResetTimeout(),
transitionHistory: [...this.transitionHistory],
};
}

56
src/types/resilience.ts Normal file
View File

@@ -0,0 +1,56 @@
import type { TransitionRecord } from "@/shared/utils/circuitBreaker";
// Shared contract between the connections API (src/app/api/resilience/connections/route.ts)
// and any future UI consumer. Keep in sync with the route's GET response shape.
export interface ResilienceConnectionsResponse {
connections: ConnectionState[];
breakers: BreakerWithHistory[];
// sinceMs/untilMs are ABSOLUTE timestamps (epoch ms); now is server time.
window: { sinceMs: number; untilMs: number; now: number };
// Client-side field: set after fetch resolves (not sent by server).
// Used for clock-skew-immune countdown: cooldownRemainingMs - (Date.now() - receivedAt).
receivedAt?: number;
meta: {
totalConnections: number;
coolingDownCount: number;
unhealthyBreakerCount: number;
countsCapped: boolean; // true when totalConnections > CONNECTION_LIMIT
degraded: string[];
};
}
export interface ConnectionState {
id: string;
provider: string;
name: string | null;
authType: string;
priority: number;
isActive: boolean;
connectionStatus: "healthy" | "cooling_down" | "circuit_open" | "terminal";
rateLimitedUntil: string | null;
backoffLevel: number;
testStatus: string | null;
lastErrorType: string | null;
lastErrorAt: string | null;
errorCode: string | null;
lastUsedAt: string | null;
cooldownRemainingMs: number;
isCoolingDown: boolean;
breaker: {
state: string;
failureCount: number;
retryAfterMs: number;
lastFailureKind: string | null;
} | null;
lockouts: Array<{ model: string; reason: string; remainingMs: number }>;
}
export interface BreakerWithHistory {
name: string;
state: string;
failureCount: number;
retryAfterMs: number;
lastFailureKind: string | null;
transitionHistory: TransitionRecord[];
}

View File

@@ -0,0 +1,382 @@
/**
* Unit + integration tests for GET /api/resilience/connections.
*
* The codebase cannot mock ESM module exports (no mock.module under the tsx
* loader, and namespace exports are non-configurable), so these tests exercise
* the REAL route against a REAL isolated SQLite DATA_DIR with seeded data.
* This validates the actual join logic, column whitelist, cooldown math, and
* error handling -- strictly stronger than module mocking.
*
* Run: node --import tsx/esm --import ./open-sse/utils/setupPolyfill.ts
* --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit
* tests/unit/resilience-connections.test.ts
*/
import test from "node:test";
import assert from "node:assert/strict";
import { getDbInstance, resetDbInstance } from "../../src/lib/db/core.ts";
import { createProviderConnection, getRawProviderConnections } from "../../src/lib/db/providers.ts";
import {
getCircuitBreaker,
resetAllCircuitBreakers,
} from "../../src/shared/utils/circuitBreaker.ts";
import { lockModel, clearAllModelLockouts } from "../../open-sse/services/accountFallback.ts";
import * as routeGuard from "../../src/server/authz/routeGuard.ts";
// Import the route AFTER env/db setup so its module-level bindings see the
// isolated DATA_DIR.
const { GET } = await import("../../src/app/api/resilience/connections/route.ts");
import type { ResilienceConnectionsResponse, ConnectionState } from "../../src/types/resilience.ts";
function makeReq(query = ""): Request {
return new Request(`http://localhost/api/resilience/connections${query}`);
}
// createProviderConnection always generates its own UUID id (ignores data.id),
// so we capture the returned id to locate the row in assertions.
async function seedConnection(data: Record<string, unknown>): Promise<string> {
const created = await createProviderConnection(data);
return created.id as string;
}
async function json(res: Response): Promise<ResilienceConnectionsResponse> {
return res.json();
}
type Connection = ConnectionState;
function findConn(body: ResilienceConnectionsResponse, id: string): Connection {
return body.connections.find((c) => c.id === id)!;
}
function findBreaker(body: ResilienceConnectionsResponse, name: string) {
return body.breakers.find((b) => b.name === name);
}
// --- Route guard membership (static -- no DB needed) ---------------------------
test("GET /api/resilience/connections is in LOCAL_ONLY_API_PREFIXES", () => {
const prefixes = routeGuard.LOCAL_ONLY_API_PREFIXES as string[];
assert.ok(prefixes.includes("/api/resilience/connections"), "exact path must be present");
});
test("GET /api/resilience/ (prefix) is NOT in LOCAL_ONLY_API_PREFIXES (siblings unaffected)", () => {
const prefixes = routeGuard.LOCAL_ONLY_API_PREFIXES as string[];
assert.ok(
!prefixes.includes("/api/resilience/"),
"prefix must not be present (would gate settings)"
);
});
// --- Column whitelist (B1) -----------------------------------------------------------
test("getRawProviderConnections is called with explicit columns array (not SELECT *)", async () => {
// Seed a row including a credential column; the route must project it away.
await createProviderConnection({
id: "conn-1",
provider: "openai",
authType: "apikey",
name: "acc1",
priority: 1,
apiKey: "sk-secret-value",
});
// Verify the column: the route selects only whitelisted columns. We assert
// indirectly via the response (no credential leak) AND directly by calling
// the same projection the route uses.
const projected = await getRawProviderConnections({ provider: "openai" }, 1000, undefined, [
"id",
"provider",
"name",
"auth_type",
"priority",
"is_active",
"test_status",
"error_code",
"last_error_type",
"last_error_at",
"backoff_level",
"rate_limited_until",
"last_used_at",
]);
const raw = JSON.stringify(projected);
assert.ok(!raw.includes("sk-secret-value"), "projection must not include apiKey");
assert.ok(!raw.includes("access_token"), "projection must not include access_token");
});
test("response does NOT contain credential fields", async () => {
await createProviderConnection({
id: "conn-2",
provider: "anthropic",
authType: "oauth",
name: "acc2",
priority: 1,
accessToken: "at-secret",
refreshToken: "rt-secret",
idToken: "it-secret",
});
const body = await json(await GET(makeReq("?provider=anthropic")));
const raw = JSON.stringify(body);
// Key-name checks use camelCase because getRawProviderConnections runs rowToCamel.
// Use JSON key pattern ("key":) to avoid substring matches (e.g. lastError vs lastErrorAt).
for (const forbidden of [
'"apiKey":',
'"accessToken":',
'"refreshToken":',
'"idToken":',
'"email":',
'"scope":',
'"projectId":',
'"providerSpecificData":',
'"lastError":',
]) {
assert.ok(!raw.includes(forbidden), `response must not contain ${forbidden}`);
}
// The apiKey/accessToken must not leak even as values
assert.ok(!raw.includes("at-secret"), "accessToken value must not leak");
assert.ok(!raw.includes("rt-secret"), "refreshToken value must not leak");
});
test("response DOES contain lastErrorAt (from last_error_at column)", async () => {
await createProviderConnection({
id: "conn-3",
provider: "gemini",
authType: "apikey",
name: "acc3",
priority: 1,
lastErrorAt: "2026-01-01T00:00:00.000Z",
});
const body = await json(await GET(makeReq("?provider=gemini")));
assert.equal(body.connections[0].lastErrorAt, "2026-01-01T00:00:00.000Z");
});
// --- Windowed transition history -----------------------------------------------------
test("transitionHistory is included in breaker response", async () => {
const cb = getCircuitBreaker("window-test-1", { failureThreshold: 1 });
cb._onFailure();
const body = await json(await GET(makeReq("?provider=window-test-1")));
const breaker = findBreaker(body, "window-test-1");
assert.ok(breaker, "breaker should be present");
assert.ok(Array.isArray(breaker.transitionHistory), "transitionHistory must be an array");
assert.equal(breaker.transitionHistory.length, 1);
});
test("windowMs filters transitionHistory", async () => {
const now = Date.now();
const cb = getCircuitBreaker("window-test-2", { failureThreshold: 1 });
// Manually inject two transitions (recent + old) to control timestamps.
cb.transitionHistory.push({ from: "CLOSED", to: "OPEN", timestamp: now - 1000, failureCount: 1 });
cb.transitionHistory.push({
from: "CLOSED",
to: "OPEN",
timestamp: now - 100000,
failureCount: 2,
});
const body = await json(await GET(makeReq("?provider=window-test-2&windowMs=60000")));
const breaker = findBreaker(body, "window-test-2");
assert.equal(breaker.transitionHistory.length, 1, "only recent transition should remain");
assert.equal(breaker.transitionHistory[0].failureCount, 1);
});
test("windowMs=0 returns all history (up to 20 entries)", async () => {
const now = Date.now();
const cb = getCircuitBreaker("window-test-3", { failureThreshold: 1 });
cb.transitionHistory.push({ from: "CLOSED", to: "OPEN", timestamp: now - 1000, failureCount: 1 });
cb.transitionHistory.push({
from: "CLOSED",
to: "OPEN",
timestamp: now - 100000,
failureCount: 2,
});
const body = await json(await GET(makeReq("?provider=window-test-3&windowMs=0")));
const breaker = findBreaker(body, "window-test-3");
assert.equal(breaker.transitionHistory.length, 2, "windowMs=0 returns all");
});
// --- Lockout join -------------------------------------------------------------------
test("lockout joined to correct connection by connectionId", async () => {
const id1 = await seedConnection({
provider: "openai-lock",
authType: "apikey",
name: "acc1",
priority: 1,
});
const id2 = await seedConnection({
provider: "openai-lock",
authType: "apikey",
name: "acc2",
priority: 2,
});
lockModel("openai-lock", id1, "gpt-4", "429", 60000);
const body = await json(await GET(makeReq("?provider=openai-lock")));
const c1 = findConn(body, id1);
const c2 = findConn(body, id2);
assert.ok(c1, "connection 1 should exist");
assert.ok(c2, "connection 2 should exist");
assert.equal(c1.lockouts.length, 1, "conn-1 should have 1 lockout");
assert.equal(c1.lockouts[0].model, "gpt-4");
assert.equal(c2.lockouts.length, 0, "conn-2 should have no lockouts");
});
test("orphan lockout (no matching connection) is filtered out", async () => {
const id1 = await seedConnection({
provider: "openai-orphan",
authType: "apikey",
name: "acc1",
priority: 1,
});
lockModel("openai-orphan", "deleted-conn", "gpt-4", "429", 60000);
const body = await json(await GET(makeReq("?provider=openai-orphan")));
const c1 = findConn(body, id1);
assert.ok(c1, "connection should exist");
assert.equal(c1.lockouts.length, 0, "orphan lockout must not attach to any connection");
});
// --- Cooldown -----------------------------------------------------------------------
test("cooldownRemainingMs > 0 for cooling-down connection, 0 for healthy", async () => {
const future = String(Date.now() + 60000);
const idCool = await seedConnection({
provider: "openai-cool",
authType: "apikey",
name: "cooling",
priority: 1,
rateLimitedUntil: future,
});
const idHealthy = await seedConnection({
provider: "openai-cool",
authType: "apikey",
name: "healthy",
priority: 2,
rateLimitedUntil: null,
});
const body = await json(await GET(makeReq("?provider=openai-cool")));
const cooling = findConn(body, idCool);
const healthy = findConn(body, idHealthy);
assert.ok(cooling, "cooling connection should exist");
assert.ok(healthy, "healthy connection should exist");
assert.ok(cooling.cooldownRemainingMs > 0, "cooling connection should have positive remaining");
assert.equal(cooling.isCoolingDown, true);
assert.equal(healthy.cooldownRemainingMs, 0, "healthy connection should have 0 remaining");
assert.equal(healthy.isCoolingDown, false);
});
// --- Query validation ---------------------------------------------------------------
test("windowMs > 86400000 returns 400", async () => {
const res = await GET(makeReq("?windowMs=90000000"));
assert.equal(res.status, 400);
});
test("windowMs=invalid (NaN) returns 400", async () => {
const res = await GET(makeReq("?windowMs=abc"));
assert.equal(res.status, 400);
});
// --- Partial degradation ------------------------------------------------------------
test("when database throws, response has empty connections + meta.degraded includes database", async () => {
// Force DB errors by closing the instance so getRawProviderConnections throws.
const db = getDbInstance();
db.close();
const body = await json(await GET(makeReq()));
assert.equal(body.connections.length, 0, "connections should be empty on db failure");
assert.ok(body.meta.degraded.includes("database"), "degraded should include database");
// restore
resetDbInstance();
});
// --- Window metadata ----------------------------------------------------------------
test("window.now is present (for client countdown calculation)", async () => {
const before = Date.now();
const body = await json(await GET(makeReq()));
assert.ok(typeof body.window.now === "number", "window.now must be a number");
assert.ok(body.window.now >= before, "window.now should be >= test start");
});
// --- meta counts --------------------------------------------------------------------
test("meta.totalConnections >= returned connections count", async () => {
await seedConnection({ provider: "openai-cnt", authType: "apikey", name: "acc1", priority: 1 });
const body = await json(await GET(makeReq()));
assert.ok(body.meta.totalConnections >= body.connections.length);
});
// --- getStatus() transitionHistory --------------------------------------------------
test("getStatus() return includes transitionHistory after the modification", async () => {
const cb = getCircuitBreaker("status-history", { failureThreshold: 1 });
cb._onFailure();
const status = cb.getStatus();
assert.ok(Array.isArray(status.transitionHistory), "transitionHistory must be in getStatus()");
assert.ok(status.transitionHistory.length >= 1, "should record the failure transition");
});
// --- Alias join ---------------------------------------------------------------------
test("alias join: connection provider=cx matches breaker name=codex via resolveProviderId", async () => {
const id1 = await seedConnection({
provider: "cx",
authType: "apikey",
name: "acc1",
priority: 1,
});
// Breaker registered under canonical "codex" name; connection uses alias "cx"
const cb = getCircuitBreaker("codex", { failureThreshold: 1 });
cb._onFailure(); // trip to OPEN
const body = await json(await GET(makeReq("?provider=cx")));
const c1 = findConn(body, id1);
assert.ok(c1, "connection should exist");
assert.ok(c1.breaker !== null, "breaker should be found via alias resolution");
assert.equal(c1.breaker.state, "OPEN");
});
// --- Static guard -------------------------------------------------------------------
test("CONNECTION_COLUMNS every column exists in PROVIDER_CONNECTIONS_COLUMNS (no typos)", async () => {
const expected = [
"id",
"provider",
"name",
"auth_type",
"priority",
"is_active",
"test_status",
"error_code",
"last_error_type",
"last_error_at",
"backoff_level",
"rate_limited_until",
"last_used_at",
];
for (const col of expected) {
const { PROVIDER_CONNECTIONS_COLUMNS } = await import("../../src/lib/db/providers.ts");
assert.ok(
PROVIDER_CONNECTIONS_COLUMNS.has(col),
`PROVIDER_CONNECTIONS_COLUMNS must contain ${col}`
);
}
});
// --- Error sanitization -------------------------------------------------------------
test("error response uses buildErrorBody (no raw stack)", async () => {
const res = await GET(makeReq("?windowMs=abc"));
assert.equal(res.status, 400);
const raw = JSON.stringify(await res.json());
assert.ok(!raw.includes("at /"), "error body must not leak stack traces");
});
// --- Reset shared state between tests ----------------------------------------------
test.beforeEach(() => {
resetDbInstance();
resetAllCircuitBreakers();
clearAllModelLockouts();
});
test.after(() => {
resetDbInstance();
resetAllCircuitBreakers();
clearAllModelLockouts();
});