mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-09-20 13:52:28 +03:00
2463781e00
Boarded together with Phases 2, 4, 5 (#12051, #12053, #12055) and validated in one combined worktree: typecheck:core, check:file-size, check:changelog-integrity, check:complexity, check:cognitive-complexity, check:cycles, check-db-rules all green. Mechanical import-path migration only, no behavior change. Thanks for the phased, well-tested cleanup.
761 lines
26 KiB
TypeScript
761 lines
26 KiB
TypeScript
import { createHash } from "crypto";
|
|
|
|
import { getProviderConnections, updateProviderConnection } from "@/lib/db/providers";
|
|
import { getCachedProviderConnectionById } from "@/lib/db/readCache";
|
|
import { clearProviderFailure, clearModelLock } from "@omniroute/open-sse/services/accountFallback";
|
|
import { resolveProviderAlias } from "@omniroute/open-sse/services/model";
|
|
|
|
type JsonRecord = Record<string, unknown>;
|
|
|
|
export type ProviderAutopilotSeverity = "info" | "warning" | "critical";
|
|
export type ProviderAutopilotState = "healthy" | "degraded" | "down";
|
|
export type ProviderAutopilotStatus = "healthy" | "warning" | "critical";
|
|
|
|
export type ProviderAutopilotActionType =
|
|
| "clear_provider_breaker"
|
|
| "clear_connection_cooldown"
|
|
| "clear_stale_connection_error"
|
|
| "clear_model_lockout"
|
|
| "reactivate_connection"
|
|
| "deactivate_connection";
|
|
|
|
export interface ProviderAutopilotTarget {
|
|
provider: string;
|
|
connectionId?: string;
|
|
model?: string;
|
|
}
|
|
|
|
export interface ProviderAutopilotAction {
|
|
type: ProviderAutopilotActionType;
|
|
label: string;
|
|
risk: "low" | "medium" | "high";
|
|
requiresConfirmation: boolean;
|
|
target: ProviderAutopilotTarget;
|
|
preconditionsHash: string;
|
|
}
|
|
|
|
export interface ProviderAutopilotIssue {
|
|
id: string;
|
|
severity: ProviderAutopilotSeverity;
|
|
kind:
|
|
| "provider_circuit_open"
|
|
| "provider_circuit_half_open"
|
|
| "connection_cooldown"
|
|
| "stale_connection_error"
|
|
| "terminal_connection_error"
|
|
| "inactive_connection"
|
|
| "model_lockout"
|
|
| "quota_monitor_warning";
|
|
title: string;
|
|
recommendation: string;
|
|
target: ProviderAutopilotTarget;
|
|
evidence: JsonRecord;
|
|
actions: ProviderAutopilotAction[];
|
|
}
|
|
|
|
export interface ProviderAutopilotProvider {
|
|
provider: string;
|
|
state: ProviderAutopilotState;
|
|
score: number;
|
|
signals: {
|
|
circuitBreaker: JsonRecord | null;
|
|
connections: {
|
|
total: number;
|
|
active: number;
|
|
inactive: number;
|
|
cooldown: number;
|
|
terminal: number;
|
|
staleErrors: number;
|
|
};
|
|
modelLockouts: number;
|
|
quotaMonitor: JsonRecord | null;
|
|
};
|
|
issues: ProviderAutopilotIssue[];
|
|
}
|
|
|
|
export interface ProviderAutopilotReport {
|
|
status: ProviderAutopilotStatus;
|
|
checkedAt: string;
|
|
summary: {
|
|
providerCount: number;
|
|
connectionCount: number;
|
|
healthyCount: number;
|
|
issueCount: number;
|
|
actionableCount: number;
|
|
};
|
|
providers: ProviderAutopilotProvider[];
|
|
}
|
|
|
|
export interface ProviderAutopilotOptions {
|
|
provider?: string | null;
|
|
includeHealthy?: boolean;
|
|
includeActions?: boolean;
|
|
}
|
|
|
|
export interface ExecuteProviderAutopilotActionInput {
|
|
type: ProviderAutopilotActionType;
|
|
target: ProviderAutopilotTarget;
|
|
preconditionsHash: string;
|
|
dryRun?: boolean;
|
|
confirm?: boolean;
|
|
}
|
|
|
|
const TERMINAL_STATUSES = new Set(["banned", "expired", "credits_exhausted"]);
|
|
const OPEN_BREAKER_STATES = new Set(["OPEN", "HALF_OPEN"]);
|
|
|
|
function toRecord(value: unknown): JsonRecord {
|
|
return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {};
|
|
}
|
|
|
|
function toString(value: unknown): string | null {
|
|
return typeof value === "string" && value.trim() ? value.trim() : null;
|
|
}
|
|
|
|
function canonicalProviderId(value: unknown): string | null {
|
|
const provider = toString(value);
|
|
return provider ? (resolveProviderAlias(provider) ?? provider) : null;
|
|
}
|
|
|
|
function toNumber(value: unknown): number | null {
|
|
if (typeof value === "number" && Number.isFinite(value)) return value;
|
|
if (typeof value === "string" && value.trim()) {
|
|
const parsed = Number(value);
|
|
if (Number.isFinite(parsed)) return parsed;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function parseTimeMs(value: unknown): number | null {
|
|
if (typeof value === "number" && Number.isFinite(value)) return value;
|
|
if (typeof value === "string" && value.trim()) {
|
|
const numeric = Number(value);
|
|
if (Number.isFinite(numeric)) return numeric;
|
|
const parsed = new Date(value).getTime();
|
|
if (Number.isFinite(parsed)) return parsed;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function stableStringify(value: unknown): string {
|
|
if (value === null || typeof value !== "object") return JSON.stringify(value);
|
|
if (Array.isArray(value)) return `[${value.map((entry) => stableStringify(entry)).join(",")}]`;
|
|
const record = value as JsonRecord;
|
|
return `{${Object.keys(record)
|
|
.sort()
|
|
.map((key) => `${JSON.stringify(key)}:${stableStringify(record[key])}`)
|
|
.join(",")}}`;
|
|
}
|
|
|
|
function hashPreconditions(input: unknown): string {
|
|
return createHash("sha256").update(stableStringify(input)).digest("hex").slice(0, 16);
|
|
}
|
|
|
|
function stableEvidenceForHash(evidence: JsonRecord): JsonRecord {
|
|
const stable: JsonRecord = { ...evidence };
|
|
delete stable.remainingMs;
|
|
delete stable.retryAfterMs;
|
|
return stable;
|
|
}
|
|
|
|
function targetMatches(left: ProviderAutopilotTarget, right: ProviderAutopilotTarget): boolean {
|
|
return (
|
|
left.provider === right.provider &&
|
|
(left.connectionId ?? null) === (right.connectionId ?? null) &&
|
|
(left.model ?? null) === (right.model ?? null)
|
|
);
|
|
}
|
|
|
|
function action(
|
|
type: ProviderAutopilotActionType,
|
|
label: string,
|
|
risk: ProviderAutopilotAction["risk"],
|
|
target: ProviderAutopilotTarget,
|
|
evidence: JsonRecord
|
|
): ProviderAutopilotAction {
|
|
return {
|
|
type,
|
|
label,
|
|
risk,
|
|
requiresConfirmation: true,
|
|
target,
|
|
preconditionsHash: hashPreconditions({
|
|
type,
|
|
target,
|
|
evidence: stableEvidenceForHash(evidence),
|
|
}),
|
|
};
|
|
}
|
|
|
|
function issueId(kind: ProviderAutopilotIssue["kind"], target: ProviderAutopilotTarget): string {
|
|
return ["pha", kind, target.provider, target.connectionId, target.model]
|
|
.filter(Boolean)
|
|
.join("_")
|
|
.replace(/[^a-zA-Z0-9_.:-]/g, "_");
|
|
}
|
|
|
|
function sanitizeConnectionLabel(connection: JsonRecord): string {
|
|
const label =
|
|
toString(connection.name) ||
|
|
toString(connection.displayName) ||
|
|
toString(connection.email) ||
|
|
toString(connection.id) ||
|
|
"connection";
|
|
const masked = label.includes("@") ? label.replace(/^(.).+(@.+)$/, "$1***$2") : label;
|
|
return masked.length > 80 ? `${masked.slice(0, 77)}…` : masked;
|
|
}
|
|
|
|
function isTerminalConnection(connection: JsonRecord): boolean {
|
|
const status = toString(connection.testStatus)?.toLowerCase();
|
|
const errorType = toString(connection.lastErrorType)?.toLowerCase();
|
|
return Boolean(
|
|
(status && TERMINAL_STATUSES.has(status)) || (errorType && TERMINAL_STATUSES.has(errorType))
|
|
);
|
|
}
|
|
|
|
function isConnectionInCooldown(connection: JsonRecord, now: number): boolean {
|
|
const until = parseTimeMs(connection.rateLimitedUntil);
|
|
return Boolean(until && until > now);
|
|
}
|
|
|
|
function hasStaleConnectionError(connection: JsonRecord, now: number): boolean {
|
|
if (isTerminalConnection(connection)) return false;
|
|
if (isConnectionInCooldown(connection, now)) return false;
|
|
const until = parseTimeMs(connection.rateLimitedUntil);
|
|
const status = toString(connection.testStatus)?.toLowerCase();
|
|
return Boolean(
|
|
toString(connection.lastError) ||
|
|
toString(connection.lastErrorType) ||
|
|
toString(connection.errorCode) ||
|
|
(until && until <= now) ||
|
|
status === "unavailable" ||
|
|
status === "error"
|
|
);
|
|
}
|
|
|
|
function providerFromLockout(lockout: JsonRecord): string | null {
|
|
return toString(lockout.provider);
|
|
}
|
|
|
|
function buildConnectionClearPatch(): JsonRecord {
|
|
return {
|
|
testStatus: "active",
|
|
errorCode: null,
|
|
lastError: null,
|
|
lastErrorAt: null,
|
|
lastErrorType: null,
|
|
lastErrorSource: null,
|
|
backoffLevel: 0,
|
|
rateLimitedUntil: null,
|
|
};
|
|
}
|
|
|
|
export async function buildProviderHealthAutopilotReport(
|
|
options: ProviderAutopilotOptions = {}
|
|
): Promise<ProviderAutopilotReport> {
|
|
const now = Date.now();
|
|
const checkedAt = new Date(now).toISOString();
|
|
const includeHealthy = options.includeHealthy === true;
|
|
const includeActions = options.includeActions !== false;
|
|
const providerFilter = canonicalProviderId(options.provider);
|
|
|
|
const [{ getAllCircuitBreakerStatuses }, { getAllModelLockouts }, quotaMonitor] =
|
|
await Promise.all([
|
|
import("@/shared/utils/circuitBreaker"),
|
|
import("@omniroute/open-sse/services/accountFallback"),
|
|
import("@omniroute/open-sse/services/quotaMonitor.ts").catch(() => null),
|
|
]);
|
|
|
|
// Connections generally use canonical ids, while breakers, lockouts, and quota
|
|
// snapshots can retain the alias used at dispatch time. Normalize the aggregation
|
|
// key, but preserve each raw source id for actions that must mutate runtime state.
|
|
const connections = ((await getProviderConnections({})) as JsonRecord[]).filter((connection) => {
|
|
const provider = canonicalProviderId(connection.provider);
|
|
return provider && (!providerFilter || provider === providerFilter);
|
|
});
|
|
const breakers = getAllCircuitBreakerStatuses().filter((breaker) => {
|
|
const name = toString((breaker as JsonRecord).name);
|
|
const provider = canonicalProviderId(name);
|
|
if (!name || !provider || name.startsWith("test-") || name.startsWith("test_")) return false;
|
|
return !providerFilter || provider === providerFilter;
|
|
});
|
|
const lockouts = (getAllModelLockouts() as JsonRecord[]).filter((lockout) => {
|
|
const provider = canonicalProviderId(providerFromLockout(lockout));
|
|
return provider && (!providerFilter || provider === providerFilter);
|
|
});
|
|
const quotaSnapshots = quotaMonitor?.getQuotaMonitorSnapshots
|
|
? (quotaMonitor.getQuotaMonitorSnapshots() as JsonRecord[]).filter((snapshot) => {
|
|
const provider = canonicalProviderId(snapshot.provider);
|
|
return provider && (!providerFilter || provider === providerFilter);
|
|
})
|
|
: [];
|
|
|
|
const providerIds = new Set<string>();
|
|
for (const connection of connections) {
|
|
const provider = canonicalProviderId(connection.provider);
|
|
if (provider) providerIds.add(provider);
|
|
}
|
|
for (const breaker of breakers) {
|
|
const provider = canonicalProviderId((breaker as JsonRecord).name);
|
|
if (provider) providerIds.add(provider);
|
|
}
|
|
for (const lockout of lockouts) {
|
|
const provider = canonicalProviderId(providerFromLockout(lockout));
|
|
if (provider) providerIds.add(provider);
|
|
}
|
|
for (const snapshot of quotaSnapshots) {
|
|
const provider = canonicalProviderId(snapshot.provider);
|
|
if (provider) providerIds.add(provider);
|
|
}
|
|
if (providerFilter) providerIds.add(providerFilter);
|
|
|
|
const providers: ProviderAutopilotProvider[] = [];
|
|
for (const provider of [...providerIds].sort()) {
|
|
const providerConnections = connections.filter(
|
|
(connection) => canonicalProviderId(connection.provider) === provider
|
|
);
|
|
const breaker = breakers.find(
|
|
(entry) => canonicalProviderId((entry as JsonRecord).name) === provider
|
|
) as JsonRecord | undefined;
|
|
const providerLockouts = lockouts.filter(
|
|
(lockout) => canonicalProviderId(providerFromLockout(lockout)) === provider
|
|
);
|
|
const providerQuota = quotaSnapshots.filter(
|
|
(snapshot) => canonicalProviderId(snapshot.provider) === provider
|
|
);
|
|
const issues: ProviderAutopilotIssue[] = [];
|
|
|
|
if (breaker && OPEN_BREAKER_STATES.has(String(breaker.state))) {
|
|
const target = { provider: toString(breaker.name) ?? provider };
|
|
const evidence = {
|
|
state: breaker.state,
|
|
failureCount: toNumber(breaker.failureCount) ?? 0,
|
|
retryAfterMs: toNumber(breaker.retryAfterMs) ?? 0,
|
|
lastFailureTime: breaker.lastFailureTime ?? null,
|
|
};
|
|
const isOpen = breaker.state === "OPEN";
|
|
issues.push({
|
|
id: issueId(isOpen ? "provider_circuit_open" : "provider_circuit_half_open", target),
|
|
severity: isOpen ? "critical" : "warning",
|
|
kind: isOpen ? "provider_circuit_open" : "provider_circuit_half_open",
|
|
title: isOpen ? "Provider circuit breaker is open" : "Provider is probing recovery",
|
|
recommendation: isOpen
|
|
? "Verify upstream recovery, then reset the provider circuit breaker or wait for the retry window."
|
|
: "Let the next probe complete or reset the breaker after manual verification.",
|
|
target,
|
|
evidence,
|
|
actions: includeActions
|
|
? [action("clear_provider_breaker", "Reset provider breaker", "medium", target, evidence)]
|
|
: [],
|
|
});
|
|
}
|
|
|
|
for (const connection of providerConnections) {
|
|
const connectionId = toString(connection.id);
|
|
if (!connectionId) continue;
|
|
const target = { provider: toString(connection.provider) ?? provider, connectionId };
|
|
const label = sanitizeConnectionLabel(connection);
|
|
const cooldownUntil = parseTimeMs(connection.rateLimitedUntil);
|
|
const terminal = isTerminalConnection(connection);
|
|
const evidence = {
|
|
connectionId,
|
|
label,
|
|
testStatus: connection.testStatus ?? null,
|
|
lastErrorType: connection.lastErrorType ?? null,
|
|
lastErrorAt: connection.lastErrorAt ?? null,
|
|
lastErrorHash: toString(connection.lastError)
|
|
? hashPreconditions(toString(connection.lastError))
|
|
: null,
|
|
errorCode: connection.errorCode ?? null,
|
|
backoffLevel: connection.backoffLevel ?? null,
|
|
updatedAt: connection.updatedAt ?? null,
|
|
rateLimitedUntil: connection.rateLimitedUntil ?? null,
|
|
remainingMs: cooldownUntil ? Math.max(0, cooldownUntil - now) : 0,
|
|
isActive: connection.isActive !== false,
|
|
};
|
|
|
|
if (terminal) {
|
|
issues.push({
|
|
id: issueId("terminal_connection_error", target),
|
|
severity: "critical",
|
|
kind: "terminal_connection_error",
|
|
title: `${label} has a terminal account state`,
|
|
recommendation:
|
|
"Check billing, re-authenticate, or replace the credential before reactivating it.",
|
|
target,
|
|
evidence,
|
|
actions: [],
|
|
});
|
|
} else if (isConnectionInCooldown(connection, now)) {
|
|
issues.push({
|
|
id: issueId("connection_cooldown", target),
|
|
severity: "warning",
|
|
kind: "connection_cooldown",
|
|
title: `${label} is in temporary cooldown`,
|
|
recommendation:
|
|
"Wait for the upstream retry window or clear the cooldown after validating recovery.",
|
|
target,
|
|
evidence,
|
|
actions: includeActions
|
|
? [
|
|
action(
|
|
"clear_connection_cooldown",
|
|
"Clear connection cooldown",
|
|
"medium",
|
|
target,
|
|
evidence
|
|
),
|
|
action(
|
|
"deactivate_connection",
|
|
"Disable this connection",
|
|
"medium",
|
|
target,
|
|
evidence
|
|
),
|
|
]
|
|
: [],
|
|
});
|
|
} else if (hasStaleConnectionError(connection, now)) {
|
|
issues.push({
|
|
id: issueId("stale_connection_error", target),
|
|
severity: "info",
|
|
kind: "stale_connection_error",
|
|
title: `${label} has stale error state`,
|
|
recommendation:
|
|
"Clear stale error fields so the connection is eligible for normal routing again.",
|
|
target,
|
|
evidence,
|
|
actions: includeActions
|
|
? [
|
|
action(
|
|
"clear_stale_connection_error",
|
|
"Clear stale error state",
|
|
"low",
|
|
target,
|
|
evidence
|
|
),
|
|
]
|
|
: [],
|
|
});
|
|
}
|
|
|
|
if (connection.isActive === false && !terminal) {
|
|
issues.push({
|
|
id: issueId("inactive_connection", target),
|
|
severity: "info",
|
|
kind: "inactive_connection",
|
|
title: `${label} is disabled`,
|
|
recommendation: "Reactivate only if this was not intentionally disabled.",
|
|
target,
|
|
evidence,
|
|
actions: includeActions
|
|
? [action("reactivate_connection", "Reactivate connection", "medium", target, evidence)]
|
|
: [],
|
|
});
|
|
}
|
|
}
|
|
|
|
for (const lockout of providerLockouts) {
|
|
const connectionId = toString(lockout.connectionId);
|
|
const model = toString(lockout.model);
|
|
if (!connectionId || !model) continue;
|
|
const connection = providerConnections.find((entry) => entry.id === connectionId);
|
|
const terminalConnection = connection ? isTerminalConnection(connection) : false;
|
|
const target = {
|
|
provider: providerFromLockout(lockout) ?? provider,
|
|
connectionId,
|
|
model,
|
|
};
|
|
const evidence = {
|
|
reason: lockout.reason ?? null,
|
|
remainingMs: toNumber(lockout.remainingMs) ?? 0,
|
|
failureCount: toNumber(lockout.failureCount) ?? 0,
|
|
lockedAt: lockout.lockedAt ?? null,
|
|
until: lockout.until ?? null,
|
|
connectionExists: Boolean(connection),
|
|
terminalConnection,
|
|
};
|
|
issues.push({
|
|
id: issueId("model_lockout", target),
|
|
severity: terminalConnection || !connection ? "info" : "warning",
|
|
kind: "model_lockout",
|
|
title: `${model} is locked out for one connection`,
|
|
recommendation:
|
|
terminalConnection || !connection
|
|
? "Resolve the connection state before clearing model-level lockouts."
|
|
: "Clear the model lockout after confirming the model quota or availability recovered.",
|
|
target,
|
|
evidence,
|
|
actions:
|
|
includeActions && connection && !terminalConnection
|
|
? [action("clear_model_lockout", "Clear model lockout", "medium", target, evidence)]
|
|
: [],
|
|
});
|
|
}
|
|
|
|
for (const snapshot of providerQuota) {
|
|
const status = toString(snapshot.status);
|
|
if (!status || !["warning", "exhausted", "error"].includes(status)) continue;
|
|
const connectionId = toString(snapshot.accountId) ?? undefined;
|
|
const sessionId = toString(snapshot.sessionId) ?? undefined;
|
|
const target = {
|
|
provider: toString(snapshot.provider) ?? provider,
|
|
...(connectionId ? { connectionId } : {}),
|
|
};
|
|
issues.push({
|
|
id: issueId("quota_monitor_warning", {
|
|
...target,
|
|
...(sessionId ? { model: sessionId } : {}),
|
|
}),
|
|
severity: status === "warning" ? "warning" : "critical",
|
|
kind: "quota_monitor_warning",
|
|
title: `Quota monitor reports ${status}`,
|
|
recommendation:
|
|
"Review quota usage and rotate traffic to another healthy connection if needed.",
|
|
target,
|
|
evidence: {
|
|
status,
|
|
lastQuotaPercent: snapshot.lastQuotaPercent ?? null,
|
|
consecutiveFailures: snapshot.consecutiveFailures ?? null,
|
|
},
|
|
actions: [],
|
|
});
|
|
}
|
|
|
|
const activeConnections = providerConnections.filter(
|
|
(connection) => connection.isActive !== false
|
|
);
|
|
const cooldownCount = providerConnections.filter((connection) =>
|
|
isConnectionInCooldown(connection, now)
|
|
).length;
|
|
const terminalCount = providerConnections.filter(isTerminalConnection).length;
|
|
const staleErrors = providerConnections.filter((connection) =>
|
|
hasStaleConnectionError(connection, now)
|
|
).length;
|
|
const breakerPenalty =
|
|
breaker?.state === "OPEN" ? 0.35 : breaker?.state === "HALF_OPEN" ? 0.2 : 0;
|
|
const total = Math.max(1, providerConnections.length);
|
|
const score = Math.max(
|
|
0,
|
|
Math.min(
|
|
1,
|
|
1 -
|
|
breakerPenalty -
|
|
(cooldownCount / total) * 0.25 -
|
|
(terminalCount / total) * 0.35 -
|
|
Math.min(0.2, providerLockouts.length * 0.05) -
|
|
Math.min(0.1, staleErrors * 0.03)
|
|
)
|
|
);
|
|
const hasCritical = issues.some((issue) => issue.severity === "critical");
|
|
const state: ProviderAutopilotState =
|
|
hasCritical || (providerConnections.length > 0 && activeConnections.length === 0)
|
|
? "down"
|
|
: issues.length > 0
|
|
? "degraded"
|
|
: "healthy";
|
|
|
|
const item: ProviderAutopilotProvider = {
|
|
provider,
|
|
state,
|
|
score: Number(score.toFixed(2)),
|
|
signals: {
|
|
circuitBreaker: breaker
|
|
? {
|
|
state: breaker.state,
|
|
failureCount: breaker.failureCount ?? 0,
|
|
retryAfterMs: breaker.retryAfterMs ?? 0,
|
|
}
|
|
: null,
|
|
connections: {
|
|
total: providerConnections.length,
|
|
active: activeConnections.length,
|
|
inactive: providerConnections.filter((connection) => connection.isActive === false)
|
|
.length,
|
|
cooldown: cooldownCount,
|
|
terminal: terminalCount,
|
|
staleErrors,
|
|
},
|
|
modelLockouts: providerLockouts.length,
|
|
quotaMonitor: providerQuota.length
|
|
? {
|
|
warning: providerQuota.filter((snapshot) => snapshot.status === "warning").length,
|
|
exhausted: providerQuota.filter((snapshot) => snapshot.status === "exhausted").length,
|
|
errors: providerQuota.filter((snapshot) => snapshot.status === "error").length,
|
|
}
|
|
: null,
|
|
},
|
|
issues,
|
|
};
|
|
|
|
if (includeHealthy || item.issues.length > 0) providers.push(item);
|
|
}
|
|
|
|
const issueCount = providers.reduce((count, provider) => count + provider.issues.length, 0);
|
|
const actionableCount = providers.reduce(
|
|
(count, provider) =>
|
|
count + provider.issues.reduce((sum, issue) => sum + issue.actions.length, 0),
|
|
0
|
|
);
|
|
const healthyCount = providers.filter((provider) => provider.state === "healthy").length;
|
|
const status: ProviderAutopilotStatus = providers.some((provider) => provider.state === "down")
|
|
? "critical"
|
|
: providers.some((provider) => provider.state === "degraded")
|
|
? "warning"
|
|
: "healthy";
|
|
|
|
return {
|
|
status,
|
|
checkedAt,
|
|
summary: {
|
|
providerCount: providers.length,
|
|
connectionCount: connections.length,
|
|
healthyCount,
|
|
issueCount,
|
|
actionableCount,
|
|
},
|
|
providers,
|
|
};
|
|
}
|
|
|
|
function findAction(
|
|
report: ProviderAutopilotReport,
|
|
input: ExecuteProviderAutopilotActionInput
|
|
): ProviderAutopilotAction | null {
|
|
for (const provider of report.providers) {
|
|
for (const issue of provider.issues) {
|
|
const match = issue.actions.find(
|
|
(candidate) =>
|
|
candidate.type === input.type && targetMatches(candidate.target, input.target)
|
|
);
|
|
if (match) return match;
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
export async function executeProviderHealthAutopilotAction(
|
|
input: ExecuteProviderAutopilotActionInput
|
|
) {
|
|
const provider = toString(input.target.provider);
|
|
if (!provider) {
|
|
return { status: 400, body: { success: false, error: "provider is required" } };
|
|
}
|
|
if (!input.confirm && !input.dryRun) {
|
|
return { status: 400, body: { success: false, error: "confirm must be true" } };
|
|
}
|
|
|
|
const report = await buildProviderHealthAutopilotReport({
|
|
provider,
|
|
includeHealthy: true,
|
|
includeActions: true,
|
|
});
|
|
const matchingAction = findAction(report, input);
|
|
if (!matchingAction) {
|
|
return { status: 404, body: { success: false, error: "action is not currently applicable" } };
|
|
}
|
|
if (matchingAction.preconditionsHash !== input.preconditionsHash) {
|
|
return {
|
|
status: 409,
|
|
body: {
|
|
success: false,
|
|
error: "autopilot observation changed; refresh before applying this action",
|
|
currentPreconditionsHash: matchingAction.preconditionsHash,
|
|
},
|
|
};
|
|
}
|
|
|
|
if (input.dryRun) {
|
|
return {
|
|
status: 200,
|
|
body: {
|
|
success: true,
|
|
dryRun: true,
|
|
action: input.type,
|
|
target: matchingAction.target,
|
|
checkedAt: new Date().toISOString(),
|
|
},
|
|
};
|
|
}
|
|
|
|
let changed: JsonRecord = {};
|
|
switch (input.type) {
|
|
case "clear_provider_breaker":
|
|
clearProviderFailure(provider);
|
|
changed = { circuitBreaker: "CLOSED" };
|
|
break;
|
|
case "clear_connection_cooldown":
|
|
case "clear_stale_connection_error": {
|
|
const connectionId = toString(input.target.connectionId);
|
|
if (!connectionId) {
|
|
return { status: 400, body: { success: false, error: "connectionId is required" } };
|
|
}
|
|
const connection = (await getCachedProviderConnectionById(connectionId)) as JsonRecord | null;
|
|
if (!connection || connection.provider !== provider) {
|
|
return { status: 404, body: { success: false, error: "connection not found" } };
|
|
}
|
|
if (isTerminalConnection(connection)) {
|
|
return { status: 409, body: { success: false, error: "terminal connection state" } };
|
|
}
|
|
await updateProviderConnection(connectionId, buildConnectionClearPatch());
|
|
changed = buildConnectionClearPatch();
|
|
break;
|
|
}
|
|
case "clear_model_lockout": {
|
|
const connectionId = toString(input.target.connectionId);
|
|
const model = toString(input.target.model);
|
|
if (!connectionId || !model) {
|
|
return {
|
|
status: 400,
|
|
body: { success: false, error: "connectionId and model are required" },
|
|
};
|
|
}
|
|
const connection = (await getCachedProviderConnectionById(connectionId)) as JsonRecord | null;
|
|
if (!connection || connection.provider !== provider) {
|
|
return { status: 404, body: { success: false, error: "connection not found" } };
|
|
}
|
|
if (isTerminalConnection(connection)) {
|
|
return { status: 409, body: { success: false, error: "terminal connection state" } };
|
|
}
|
|
const removed = clearModelLock(provider, connectionId, model);
|
|
if (!removed) {
|
|
return {
|
|
status: 409,
|
|
body: { success: false, error: "model lockout changed; refresh before retrying" },
|
|
};
|
|
}
|
|
changed = { removed };
|
|
break;
|
|
}
|
|
case "reactivate_connection":
|
|
case "deactivate_connection": {
|
|
const connectionId = toString(input.target.connectionId);
|
|
if (!connectionId) {
|
|
return { status: 400, body: { success: false, error: "connectionId is required" } };
|
|
}
|
|
const connection = (await getCachedProviderConnectionById(connectionId)) as JsonRecord | null;
|
|
if (!connection || connection.provider !== provider) {
|
|
return { status: 404, body: { success: false, error: "connection not found" } };
|
|
}
|
|
if (input.type === "reactivate_connection" && isTerminalConnection(connection)) {
|
|
return { status: 409, body: { success: false, error: "terminal connection state" } };
|
|
}
|
|
const isActive = input.type === "reactivate_connection";
|
|
await updateProviderConnection(connectionId, { isActive });
|
|
changed = { isActive };
|
|
break;
|
|
}
|
|
}
|
|
|
|
return {
|
|
status: 200,
|
|
body: {
|
|
success: true,
|
|
dryRun: false,
|
|
action: input.type,
|
|
target: matchingAction.target,
|
|
changed,
|
|
checkedAt: new Date().toISOString(),
|
|
},
|
|
};
|
|
}
|