mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-01 04:42:10 +03:00
feat(dashboard): Combo Studio connection-cooldown badge (U1b Slice 2) (#4068)
Integrado em release/v3.8.28. Combo Studio connection-cooldown badge (U1b Slice 2 / F5.1).
This commit is contained in:
committed by
GitHub
parent
9a77456890
commit
911567758d
@@ -7,9 +7,11 @@ import {
|
||||
comboRunToFlow,
|
||||
reduceComboEvent,
|
||||
enrichRunWithBreakers,
|
||||
enrichRunWithConnectionCooldown,
|
||||
type ComboRunModel,
|
||||
type ComboEventInput,
|
||||
type ProviderBreakerSnapshot,
|
||||
type ConnectionCooldownSnapshot,
|
||||
} from "./comboFlowModel";
|
||||
import { aggregateComboEventsToSets } from "./fleetAggregation";
|
||||
import { StrategyNode } from "./nodes/StrategyNode";
|
||||
@@ -188,6 +190,13 @@ export interface ComboLiveStudioProps {
|
||||
* absent → no breaker badges (graceful).
|
||||
*/
|
||||
providerHealth?: Record<string, ProviderBreakerSnapshot> | null;
|
||||
/**
|
||||
* Per-provider connection-cooldown snapshot (`connectionHealth` from
|
||||
* GET /api/monitoring/health). When supplied, the cascade overlays the REAL
|
||||
* cooldown state (cooldown 2/3 · 28s) onto each target — U1b Slice 2. Optional;
|
||||
* absent → no cooldown badges (graceful).
|
||||
*/
|
||||
connectionHealth?: Record<string, ConnectionCooldownSnapshot> | null;
|
||||
}
|
||||
|
||||
// ── Main component ────────────────────────────────────────────────────────
|
||||
@@ -213,6 +222,7 @@ export function ComboLiveStudio({
|
||||
combos: combosProp,
|
||||
isConnected = true,
|
||||
providerHealth,
|
||||
connectionHealth,
|
||||
}: ComboLiveStudioProps) {
|
||||
const [mode, setMode] = useState<"single" | "fleet">("single");
|
||||
const [selectedCombo, setSelectedCombo] = useState<string>("");
|
||||
@@ -245,8 +255,13 @@ export function ComboLiveStudio({
|
||||
null
|
||||
);
|
||||
}
|
||||
return enrichRunWithBreakers(baseRun, providerHealth);
|
||||
}, [runProp, selectedCombo, comboEvents, providerHealth]);
|
||||
// Compose overlays: breaker state first, then connection cooldown. Both are
|
||||
// pure no-ops when their health map is absent, and they touch disjoint fields.
|
||||
return enrichRunWithConnectionCooldown(
|
||||
enrichRunWithBreakers(baseRun, providerHealth),
|
||||
connectionHealth
|
||||
);
|
||||
}, [runProp, selectedCombo, comboEvents, providerHealth, connectionHealth]);
|
||||
|
||||
// Build ReactFlow graph from the current run
|
||||
const { nodes, edges } = useMemo(() => {
|
||||
|
||||
@@ -50,6 +50,13 @@ export interface TargetNodeModel {
|
||||
cbState?: CbState;
|
||||
/** Milliseconds until the breaker allows a probe again (when cbState is set). */
|
||||
cbRetryAfterMs?: number;
|
||||
/** Connections for this provider currently in cooldown (U1b Slice 2).
|
||||
* Only set when at least one connection is cooling down. */
|
||||
cooldownCount?: number;
|
||||
/** Total connections configured for this provider (set with cooldownCount). */
|
||||
cooldownTotal?: number;
|
||||
/** Milliseconds until the first cooling connection recovers (the soonest). */
|
||||
cooldownRetryAfterMs?: number;
|
||||
}
|
||||
|
||||
export interface ComboRunModel {
|
||||
@@ -307,6 +314,9 @@ export function comboRunToFlow(run: ComboRunModel): { nodes: Node[]; edges: Edge
|
||||
targetIndex: t.targetIndex,
|
||||
cbState: t.cbState,
|
||||
cbRetryAfterMs: t.cbRetryAfterMs,
|
||||
cooldownCount: t.cooldownCount,
|
||||
cooldownTotal: t.cooldownTotal,
|
||||
cooldownRetryAfterMs: t.cooldownRetryAfterMs,
|
||||
},
|
||||
});
|
||||
|
||||
@@ -415,3 +425,76 @@ export function enrichRunWithBreakers(
|
||||
|
||||
return changed ? { ...run, targets } : run;
|
||||
}
|
||||
|
||||
// ── enrichRunWithConnectionCooldown (U1b Slice 2) ─────────────────────────────
|
||||
|
||||
/**
|
||||
* Per-provider connection-cooldown summary, as exposed by GET /api/monitoring/health
|
||||
* (`connectionHealth[provider]`). Only providers with at least one cooling connection
|
||||
* appear in the map. Only the fields the cascade badge consumes.
|
||||
*/
|
||||
export interface ConnectionCooldownSnapshot {
|
||||
coolingDown?: number;
|
||||
total?: number;
|
||||
soonestRetryAfterMs?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Overlay real per-provider connection-cooldown state onto a combo run's targets
|
||||
* (U1b Slice 2). Returns a new run (pure) only when something changed; otherwise the
|
||||
* same reference. Composes with {@link enrichRunWithBreakers} — it only touches the
|
||||
* `cooldown*` fields, leaving `cbState`/`cbRetryAfterMs` intact.
|
||||
*
|
||||
* A badge is attached only when the provider has ≥1 connection cooling down; a zero,
|
||||
* unknown, or absent summary clears any stale cooldown fields so the cascade reflects
|
||||
* recovery. Provider lookup is by `target.provider`.
|
||||
*
|
||||
* @param run current combo run model (or null)
|
||||
* @param connectionHealth `connectionHealth` map from /api/monitoring/health
|
||||
*/
|
||||
export function enrichRunWithConnectionCooldown(
|
||||
run: ComboRunModel | null,
|
||||
connectionHealth: Record<string, ConnectionCooldownSnapshot> | null | undefined
|
||||
): ComboRunModel | null {
|
||||
if (!run) return null;
|
||||
if (!connectionHealth) return run;
|
||||
|
||||
let changed = false;
|
||||
const targets = run.targets.map((target) => {
|
||||
const snapshot = connectionHealth[target.provider];
|
||||
const coolingDown = typeof snapshot?.coolingDown === "number" ? snapshot.coolingDown : 0;
|
||||
|
||||
if (coolingDown > 0) {
|
||||
const total = typeof snapshot?.total === "number" ? snapshot.total : coolingDown;
|
||||
const retryAfterMs = snapshot?.soonestRetryAfterMs;
|
||||
if (
|
||||
target.cooldownCount === coolingDown &&
|
||||
target.cooldownTotal === total &&
|
||||
target.cooldownRetryAfterMs === retryAfterMs
|
||||
) {
|
||||
return target;
|
||||
}
|
||||
changed = true;
|
||||
return {
|
||||
...target,
|
||||
cooldownCount: coolingDown,
|
||||
cooldownTotal: total,
|
||||
cooldownRetryAfterMs: retryAfterMs,
|
||||
};
|
||||
}
|
||||
|
||||
// No cooldown → strip any stale cooldown badge.
|
||||
if (
|
||||
target.cooldownCount !== undefined ||
|
||||
target.cooldownTotal !== undefined ||
|
||||
target.cooldownRetryAfterMs !== undefined
|
||||
) {
|
||||
changed = true;
|
||||
const { cooldownCount: _c, cooldownTotal: _t, cooldownRetryAfterMs: _r, ...rest } = target;
|
||||
return rest;
|
||||
}
|
||||
return target;
|
||||
});
|
||||
|
||||
return changed ? { ...run, targets } : run;
|
||||
}
|
||||
|
||||
@@ -51,14 +51,29 @@ const CB_BADGE_COLORS: Partial<Record<CbState, string>> = {
|
||||
DEGRADED: FLOW_EDGE_COLORS.last,
|
||||
};
|
||||
|
||||
/** "CB: OPEN · 41s" — the retry hint is omitted when unknown/elapsed. */
|
||||
function formatCbBadge(cbState: CbState, retryAfterMs?: number): string {
|
||||
/** Connection-cooldown badge colour (U1b Slice 2) — amber: a partial/recovering state. */
|
||||
const COOLDOWN_BADGE_COLOR = FLOW_EDGE_COLORS.last;
|
||||
|
||||
/** Format a relative duration as "28s" or "1m05s"; "" when unknown/elapsed. */
|
||||
function formatRetryHint(retryAfterMs?: number): string {
|
||||
if (typeof retryAfterMs !== "number" || !Number.isFinite(retryAfterMs) || retryAfterMs <= 0) {
|
||||
return `CB: ${cbState}`;
|
||||
return "";
|
||||
}
|
||||
const seconds = Math.round(retryAfterMs / 1000);
|
||||
const hint = seconds >= 60 ? `${Math.floor(seconds / 60)}m${seconds % 60}s` : `${seconds}s`;
|
||||
return `CB: ${cbState} · ${hint}`;
|
||||
return seconds >= 60 ? `${Math.floor(seconds / 60)}m${seconds % 60}s` : `${seconds}s`;
|
||||
}
|
||||
|
||||
/** "CB: OPEN · 41s" — the retry hint is omitted when unknown/elapsed. */
|
||||
function formatCbBadge(cbState: CbState, retryAfterMs?: number): string {
|
||||
const hint = formatRetryHint(retryAfterMs);
|
||||
return hint ? `CB: ${cbState} · ${hint}` : `CB: ${cbState}`;
|
||||
}
|
||||
|
||||
/** "cooldown 2/3 · 28s" — N of M connections cooling down; retry hint omitted when unknown. */
|
||||
function formatCooldownBadge(count: number, total?: number, retryAfterMs?: number): string {
|
||||
const ratio = typeof total === "number" && total > 0 ? `${count}/${total}` : `${count}`;
|
||||
const hint = formatRetryHint(retryAfterMs);
|
||||
return hint ? `cooldown ${ratio} · ${hint}` : `cooldown ${ratio}`;
|
||||
}
|
||||
|
||||
// ── Node data shape ───────────────────────────────────────────────────────
|
||||
@@ -75,6 +90,10 @@ export interface ProviderCascadeNodeData {
|
||||
/** Real circuit-breaker state for this provider (U1b); only set when non-healthy. */
|
||||
cbState?: CbState;
|
||||
cbRetryAfterMs?: number;
|
||||
/** Connection-cooldown summary for this provider (U1b Slice 2); set when ≥1 cooling. */
|
||||
cooldownCount?: number;
|
||||
cooldownTotal?: number;
|
||||
cooldownRetryAfterMs?: number;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
@@ -92,8 +111,19 @@ export interface ProviderCascadeNodeData {
|
||||
* Has Left (target) and Right (source) Handles for the cascade flow.
|
||||
*/
|
||||
export function ProviderCascadeNode({ data }: NodeProps) {
|
||||
const { provider, model, state, latencyMs, failKind, targetIndex, cbState, cbRetryAfterMs } =
|
||||
data as ProviderCascadeNodeData;
|
||||
const {
|
||||
provider,
|
||||
model,
|
||||
state,
|
||||
latencyMs,
|
||||
failKind,
|
||||
targetIndex,
|
||||
cbState,
|
||||
cbRetryAfterMs,
|
||||
cooldownCount,
|
||||
cooldownTotal,
|
||||
cooldownRetryAfterMs,
|
||||
} = data as ProviderCascadeNodeData;
|
||||
|
||||
const borderColor = getStateBorderColor(state as TargetState);
|
||||
const glow = getStateGlow(state as TargetState);
|
||||
@@ -186,6 +216,24 @@ export function ProviderCascadeNode({ data }: NodeProps) {
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Footer: real connection-cooldown state (U1b Slice 2) — N of M connections
|
||||
for this provider are in cooldown. Independent of target state and of the
|
||||
provider breaker (a key can be cooling while the provider breaker is closed). */}
|
||||
{typeof cooldownCount === "number" && cooldownCount > 0 && (
|
||||
<div className="px-2.5 pb-2">
|
||||
<span
|
||||
className="text-[9px] font-semibold px-1.5 py-0.5 rounded"
|
||||
style={{
|
||||
backgroundColor: `${COOLDOWN_BADGE_COLOR}20`,
|
||||
color: COOLDOWN_BADGE_COLOR,
|
||||
}}
|
||||
data-testid="cooldown-badge"
|
||||
>
|
||||
{formatCooldownBadge(cooldownCount, cooldownTotal, cooldownRetryAfterMs)}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ import { useProviderBreakerHealth } from "@/hooks/useProviderBreakerHealth";
|
||||
*/
|
||||
export default function ComboLiveStudioPage() {
|
||||
const { comboEvents, activeCombos, isConnected } = useLiveComboStatus();
|
||||
const providerHealth = useProviderBreakerHealth();
|
||||
const { providerHealth, connectionHealth } = useProviderBreakerHealth();
|
||||
|
||||
return (
|
||||
<div className="p-4 h-[calc(100dvh-6rem)] min-h-[480px]">
|
||||
@@ -26,6 +26,7 @@ export default function ComboLiveStudioPage() {
|
||||
combos={[...activeCombos]}
|
||||
isConnected={isConnected}
|
||||
providerHealth={providerHealth}
|
||||
connectionHealth={connectionHealth}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -1,24 +1,33 @@
|
||||
"use client";
|
||||
|
||||
import { useEffect, useState } from "react";
|
||||
import type { ProviderBreakerSnapshot } from "@/app/(dashboard)/dashboard/combos/live/comboFlowModel";
|
||||
import type {
|
||||
ProviderBreakerSnapshot,
|
||||
ConnectionCooldownSnapshot,
|
||||
} from "@/app/(dashboard)/dashboard/combos/live/comboFlowModel";
|
||||
|
||||
const DEFAULT_POLL_MS = 5000;
|
||||
|
||||
export interface ResilienceHealthSnapshot {
|
||||
/** Per-provider circuit-breaker state (`providerHealth`). */
|
||||
providerHealth: Record<string, ProviderBreakerSnapshot>;
|
||||
/** Per-provider connection-cooldown summary (`connectionHealth`). */
|
||||
connectionHealth: Record<string, ConnectionCooldownSnapshot>;
|
||||
}
|
||||
|
||||
const EMPTY: ResilienceHealthSnapshot = { providerHealth: {}, connectionHealth: {} };
|
||||
|
||||
/**
|
||||
* Polls `GET /api/monitoring/health` and exposes its per-provider circuit-breaker
|
||||
* snapshot (`providerHealth: { [provider]: { state, retryAfterMs } }`).
|
||||
* Polls `GET /api/monitoring/health` and exposes the resilience overlays the Combo
|
||||
* Live Studio consumes (U1b): per-provider circuit-breaker state (`providerHealth`)
|
||||
* and per-provider connection-cooldown summary (`connectionHealth`).
|
||||
*
|
||||
* Fail-soft by design: any network/parse error keeps the last known map (or the
|
||||
* empty default), so the Combo Live Studio (U1b) simply shows no breaker badges
|
||||
* instead of breaking. Polls every `pollMs` and on mount.
|
||||
* Fail-soft by design: any network/parse error keeps the last known snapshot (or the
|
||||
* empty default), so the cascade simply shows no resilience badges instead of breaking.
|
||||
* One poll covers both overlays. Polls every `pollMs` and on mount.
|
||||
*/
|
||||
export function useProviderBreakerHealth(
|
||||
pollMs = DEFAULT_POLL_MS
|
||||
): Record<string, ProviderBreakerSnapshot> {
|
||||
const [providerHealth, setProviderHealth] = useState<
|
||||
Record<string, ProviderBreakerSnapshot>
|
||||
>({});
|
||||
export function useProviderBreakerHealth(pollMs = DEFAULT_POLL_MS): ResilienceHealthSnapshot {
|
||||
const [snapshot, setSnapshot] = useState<ResilienceHealthSnapshot>(EMPTY);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
@@ -29,10 +38,19 @@ export function useProviderBreakerHealth(
|
||||
if (!res.ok) return;
|
||||
const json = (await res.json()) as {
|
||||
providerHealth?: Record<string, ProviderBreakerSnapshot>;
|
||||
connectionHealth?: Record<string, ConnectionCooldownSnapshot>;
|
||||
};
|
||||
if (!cancelled && json && typeof json.providerHealth === "object" && json.providerHealth) {
|
||||
setProviderHealth(json.providerHealth);
|
||||
}
|
||||
if (cancelled || !json || typeof json !== "object") return;
|
||||
setSnapshot({
|
||||
providerHealth:
|
||||
typeof json.providerHealth === "object" && json.providerHealth
|
||||
? json.providerHealth
|
||||
: {},
|
||||
connectionHealth:
|
||||
typeof json.connectionHealth === "object" && json.connectionHealth
|
||||
? json.connectionHealth
|
||||
: {},
|
||||
});
|
||||
} catch {
|
||||
// Fail-soft: keep the previous snapshot; cascade degrades to no badges.
|
||||
}
|
||||
@@ -46,5 +64,5 @@ export function useProviderBreakerHealth(
|
||||
};
|
||||
}, [pollMs]);
|
||||
|
||||
return providerHealth;
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
@@ -70,7 +70,7 @@ interface BuildHealthPayloadOptions {
|
||||
appVersion: string;
|
||||
catalogCount?: number;
|
||||
settings: { setupComplete?: boolean } | null | undefined;
|
||||
connections: Array<{ provider?: string; isActive?: boolean | null }>;
|
||||
connections: Array<{ provider?: string; isActive?: boolean | null; rateLimitedUntil?: unknown }>;
|
||||
circuitBreakers: CircuitBreakerStatus[];
|
||||
rateLimitStatus: JsonRecord;
|
||||
learnedLimits: JsonRecord;
|
||||
@@ -141,6 +141,76 @@ export function buildTelemetryPayload({
|
||||
};
|
||||
}
|
||||
|
||||
/** Per-provider connection-cooldown summary, exposed as `connectionHealth[provider]`. */
|
||||
export interface ConnectionCooldownSummary {
|
||||
/** Connections currently in cooldown (future `rateLimitedUntil`). Always > 0 when present. */
|
||||
coolingDown: number;
|
||||
/** Total connections configured for the provider. */
|
||||
total: number;
|
||||
/** Relative ms until the first cooling connection recovers (the soonest). */
|
||||
soonestRetryAfterMs: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a connection's `rateLimitedUntil` to an absolute epoch (ms). Mirrors the
|
||||
* canonical `cooldownUntilMs` (open-sse/services/accountFallback.ts, #3954) — kept
|
||||
* inline so this monitoring util stays decoupled from the heavy executor module.
|
||||
* Accepts ISO strings, Date objects, and numeric-epoch strings (the SQLite
|
||||
* TEXT-affinity case where `new Date(...)` would yield NaN).
|
||||
*/
|
||||
function parseCooldownUntilMs(value: unknown): number {
|
||||
if (value === null || value === undefined || value === "") return NaN;
|
||||
if (value instanceof Date) return value.getTime();
|
||||
if (typeof value === "number") return value;
|
||||
if (typeof value !== "string") return NaN;
|
||||
const raw = value.trim();
|
||||
if (/^\d+(\.\d+)?$/.test(raw)) return Number(raw);
|
||||
return new Date(raw).getTime();
|
||||
}
|
||||
|
||||
/**
|
||||
* Aggregate per-connection cooldown state into a per-provider summary. Only providers
|
||||
* with at least one connection still cooling down (future `rateLimitedUntil`) appear in
|
||||
* the result — mirroring `providerHealth`, which only carries non-healthy breakers — so
|
||||
* the cascade overlay attaches a badge only when there is something to show.
|
||||
*
|
||||
* `nowMs` is injected (not read from the clock here) to keep the function pure/testable.
|
||||
*/
|
||||
export function summarizeConnectionCooldown(
|
||||
connections: Array<{ provider?: string; rateLimitedUntil?: unknown }>,
|
||||
nowMs: number
|
||||
): Record<string, ConnectionCooldownSummary> {
|
||||
const byProvider: Record<string, { total: number; coolingDown: number; soonestUntil: number }> =
|
||||
{};
|
||||
for (const connection of connections) {
|
||||
const provider = connection?.provider;
|
||||
if (!provider) continue;
|
||||
const bucket = (byProvider[provider] ??= {
|
||||
total: 0,
|
||||
coolingDown: 0,
|
||||
soonestUntil: Infinity,
|
||||
});
|
||||
bucket.total += 1;
|
||||
const until = parseCooldownUntilMs(connection.rateLimitedUntil);
|
||||
if (Number.isFinite(until) && until > nowMs) {
|
||||
bucket.coolingDown += 1;
|
||||
if (until < bucket.soonestUntil) bucket.soonestUntil = until;
|
||||
}
|
||||
}
|
||||
|
||||
const summary: Record<string, ConnectionCooldownSummary> = {};
|
||||
for (const [provider, bucket] of Object.entries(byProvider)) {
|
||||
if (bucket.coolingDown <= 0) continue;
|
||||
summary[provider] = {
|
||||
coolingDown: bucket.coolingDown,
|
||||
total: bucket.total,
|
||||
soonestRetryAfterMs:
|
||||
bucket.soonestUntil === Infinity ? 0 : Math.max(0, bucket.soonestUntil - nowMs),
|
||||
};
|
||||
}
|
||||
return summary;
|
||||
}
|
||||
|
||||
export function buildHealthPayload({
|
||||
appVersion,
|
||||
catalogCount = 0,
|
||||
@@ -196,6 +266,8 @@ export function buildHealthPayload({
|
||||
};
|
||||
}
|
||||
|
||||
const connectionHealth = summarizeConnectionCooldown(connections, Date.now());
|
||||
|
||||
const configuredProviders = new Set(
|
||||
connections.map((connection) => connection.provider).filter(Boolean)
|
||||
);
|
||||
@@ -232,6 +304,7 @@ export function buildHealthPayload({
|
||||
},
|
||||
providerBreakers,
|
||||
providerHealth,
|
||||
connectionHealth,
|
||||
providerSummary: {
|
||||
catalogCount,
|
||||
configuredCount: configuredProviders.size,
|
||||
|
||||
97
tests/unit/lib/connection-cooldown-summary.test.ts
Normal file
97
tests/unit/lib/connection-cooldown-summary.test.ts
Normal file
@@ -0,0 +1,97 @@
|
||||
/**
|
||||
* TDD for F5.1 (Combo U1b Slice 2) — summarizeConnectionCooldown: aggregates the
|
||||
* per-connection cooldown state (`rateLimitedUntil`) into a per-provider summary the
|
||||
* cascade can badge, exposed by GET /api/monitoring/health as `connectionHealth`.
|
||||
*
|
||||
* Parsing of rateLimitedUntil is delegated to cooldownUntilMs (#3954) — it accepts
|
||||
* ISO strings, Date objects, AND numeric epoch strings (the SQLite TEXT-affinity case).
|
||||
*
|
||||
* Run: node --import tsx/esm --test tests/unit/monitoring/connection-cooldown-summary.test.ts
|
||||
*/
|
||||
import { describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import { summarizeConnectionCooldown } from "../../../src/lib/monitoring/observability.ts";
|
||||
|
||||
const NOW = 1_000_000_000_000;
|
||||
|
||||
describe("summarizeConnectionCooldown", () => {
|
||||
it("returns an empty map for no connections", () => {
|
||||
assert.deepEqual(summarizeConnectionCooldown([], NOW), {});
|
||||
});
|
||||
|
||||
it("omits providers whose connections are all available (no future rateLimitedUntil)", () => {
|
||||
const out = summarizeConnectionCooldown(
|
||||
[
|
||||
{ provider: "openai", rateLimitedUntil: null },
|
||||
{ provider: "openai", rateLimitedUntil: new Date(NOW - 5000).toISOString() }, // past → expired
|
||||
],
|
||||
NOW
|
||||
);
|
||||
assert.deepEqual(out, {});
|
||||
});
|
||||
|
||||
it("reports coolingDown / total / soonestRetryAfterMs for a provider with cooling connections", () => {
|
||||
const out = summarizeConnectionCooldown(
|
||||
[
|
||||
{ provider: "anthropic", rateLimitedUntil: new Date(NOW + 28_000).toISOString() },
|
||||
{ provider: "anthropic", rateLimitedUntil: new Date(NOW + 60_000).toISOString() },
|
||||
{ provider: "anthropic", rateLimitedUntil: null }, // available
|
||||
],
|
||||
NOW
|
||||
);
|
||||
assert.ok(out.anthropic);
|
||||
assert.equal(out.anthropic.coolingDown, 2);
|
||||
assert.equal(out.anthropic.total, 3);
|
||||
assert.equal(
|
||||
out.anthropic.soonestRetryAfterMs,
|
||||
28_000,
|
||||
"soonest = the connection that recovers first"
|
||||
);
|
||||
});
|
||||
|
||||
it("parses numeric-epoch-string rateLimitedUntil (SQLite TEXT-affinity, #3954)", () => {
|
||||
const out = summarizeConnectionCooldown(
|
||||
[{ provider: "glm", rateLimitedUntil: String(NOW + 15_000) }],
|
||||
NOW
|
||||
);
|
||||
assert.ok(out.glm);
|
||||
assert.equal(out.glm.coolingDown, 1);
|
||||
assert.equal(out.glm.soonestRetryAfterMs, 15_000);
|
||||
});
|
||||
|
||||
it("accepts a raw epoch number too", () => {
|
||||
const out = summarizeConnectionCooldown(
|
||||
[{ provider: "glm", rateLimitedUntil: NOW + 9000 }],
|
||||
NOW
|
||||
);
|
||||
assert.equal(out.glm?.soonestRetryAfterMs, 9000);
|
||||
});
|
||||
|
||||
it("groups connections per provider independently", () => {
|
||||
const out = summarizeConnectionCooldown(
|
||||
[
|
||||
{ provider: "openai", rateLimitedUntil: new Date(NOW + 10_000).toISOString() },
|
||||
{ provider: "anthropic", rateLimitedUntil: new Date(NOW + 40_000).toISOString() },
|
||||
{ provider: "openai", rateLimitedUntil: null },
|
||||
],
|
||||
NOW
|
||||
);
|
||||
assert.equal(out.openai.coolingDown, 1);
|
||||
assert.equal(out.openai.total, 2);
|
||||
assert.equal(out.anthropic.coolingDown, 1);
|
||||
assert.equal(out.anthropic.total, 1);
|
||||
});
|
||||
|
||||
it("ignores connections without a provider and never returns negative retry", () => {
|
||||
const out = summarizeConnectionCooldown(
|
||||
[
|
||||
{ rateLimitedUntil: new Date(NOW + 5000).toISOString() }, // no provider
|
||||
{ provider: "x", rateLimitedUntil: new Date(NOW + 1).toISOString() },
|
||||
],
|
||||
NOW
|
||||
);
|
||||
assert.equal(Object.keys(out).length, 1);
|
||||
assert.ok(out.x.soonestRetryAfterMs >= 0);
|
||||
});
|
||||
});
|
||||
125
tests/unit/ui/comboFlowModel-cooldown.test.ts
Normal file
125
tests/unit/ui/comboFlowModel-cooldown.test.ts
Normal file
@@ -0,0 +1,125 @@
|
||||
/**
|
||||
* tests/unit/ui/comboFlowModel-cooldown.test.ts
|
||||
*
|
||||
* TDD for F5.1 (Combo U1b Slice 2) — enrichRunWithConnectionCooldown: overlays the
|
||||
* real per-provider connection-cooldown summary (from GET /api/monitoring/health →
|
||||
* connectionHealth[provider]) onto a combo run's targets, so the cascade can badge
|
||||
* "cooldown 2/3 · 28s" alongside the circuit-breaker badge.
|
||||
*
|
||||
* Mirrors comboFlowModel-breakers.test.ts (Slice 1).
|
||||
* Run: node --import tsx/esm --test tests/unit/ui/comboFlowModel-cooldown.test.ts
|
||||
*/
|
||||
|
||||
import { describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import {
|
||||
enrichRunWithConnectionCooldown,
|
||||
comboRunToFlow,
|
||||
type ComboRunModel,
|
||||
type TargetNodeModel,
|
||||
} from "../../../src/app/(dashboard)/dashboard/combos/live/comboFlowModel.ts";
|
||||
|
||||
function mkRun(targets: Array<Partial<TargetNodeModel> & { provider: string }>): ComboRunModel {
|
||||
return {
|
||||
comboName: "c",
|
||||
strategy: "priority",
|
||||
outcome: "running",
|
||||
startedAt: 0,
|
||||
targets: targets.map((t, i) => ({
|
||||
targetIndex: i,
|
||||
provider: t.provider,
|
||||
model: t.model ?? "m",
|
||||
state: t.state ?? "idle",
|
||||
...t,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
describe("enrichRunWithConnectionCooldown", () => {
|
||||
it("returns null for a null run", () => {
|
||||
assert.equal(enrichRunWithConnectionCooldown(null, {}), null);
|
||||
});
|
||||
|
||||
it("returns the same run reference when no health map is provided", () => {
|
||||
const run = mkRun([{ provider: "openai" }]);
|
||||
assert.equal(enrichRunWithConnectionCooldown(run, null), run);
|
||||
assert.equal(enrichRunWithConnectionCooldown(run, undefined), run);
|
||||
});
|
||||
|
||||
it("attaches cooldown count/total/retry for a provider with cooling connections", () => {
|
||||
const run = mkRun([{ provider: "anthropic" }, { provider: "openai" }]);
|
||||
const out = enrichRunWithConnectionCooldown(run, {
|
||||
anthropic: { coolingDown: 2, total: 3, soonestRetryAfterMs: 28_000 },
|
||||
});
|
||||
assert.ok(out);
|
||||
assert.equal(out.targets[0].cooldownCount, 2);
|
||||
assert.equal(out.targets[0].cooldownTotal, 3);
|
||||
assert.equal(out.targets[0].cooldownRetryAfterMs, 28_000);
|
||||
assert.equal(
|
||||
out.targets[1].cooldownCount,
|
||||
undefined,
|
||||
"provider with no cooldown gets no badge"
|
||||
);
|
||||
});
|
||||
|
||||
it("does not attach a badge when coolingDown is 0 or absent", () => {
|
||||
const run = mkRun([{ provider: "a" }, { provider: "b" }]);
|
||||
const out = enrichRunWithConnectionCooldown(run, {
|
||||
a: { coolingDown: 0, total: 2, soonestRetryAfterMs: 0 },
|
||||
b: {},
|
||||
});
|
||||
assert.equal(out?.targets[0].cooldownCount, undefined);
|
||||
assert.equal(out?.targets[1].cooldownCount, undefined);
|
||||
});
|
||||
|
||||
it("does not mutate the input run (pure)", () => {
|
||||
const run = mkRun([{ provider: "openai" }]);
|
||||
enrichRunWithConnectionCooldown(run, {
|
||||
openai: { coolingDown: 1, total: 1, soonestRetryAfterMs: 5000 },
|
||||
});
|
||||
assert.equal(run.targets[0].cooldownCount, undefined, "original run must be untouched");
|
||||
});
|
||||
|
||||
it("strips a stale cooldown badge when the provider's connections recover", () => {
|
||||
const run = mkRun([
|
||||
{ provider: "openai", cooldownCount: 2, cooldownTotal: 3, cooldownRetryAfterMs: 9000 },
|
||||
]);
|
||||
const out = enrichRunWithConnectionCooldown(run, {
|
||||
openai: { coolingDown: 0, total: 3, soonestRetryAfterMs: 0 },
|
||||
});
|
||||
assert.equal(out?.targets[0].cooldownCount, undefined);
|
||||
assert.equal(out?.targets[0].cooldownTotal, undefined);
|
||||
assert.equal(out?.targets[0].cooldownRetryAfterMs, undefined);
|
||||
});
|
||||
|
||||
it("comboRunToFlow carries the cooldown fields into the target node data", () => {
|
||||
const run = mkRun([{ provider: "anthropic" }, { provider: "openai" }]);
|
||||
const enriched = enrichRunWithConnectionCooldown(run, {
|
||||
anthropic: { coolingDown: 1, total: 2, soonestRetryAfterMs: 12_000 },
|
||||
});
|
||||
const { nodes } = comboRunToFlow(enriched as ComboRunModel);
|
||||
|
||||
const target0 = nodes.find((n) => n.id === "target-0");
|
||||
assert.equal(target0?.data.cooldownCount, 1);
|
||||
assert.equal(target0?.data.cooldownTotal, 2);
|
||||
assert.equal(target0?.data.cooldownRetryAfterMs, 12_000);
|
||||
|
||||
const target1 = nodes.find((n) => n.id === "target-1");
|
||||
assert.equal(target1?.data.cooldownCount, undefined, "healthy provider node has no cooldown");
|
||||
});
|
||||
|
||||
it("composes with enrichRunWithBreakers without clobbering cbState", async () => {
|
||||
const { enrichRunWithBreakers } =
|
||||
await import("../../../src/app/(dashboard)/dashboard/combos/live/comboFlowModel.ts");
|
||||
const run = mkRun([{ provider: "anthropic" }]);
|
||||
const withBreaker = enrichRunWithBreakers(run, {
|
||||
anthropic: { state: "OPEN", retryAfterMs: 41_000 },
|
||||
});
|
||||
const both = enrichRunWithConnectionCooldown(withBreaker, {
|
||||
anthropic: { coolingDown: 1, total: 2, soonestRetryAfterMs: 5000 },
|
||||
});
|
||||
assert.equal(both?.targets[0].cbState, "OPEN", "breaker overlay survives the cooldown overlay");
|
||||
assert.equal(both?.targets[0].cooldownCount, 1);
|
||||
});
|
||||
});
|
||||
@@ -202,18 +202,18 @@ describe("ProviderCascadeNode", () => {
|
||||
{...makeNodeProps({ state: "idle", cbState: "HALF_OPEN", cbRetryAfterMs: 5000 })}
|
||||
/>
|
||||
);
|
||||
expect(
|
||||
container.querySelector("[data-testid='cb-state-badge']")?.textContent
|
||||
).toContain("CB: HALF_OPEN");
|
||||
expect(container.querySelector("[data-testid='cb-state-badge']")?.textContent).toContain(
|
||||
"CB: HALF_OPEN"
|
||||
);
|
||||
});
|
||||
|
||||
it("omits the retry hint when cbRetryAfterMs is absent", () => {
|
||||
const container = mount(
|
||||
<ProviderCascadeNode {...makeNodeProps({ state: "skipped", cbState: "DEGRADED" })} />
|
||||
);
|
||||
expect(
|
||||
container.querySelector("[data-testid='cb-state-badge']")?.textContent?.trim()
|
||||
).toBe("CB: DEGRADED");
|
||||
expect(container.querySelector("[data-testid='cb-state-badge']")?.textContent?.trim()).toBe(
|
||||
"CB: DEGRADED"
|
||||
);
|
||||
});
|
||||
|
||||
it("does NOT show the CB badge when cbState is absent", () => {
|
||||
@@ -222,4 +222,77 @@ describe("ProviderCascadeNode", () => {
|
||||
);
|
||||
expect(container.querySelector("[data-testid='cb-state-badge']")).toBeNull();
|
||||
});
|
||||
|
||||
// ── U1b Slice 2: connection-cooldown badge ───────────────────────────────
|
||||
|
||||
it("shows the cooldown badge with count/total + retry hint when connections are cooling", () => {
|
||||
const container = mount(
|
||||
<ProviderCascadeNode
|
||||
{...makeNodeProps({
|
||||
state: "idle",
|
||||
cooldownCount: 2,
|
||||
cooldownTotal: 3,
|
||||
cooldownRetryAfterMs: 28000,
|
||||
})}
|
||||
/>
|
||||
);
|
||||
const badge = container.querySelector("[data-testid='cooldown-badge']");
|
||||
expect(badge).toBeTruthy();
|
||||
expect(badge?.textContent).toContain("cooldown 2/3");
|
||||
expect(badge?.textContent).toContain("28s");
|
||||
});
|
||||
|
||||
it("shows the cooldown badge independent of target state (succeeded target)", () => {
|
||||
const container = mount(
|
||||
<ProviderCascadeNode
|
||||
{...makeNodeProps({ state: "succeeded", cooldownCount: 1, cooldownTotal: 4 })}
|
||||
/>
|
||||
);
|
||||
expect(container.querySelector("[data-testid='cooldown-badge']")?.textContent).toContain(
|
||||
"cooldown 1/4"
|
||||
);
|
||||
});
|
||||
|
||||
it("omits the retry hint when cooldownRetryAfterMs is absent", () => {
|
||||
const container = mount(
|
||||
<ProviderCascadeNode
|
||||
{...makeNodeProps({ state: "idle", cooldownCount: 1, cooldownTotal: 2 })}
|
||||
/>
|
||||
);
|
||||
expect(container.querySelector("[data-testid='cooldown-badge']")?.textContent?.trim()).toBe(
|
||||
"cooldown 1/2"
|
||||
);
|
||||
});
|
||||
|
||||
it("does NOT show the cooldown badge when cooldownCount is absent or zero", () => {
|
||||
const absent = mount(<ProviderCascadeNode {...makeNodeProps({ state: "idle" })} />);
|
||||
expect(absent.querySelector("[data-testid='cooldown-badge']")).toBeNull();
|
||||
const zero = mount(
|
||||
<ProviderCascadeNode
|
||||
{...makeNodeProps({ state: "idle", cooldownCount: 0, cooldownTotal: 3 })}
|
||||
/>
|
||||
);
|
||||
expect(zero.querySelector("[data-testid='cooldown-badge']")).toBeNull();
|
||||
});
|
||||
|
||||
it("shows both the CB badge and the cooldown badge together", () => {
|
||||
const container = mount(
|
||||
<ProviderCascadeNode
|
||||
{...makeNodeProps({
|
||||
state: "skipped",
|
||||
cbState: "OPEN",
|
||||
cbRetryAfterMs: 41000,
|
||||
cooldownCount: 1,
|
||||
cooldownTotal: 2,
|
||||
cooldownRetryAfterMs: 9000,
|
||||
})}
|
||||
/>
|
||||
);
|
||||
expect(container.querySelector("[data-testid='cb-state-badge']")?.textContent).toContain(
|
||||
"CB: OPEN"
|
||||
);
|
||||
expect(container.querySelector("[data-testid='cooldown-badge']")?.textContent).toContain(
|
||||
"cooldown 1/2"
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user