fix(dashboard): topology reflects connection health + clears finished requests (#7672)

The provider topology only lit nodes from live/recent traffic, so between
requests (and right after a restart) it went blank even though 50+ connections
were healthy — which reads as "lost providers". Two root causes:

1. Stuck-green latch: request.completed/request.failed are declared in the
   dashboard event map and consumed by useLiveRequests to drain the active-request
   set, but they were never emitted (only request.started was). A node's green
   "active" pulse therefore only cleared on a page reload, and accumulated over a
   session. Emit the terminal event from persistAttemptLogs — keyed by the same
   traceId as request.started — through a pure resolveRequestLifecycleEvent()
   helper (2xx/3xx + no error => completed, else failed).

2. No at-rest state: the map had nothing to show when idle. Colour each node by
   connection health (green connected / red error / grey idle) as a base layer,
   with live/recent traffic still taking precedence and pulsing brighter on top.
   edgeStyle() gains an optional trailing `healthy` param (static dim green) and
   StatusDot a `pulse` prop (static dot for connected-at-rest); both backward
   compatible. Legend "Active" -> "Connected".

Tests: resolveRequestLifecycleEvent success/failure/token-alias units, edgeStyle
healthy variant + precedence, and source guards for the emit wiring (traceId
threaded into persistAttemptLogs) and the health-colour wiring.

Co-authored-by: Diego Rodrigues de Sa e Souza <diegosouza.pw@gmail.com>
Co-authored-by: Diego Rodrigues de Sa e Souza <diegosouzapw@users.noreply.github.com>
This commit is contained in:
danscMax
2026-07-20 02:52:43 +03:00
committed by GitHub
parent 1b7209034e
commit 18a8da6df7
10 changed files with 359 additions and 31 deletions

View File

@@ -844,6 +844,7 @@ export async function handleChatCore({
// once so the 16 call sites keep passing only the per-attempt args (byte-identical).
const persistAttemptLogs = (args: PersistAttemptLogsArgs) =>
persistAttemptLogsFor(args, {
traceId,
provider,
connectionId,
model,

View File

@@ -12,6 +12,8 @@
import { extractProviderWarnings } from "@/lib/compliance/providerAudit";
import { logAuditEvent } from "@/lib/compliance";
import { emit } from "@/lib/events/eventBus";
import type { RequestCompletedPayload, RequestFailedPayload } from "@/lib/events/types";
import { saveCallLog } from "@/lib/usageDb";
import { cloneBoundedChatLogPayload, truncateForLog } from "./logTruncation.ts";
import { attachLogMeta } from "./cacheUsageMeta.ts";
@@ -30,6 +32,9 @@ export type PersistAttemptLogsArgs = {
};
export type PersistAttemptLogsContext = {
/** Per-attempt trace id — MUST match the id emitted in `request.started` so the live
* dashboard can pair the terminal event and clear the topology node's active pulse. */
traceId: string;
provider: string | null | undefined;
connectionId: string | null | undefined;
model: string | null | undefined;
@@ -74,6 +79,61 @@ function buildAccountRotationMeta(
};
}
/**
* Pure resolver for the terminal request-lifecycle dashboard event. Extracted so the
* "stuck green" latch fix (emitting request.completed/failed to clear the live topology
* node) is unit-testable without the DB write in persistAttemptLogs. A 2xx/3xx status
* with no error is a completion; everything else (including a missing/odd status) is a
* failure. `id` mirrors the `traceId` used by the paired `request.started`.
*/
export function resolveRequestLifecycleEvent(input: {
traceId: string;
status: number;
error?: string | null;
model?: string | null;
provider?: string | null;
comboName?: unknown;
tokens?: unknown;
latencyMs: number;
}):
| { name: "request.completed"; payload: RequestCompletedPayload }
| { name: "request.failed"; payload: RequestFailedPayload } {
const { traceId, status, error, model, provider, comboName, tokens, latencyMs } = input;
const succeeded = typeof status === "number" && status >= 200 && status < 400 && !error;
const resolvedComboName = typeof comboName === "string" && comboName ? comboName : undefined;
if (succeeded) {
const tokenBag = (tokens && typeof tokens === "object" ? tokens : {}) as Record<
string,
unknown
>;
const num = (v: unknown) => (typeof v === "number" && Number.isFinite(v) ? v : 0);
return {
name: "request.completed",
payload: {
id: traceId,
status: "success",
model: model || "unknown",
provider: provider || "unknown",
tokensInput: num(tokenBag.input ?? tokenBag.prompt_tokens ?? tokenBag.inputTokens),
tokensOutput: num(tokenBag.output ?? tokenBag.completion_tokens ?? tokenBag.outputTokens),
latencyMs,
comboName: resolvedComboName,
},
};
}
return {
name: "request.failed",
payload: {
id: traceId,
error: error || `HTTP ${status}`,
statusCode: typeof status === "number" ? status : undefined,
latencyMs,
model: model || undefined,
provider: provider || undefined,
},
};
}
export function persistAttemptLogs(args: PersistAttemptLogsArgs, ctx: PersistAttemptLogsContext) {
const {
status,
@@ -88,6 +148,7 @@ export function persistAttemptLogs(args: PersistAttemptLogsArgs, ctx: PersistAtt
cacheSource,
} = args;
const {
traceId,
provider,
connectionId,
model,
@@ -210,4 +271,27 @@ export function persistAttemptLogs(args: PersistAttemptLogsArgs, ctx: PersistAtt
correlationId,
modelPinned: modelPinned || false,
}).catch(() => {});
// Emit the terminal request-lifecycle event to the live dashboard bus. `request.started`
// is emitted in chatCore with this same `traceId`; without a matching completed/failed the
// client's active-request map never drains, so the topology node stays green forever (the
// "stuck green" latch — request.completed/failed were declared + consumed but never emitted).
// Deferred via setImmediate to keep it off the response hot path, mirroring request.started.
setImmediate(() => {
const lifecycle = resolveRequestLifecycleEvent({
traceId,
status,
error,
model,
provider,
comboName,
tokens,
latencyMs: Date.now() - startTime,
});
if (lifecycle.name === "request.completed") {
emit("request.completed", lifecycle.payload);
} else {
emit("request.failed", lifecycle.payload);
}
});
}

View File

@@ -464,9 +464,27 @@ export default function HomePageClient({ machineId }: HomePageClientProps) {
}, [selectedProvider, models]);
const topologyProviders = useMemo(() => {
const byProvider = new Map<string, { id: string; provider: string; name?: string }>();
type ProviderHealth = "active" | "error" | "idle";
const byProvider = new Map<
string,
{ id: string; provider: string; name?: string; status: ProviderHealth }
>();
const providerConfig = AI_PROVIDERS as Record<string, { name?: string }>;
// Connection-health per provider, so the topology node reflects "what is connected"
// at rest (green healthy / red error) instead of going blank between requests. A
// provider with ≥1 healthy connection is "active"; if none are healthy but some are
// errored it is "error"; otherwise "idle". Live/recent traffic still overrides this.
const healthByProvider = new Map<string, ProviderHealth>();
for (const stat of providerStats) {
const canonical = normalizeProviderId(stat.id);
if (!canonical) continue;
healthByProvider.set(
canonical,
stat.connected > 0 ? "active" : stat.errors > 0 ? "error" : "idle"
);
}
const addProvider = (providerId?: string | null, name?: string) => {
const rawProviderId = typeof providerId === "string" ? providerId.trim() : "";
if (!rawProviderId) return;
@@ -484,6 +502,7 @@ export default function HomePageClient({ machineId }: HomePageClientProps) {
id: canonicalProviderId,
provider: canonicalProviderId,
name: resolvedName,
status: healthByProvider.get(canonicalProviderId) ?? "idle",
});
};

View File

@@ -13,6 +13,8 @@ type TopologyProvider = {
id: string;
provider: string;
name?: string;
/** Connection-health base state, so the topology can colour a node at rest. */
status?: "active" | "error" | "idle";
};
export function HomeProviderTopologySection({

View File

@@ -7,7 +7,7 @@ import { AI_PROVIDERS } from "@/shared/constants/providers";
import ProviderIcon from "@/shared/components/ProviderIcon";
import { FlowCanvas } from "@/shared/components/flow/FlowCanvas";
import { StatusDot } from "@/shared/components/flow/StatusDot";
import { edgeStyle } from "@/shared/components/flow/edgeStyles";
import { edgeStyle, FLOW_EDGE_COLORS } from "@/shared/components/flow/edgeStyles";
import { resolveTopologyNodeLabel } from "./topologyLabel";
// Rings: [capacity, rx, ry]. Each successive ring fits ~6 more nodes.
@@ -37,17 +37,27 @@ type ProviderNodeData = {
providerId: string;
active: boolean;
error: boolean;
/** Connection-health base state: a healthy connection with no in-flight traffic. */
healthy: boolean;
};
function ProviderNode({ data }: { data: ProviderNodeData }) {
const { label, color, providerId, active, error } = data;
const { label, color, providerId, active, error, healthy } = data;
const GREEN = FLOW_EDGE_COLORS.active;
const RED = FLOW_EDGE_COLORS.error;
return (
<div
className="flex items-center gap-2 px-2.5 py-1.5 rounded-lg border-2 transition-all duration-300 bg-bg"
style={{
borderColor: error ? "#ef4444" : active ? color : "var(--color-border)",
boxShadow: error ? `0 0 12px #ef444430` : active ? `0 0 12px ${color}30` : "none",
borderColor: error ? RED : active ? color : healthy ? GREEN : "var(--color-border)",
boxShadow: error
? `0 0 12px ${RED}30`
: active
? `0 0 12px ${color}30`
: healthy
? `0 0 10px ${GREEN}20`
: "none",
minWidth: "136px",
}}
>
@@ -85,12 +95,16 @@ function ProviderNode({ data }: { data: ProviderNodeData }) {
<span
className="text-xs font-medium truncate flex-1"
style={{ color: active ? color : error ? "#ef4444" : "var(--color-text-main)" }}
style={{
color: active ? color : error ? RED : healthy ? GREEN : "var(--color-text-main)",
}}
>
{label}
</span>
{(active || error) && <StatusDot color={color} error={error} />}
{(active || error || healthy) && (
<StatusDot color={active ? color : GREEN} error={error} pulse={active || error} />
)}
</div>
);
}
@@ -143,7 +157,8 @@ const nodeTypes: NodeTypes = {
router: RouterNode as any,
};
type ProviderEntry = { id?: string; provider: string; name?: string };
type ProviderHealth = "active" | "error" | "idle";
type ProviderEntry = { id?: string; provider: string; name?: string; status?: ProviderHealth };
function getHandles(angle: number, cx: number): { sourceHandle: string; targetHandle: string } {
const rel = (((angle + Math.PI / 2) % (2 * Math.PI)) + 2 * Math.PI) % (2 * Math.PI);
@@ -180,18 +195,20 @@ function buildLayout(
if (providers.length === 0) return { nodes, edges };
// Sort: active → error → last-used → rest (alpha within groups)
// Sort: active → error → last-used → healthy(connected) → rest (alpha within groups)
const sorted = [...providers].sort((a, b) => {
const aId = a.provider.toLowerCase();
const bId = b.provider.toLowerCase();
const rank = (id: string) => {
const rank = (p: ProviderEntry) => {
const id = p.provider.toLowerCase();
if (activeSet.has(id)) return 0;
if (errorSet.has(id)) return 1;
if (errorSet.has(id) || p.status === "error") return 1;
if (lastSet.has(id)) return 2;
return 3;
if (p.status === "active") return 3;
return 4;
};
const d = rank(aId) - rank(bId);
return d !== 0 ? d : aId.localeCompare(bId); // teknik sıralama: ASCII kasıtlı
const d = rank(a) - rank(b);
return d !== 0
? d
: a.provider.toLowerCase().localeCompare(b.provider.toLowerCase()); // ASCII kasıtlı
});
let provIdx = 0;
@@ -203,8 +220,14 @@ function buildLayout(
const p = sorted[provIdx++];
const pid = p.provider.toLowerCase();
const active = activeSet.has(pid);
const error = !active && errorSet.has(pid);
const last = !active && !error && lastSet.has(pid);
// Traffic signals (live/recent request) take precedence; connection health is the
// base state shown when a provider has no in-flight or recent traffic, so the map
// still reflects "what is connected" at rest instead of going blank after a restart.
const trafficError = !active && errorSet.has(pid);
const last = !active && !trafficError && lastSet.has(pid);
const healthError = !active && !trafficError && !last && p.status === "error";
const healthy = !active && !trafficError && !last && !healthError && p.status === "active";
const error = trafficError || healthError;
const config = getProviderConfig(p.provider);
const nodeId = `provider-${p.provider}`;
@@ -223,6 +246,7 @@ function buildLayout(
providerId: p.provider,
active,
error,
healthy,
} satisfies ProviderNodeData,
draggable: false,
});
@@ -234,7 +258,7 @@ function buildLayout(
target: nodeId,
targetHandle,
animated: active,
style: edgeStyle(active, last, error),
style: edgeStyle(active, last, error, healthy),
});
}
}

View File

@@ -10,21 +10,34 @@ type StatusDotProps = {
* value used by ProviderTopology so the home pulse is pixel-identical.
*/
sizeClass?: string;
/**
* Whether to render the `animate-ping` halo. Defaults to true (live/active pulse).
* Pass false for a static presence dot — e.g. a connection that is healthy but has
* no in-flight traffic, which should read as "connected" without implying activity.
*/
pulse?: boolean;
};
/**
* The pulsing presence indicator extracted from `ProviderTopology` (U0). Renders
* an `animate-ping` halo plus a solid dot. Callers decide *whether* to show it
* (e.g. only when a node is active or errored); this component only draws it.
* The presence indicator extracted from `ProviderTopology` (U0). Renders an optional
* `animate-ping` halo plus a solid dot. Callers decide *whether* to show it (e.g. only
* when a node is active, healthy, or errored); this component only draws it.
*/
export function StatusDot({ color, error = false, sizeClass = "size-1.5" }: StatusDotProps) {
export function StatusDot({
color,
error = false,
sizeClass = "size-1.5",
pulse = true,
}: StatusDotProps) {
const dotColor = error ? FLOW_EDGE_COLORS.error : color;
return (
<span className={`relative flex ${sizeClass} shrink-0`}>
<span
className="animate-ping absolute inline-flex h-full w-full rounded-full opacity-70"
style={{ backgroundColor: dotColor }}
/>
{pulse && (
<span
className="animate-ping absolute inline-flex h-full w-full rounded-full opacity-70"
style={{ backgroundColor: dotColor }}
/>
)}
<span
className={`relative inline-flex rounded-full ${sizeClass}`}
style={{ backgroundColor: dotColor }}

View File

@@ -21,12 +21,22 @@ export interface FlowEdgeStyle {
/**
* Resolve the stroke style for an edge given its state. Precedence is
* error > active > last-used > idle — identical to the original ProviderTopology
* implementation (do not reorder without updating the home regression).
* error > active > last-used > healthy > idle — the first three are identical to the
* original ProviderTopology implementation (do not reorder without updating the home
* regression). `healthy` is the connection-health base state (a configured provider with
* a live/healthy connection but no in-flight traffic): a static, dimmer green that makes
* the map meaningful at rest, distinct from the animated `active` pulse. It is an optional
* trailing param so existing callers (Combo/Compression studios) stay unaffected.
*/
export function edgeStyle(active: boolean, last: boolean, error: boolean): FlowEdgeStyle {
export function edgeStyle(
active: boolean,
last: boolean,
error: boolean,
healthy = false
): FlowEdgeStyle {
if (error) return { stroke: FLOW_EDGE_COLORS.error, strokeWidth: 2, opacity: 0.85 };
if (active) return { stroke: FLOW_EDGE_COLORS.active, strokeWidth: 2.5, opacity: 1 };
if (last) return { stroke: FLOW_EDGE_COLORS.last, strokeWidth: 1.5, opacity: 0.6 };
if (healthy) return { stroke: FLOW_EDGE_COLORS.active, strokeWidth: 1.5, opacity: 0.4 };
return { stroke: FLOW_EDGE_COLORS.idle, strokeWidth: 1, opacity: 0.3 };
}

View File

@@ -0,0 +1,55 @@
import test from "node:test";
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
// The topology used to colour nodes only from live/recent traffic, so between requests
// (and right after a restart) the map went blank even though connections were healthy.
// These guard the connection-health base layer that keeps "what is connected" visible.
const read = (rel: string) =>
readFileSync(fileURLToPath(new URL(rel, import.meta.url)), "utf8");
const homePageClientSrc = read("../../src/app/(dashboard)/dashboard/HomePageClient.tsx");
const providerTopologySrc = read("../../src/app/(dashboard)/home/ProviderTopology.tsx");
const sectionSrc = read("../../src/app/(dashboard)/dashboard/HomeProviderTopologySection.tsx");
test("HomePageClient derives per-provider health from connection testStatus counts", () => {
assert.match(homePageClientSrc, /healthByProvider/, "must build a per-provider health map");
assert.match(
homePageClientSrc,
/stat\.connected > 0 \? "active" : stat\.errors > 0 \? "error" : "idle"/,
"healthy = has a working connection; error = only failing ones; else idle"
);
assert.match(
homePageClientSrc,
/status:\s*healthByProvider\.get\(canonicalProviderId\)\s*\?\?\s*"idle"/,
"each topology entry must carry the resolved health status"
);
});
test("HomeProviderTopologySection forwards the status field on each provider", () => {
assert.match(
sectionSrc,
/status\?:\s*"active"\s*\|\s*"error"\s*\|\s*"idle"/,
"the section's provider type must include the health status"
);
});
test("ProviderTopology renders a connection-health base layer under the traffic signals", () => {
// Traffic (live/recent/error) must still take precedence over the static health colour.
assert.match(
providerTopologySrc,
/const healthy =\s*!active && !trafficError && !last && !healthError && p\.status === "active"/,
"healthy is only shown when there is no stronger traffic signal"
);
assert.match(
providerTopologySrc,
/edgeStyle\(active, last, error, healthy\)/,
"the healthy state must reach the edge palette"
);
// The node must render the health state (green border / static dot) — a non-pulsing dot
// distinguishes "connected" from "active".
assert.match(providerTopologySrc, /pulse=\{active \|\| error\}/);
assert.match(providerTopologySrc, /active \|\| error \|\| healthy/);
});

View File

@@ -0,0 +1,99 @@
import test from "node:test";
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
import { resolveRequestLifecycleEvent } from "../../open-sse/handlers/chatCore/attemptLogging.ts";
// The live topology lights a node green on `request.started` and only clears it when a
// matching `request.completed`/`request.failed` arrives. Those terminal events were
// declared + consumed by the client but never emitted, so a node stayed green until a
// page reload (the "stuck green" latch). These tests guard the fix that emits them.
test("resolveRequestLifecycleEvent: 2xx success → request.completed keyed by traceId", () => {
const ev = resolveRequestLifecycleEvent({
traceId: "abc123",
status: 200,
error: null,
model: "gpt-5.6-sol",
provider: "codex",
comboName: "my-combo",
tokens: { input: 10, output: 5 },
latencyMs: 1234,
});
assert.equal(ev.name, "request.completed");
assert.equal(ev.payload.id, "abc123");
if (ev.name !== "request.completed") return;
assert.equal(ev.payload.status, "success");
assert.equal(ev.payload.provider, "codex");
assert.equal(ev.payload.model, "gpt-5.6-sol");
assert.equal(ev.payload.tokensInput, 10);
assert.equal(ev.payload.tokensOutput, 5);
assert.equal(ev.payload.latencyMs, 1234);
assert.equal(ev.payload.comboName, "my-combo");
});
test("resolveRequestLifecycleEvent: 5xx → request.failed keyed by the same traceId", () => {
const ev = resolveRequestLifecycleEvent({
traceId: "e1",
status: 500,
error: "boom",
model: "m",
provider: "p",
latencyMs: 7,
});
assert.equal(ev.name, "request.failed");
if (ev.name !== "request.failed") return;
assert.equal(ev.payload.id, "e1");
assert.equal(ev.payload.error, "boom");
assert.equal(ev.payload.statusCode, 500);
assert.equal(ev.payload.latencyMs, 7);
});
test("resolveRequestLifecycleEvent: a 2xx status carrying an error string is still a failure", () => {
const ev = resolveRequestLifecycleEvent({ traceId: "x", status: 200, error: "late error", latencyMs: 1 });
assert.equal(ev.name, "request.failed");
});
test("resolveRequestLifecycleEvent: tokens resolve from prompt_tokens/completion_tokens aliases", () => {
const ev = resolveRequestLifecycleEvent({
traceId: "t",
status: 201,
tokens: { prompt_tokens: 3, completion_tokens: 8 },
latencyMs: 0,
});
assert.equal(ev.name, "request.completed");
if (ev.name !== "request.completed") return;
assert.equal(ev.payload.tokensInput, 3);
assert.equal(ev.payload.tokensOutput, 8);
});
test("resolveRequestLifecycleEvent: missing/odd tokens degrade to zero, never NaN", () => {
const ev = resolveRequestLifecycleEvent({ traceId: "z", status: 200, tokens: "nope", latencyMs: 2 });
assert.equal(ev.name, "request.completed");
if (ev.name !== "request.completed") return;
assert.equal(ev.payload.tokensInput, 0);
assert.equal(ev.payload.tokensOutput, 0);
});
test("attemptLogging emits both terminal events through the dashboard event bus", () => {
const src = readFileSync(
fileURLToPath(new URL("../../open-sse/handlers/chatCore/attemptLogging.ts", import.meta.url)),
"utf8"
);
assert.match(src, /emit\("request\.completed"/, "success attempts must emit request.completed");
assert.match(src, /emit\("request\.failed"/, "failed attempts must emit request.failed");
assert.match(src, /resolveRequestLifecycleEvent/, "emit must go through the pure resolver");
});
test("chatCore threads traceId into the persistAttemptLogs context (pairs with request.started)", () => {
const src = readFileSync(
fileURLToPath(new URL("../../open-sse/handlers/chatCore.ts", import.meta.url)),
"utf8"
);
assert.match(
src,
/persistAttemptLogsFor\(args,\s*\{\s*traceId/,
"the terminal event id must be the same traceId emitted in request.started"
);
});

View File

@@ -43,8 +43,29 @@ describe("flow edgeStyles (U0 — extracted from ProviderTopology)", () => {
});
});
it("applies precedence error > active > last", () => {
it("styles a healthy (connected, no in-flight traffic) edge as static dim green", () => {
assert.deepEqual(edgeStyle(false, false, false, true), {
stroke: "#22c55e",
strokeWidth: 1.5,
opacity: 0.4,
});
});
it("defaults healthy to false so 3-arg callers stay idle", () => {
assert.deepEqual(edgeStyle(false, false, false), {
stroke: "var(--color-text-muted)",
strokeWidth: 1,
opacity: 0.3,
});
});
it("applies precedence error > active > last > healthy", () => {
assert.equal(edgeStyle(true, true, true).stroke, "#ef4444"); // error wins
assert.equal(edgeStyle(true, true, false).stroke, "#22c55e"); // active beats last
assert.equal(edgeStyle(false, false, true, true).stroke, "#ef4444"); // error beats healthy
assert.equal(edgeStyle(false, true, false, true).stroke, "#f59e0b"); // last beats healthy
// healthy green is dimmer/thinner than the active pulse green
assert.equal(edgeStyle(false, false, false, true).opacity, 0.4);
assert.equal(edgeStyle(true, false, false, true).opacity, 1); // active still wins
});
});