diff --git a/changelog.d/fixes/13089-combo-live-roundrobin-missing-events.md b/changelog.d/fixes/13089-combo-live-roundrobin-missing-events.md new file mode 100644 index 0000000000..a873d3b894 --- /dev/null +++ b/changelog.d/fixes/13089-combo-live-roundrobin-missing-events.md @@ -0,0 +1 @@ +- **fix(routing):** round-robin combos now show up in Combo Studio's Live dashboard — they were completing successfully but never publishing the attempt/success/failure events the dashboard listens for (#13089) — thanks @adityadwi21 diff --git a/config/quality/file-size-baseline.json b/config/quality/file-size-baseline.json index 78a27ebb4c..b7d516ebe7 100644 --- a/config/quality/file-size-baseline.json +++ b/config/quality/file-size-baseline.json @@ -503,6 +503,7 @@ "open-sse/services/autoCombo/virtualFactory.ts": 1230, "open-sse/services/combo/roundRobinCombo.ts": 1213 }, + "_rebaseline_2026_09_15_roundrobin_dashboard_events": "Fix #13089 (Combo Studio Live dashboard shows an empty backlog for round-robin combos): open-sse/services/combo/roundRobinCombo.ts 1205->1213. Round-robin is the only combo strategy that bypasses handleComboChat/executeTargetAttempt.ts, the path that publishes the combo.target.attempt/succeeded/failed EventBus events the Live dashboard listens for — so round-robin completions never showed up. The new call-site wiring (createRRDashboardEvents(...) instantiated once per target, one-line .attempt()/.succeeded()/.failed() calls at the 6 existing dispatch/outcome points) is the emitter logic actually extracted into a new module, open-sse/services/combo/rrDashboardEvents.ts — this is the minimum irreducible footprint for wiring 6 required call sites into 6 fixed control-flow points of the frozen file. Covered by tests/unit/issue-13089-roundrobin-live-ws-events.test.ts (2 tests: success + failure paths).", "_rebaseline_base_2026_08_10_proxyfetch": "Base-red fix (green-prs sweep, issue #9985): open-sse/utils/proxyFetch.ts 1207 > cap 1000 — new proxied-TLS fetch helper introduced by the Fal reference-image work. Owner-authorized quick rebaseline to green; structural slim tracked for v3.9.0.", "_rebaseline_2026_07_27_v3849_train2": "Merge-train 2 (7 PRs) — owner-approved 2026-07-27. Single entry: chatCore.ts 4955->5006 (#8595, Responses multi-turn image compaction before the context hard-reject). Genuine irreducible growth at the existing compaction chokepoint in handleChatCore — the PR adds a last-resort retry against the concrete budget plus the estimateFinalInputTokens helper, both wired at the pre-existing call site rather than a new branch. Covered by tests/unit/8560-responses-image-compaction.test.ts (4 tests).", "_rebaseline_2026_07_27_v3849_train3": "Merge-train 3 (13 PRs) — owner-approved 2026-07-27. Both entries are genuine irreducible growth at existing chokepoints, not new branches: src/lib/db/apiKeys.ts 1518->1529 (#8805 cx/* ≡ codex/* API-key model permissions); open-sse/handlers/chatCore.ts 5006->5020 (#8806 real response payload into plugin onResponse hooks). Covered by tests/unit/db-apiKeys-crud.test.ts (4 new cases) and the two plugin-hook test files updated in #8806 respectively.", diff --git a/open-sse/services/combo/roundRobinCombo.ts b/open-sse/services/combo/roundRobinCombo.ts index be973e57d0..3b728188a9 100644 --- a/open-sse/services/combo/roundRobinCombo.ts +++ b/open-sse/services/combo/roundRobinCombo.ts @@ -100,6 +100,7 @@ import { import { applyComboTargetExhaustion } from "./targetExhaustion.ts"; import { isRetryAfterEligibleStatus } from "./unavailableRetryGate.ts"; import { isRecord } from "./comboData.ts"; +import { createRRDashboardEvents } from "./rrDashboardEvents.ts"; import { attemptCompatRejectedFallback } from "./comboCompatFallback.ts"; import { applyRequestTagRouting } from "./autoStrategy.ts"; import { @@ -485,6 +486,7 @@ export async function handleRoundRobinCombo({ const target = filteredTargets[modelIndex]; const modelStr = target.modelStr; const provider = target.provider; + const rrEvents = createRRDashboardEvents(combo.name, modelIndex, provider, modelStr); const profile = await getRuntimeProviderProfile(provider); const semaphoreKey = `combo:${combo.name}:${target.executionKey}`; const allowRateLimitedConnection = @@ -646,6 +648,7 @@ export async function handleRoundRobinCombo({ fingerprint: resolveTargetFingerprint(target) ?? "", }); + rrEvents.attempt(); const result = await Promise.race([ handleSingleModel(attemptBody, modelStr, { ...targetForAttempt, @@ -709,6 +712,7 @@ export async function handleRoundRobinCombo({ rrSelectedConnectionId || target.connectionId ); } + rrEvents.failed(`Quality: ${quality.reason}`, Date.now() - startTime); recordComboRequest(combo.name, modelStr, { success: false, latencyMs: Date.now() - startTime, @@ -735,6 +739,7 @@ export async function handleRoundRobinCombo({ "COMBO-RR", `${modelStr} succeeded (${latencyMs}ms, ${fallbackCount} fallbacks)` ); + rrEvents.succeeded(latencyMs); recordComboRequest(combo.name, modelStr, { success: true, latencyMs, @@ -844,6 +849,7 @@ export async function handleRoundRobinCombo({ "COMBO-RR", `Client disconnected (499) during ${modelStr} — stopping combo loop` ); + rrEvents.failed("Client disconnected", Date.now() - startTime); recordComboRequest(combo.name, modelStr, { success: false, latencyMs: Date.now() - startTime, @@ -884,6 +890,7 @@ export async function handleRoundRobinCombo({ "COMBO-RR", `Local rate-limit queue capacity reached for ${modelStr} — returning without upstream fallback` ); + rrEvents.failed(errorText || "Local queue full", Date.now() - startTime); recordComboRequest(combo.name, modelStr, { success: false, latencyMs: Date.now() - startTime, @@ -1023,6 +1030,7 @@ export async function handleRoundRobinCombo({ } // Done with this model + rrEvents.failed(errorText || `HTTP ${result.status}`, Date.now() - startTime); recordComboRequest(combo.name, modelStr, { success: false, latencyMs: Date.now() - startTime, diff --git a/open-sse/services/combo/rrDashboardEvents.ts b/open-sse/services/combo/rrDashboardEvents.ts new file mode 100644 index 0000000000..55bf0b703e --- /dev/null +++ b/open-sse/services/combo/rrDashboardEvents.ts @@ -0,0 +1,43 @@ +/** + * Dashboard EventBus emitters for the round-robin combo loop (#13089). + * + * `roundRobinCombo.ts` is frozen at its file-size cap (#12884), so the + * `combo.target.attempt` / `combo.target.succeeded` / `combo.target.failed` + * publishing logic lives here — a factory bound to one target's identity so + * each call site in the frozen file is a single line. + * + * @internal — not part of the public combo.ts barrel. + */ +import { emit } from "../../../src/lib/events/eventBus"; + +export interface RRDashboardEvents { + attempt(): void; + succeeded(latencyMs: number): void; + failed(error: string, latencyMs: number): void; +} + +export function createRRDashboardEvents( + comboName: string, + targetIndex: number, + provider: string, + model: string +): RRDashboardEvents { + return { + attempt() { + emit("combo.target.attempt", { + comboName, + targetIndex, + provider, + model, + timestamp: Date.now(), + strategy: "round-robin", + }); + }, + succeeded(latencyMs) { + emit("combo.target.succeeded", { comboName, targetIndex, provider, model, latencyMs }); + }, + failed(error, latencyMs) { + emit("combo.target.failed", { comboName, targetIndex, provider, model, error, latencyMs }); + }, + }; +} diff --git a/tests/unit/issue-13089-roundrobin-live-ws-events.test.ts b/tests/unit/issue-13089-roundrobin-live-ws-events.test.ts new file mode 100644 index 0000000000..b2d65f7e62 --- /dev/null +++ b/tests/unit/issue-13089-roundrobin-live-ws-events.test.ts @@ -0,0 +1,140 @@ +/** + * Repro for #13089 — Combo Studio Live dashboard shows empty backlog despite + * successful (or failed) chat completions routed through a round-robin combo. + * + * Round-robin bypasses `handleComboChat` -> `executeTargetAttempt` (the path that + * publishes `combo.target.attempt` / `combo.target.succeeded` / `combo.target.failed` + * on the dashboard EventBus) and dispatches through its own loop in + * `open-sse/services/combo/roundRobinCombo.ts`, which never emitted those events. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-13089-")); +const ORIGINAL_DATA_DIR = process.env.DATA_DIR; +process.env.DATA_DIR = TEST_DATA_DIR; + +const { handleComboChat } = await import("../../open-sse/services/combo.ts"); +const rrState = await import("../../open-sse/services/combo/rrState.ts"); +const dbCore = await import("../../src/lib/db/core.ts"); +const eventBus = await import("../../src/lib/events/eventBus.ts"); + +function makeLog() { + return { info() {}, warn() {}, debug() {}, error() {} }; +} + +function rrCombo(name: string, maxRetries = 0) { + return { + name, + strategy: "round-robin", + config: { maxRetries, disableSessionStickiness: true }, + models: [ + { + kind: "model", + provider: "codex", + providerId: "codex", + model: "m-a", + connectionId: "conn-A", + id: `${name}-0`, + }, + ], + }; +} + +test.beforeEach(() => { + rrState.rrCounters.clear(); + rrState.rrStickyTargets.clear(); +}); + +test.after(() => { + try { + dbCore.resetDbInstance?.(); + } catch { + /* ignore */ + } + if (ORIGINAL_DATA_DIR === undefined) delete process.env.DATA_DIR; + else process.env.DATA_DIR = ORIGINAL_DATA_DIR; + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); +}); + +test("#13089: a successful round-robin combo completion publishes combo.target.succeeded on the dashboard EventBus", async () => { + const combo = rrCombo("rr13089-success"); + + const seen: Array<{ event: string; payload: unknown }> = []; + const unsubscribeAttempt = eventBus.on("combo.target.attempt", (payload) => { + seen.push({ event: "combo.target.attempt", payload }); + }); + const unsubscribeSucceeded = eventBus.on("combo.target.succeeded", (payload) => { + seen.push({ event: "combo.target.succeeded", payload }); + }); + + try { + const response = await handleComboChat({ + body: { model: combo.name, messages: [{ role: "user", content: "hi" }], stream: false }, + combo, + allCombos: [combo], + isModelAvailable: async () => true, + relayOptions: undefined, + signal: undefined, + settings: {}, + log: makeLog(), + handleSingleModel: async (_b, modelStr, _target) => { + return Response.json({ + choices: [{ message: { role: "assistant", content: modelStr } }], + }); + }, + }); + + assert.equal(response.status, 200, "the combo-routed completion itself must succeed"); + assert.ok( + seen.some((e) => e.event === "combo.target.attempt"), + `expected a "combo.target.attempt" EventBus event, but none was published (saw: ${JSON.stringify(seen)}).` + ); + assert.ok( + seen.some((e) => e.event === "combo.target.succeeded"), + `expected a "combo.target.succeeded" EventBus event after a successful round-robin ` + + `combo completion, but none was published (saw: ${JSON.stringify(seen)}). This is why ` + + `Combo Studio -> Live never shows round-robin combo executions (#13089).` + ); + } finally { + unsubscribeAttempt(); + unsubscribeSucceeded(); + } +}); + +test("#13089: a failing round-robin combo target publishes combo.target.failed on the dashboard EventBus", async () => { + const combo = rrCombo("rr13089-failure"); + + const seen: Array<{ event: string; payload: unknown }> = []; + const unsubscribeFailed = eventBus.on("combo.target.failed", (payload) => { + seen.push({ event: "combo.target.failed", payload }); + }); + + try { + const response = await handleComboChat({ + body: { model: combo.name, messages: [{ role: "user", content: "hi" }], stream: false }, + combo, + allCombos: [combo], + isModelAvailable: async () => true, + relayOptions: undefined, + signal: undefined, + settings: {}, + log: makeLog(), + handleSingleModel: async () => { + return Response.json({ error: { message: "rate limited" } }, { status: 429 }); + }, + }); + + assert.equal(response.status, 429, "the exhausted combo must surface the upstream failure"); + assert.ok( + seen.some((e) => e.event === "combo.target.failed"), + `expected a "combo.target.failed" EventBus event after an exhausted round-robin ` + + `combo target, but none was published (saw: ${JSON.stringify(seen)}).` + ); + } finally { + unsubscribeFailed(); + } +});