From 7ef34c22461ee050828322d6fa2e6568fc6026a1 Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza <8016841+diegosouzapw@users.noreply.github.com> Date: Sun, 21 Jun 2026 20:01:55 -0300 Subject: [PATCH] fix(combo): extract dynamic connection ID from headers on success path for precise lockout decay and success telemetry (#4550) (#4581) Integrated into release/v3.8.33 (cherry-pick of #4550 with stale reverts dropped; @Chewji9875 credited in CHANGELOG) --- CHANGELOG.md | 1 + config/quality/file-size-baseline.json | 4 +- open-sse/services/combo.ts | 94 +++-- src/sse/handlers/chat.ts | 11 +- src/sse/handlers/chatHelpers.ts | 20 + .../combo-selected-connection-success.test.ts | 379 ++++++++++++++++++ 6 files changed, 473 insertions(+), 36 deletions(-) create mode 100644 tests/unit/combo-selected-connection-success.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 4abc7a10ed..802ed1c6cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ _In development — bullets added per PR; finalized at release._ - **fix(antigravity): reasoning/thinking models no longer 400 with `oneOf at '/' not met`** — the Cloud Code envelope passthrough also leaked the Claude/OpenAI-native thinking fields (`thinking`, `reasoning_effort`, `reasoning`, `enable_thinking`, `thinking_budget`) the unified thinking adapter sets at the body root; Google rejected them with `400 Bad input: oneOf at '/' not met`. The whole thinking family is now stripped before the envelope is built; Gemini's own `generationConfig.thinkingConfig` is unaffected. (port from 9router#1926 — thanks @theseven99 / @diegosouzapw) - **fix(integration): restore the codex and memory pipeline contracts** — realigns the CLI fingerprint + memory-tools contracts so the codex and memory pipelines pass their integration checks again. ([#4474](https://github.com/diegosouzapw/OmniRoute/pull/4474) — thanks @KooshaPari) - **perf(quota): stop writing redundant `quota_snapshots` rows from idle connections** — the 60s background refresh persisted a snapshot for every window of every connection regardless of change, generating 400K+ rows/day from idle accounts. `setQuotaCache` now skips the write when a window's `remaining_percentage`/`is_exhausted` is unchanged from the last cached observation; the first observation and every real change still persist. ([#4438](https://github.com/diegosouzapw/OmniRoute/issues/4438) — thanks @oyi77) +- **fix(combo): attribute lockout decay & success telemetry to the dynamically-selected connection** — on the combo success path the actual connection chosen by dynamic account-selection is now read from the `X-OmniRoute-Selected-Connection-Id` response header (instead of the often-empty static `target.connectionId`), so model-lockout decay, `recordProviderSuccess`, LKGP and success/failure telemetry attribute to the right connection on both the priority and round-robin paths. The pre-screen "unavailable" snapshot is also no longer a permanent skip — availability is re-checked on each retry since connection cooldowns can expire mid-request. ([#4550](https://github.com/diegosouzapw/OmniRoute/pull/4550) — thanks @Chewji9875) ### 📝 Maintenance diff --git a/config/quality/file-size-baseline.json b/config/quality/file-size-baseline.json index 574cd53bf7..cd641e112e 100644 --- a/config/quality/file-size-baseline.json +++ b/config/quality/file-size-baseline.json @@ -130,7 +130,7 @@ "open-sse/services/batchProcessor.ts": 828, "open-sse/services/browserBackedChat.ts": 850, "open-sse/services/claudeCodeCompatible.ts": 1202, - "open-sse/services/combo.ts": 2955, + "open-sse/services/combo.ts": 2991, "open-sse/services/rateLimitManager.ts": 1035, "open-sse/services/tokenRefresh.ts": 1997, "open-sse/services/usage.ts": 3450, @@ -205,7 +205,7 @@ "src/shared/constants/sidebarVisibility.ts": 1100, "src/shared/services/cliRuntime.ts": 1090, "src/shared/validation/schemas.ts": 2523, - "src/sse/handlers/chat.ts": 1515, + "src/sse/handlers/chat.ts": 1516, "src/sse/services/auth.ts": 2279 }, "testCap": 800, diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index 1b85b98821..3645510dc8 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -1501,18 +1501,11 @@ export async function handleComboChat({ return null; } - // Pre-screen may have already determined this target unavailable (e.g. - // circuit-breaker OPEN at resolve time). Skip immediately in that case. - // For targets pre-screened as "available" we still call isModelAvailable - // below because connection cooldowns (rateLimitedUntil) can change - // mid-request after a same-provider failure — the pre-screen snapshot is - // stale by the time we reach the 2nd/3rd same-provider target. - const preCheckedAvailable = preScreenEntry?.available ?? null; - if (preCheckedAvailable === false) { - log.info("COMBO", `Skipping ${modelStr} — pre-screen marked unavailable`); - if (i > 0) fallbackCount++; - return null; - } + // Pre-screen snapshot is NOT used as a permanent skip — availability + // is always re-checked via isModelAvailable below because connection + // cooldowns can expire between setTry retries, making a previously + // unavailable target available again. Circuit-breaker-OPEN providers + // are already caught by the dedicated breaker check above. if (isModelAvailable) { const available = await isModelAvailable(modelStr, targetForAttempt); if (!available) { @@ -1690,6 +1683,12 @@ export async function handleComboChat({ // Success — validate response quality before returning if (result.ok) { + const selectedConnectionId = + result.headers?.get("X-OmniRoute-Selected-Connection-Id") || + result.headers?.get("x-omniroute-selected-connection-id") || + undefined; + const effectiveConnectionId = selectedConnectionId || target.connectionId || ""; + const quality = await validateResponseQuality(result, clientRequestedStream, log); if (!quality.valid) { log.warn( @@ -1743,11 +1742,7 @@ export async function handleComboChat({ // Success decay: a healthy response walks the model's lockout failure // count back down (and eventually clears an expired lockout entirely). if (provider && rawModel) { - const dcResult = decayModelFailureCount( - provider, - target.connectionId || "", - rawModel - ); + const dcResult = decayModelFailureCount(provider, effectiveConnectionId, rawModel); if (dcResult.cleared) { log.info("COMBO", `Model ${modelStr} fully recovered — lockout cleared`); } else if (dcResult.newFailureCount > 0) { @@ -1781,7 +1776,7 @@ export async function handleComboChat({ // Reset cooldown on success if (provider && provider !== "unknown") { - recordProviderSuccess(provider, target.connectionId ?? undefined); + recordProviderSuccess(provider, effectiveConnectionId || undefined); } if (strategy === "weighted" && stickyWeightedLimit > 1) { const stickySuccessKey = getWeightedStepKeyForTarget(target); @@ -1798,7 +1793,7 @@ export async function handleComboChat({ typeof target.label === "string" && target.label.trim().length > 0 ? target.label.trim() : "", - accountId: target.connectionId ?? "", + accountId: effectiveConnectionId ?? "", latencyMs, fallbackCount, }); @@ -1909,7 +1904,7 @@ export async function handleComboChat({ // Record last known good provider (LKGP) for this combo/model (#919) if (provider) { - const connId = target.connectionId || undefined; + const connId = effectiveConnectionId || undefined; void (async () => { try { const { setLKGP } = await import("../../src/lib/localDb"); @@ -2035,10 +2030,17 @@ export async function handleComboChat({ structuredError ); const { cooldownMs } = fallbackResult; + const selectedConnectionId = + result.headers?.get("X-OmniRoute-Selected-Connection-Id") || + result.headers?.get("x-omniroute-selected-connection-id") || + undefined; + const targetWithConnection = selectedConnectionId + ? { ...target, connectionId: selectedConnectionId } + : target; // #1731 / #1731v2: classify the upstream error and update the exhaustion sets // (shared with handleRoundRobinCombo). Returns whether the provider is fully exhausted. - const providerExhausted = applyComboTargetExhaustion(target, { + const providerExhausted = applyComboTargetExhaustion(targetWithConnection, { result, fallbackResult, errorText, @@ -2114,7 +2116,7 @@ export async function handleComboChat({ skipProviderBreaker: fallbackResult.skipProviderBreaker, }) ) { - recordProviderFailure(provider, log, target.connectionId, profile); + recordProviderFailure(provider, log, targetWithConnection.connectionId, profile); } // Check if this is a transient error worth retrying on same model. @@ -2133,7 +2135,7 @@ export async function handleComboChat({ if (mlSettings.enabled && mlSettings.errorCodes.includes(result.status)) { recordModelLockoutFailure( provider, - target.connectionId || "", + targetWithConnection.connectionId || "", rawModel, classifyLockoutReason(result.status), result.status, @@ -2176,7 +2178,7 @@ export async function handleComboChat({ if (mlSettings.enabled && mlSettings.errorCodes.includes(result.status)) { recordModelLockoutFailure( provider, - target.connectionId || "", + targetWithConnection.connectionId || "", rawModel, classifyLockoutReason(result.status), result.status, @@ -2193,7 +2195,11 @@ export async function handleComboChat({ log.warn("COMBO", `Model ${modelStr} failed, trying next`, { status: result.status }); if (resilienceSettings.providerCooldown.enabled && provider && provider !== "unknown") { - recordProviderCooldown(provider, target.connectionId ?? undefined, resilienceSettings); + recordProviderCooldown( + provider, + targetWithConnection.connectionId ?? undefined, + resilienceSettings + ); } const fallbackWaitMs = @@ -2676,8 +2682,27 @@ async function handleRoundRobinCombo({ }); recordedAttempts++; + const selectedConnectionId = + result.headers?.get("X-OmniRoute-Selected-Connection-Id") || + result.headers?.get("x-omniroute-selected-connection-id") || + undefined; + const effectiveConnectionId = selectedConnectionId || target.connectionId || ""; + + const rawModel = parseModel(modelStr).model || modelStr; + if (provider && rawModel) { + const dcResult = decayModelFailureCount(provider, effectiveConnectionId, rawModel); + if (dcResult.cleared) { + log.info("COMBO-RR", `Model ${modelStr} fully recovered — lockout cleared`); + } else if (dcResult.newFailureCount > 0) { + log.debug?.( + "COMBO-RR", + `Model ${modelStr} decayed to failureCount=${dcResult.newFailureCount}` + ); + } + } + if (provider && provider !== "unknown") { - recordProviderSuccess(provider, target.connectionId ?? undefined); + recordProviderSuccess(provider, effectiveConnectionId || undefined); } if (stickyRoundRobinEnabled) { @@ -2685,7 +2710,7 @@ async function handleRoundRobinCombo({ } if (provider) { - const connId = target.connectionId || undefined; + const connId = effectiveConnectionId || undefined; void (async () => { try { const { setLKGP } = await import("../../src/lib/localDb"); @@ -2810,6 +2835,13 @@ async function handleRoundRobinCombo({ structuredError ); const { cooldownMs } = fallbackResult; + const selectedConnectionId = + result.headers?.get("X-OmniRoute-Selected-Connection-Id") || + result.headers?.get("x-omniroute-selected-connection-id") || + undefined; + const targetWithConnection = selectedConnectionId + ? { ...target, connectionId: selectedConnectionId } + : target; const isAllAccountsRateLimited = isAllAccountsRateLimitedResponse( result.status, @@ -2823,7 +2855,7 @@ async function handleRoundRobinCombo({ // combo from trying another target for the same provider in this request. // #1731 / #1731v2: classify the upstream error and update the exhaustion sets // (shared with handleComboChat). Returns whether the provider is fully exhausted. - const providerExhausted = applyComboTargetExhaustion(target, { + const providerExhausted = applyComboTargetExhaustion(targetWithConnection, { result, fallbackResult, errorText, @@ -2879,7 +2911,11 @@ async function handleRoundRobinCombo({ log.warn("COMBO-RR", `${modelStr} failed, trying next model`, { status: result.status }); if (resilienceSettings.providerCooldown.enabled && provider && provider !== "unknown") { - recordProviderCooldown(provider, target.connectionId ?? undefined, resilienceSettings); + recordProviderCooldown( + provider, + targetWithConnection.connectionId ?? undefined, + resilienceSettings + ); } const fallbackWaitMs = diff --git a/src/sse/handlers/chat.ts b/src/sse/handlers/chat.ts index 3101069c23..6ae3715699 100644 --- a/src/sse/handlers/chat.ts +++ b/src/sse/handlers/chat.ts @@ -66,6 +66,7 @@ import { safeLogEvents, shouldRetryStreamEarlyEof, withSessionHeader, + withSelectedConnectionHeader, } from "./chatHelpers"; import { connectionHasExtraKeys } from "@omniroute/open-sse/services/apiKeyRotator.ts"; @@ -1219,7 +1220,7 @@ async function handleSingleModelChat( // Stream readiness timeout is an upstream stall after an HTTP response was received, // not an account/quota failure. Do NOT mark the account unavailable here. - return result.response; + return withSelectedConnectionHeader(result.response, credentials?.connectionId); } if (isAntigravityStreamReadinessFailure) { @@ -1263,7 +1264,7 @@ async function handleSingleModelChat( continue; } - return result.response; + return withSelectedConnectionHeader(result.response, credentials?.connectionId); } const isAntigravityPreResponseTimeout = @@ -1313,14 +1314,14 @@ async function handleSingleModelChat( continue; } - return result.response; + return withSelectedConnectionHeader(result.response, credentials?.connectionId); } if (result.errorType === "account_semaphore_capacity") { // Local concurrency pressure is not an upstream quota failure. Prefer another // account when possible; pinned combo steps fall through to combo orchestration. if (hasForcedConnection) { - return result.response; + return withSelectedConnectionHeader(result.response, credentials?.connectionId); } log.warn( @@ -1508,7 +1509,7 @@ async function handleSingleModelChat( breaker._onFailure(); } - return result.response; + return withSelectedConnectionHeader(result.response, credentials?.connectionId); } } } diff --git a/src/sse/handlers/chatHelpers.ts b/src/sse/handlers/chatHelpers.ts index 7de7672207..e8abf6533e 100644 --- a/src/sse/handlers/chatHelpers.ts +++ b/src/sse/handlers/chatHelpers.ts @@ -758,3 +758,23 @@ export function withSessionHeader(response: Response, sessionId: string | null): return cloned; } } + +export function withSelectedConnectionHeader( + response: Response, + connectionId: string | null | undefined +): Response { + if (!response || !connectionId) return response; + + try { + response.headers.set("X-OmniRoute-Selected-Connection-Id", connectionId); + return response; + } catch { + const cloned = new Response(response.body, { + status: response.status, + statusText: response.statusText, + headers: response.headers, + }); + cloned.headers.set("X-OmniRoute-Selected-Connection-Id", connectionId); + return cloned; + } +} diff --git a/tests/unit/combo-selected-connection-success.test.ts b/tests/unit/combo-selected-connection-success.test.ts new file mode 100644 index 0000000000..2e986639c7 --- /dev/null +++ b/tests/unit/combo-selected-connection-success.test.ts @@ -0,0 +1,379 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import test, { describe, beforeEach } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-combo-sel-conn-")); +const ORIGINAL_DATA_DIR = process.env.DATA_DIR; +process.env.DATA_DIR = TEST_DATA_DIR; + +const { handleComboChat } = await import("../../open-sse/services/combo.ts"); +const { + recordModelLockoutFailure, + getModelLockoutInfo, + clearAllModelLockouts, + decayModelFailureCount, +} = await import("../../open-sse/services/accountFallback.ts"); +const { recordProviderCooldown, isProviderInCooldown, recordProviderSuccess, clearCooldownState } = + await import("../../open-sse/services/providerCooldownTracker.ts"); +const { resolveResilienceSettings } = await import("../../src/lib/resilience/settings.ts"); +const core = await import("../../src/lib/db/core.ts"); +const settingsDb = await import("../../src/lib/db/settings.ts"); +const providersDb = await import("../../src/lib/db/providers.ts"); + +const settings = resolveResilienceSettings({ + resilienceSettings: { + providerCooldown: { + enabled: true, + minRetryCooldownMs: 5000, + maxRetryCooldownMs: 300000, + }, + }, +}); + +function createLog() { + return { + info: (tag: any, msg: any) => console.log(`[INFO][${tag}] ${msg}`), + warn: (tag: any, msg: any) => console.log(`[WARN][${tag}] ${msg}`), + error: (tag: any, msg: any) => console.log(`[ERROR][${tag}] ${msg}`), + debug: (tag: any, msg: any) => console.log(`[DEBUG][${tag}] ${msg}`), + }; +} + +async function cleanupTestDataDir() { + let lastError; + for (let attempt = 0; attempt < 5; attempt += 1) { + try { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + return; + } catch (error: any) { + lastError = error; + await new Promise((resolve) => setTimeout(resolve, 25)); + } + } + if (lastError) { + throw lastError; + } +} + +test.after(async () => { + await cleanupTestDataDir(); + process.env.DATA_DIR = ORIGINAL_DATA_DIR; +}); + +beforeEach(async () => { + clearAllModelLockouts(); + clearCooldownState(); + settingsDb.clearAllLKGP(); +}); + +describe("combo selected connection success handling", () => { + test("priority strategy correctly extracts dynamic connection ID from success response headers and decays lockout, resets provider cooldown, and updates LKGP", async () => { + const comboName = "test-combo-priority"; + const modelStr = "openai/gpt-4"; + const provider = "openai"; + const dynamicConnId = "conn-dynamic-123"; + + await providersDb.createProviderConnection({ + provider, + authType: "apikey", + name: "OpenAI Test", + apiKey: "sk-test", + }); + + // 1. Populate lockout failure count = 4 + recordModelLockoutFailure( + provider, + dynamicConnId, + "gpt-4", + "rate_limit_exceeded", + 429, + 120_000, + null, + { exactCooldownMs: 60_000 } + ); + for (let i = 0; i < 3; i++) { + recordModelLockoutFailure( + provider, + dynamicConnId, + "gpt-4", + "rate_limit_exceeded", + 429, + 120_000, + null, + { exactCooldownMs: 60_000 } + ); + } + + // Verify initial failureCount is 4 + const initialLockout = getModelLockoutInfo(provider, dynamicConnId, "gpt-4"); + assert.equal(initialLockout?.failureCount, 4); + + // 2. Record provider cooldown + recordProviderCooldown(provider, dynamicConnId, settings); + assert.ok(isProviderInCooldown(provider, dynamicConnId, settings)); + + // 3. Invoke handleComboChat + const result = await handleComboChat({ + body: { stream: false }, + combo: { + name: comboName, + strategy: "priority", + models: [modelStr], + config: { maxRetries: 0, concurrencyPerModel: 1, queueTimeoutMs: 1000 }, + }, + handleSingleModel: async () => { + return new Response(JSON.stringify({ choices: [{ message: { content: "ok" } }] }), { + status: 200, + headers: { + "content-type": "application/json", + "X-OmniRoute-Selected-Connection-Id": dynamicConnId, + }, + }); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: null, + allCombos: null, + }); + + assert.equal(result.ok, true); + + // 4. Assertions + // A. Dynamic connection-level failure count decay (halved to 2) + const decayCheck = decayModelFailureCount(provider, dynamicConnId, "gpt-4"); + assert.equal( + decayCheck.newFailureCount, + 1, + "failure count should decay from 4 to 2, and now to 1" + ); + + // B. Dynamic connection-level provider success tracking (not in cooldown anymore) + assert.equal( + isProviderInCooldown(provider, dynamicConnId, settings), + false, + "provider cooldown should be cleared on success" + ); + + // C. Correct LKGP record updated with the dynamic connection ID (setLKGP is called) + let persisted: any = null; + for (let i = 0; i < 20; i++) { + persisted = await settingsDb.getLKGP(comboName, comboName); + if (persisted?.connectionId === dynamicConnId) { + break; + } + await new Promise((resolve) => setTimeout(resolve, 10)); + } + assert.equal(persisted?.provider, provider); + assert.equal( + persisted?.connectionId, + dynamicConnId, + "LKGP connectionId must be the dynamic connection ID" + ); + }); + + test("priority strategy with lowercase selected connection ID header correctly decays lockout, resets provider cooldown, and updates LKGP", async () => { + const comboName = "test-combo-priority-lc"; + const modelStr = "openai/gpt-4"; + const provider = "openai"; + const dynamicConnId = "conn-dynamic-123-lc"; + + await providersDb.createProviderConnection({ + provider, + authType: "apikey", + name: "OpenAI Test LC", + apiKey: "sk-test", + }); + + // 1. Populate lockout failure count = 4 + recordModelLockoutFailure( + provider, + dynamicConnId, + "gpt-4", + "rate_limit_exceeded", + 429, + 120_000, + null, + { exactCooldownMs: 60_000 } + ); + for (let i = 0; i < 3; i++) { + recordModelLockoutFailure( + provider, + dynamicConnId, + "gpt-4", + "rate_limit_exceeded", + 429, + 120_000, + null, + { exactCooldownMs: 60_000 } + ); + } + + // Verify initial failureCount is 4 + const initialLockout = getModelLockoutInfo(provider, dynamicConnId, "gpt-4"); + assert.equal(initialLockout?.failureCount, 4); + + // 2. Record provider cooldown + recordProviderCooldown(provider, dynamicConnId, settings); + assert.ok(isProviderInCooldown(provider, dynamicConnId, settings)); + + // 3. Invoke handleComboChat + const result = await handleComboChat({ + body: { stream: false }, + combo: { + name: comboName, + strategy: "priority", + models: [modelStr], + config: { maxRetries: 0, concurrencyPerModel: 1, queueTimeoutMs: 1000 }, + }, + handleSingleModel: async () => { + return new Response(JSON.stringify({ choices: [{ message: { content: "ok" } }] }), { + status: 200, + headers: { + "content-type": "application/json", + "x-omniroute-selected-connection-id": dynamicConnId, + }, + }); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: null, + allCombos: null, + }); + + assert.equal(result.ok, true); + + // 4. Assertions + const decayCheck = decayModelFailureCount(provider, dynamicConnId, "gpt-4"); + assert.equal( + decayCheck.newFailureCount, + 1, + "failure count should decay from 4 to 2, and now to 1" + ); + + assert.equal( + isProviderInCooldown(provider, dynamicConnId, settings), + false, + "provider cooldown should be cleared on success" + ); + + let persisted: any = null; + for (let i = 0; i < 20; i++) { + persisted = await settingsDb.getLKGP(comboName, comboName); + if (persisted?.connectionId === dynamicConnId) { + break; + } + await new Promise((resolve) => setTimeout(resolve, 10)); + } + assert.equal(persisted?.provider, provider); + assert.equal( + persisted?.connectionId, + dynamicConnId, + "LKGP connectionId must be the dynamic connection ID" + ); + }); + + test("round-robin strategy correctly extracts dynamic connection ID from success response headers and decays lockout, resets provider cooldown, and updates LKGP", async () => { + const comboName = "test-combo-rr"; + const modelStr = "openai/gpt-4"; + const provider = "openai"; + const dynamicConnId = "conn-dynamic-123-rr"; + + await providersDb.createProviderConnection({ + provider, + authType: "apikey", + name: "OpenAI Test RR", + apiKey: "sk-test", + }); + + // 1. Populate lockout failure count = 4 + recordModelLockoutFailure( + provider, + dynamicConnId, + "gpt-4", + "rate_limit_exceeded", + 429, + 120_000, + null, + { exactCooldownMs: 60_000 } + ); + for (let i = 0; i < 3; i++) { + recordModelLockoutFailure( + provider, + dynamicConnId, + "gpt-4", + "rate_limit_exceeded", + 429, + 120_000, + null, + { exactCooldownMs: 60_000 } + ); + } + + // Verify initial failureCount is 4 + const initialLockout = getModelLockoutInfo(provider, dynamicConnId, "gpt-4"); + assert.equal(initialLockout?.failureCount, 4); + + // 2. Record provider cooldown + recordProviderCooldown(provider, dynamicConnId, settings); + assert.ok(isProviderInCooldown(provider, dynamicConnId, settings)); + + // 3. Invoke handleComboChat with round-robin strategy + const result = await handleComboChat({ + body: { stream: false }, + combo: { + name: comboName, + strategy: "round-robin", + models: [modelStr], + config: { maxRetries: 0, concurrencyPerModel: 1, queueTimeoutMs: 1000 }, + }, + handleSingleModel: async () => { + return new Response(JSON.stringify({ choices: [{ message: { content: "ok" } }] }), { + status: 200, + headers: { + "content-type": "application/json", + "X-OmniRoute-Selected-Connection-Id": dynamicConnId, + }, + }); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: null, + allCombos: null, + }); + + assert.equal(result.ok, true); + + // 4. Assertions + const decayCheck = decayModelFailureCount(provider, dynamicConnId, "gpt-4"); + assert.equal( + decayCheck.newFailureCount, + 1, + "failure count should decay from 4 to 2, and now to 1" + ); + + assert.equal( + isProviderInCooldown(provider, dynamicConnId, settings), + false, + "provider cooldown should be cleared on success" + ); + + let persisted: any = null; + for (let i = 0; i < 20; i++) { + persisted = await settingsDb.getLKGP(comboName, comboName); + if (persisted?.connectionId === dynamicConnId) { + break; + } + await new Promise((resolve) => setTimeout(resolve, 10)); + } + assert.equal(persisted?.provider, provider); + assert.equal( + persisted?.connectionId, + dynamicConnId, + "LKGP connectionId must be the dynamic connection ID" + ); + }); +});