fix(combo): extract dynamic connection ID from headers on success path for precise lockout decay and success telemetry (#4550) (#4581)

Integrated into release/v3.8.33 (cherry-pick of #4550 with stale reverts dropped; @Chewji9875 credited in CHANGELOG)
This commit is contained in:
Diego Rodrigues de Sa e Souza
2026-06-21 20:01:55 -03:00
committed by GitHub
parent 09d8a4aa8c
commit 7ef34c2246
6 changed files with 473 additions and 36 deletions

View File

@@ -18,6 +18,7 @@ _In development — bullets added per PR; finalized at release._
- **fix(antigravity): reasoning/thinking models no longer 400 with `oneOf at '/' not met`** — the Cloud Code envelope passthrough also leaked the Claude/OpenAI-native thinking fields (`thinking`, `reasoning_effort`, `reasoning`, `enable_thinking`, `thinking_budget`) the unified thinking adapter sets at the body root; Google rejected them with `400 Bad input: oneOf at '/' not met`. The whole thinking family is now stripped before the envelope is built; Gemini's own `generationConfig.thinkingConfig` is unaffected. (port from 9router#1926 — thanks @theseven99 / @diegosouzapw)
- **fix(integration): restore the codex and memory pipeline contracts** — realigns the CLI fingerprint + memory-tools contracts so the codex and memory pipelines pass their integration checks again. ([#4474](https://github.com/diegosouzapw/OmniRoute/pull/4474) — thanks @KooshaPari)
- **perf(quota): stop writing redundant `quota_snapshots` rows from idle connections** — the 60s background refresh persisted a snapshot for every window of every connection regardless of change, generating 400K+ rows/day from idle accounts. `setQuotaCache` now skips the write when a window's `remaining_percentage`/`is_exhausted` is unchanged from the last cached observation; the first observation and every real change still persist. ([#4438](https://github.com/diegosouzapw/OmniRoute/issues/4438) — thanks @oyi77)
- **fix(combo): attribute lockout decay & success telemetry to the dynamically-selected connection** — on the combo success path the actual connection chosen by dynamic account-selection is now read from the `X-OmniRoute-Selected-Connection-Id` response header (instead of the often-empty static `target.connectionId`), so model-lockout decay, `recordProviderSuccess`, LKGP and success/failure telemetry attribute to the right connection on both the priority and round-robin paths. The pre-screen "unavailable" snapshot is also no longer a permanent skip — availability is re-checked on each retry since connection cooldowns can expire mid-request. ([#4550](https://github.com/diegosouzapw/OmniRoute/pull/4550) — thanks @Chewji9875)
### 📝 Maintenance

View File

@@ -130,7 +130,7 @@
"open-sse/services/batchProcessor.ts": 828,
"open-sse/services/browserBackedChat.ts": 850,
"open-sse/services/claudeCodeCompatible.ts": 1202,
"open-sse/services/combo.ts": 2955,
"open-sse/services/combo.ts": 2991,
"open-sse/services/rateLimitManager.ts": 1035,
"open-sse/services/tokenRefresh.ts": 1997,
"open-sse/services/usage.ts": 3450,
@@ -205,7 +205,7 @@
"src/shared/constants/sidebarVisibility.ts": 1100,
"src/shared/services/cliRuntime.ts": 1090,
"src/shared/validation/schemas.ts": 2523,
"src/sse/handlers/chat.ts": 1515,
"src/sse/handlers/chat.ts": 1516,
"src/sse/services/auth.ts": 2279
},
"testCap": 800,

View File

@@ -1501,18 +1501,11 @@ export async function handleComboChat({
return null;
}
// Pre-screen may have already determined this target unavailable (e.g.
// circuit-breaker OPEN at resolve time). Skip immediately in that case.
// For targets pre-screened as "available" we still call isModelAvailable
// below because connection cooldowns (rateLimitedUntil) can change
// mid-request after a same-provider failure — the pre-screen snapshot is
// stale by the time we reach the 2nd/3rd same-provider target.
const preCheckedAvailable = preScreenEntry?.available ?? null;
if (preCheckedAvailable === false) {
log.info("COMBO", `Skipping ${modelStr} — pre-screen marked unavailable`);
if (i > 0) fallbackCount++;
return null;
}
// Pre-screen snapshot is NOT used as a permanent skip — availability
// is always re-checked via isModelAvailable below because connection
// cooldowns can expire between setTry retries, making a previously
// unavailable target available again. Circuit-breaker-OPEN providers
// are already caught by the dedicated breaker check above.
if (isModelAvailable) {
const available = await isModelAvailable(modelStr, targetForAttempt);
if (!available) {
@@ -1690,6 +1683,12 @@ export async function handleComboChat({
// Success — validate response quality before returning
if (result.ok) {
const selectedConnectionId =
result.headers?.get("X-OmniRoute-Selected-Connection-Id") ||
result.headers?.get("x-omniroute-selected-connection-id") ||
undefined;
const effectiveConnectionId = selectedConnectionId || target.connectionId || "";
const quality = await validateResponseQuality(result, clientRequestedStream, log);
if (!quality.valid) {
log.warn(
@@ -1743,11 +1742,7 @@ export async function handleComboChat({
// Success decay: a healthy response walks the model's lockout failure
// count back down (and eventually clears an expired lockout entirely).
if (provider && rawModel) {
const dcResult = decayModelFailureCount(
provider,
target.connectionId || "",
rawModel
);
const dcResult = decayModelFailureCount(provider, effectiveConnectionId, rawModel);
if (dcResult.cleared) {
log.info("COMBO", `Model ${modelStr} fully recovered — lockout cleared`);
} else if (dcResult.newFailureCount > 0) {
@@ -1781,7 +1776,7 @@ export async function handleComboChat({
// Reset cooldown on success
if (provider && provider !== "unknown") {
recordProviderSuccess(provider, target.connectionId ?? undefined);
recordProviderSuccess(provider, effectiveConnectionId || undefined);
}
if (strategy === "weighted" && stickyWeightedLimit > 1) {
const stickySuccessKey = getWeightedStepKeyForTarget(target);
@@ -1798,7 +1793,7 @@ export async function handleComboChat({
typeof target.label === "string" && target.label.trim().length > 0
? target.label.trim()
: "",
accountId: target.connectionId ?? "",
accountId: effectiveConnectionId ?? "",
latencyMs,
fallbackCount,
});
@@ -1909,7 +1904,7 @@ export async function handleComboChat({
// Record last known good provider (LKGP) for this combo/model (#919)
if (provider) {
const connId = target.connectionId || undefined;
const connId = effectiveConnectionId || undefined;
void (async () => {
try {
const { setLKGP } = await import("../../src/lib/localDb");
@@ -2035,10 +2030,17 @@ export async function handleComboChat({
structuredError
);
const { cooldownMs } = fallbackResult;
const selectedConnectionId =
result.headers?.get("X-OmniRoute-Selected-Connection-Id") ||
result.headers?.get("x-omniroute-selected-connection-id") ||
undefined;
const targetWithConnection = selectedConnectionId
? { ...target, connectionId: selectedConnectionId }
: target;
// #1731 / #1731v2: classify the upstream error and update the exhaustion sets
// (shared with handleRoundRobinCombo). Returns whether the provider is fully exhausted.
const providerExhausted = applyComboTargetExhaustion(target, {
const providerExhausted = applyComboTargetExhaustion(targetWithConnection, {
result,
fallbackResult,
errorText,
@@ -2114,7 +2116,7 @@ export async function handleComboChat({
skipProviderBreaker: fallbackResult.skipProviderBreaker,
})
) {
recordProviderFailure(provider, log, target.connectionId, profile);
recordProviderFailure(provider, log, targetWithConnection.connectionId, profile);
}
// Check if this is a transient error worth retrying on same model.
@@ -2133,7 +2135,7 @@ export async function handleComboChat({
if (mlSettings.enabled && mlSettings.errorCodes.includes(result.status)) {
recordModelLockoutFailure(
provider,
target.connectionId || "",
targetWithConnection.connectionId || "",
rawModel,
classifyLockoutReason(result.status),
result.status,
@@ -2176,7 +2178,7 @@ export async function handleComboChat({
if (mlSettings.enabled && mlSettings.errorCodes.includes(result.status)) {
recordModelLockoutFailure(
provider,
target.connectionId || "",
targetWithConnection.connectionId || "",
rawModel,
classifyLockoutReason(result.status),
result.status,
@@ -2193,7 +2195,11 @@ export async function handleComboChat({
log.warn("COMBO", `Model ${modelStr} failed, trying next`, { status: result.status });
if (resilienceSettings.providerCooldown.enabled && provider && provider !== "unknown") {
recordProviderCooldown(provider, target.connectionId ?? undefined, resilienceSettings);
recordProviderCooldown(
provider,
targetWithConnection.connectionId ?? undefined,
resilienceSettings
);
}
const fallbackWaitMs =
@@ -2676,8 +2682,27 @@ async function handleRoundRobinCombo({
});
recordedAttempts++;
const selectedConnectionId =
result.headers?.get("X-OmniRoute-Selected-Connection-Id") ||
result.headers?.get("x-omniroute-selected-connection-id") ||
undefined;
const effectiveConnectionId = selectedConnectionId || target.connectionId || "";
const rawModel = parseModel(modelStr).model || modelStr;
if (provider && rawModel) {
const dcResult = decayModelFailureCount(provider, effectiveConnectionId, rawModel);
if (dcResult.cleared) {
log.info("COMBO-RR", `Model ${modelStr} fully recovered — lockout cleared`);
} else if (dcResult.newFailureCount > 0) {
log.debug?.(
"COMBO-RR",
`Model ${modelStr} decayed to failureCount=${dcResult.newFailureCount}`
);
}
}
if (provider && provider !== "unknown") {
recordProviderSuccess(provider, target.connectionId ?? undefined);
recordProviderSuccess(provider, effectiveConnectionId || undefined);
}
if (stickyRoundRobinEnabled) {
@@ -2685,7 +2710,7 @@ async function handleRoundRobinCombo({
}
if (provider) {
const connId = target.connectionId || undefined;
const connId = effectiveConnectionId || undefined;
void (async () => {
try {
const { setLKGP } = await import("../../src/lib/localDb");
@@ -2810,6 +2835,13 @@ async function handleRoundRobinCombo({
structuredError
);
const { cooldownMs } = fallbackResult;
const selectedConnectionId =
result.headers?.get("X-OmniRoute-Selected-Connection-Id") ||
result.headers?.get("x-omniroute-selected-connection-id") ||
undefined;
const targetWithConnection = selectedConnectionId
? { ...target, connectionId: selectedConnectionId }
: target;
const isAllAccountsRateLimited = isAllAccountsRateLimitedResponse(
result.status,
@@ -2823,7 +2855,7 @@ async function handleRoundRobinCombo({
// combo from trying another target for the same provider in this request.
// #1731 / #1731v2: classify the upstream error and update the exhaustion sets
// (shared with handleComboChat). Returns whether the provider is fully exhausted.
const providerExhausted = applyComboTargetExhaustion(target, {
const providerExhausted = applyComboTargetExhaustion(targetWithConnection, {
result,
fallbackResult,
errorText,
@@ -2879,7 +2911,11 @@ async function handleRoundRobinCombo({
log.warn("COMBO-RR", `${modelStr} failed, trying next model`, { status: result.status });
if (resilienceSettings.providerCooldown.enabled && provider && provider !== "unknown") {
recordProviderCooldown(provider, target.connectionId ?? undefined, resilienceSettings);
recordProviderCooldown(
provider,
targetWithConnection.connectionId ?? undefined,
resilienceSettings
);
}
const fallbackWaitMs =

View File

@@ -66,6 +66,7 @@ import {
safeLogEvents,
shouldRetryStreamEarlyEof,
withSessionHeader,
withSelectedConnectionHeader,
} from "./chatHelpers";
import { connectionHasExtraKeys } from "@omniroute/open-sse/services/apiKeyRotator.ts";
@@ -1219,7 +1220,7 @@ async function handleSingleModelChat(
// Stream readiness timeout is an upstream stall after an HTTP response was received,
// not an account/quota failure. Do NOT mark the account unavailable here.
return result.response;
return withSelectedConnectionHeader(result.response, credentials?.connectionId);
}
if (isAntigravityStreamReadinessFailure) {
@@ -1263,7 +1264,7 @@ async function handleSingleModelChat(
continue;
}
return result.response;
return withSelectedConnectionHeader(result.response, credentials?.connectionId);
}
const isAntigravityPreResponseTimeout =
@@ -1313,14 +1314,14 @@ async function handleSingleModelChat(
continue;
}
return result.response;
return withSelectedConnectionHeader(result.response, credentials?.connectionId);
}
if (result.errorType === "account_semaphore_capacity") {
// Local concurrency pressure is not an upstream quota failure. Prefer another
// account when possible; pinned combo steps fall through to combo orchestration.
if (hasForcedConnection) {
return result.response;
return withSelectedConnectionHeader(result.response, credentials?.connectionId);
}
log.warn(
@@ -1508,7 +1509,7 @@ async function handleSingleModelChat(
breaker._onFailure();
}
return result.response;
return withSelectedConnectionHeader(result.response, credentials?.connectionId);
}
}
}

View File

@@ -758,3 +758,23 @@ export function withSessionHeader(response: Response, sessionId: string | null):
return cloned;
}
}
export function withSelectedConnectionHeader(
response: Response,
connectionId: string | null | undefined
): Response {
if (!response || !connectionId) return response;
try {
response.headers.set("X-OmniRoute-Selected-Connection-Id", connectionId);
return response;
} catch {
const cloned = new Response(response.body, {
status: response.status,
statusText: response.statusText,
headers: response.headers,
});
cloned.headers.set("X-OmniRoute-Selected-Connection-Id", connectionId);
return cloned;
}
}

View File

@@ -0,0 +1,379 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import test, { describe, beforeEach } from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-combo-sel-conn-"));
const ORIGINAL_DATA_DIR = process.env.DATA_DIR;
process.env.DATA_DIR = TEST_DATA_DIR;
const { handleComboChat } = await import("../../open-sse/services/combo.ts");
const {
recordModelLockoutFailure,
getModelLockoutInfo,
clearAllModelLockouts,
decayModelFailureCount,
} = await import("../../open-sse/services/accountFallback.ts");
const { recordProviderCooldown, isProviderInCooldown, recordProviderSuccess, clearCooldownState } =
await import("../../open-sse/services/providerCooldownTracker.ts");
const { resolveResilienceSettings } = await import("../../src/lib/resilience/settings.ts");
const core = await import("../../src/lib/db/core.ts");
const settingsDb = await import("../../src/lib/db/settings.ts");
const providersDb = await import("../../src/lib/db/providers.ts");
const settings = resolveResilienceSettings({
resilienceSettings: {
providerCooldown: {
enabled: true,
minRetryCooldownMs: 5000,
maxRetryCooldownMs: 300000,
},
},
});
function createLog() {
return {
info: (tag: any, msg: any) => console.log(`[INFO][${tag}] ${msg}`),
warn: (tag: any, msg: any) => console.log(`[WARN][${tag}] ${msg}`),
error: (tag: any, msg: any) => console.log(`[ERROR][${tag}] ${msg}`),
debug: (tag: any, msg: any) => console.log(`[DEBUG][${tag}] ${msg}`),
};
}
async function cleanupTestDataDir() {
let lastError;
for (let attempt = 0; attempt < 5; attempt += 1) {
try {
core.resetDbInstance();
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
return;
} catch (error: any) {
lastError = error;
await new Promise((resolve) => setTimeout(resolve, 25));
}
}
if (lastError) {
throw lastError;
}
}
test.after(async () => {
await cleanupTestDataDir();
process.env.DATA_DIR = ORIGINAL_DATA_DIR;
});
beforeEach(async () => {
clearAllModelLockouts();
clearCooldownState();
settingsDb.clearAllLKGP();
});
describe("combo selected connection success handling", () => {
test("priority strategy correctly extracts dynamic connection ID from success response headers and decays lockout, resets provider cooldown, and updates LKGP", async () => {
const comboName = "test-combo-priority";
const modelStr = "openai/gpt-4";
const provider = "openai";
const dynamicConnId = "conn-dynamic-123";
await providersDb.createProviderConnection({
provider,
authType: "apikey",
name: "OpenAI Test",
apiKey: "sk-test",
});
// 1. Populate lockout failure count = 4
recordModelLockoutFailure(
provider,
dynamicConnId,
"gpt-4",
"rate_limit_exceeded",
429,
120_000,
null,
{ exactCooldownMs: 60_000 }
);
for (let i = 0; i < 3; i++) {
recordModelLockoutFailure(
provider,
dynamicConnId,
"gpt-4",
"rate_limit_exceeded",
429,
120_000,
null,
{ exactCooldownMs: 60_000 }
);
}
// Verify initial failureCount is 4
const initialLockout = getModelLockoutInfo(provider, dynamicConnId, "gpt-4");
assert.equal(initialLockout?.failureCount, 4);
// 2. Record provider cooldown
recordProviderCooldown(provider, dynamicConnId, settings);
assert.ok(isProviderInCooldown(provider, dynamicConnId, settings));
// 3. Invoke handleComboChat
const result = await handleComboChat({
body: { stream: false },
combo: {
name: comboName,
strategy: "priority",
models: [modelStr],
config: { maxRetries: 0, concurrencyPerModel: 1, queueTimeoutMs: 1000 },
},
handleSingleModel: async () => {
return new Response(JSON.stringify({ choices: [{ message: { content: "ok" } }] }), {
status: 200,
headers: {
"content-type": "application/json",
"X-OmniRoute-Selected-Connection-Id": dynamicConnId,
},
});
},
isModelAvailable: async () => true,
log: createLog(),
settings: null,
allCombos: null,
});
assert.equal(result.ok, true);
// 4. Assertions
// A. Dynamic connection-level failure count decay (halved to 2)
const decayCheck = decayModelFailureCount(provider, dynamicConnId, "gpt-4");
assert.equal(
decayCheck.newFailureCount,
1,
"failure count should decay from 4 to 2, and now to 1"
);
// B. Dynamic connection-level provider success tracking (not in cooldown anymore)
assert.equal(
isProviderInCooldown(provider, dynamicConnId, settings),
false,
"provider cooldown should be cleared on success"
);
// C. Correct LKGP record updated with the dynamic connection ID (setLKGP is called)
let persisted: any = null;
for (let i = 0; i < 20; i++) {
persisted = await settingsDb.getLKGP(comboName, comboName);
if (persisted?.connectionId === dynamicConnId) {
break;
}
await new Promise((resolve) => setTimeout(resolve, 10));
}
assert.equal(persisted?.provider, provider);
assert.equal(
persisted?.connectionId,
dynamicConnId,
"LKGP connectionId must be the dynamic connection ID"
);
});
test("priority strategy with lowercase selected connection ID header correctly decays lockout, resets provider cooldown, and updates LKGP", async () => {
const comboName = "test-combo-priority-lc";
const modelStr = "openai/gpt-4";
const provider = "openai";
const dynamicConnId = "conn-dynamic-123-lc";
await providersDb.createProviderConnection({
provider,
authType: "apikey",
name: "OpenAI Test LC",
apiKey: "sk-test",
});
// 1. Populate lockout failure count = 4
recordModelLockoutFailure(
provider,
dynamicConnId,
"gpt-4",
"rate_limit_exceeded",
429,
120_000,
null,
{ exactCooldownMs: 60_000 }
);
for (let i = 0; i < 3; i++) {
recordModelLockoutFailure(
provider,
dynamicConnId,
"gpt-4",
"rate_limit_exceeded",
429,
120_000,
null,
{ exactCooldownMs: 60_000 }
);
}
// Verify initial failureCount is 4
const initialLockout = getModelLockoutInfo(provider, dynamicConnId, "gpt-4");
assert.equal(initialLockout?.failureCount, 4);
// 2. Record provider cooldown
recordProviderCooldown(provider, dynamicConnId, settings);
assert.ok(isProviderInCooldown(provider, dynamicConnId, settings));
// 3. Invoke handleComboChat
const result = await handleComboChat({
body: { stream: false },
combo: {
name: comboName,
strategy: "priority",
models: [modelStr],
config: { maxRetries: 0, concurrencyPerModel: 1, queueTimeoutMs: 1000 },
},
handleSingleModel: async () => {
return new Response(JSON.stringify({ choices: [{ message: { content: "ok" } }] }), {
status: 200,
headers: {
"content-type": "application/json",
"x-omniroute-selected-connection-id": dynamicConnId,
},
});
},
isModelAvailable: async () => true,
log: createLog(),
settings: null,
allCombos: null,
});
assert.equal(result.ok, true);
// 4. Assertions
const decayCheck = decayModelFailureCount(provider, dynamicConnId, "gpt-4");
assert.equal(
decayCheck.newFailureCount,
1,
"failure count should decay from 4 to 2, and now to 1"
);
assert.equal(
isProviderInCooldown(provider, dynamicConnId, settings),
false,
"provider cooldown should be cleared on success"
);
let persisted: any = null;
for (let i = 0; i < 20; i++) {
persisted = await settingsDb.getLKGP(comboName, comboName);
if (persisted?.connectionId === dynamicConnId) {
break;
}
await new Promise((resolve) => setTimeout(resolve, 10));
}
assert.equal(persisted?.provider, provider);
assert.equal(
persisted?.connectionId,
dynamicConnId,
"LKGP connectionId must be the dynamic connection ID"
);
});
test("round-robin strategy correctly extracts dynamic connection ID from success response headers and decays lockout, resets provider cooldown, and updates LKGP", async () => {
const comboName = "test-combo-rr";
const modelStr = "openai/gpt-4";
const provider = "openai";
const dynamicConnId = "conn-dynamic-123-rr";
await providersDb.createProviderConnection({
provider,
authType: "apikey",
name: "OpenAI Test RR",
apiKey: "sk-test",
});
// 1. Populate lockout failure count = 4
recordModelLockoutFailure(
provider,
dynamicConnId,
"gpt-4",
"rate_limit_exceeded",
429,
120_000,
null,
{ exactCooldownMs: 60_000 }
);
for (let i = 0; i < 3; i++) {
recordModelLockoutFailure(
provider,
dynamicConnId,
"gpt-4",
"rate_limit_exceeded",
429,
120_000,
null,
{ exactCooldownMs: 60_000 }
);
}
// Verify initial failureCount is 4
const initialLockout = getModelLockoutInfo(provider, dynamicConnId, "gpt-4");
assert.equal(initialLockout?.failureCount, 4);
// 2. Record provider cooldown
recordProviderCooldown(provider, dynamicConnId, settings);
assert.ok(isProviderInCooldown(provider, dynamicConnId, settings));
// 3. Invoke handleComboChat with round-robin strategy
const result = await handleComboChat({
body: { stream: false },
combo: {
name: comboName,
strategy: "round-robin",
models: [modelStr],
config: { maxRetries: 0, concurrencyPerModel: 1, queueTimeoutMs: 1000 },
},
handleSingleModel: async () => {
return new Response(JSON.stringify({ choices: [{ message: { content: "ok" } }] }), {
status: 200,
headers: {
"content-type": "application/json",
"X-OmniRoute-Selected-Connection-Id": dynamicConnId,
},
});
},
isModelAvailable: async () => true,
log: createLog(),
settings: null,
allCombos: null,
});
assert.equal(result.ok, true);
// 4. Assertions
const decayCheck = decayModelFailureCount(provider, dynamicConnId, "gpt-4");
assert.equal(
decayCheck.newFailureCount,
1,
"failure count should decay from 4 to 2, and now to 1"
);
assert.equal(
isProviderInCooldown(provider, dynamicConnId, settings),
false,
"provider cooldown should be cleared on success"
);
let persisted: any = null;
for (let i = 0; i < 20; i++) {
persisted = await settingsDb.getLKGP(comboName, comboName);
if (persisted?.connectionId === dynamicConnId) {
break;
}
await new Promise((resolve) => setTimeout(resolve, 10));
}
assert.equal(persisted?.provider, provider);
assert.equal(
persisted?.connectionId,
dynamicConnId,
"LKGP connectionId must be the dynamic connection ID"
);
});
});