mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-06 07:12:12 +03:00
fix(combo): extract dynamic connection ID from headers on success path for precise lockout decay and success telemetry (#4550) (#4581)
Integrated into release/v3.8.33 (cherry-pick of #4550 with stale reverts dropped; @Chewji9875 credited in CHANGELOG)
This commit is contained in:
committed by
GitHub
parent
09d8a4aa8c
commit
7ef34c2246
@@ -18,6 +18,7 @@ _In development — bullets added per PR; finalized at release._
|
||||
- **fix(antigravity): reasoning/thinking models no longer 400 with `oneOf at '/' not met`** — the Cloud Code envelope passthrough also leaked the Claude/OpenAI-native thinking fields (`thinking`, `reasoning_effort`, `reasoning`, `enable_thinking`, `thinking_budget`) the unified thinking adapter sets at the body root; Google rejected them with `400 Bad input: oneOf at '/' not met`. The whole thinking family is now stripped before the envelope is built; Gemini's own `generationConfig.thinkingConfig` is unaffected. (port from 9router#1926 — thanks @theseven99 / @diegosouzapw)
|
||||
- **fix(integration): restore the codex and memory pipeline contracts** — realigns the CLI fingerprint + memory-tools contracts so the codex and memory pipelines pass their integration checks again. ([#4474](https://github.com/diegosouzapw/OmniRoute/pull/4474) — thanks @KooshaPari)
|
||||
- **perf(quota): stop writing redundant `quota_snapshots` rows from idle connections** — the 60s background refresh persisted a snapshot for every window of every connection regardless of change, generating 400K+ rows/day from idle accounts. `setQuotaCache` now skips the write when a window's `remaining_percentage`/`is_exhausted` is unchanged from the last cached observation; the first observation and every real change still persist. ([#4438](https://github.com/diegosouzapw/OmniRoute/issues/4438) — thanks @oyi77)
|
||||
- **fix(combo): attribute lockout decay & success telemetry to the dynamically-selected connection** — on the combo success path the actual connection chosen by dynamic account-selection is now read from the `X-OmniRoute-Selected-Connection-Id` response header (instead of the often-empty static `target.connectionId`), so model-lockout decay, `recordProviderSuccess`, LKGP and success/failure telemetry attribute to the right connection on both the priority and round-robin paths. The pre-screen "unavailable" snapshot is also no longer a permanent skip — availability is re-checked on each retry since connection cooldowns can expire mid-request. ([#4550](https://github.com/diegosouzapw/OmniRoute/pull/4550) — thanks @Chewji9875)
|
||||
|
||||
### 📝 Maintenance
|
||||
|
||||
|
||||
@@ -130,7 +130,7 @@
|
||||
"open-sse/services/batchProcessor.ts": 828,
|
||||
"open-sse/services/browserBackedChat.ts": 850,
|
||||
"open-sse/services/claudeCodeCompatible.ts": 1202,
|
||||
"open-sse/services/combo.ts": 2955,
|
||||
"open-sse/services/combo.ts": 2991,
|
||||
"open-sse/services/rateLimitManager.ts": 1035,
|
||||
"open-sse/services/tokenRefresh.ts": 1997,
|
||||
"open-sse/services/usage.ts": 3450,
|
||||
@@ -205,7 +205,7 @@
|
||||
"src/shared/constants/sidebarVisibility.ts": 1100,
|
||||
"src/shared/services/cliRuntime.ts": 1090,
|
||||
"src/shared/validation/schemas.ts": 2523,
|
||||
"src/sse/handlers/chat.ts": 1515,
|
||||
"src/sse/handlers/chat.ts": 1516,
|
||||
"src/sse/services/auth.ts": 2279
|
||||
},
|
||||
"testCap": 800,
|
||||
|
||||
@@ -1501,18 +1501,11 @@ export async function handleComboChat({
|
||||
return null;
|
||||
}
|
||||
|
||||
// Pre-screen may have already determined this target unavailable (e.g.
|
||||
// circuit-breaker OPEN at resolve time). Skip immediately in that case.
|
||||
// For targets pre-screened as "available" we still call isModelAvailable
|
||||
// below because connection cooldowns (rateLimitedUntil) can change
|
||||
// mid-request after a same-provider failure — the pre-screen snapshot is
|
||||
// stale by the time we reach the 2nd/3rd same-provider target.
|
||||
const preCheckedAvailable = preScreenEntry?.available ?? null;
|
||||
if (preCheckedAvailable === false) {
|
||||
log.info("COMBO", `Skipping ${modelStr} — pre-screen marked unavailable`);
|
||||
if (i > 0) fallbackCount++;
|
||||
return null;
|
||||
}
|
||||
// Pre-screen snapshot is NOT used as a permanent skip — availability
|
||||
// is always re-checked via isModelAvailable below because connection
|
||||
// cooldowns can expire between setTry retries, making a previously
|
||||
// unavailable target available again. Circuit-breaker-OPEN providers
|
||||
// are already caught by the dedicated breaker check above.
|
||||
if (isModelAvailable) {
|
||||
const available = await isModelAvailable(modelStr, targetForAttempt);
|
||||
if (!available) {
|
||||
@@ -1690,6 +1683,12 @@ export async function handleComboChat({
|
||||
|
||||
// Success — validate response quality before returning
|
||||
if (result.ok) {
|
||||
const selectedConnectionId =
|
||||
result.headers?.get("X-OmniRoute-Selected-Connection-Id") ||
|
||||
result.headers?.get("x-omniroute-selected-connection-id") ||
|
||||
undefined;
|
||||
const effectiveConnectionId = selectedConnectionId || target.connectionId || "";
|
||||
|
||||
const quality = await validateResponseQuality(result, clientRequestedStream, log);
|
||||
if (!quality.valid) {
|
||||
log.warn(
|
||||
@@ -1743,11 +1742,7 @@ export async function handleComboChat({
|
||||
// Success decay: a healthy response walks the model's lockout failure
|
||||
// count back down (and eventually clears an expired lockout entirely).
|
||||
if (provider && rawModel) {
|
||||
const dcResult = decayModelFailureCount(
|
||||
provider,
|
||||
target.connectionId || "",
|
||||
rawModel
|
||||
);
|
||||
const dcResult = decayModelFailureCount(provider, effectiveConnectionId, rawModel);
|
||||
if (dcResult.cleared) {
|
||||
log.info("COMBO", `Model ${modelStr} fully recovered — lockout cleared`);
|
||||
} else if (dcResult.newFailureCount > 0) {
|
||||
@@ -1781,7 +1776,7 @@ export async function handleComboChat({
|
||||
|
||||
// Reset cooldown on success
|
||||
if (provider && provider !== "unknown") {
|
||||
recordProviderSuccess(provider, target.connectionId ?? undefined);
|
||||
recordProviderSuccess(provider, effectiveConnectionId || undefined);
|
||||
}
|
||||
if (strategy === "weighted" && stickyWeightedLimit > 1) {
|
||||
const stickySuccessKey = getWeightedStepKeyForTarget(target);
|
||||
@@ -1798,7 +1793,7 @@ export async function handleComboChat({
|
||||
typeof target.label === "string" && target.label.trim().length > 0
|
||||
? target.label.trim()
|
||||
: "",
|
||||
accountId: target.connectionId ?? "",
|
||||
accountId: effectiveConnectionId ?? "",
|
||||
latencyMs,
|
||||
fallbackCount,
|
||||
});
|
||||
@@ -1909,7 +1904,7 @@ export async function handleComboChat({
|
||||
|
||||
// Record last known good provider (LKGP) for this combo/model (#919)
|
||||
if (provider) {
|
||||
const connId = target.connectionId || undefined;
|
||||
const connId = effectiveConnectionId || undefined;
|
||||
void (async () => {
|
||||
try {
|
||||
const { setLKGP } = await import("../../src/lib/localDb");
|
||||
@@ -2035,10 +2030,17 @@ export async function handleComboChat({
|
||||
structuredError
|
||||
);
|
||||
const { cooldownMs } = fallbackResult;
|
||||
const selectedConnectionId =
|
||||
result.headers?.get("X-OmniRoute-Selected-Connection-Id") ||
|
||||
result.headers?.get("x-omniroute-selected-connection-id") ||
|
||||
undefined;
|
||||
const targetWithConnection = selectedConnectionId
|
||||
? { ...target, connectionId: selectedConnectionId }
|
||||
: target;
|
||||
|
||||
// #1731 / #1731v2: classify the upstream error and update the exhaustion sets
|
||||
// (shared with handleRoundRobinCombo). Returns whether the provider is fully exhausted.
|
||||
const providerExhausted = applyComboTargetExhaustion(target, {
|
||||
const providerExhausted = applyComboTargetExhaustion(targetWithConnection, {
|
||||
result,
|
||||
fallbackResult,
|
||||
errorText,
|
||||
@@ -2114,7 +2116,7 @@ export async function handleComboChat({
|
||||
skipProviderBreaker: fallbackResult.skipProviderBreaker,
|
||||
})
|
||||
) {
|
||||
recordProviderFailure(provider, log, target.connectionId, profile);
|
||||
recordProviderFailure(provider, log, targetWithConnection.connectionId, profile);
|
||||
}
|
||||
|
||||
// Check if this is a transient error worth retrying on same model.
|
||||
@@ -2133,7 +2135,7 @@ export async function handleComboChat({
|
||||
if (mlSettings.enabled && mlSettings.errorCodes.includes(result.status)) {
|
||||
recordModelLockoutFailure(
|
||||
provider,
|
||||
target.connectionId || "",
|
||||
targetWithConnection.connectionId || "",
|
||||
rawModel,
|
||||
classifyLockoutReason(result.status),
|
||||
result.status,
|
||||
@@ -2176,7 +2178,7 @@ export async function handleComboChat({
|
||||
if (mlSettings.enabled && mlSettings.errorCodes.includes(result.status)) {
|
||||
recordModelLockoutFailure(
|
||||
provider,
|
||||
target.connectionId || "",
|
||||
targetWithConnection.connectionId || "",
|
||||
rawModel,
|
||||
classifyLockoutReason(result.status),
|
||||
result.status,
|
||||
@@ -2193,7 +2195,11 @@ export async function handleComboChat({
|
||||
log.warn("COMBO", `Model ${modelStr} failed, trying next`, { status: result.status });
|
||||
|
||||
if (resilienceSettings.providerCooldown.enabled && provider && provider !== "unknown") {
|
||||
recordProviderCooldown(provider, target.connectionId ?? undefined, resilienceSettings);
|
||||
recordProviderCooldown(
|
||||
provider,
|
||||
targetWithConnection.connectionId ?? undefined,
|
||||
resilienceSettings
|
||||
);
|
||||
}
|
||||
|
||||
const fallbackWaitMs =
|
||||
@@ -2676,8 +2682,27 @@ async function handleRoundRobinCombo({
|
||||
});
|
||||
recordedAttempts++;
|
||||
|
||||
const selectedConnectionId =
|
||||
result.headers?.get("X-OmniRoute-Selected-Connection-Id") ||
|
||||
result.headers?.get("x-omniroute-selected-connection-id") ||
|
||||
undefined;
|
||||
const effectiveConnectionId = selectedConnectionId || target.connectionId || "";
|
||||
|
||||
const rawModel = parseModel(modelStr).model || modelStr;
|
||||
if (provider && rawModel) {
|
||||
const dcResult = decayModelFailureCount(provider, effectiveConnectionId, rawModel);
|
||||
if (dcResult.cleared) {
|
||||
log.info("COMBO-RR", `Model ${modelStr} fully recovered — lockout cleared`);
|
||||
} else if (dcResult.newFailureCount > 0) {
|
||||
log.debug?.(
|
||||
"COMBO-RR",
|
||||
`Model ${modelStr} decayed to failureCount=${dcResult.newFailureCount}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (provider && provider !== "unknown") {
|
||||
recordProviderSuccess(provider, target.connectionId ?? undefined);
|
||||
recordProviderSuccess(provider, effectiveConnectionId || undefined);
|
||||
}
|
||||
|
||||
if (stickyRoundRobinEnabled) {
|
||||
@@ -2685,7 +2710,7 @@ async function handleRoundRobinCombo({
|
||||
}
|
||||
|
||||
if (provider) {
|
||||
const connId = target.connectionId || undefined;
|
||||
const connId = effectiveConnectionId || undefined;
|
||||
void (async () => {
|
||||
try {
|
||||
const { setLKGP } = await import("../../src/lib/localDb");
|
||||
@@ -2810,6 +2835,13 @@ async function handleRoundRobinCombo({
|
||||
structuredError
|
||||
);
|
||||
const { cooldownMs } = fallbackResult;
|
||||
const selectedConnectionId =
|
||||
result.headers?.get("X-OmniRoute-Selected-Connection-Id") ||
|
||||
result.headers?.get("x-omniroute-selected-connection-id") ||
|
||||
undefined;
|
||||
const targetWithConnection = selectedConnectionId
|
||||
? { ...target, connectionId: selectedConnectionId }
|
||||
: target;
|
||||
|
||||
const isAllAccountsRateLimited = isAllAccountsRateLimitedResponse(
|
||||
result.status,
|
||||
@@ -2823,7 +2855,7 @@ async function handleRoundRobinCombo({
|
||||
// combo from trying another target for the same provider in this request.
|
||||
// #1731 / #1731v2: classify the upstream error and update the exhaustion sets
|
||||
// (shared with handleComboChat). Returns whether the provider is fully exhausted.
|
||||
const providerExhausted = applyComboTargetExhaustion(target, {
|
||||
const providerExhausted = applyComboTargetExhaustion(targetWithConnection, {
|
||||
result,
|
||||
fallbackResult,
|
||||
errorText,
|
||||
@@ -2879,7 +2911,11 @@ async function handleRoundRobinCombo({
|
||||
log.warn("COMBO-RR", `${modelStr} failed, trying next model`, { status: result.status });
|
||||
|
||||
if (resilienceSettings.providerCooldown.enabled && provider && provider !== "unknown") {
|
||||
recordProviderCooldown(provider, target.connectionId ?? undefined, resilienceSettings);
|
||||
recordProviderCooldown(
|
||||
provider,
|
||||
targetWithConnection.connectionId ?? undefined,
|
||||
resilienceSettings
|
||||
);
|
||||
}
|
||||
|
||||
const fallbackWaitMs =
|
||||
|
||||
@@ -66,6 +66,7 @@ import {
|
||||
safeLogEvents,
|
||||
shouldRetryStreamEarlyEof,
|
||||
withSessionHeader,
|
||||
withSelectedConnectionHeader,
|
||||
} from "./chatHelpers";
|
||||
import { connectionHasExtraKeys } from "@omniroute/open-sse/services/apiKeyRotator.ts";
|
||||
|
||||
@@ -1219,7 +1220,7 @@ async function handleSingleModelChat(
|
||||
|
||||
// Stream readiness timeout is an upstream stall after an HTTP response was received,
|
||||
// not an account/quota failure. Do NOT mark the account unavailable here.
|
||||
return result.response;
|
||||
return withSelectedConnectionHeader(result.response, credentials?.connectionId);
|
||||
}
|
||||
|
||||
if (isAntigravityStreamReadinessFailure) {
|
||||
@@ -1263,7 +1264,7 @@ async function handleSingleModelChat(
|
||||
continue;
|
||||
}
|
||||
|
||||
return result.response;
|
||||
return withSelectedConnectionHeader(result.response, credentials?.connectionId);
|
||||
}
|
||||
|
||||
const isAntigravityPreResponseTimeout =
|
||||
@@ -1313,14 +1314,14 @@ async function handleSingleModelChat(
|
||||
continue;
|
||||
}
|
||||
|
||||
return result.response;
|
||||
return withSelectedConnectionHeader(result.response, credentials?.connectionId);
|
||||
}
|
||||
|
||||
if (result.errorType === "account_semaphore_capacity") {
|
||||
// Local concurrency pressure is not an upstream quota failure. Prefer another
|
||||
// account when possible; pinned combo steps fall through to combo orchestration.
|
||||
if (hasForcedConnection) {
|
||||
return result.response;
|
||||
return withSelectedConnectionHeader(result.response, credentials?.connectionId);
|
||||
}
|
||||
|
||||
log.warn(
|
||||
@@ -1508,7 +1509,7 @@ async function handleSingleModelChat(
|
||||
breaker._onFailure();
|
||||
}
|
||||
|
||||
return result.response;
|
||||
return withSelectedConnectionHeader(result.response, credentials?.connectionId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -758,3 +758,23 @@ export function withSessionHeader(response: Response, sessionId: string | null):
|
||||
return cloned;
|
||||
}
|
||||
}
|
||||
|
||||
export function withSelectedConnectionHeader(
|
||||
response: Response,
|
||||
connectionId: string | null | undefined
|
||||
): Response {
|
||||
if (!response || !connectionId) return response;
|
||||
|
||||
try {
|
||||
response.headers.set("X-OmniRoute-Selected-Connection-Id", connectionId);
|
||||
return response;
|
||||
} catch {
|
||||
const cloned = new Response(response.body, {
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
headers: response.headers,
|
||||
});
|
||||
cloned.headers.set("X-OmniRoute-Selected-Connection-Id", connectionId);
|
||||
return cloned;
|
||||
}
|
||||
}
|
||||
|
||||
379
tests/unit/combo-selected-connection-success.test.ts
Normal file
379
tests/unit/combo-selected-connection-success.test.ts
Normal file
@@ -0,0 +1,379 @@
|
||||
/* eslint-disable @typescript-eslint/no-explicit-any */
|
||||
import test, { describe, beforeEach } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-combo-sel-conn-"));
|
||||
const ORIGINAL_DATA_DIR = process.env.DATA_DIR;
|
||||
process.env.DATA_DIR = TEST_DATA_DIR;
|
||||
|
||||
const { handleComboChat } = await import("../../open-sse/services/combo.ts");
|
||||
const {
|
||||
recordModelLockoutFailure,
|
||||
getModelLockoutInfo,
|
||||
clearAllModelLockouts,
|
||||
decayModelFailureCount,
|
||||
} = await import("../../open-sse/services/accountFallback.ts");
|
||||
const { recordProviderCooldown, isProviderInCooldown, recordProviderSuccess, clearCooldownState } =
|
||||
await import("../../open-sse/services/providerCooldownTracker.ts");
|
||||
const { resolveResilienceSettings } = await import("../../src/lib/resilience/settings.ts");
|
||||
const core = await import("../../src/lib/db/core.ts");
|
||||
const settingsDb = await import("../../src/lib/db/settings.ts");
|
||||
const providersDb = await import("../../src/lib/db/providers.ts");
|
||||
|
||||
const settings = resolveResilienceSettings({
|
||||
resilienceSettings: {
|
||||
providerCooldown: {
|
||||
enabled: true,
|
||||
minRetryCooldownMs: 5000,
|
||||
maxRetryCooldownMs: 300000,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
function createLog() {
|
||||
return {
|
||||
info: (tag: any, msg: any) => console.log(`[INFO][${tag}] ${msg}`),
|
||||
warn: (tag: any, msg: any) => console.log(`[WARN][${tag}] ${msg}`),
|
||||
error: (tag: any, msg: any) => console.log(`[ERROR][${tag}] ${msg}`),
|
||||
debug: (tag: any, msg: any) => console.log(`[DEBUG][${tag}] ${msg}`),
|
||||
};
|
||||
}
|
||||
|
||||
async function cleanupTestDataDir() {
|
||||
let lastError;
|
||||
for (let attempt = 0; attempt < 5; attempt += 1) {
|
||||
try {
|
||||
core.resetDbInstance();
|
||||
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
|
||||
return;
|
||||
} catch (error: any) {
|
||||
lastError = error;
|
||||
await new Promise((resolve) => setTimeout(resolve, 25));
|
||||
}
|
||||
}
|
||||
if (lastError) {
|
||||
throw lastError;
|
||||
}
|
||||
}
|
||||
|
||||
test.after(async () => {
|
||||
await cleanupTestDataDir();
|
||||
process.env.DATA_DIR = ORIGINAL_DATA_DIR;
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
clearAllModelLockouts();
|
||||
clearCooldownState();
|
||||
settingsDb.clearAllLKGP();
|
||||
});
|
||||
|
||||
describe("combo selected connection success handling", () => {
|
||||
test("priority strategy correctly extracts dynamic connection ID from success response headers and decays lockout, resets provider cooldown, and updates LKGP", async () => {
|
||||
const comboName = "test-combo-priority";
|
||||
const modelStr = "openai/gpt-4";
|
||||
const provider = "openai";
|
||||
const dynamicConnId = "conn-dynamic-123";
|
||||
|
||||
await providersDb.createProviderConnection({
|
||||
provider,
|
||||
authType: "apikey",
|
||||
name: "OpenAI Test",
|
||||
apiKey: "sk-test",
|
||||
});
|
||||
|
||||
// 1. Populate lockout failure count = 4
|
||||
recordModelLockoutFailure(
|
||||
provider,
|
||||
dynamicConnId,
|
||||
"gpt-4",
|
||||
"rate_limit_exceeded",
|
||||
429,
|
||||
120_000,
|
||||
null,
|
||||
{ exactCooldownMs: 60_000 }
|
||||
);
|
||||
for (let i = 0; i < 3; i++) {
|
||||
recordModelLockoutFailure(
|
||||
provider,
|
||||
dynamicConnId,
|
||||
"gpt-4",
|
||||
"rate_limit_exceeded",
|
||||
429,
|
||||
120_000,
|
||||
null,
|
||||
{ exactCooldownMs: 60_000 }
|
||||
);
|
||||
}
|
||||
|
||||
// Verify initial failureCount is 4
|
||||
const initialLockout = getModelLockoutInfo(provider, dynamicConnId, "gpt-4");
|
||||
assert.equal(initialLockout?.failureCount, 4);
|
||||
|
||||
// 2. Record provider cooldown
|
||||
recordProviderCooldown(provider, dynamicConnId, settings);
|
||||
assert.ok(isProviderInCooldown(provider, dynamicConnId, settings));
|
||||
|
||||
// 3. Invoke handleComboChat
|
||||
const result = await handleComboChat({
|
||||
body: { stream: false },
|
||||
combo: {
|
||||
name: comboName,
|
||||
strategy: "priority",
|
||||
models: [modelStr],
|
||||
config: { maxRetries: 0, concurrencyPerModel: 1, queueTimeoutMs: 1000 },
|
||||
},
|
||||
handleSingleModel: async () => {
|
||||
return new Response(JSON.stringify({ choices: [{ message: { content: "ok" } }] }), {
|
||||
status: 200,
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
"X-OmniRoute-Selected-Connection-Id": dynamicConnId,
|
||||
},
|
||||
});
|
||||
},
|
||||
isModelAvailable: async () => true,
|
||||
log: createLog(),
|
||||
settings: null,
|
||||
allCombos: null,
|
||||
});
|
||||
|
||||
assert.equal(result.ok, true);
|
||||
|
||||
// 4. Assertions
|
||||
// A. Dynamic connection-level failure count decay (halved to 2)
|
||||
const decayCheck = decayModelFailureCount(provider, dynamicConnId, "gpt-4");
|
||||
assert.equal(
|
||||
decayCheck.newFailureCount,
|
||||
1,
|
||||
"failure count should decay from 4 to 2, and now to 1"
|
||||
);
|
||||
|
||||
// B. Dynamic connection-level provider success tracking (not in cooldown anymore)
|
||||
assert.equal(
|
||||
isProviderInCooldown(provider, dynamicConnId, settings),
|
||||
false,
|
||||
"provider cooldown should be cleared on success"
|
||||
);
|
||||
|
||||
// C. Correct LKGP record updated with the dynamic connection ID (setLKGP is called)
|
||||
let persisted: any = null;
|
||||
for (let i = 0; i < 20; i++) {
|
||||
persisted = await settingsDb.getLKGP(comboName, comboName);
|
||||
if (persisted?.connectionId === dynamicConnId) {
|
||||
break;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
}
|
||||
assert.equal(persisted?.provider, provider);
|
||||
assert.equal(
|
||||
persisted?.connectionId,
|
||||
dynamicConnId,
|
||||
"LKGP connectionId must be the dynamic connection ID"
|
||||
);
|
||||
});
|
||||
|
||||
test("priority strategy with lowercase selected connection ID header correctly decays lockout, resets provider cooldown, and updates LKGP", async () => {
|
||||
const comboName = "test-combo-priority-lc";
|
||||
const modelStr = "openai/gpt-4";
|
||||
const provider = "openai";
|
||||
const dynamicConnId = "conn-dynamic-123-lc";
|
||||
|
||||
await providersDb.createProviderConnection({
|
||||
provider,
|
||||
authType: "apikey",
|
||||
name: "OpenAI Test LC",
|
||||
apiKey: "sk-test",
|
||||
});
|
||||
|
||||
// 1. Populate lockout failure count = 4
|
||||
recordModelLockoutFailure(
|
||||
provider,
|
||||
dynamicConnId,
|
||||
"gpt-4",
|
||||
"rate_limit_exceeded",
|
||||
429,
|
||||
120_000,
|
||||
null,
|
||||
{ exactCooldownMs: 60_000 }
|
||||
);
|
||||
for (let i = 0; i < 3; i++) {
|
||||
recordModelLockoutFailure(
|
||||
provider,
|
||||
dynamicConnId,
|
||||
"gpt-4",
|
||||
"rate_limit_exceeded",
|
||||
429,
|
||||
120_000,
|
||||
null,
|
||||
{ exactCooldownMs: 60_000 }
|
||||
);
|
||||
}
|
||||
|
||||
// Verify initial failureCount is 4
|
||||
const initialLockout = getModelLockoutInfo(provider, dynamicConnId, "gpt-4");
|
||||
assert.equal(initialLockout?.failureCount, 4);
|
||||
|
||||
// 2. Record provider cooldown
|
||||
recordProviderCooldown(provider, dynamicConnId, settings);
|
||||
assert.ok(isProviderInCooldown(provider, dynamicConnId, settings));
|
||||
|
||||
// 3. Invoke handleComboChat
|
||||
const result = await handleComboChat({
|
||||
body: { stream: false },
|
||||
combo: {
|
||||
name: comboName,
|
||||
strategy: "priority",
|
||||
models: [modelStr],
|
||||
config: { maxRetries: 0, concurrencyPerModel: 1, queueTimeoutMs: 1000 },
|
||||
},
|
||||
handleSingleModel: async () => {
|
||||
return new Response(JSON.stringify({ choices: [{ message: { content: "ok" } }] }), {
|
||||
status: 200,
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
"x-omniroute-selected-connection-id": dynamicConnId,
|
||||
},
|
||||
});
|
||||
},
|
||||
isModelAvailable: async () => true,
|
||||
log: createLog(),
|
||||
settings: null,
|
||||
allCombos: null,
|
||||
});
|
||||
|
||||
assert.equal(result.ok, true);
|
||||
|
||||
// 4. Assertions
|
||||
const decayCheck = decayModelFailureCount(provider, dynamicConnId, "gpt-4");
|
||||
assert.equal(
|
||||
decayCheck.newFailureCount,
|
||||
1,
|
||||
"failure count should decay from 4 to 2, and now to 1"
|
||||
);
|
||||
|
||||
assert.equal(
|
||||
isProviderInCooldown(provider, dynamicConnId, settings),
|
||||
false,
|
||||
"provider cooldown should be cleared on success"
|
||||
);
|
||||
|
||||
let persisted: any = null;
|
||||
for (let i = 0; i < 20; i++) {
|
||||
persisted = await settingsDb.getLKGP(comboName, comboName);
|
||||
if (persisted?.connectionId === dynamicConnId) {
|
||||
break;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
}
|
||||
assert.equal(persisted?.provider, provider);
|
||||
assert.equal(
|
||||
persisted?.connectionId,
|
||||
dynamicConnId,
|
||||
"LKGP connectionId must be the dynamic connection ID"
|
||||
);
|
||||
});
|
||||
|
||||
test("round-robin strategy correctly extracts dynamic connection ID from success response headers and decays lockout, resets provider cooldown, and updates LKGP", async () => {
|
||||
const comboName = "test-combo-rr";
|
||||
const modelStr = "openai/gpt-4";
|
||||
const provider = "openai";
|
||||
const dynamicConnId = "conn-dynamic-123-rr";
|
||||
|
||||
await providersDb.createProviderConnection({
|
||||
provider,
|
||||
authType: "apikey",
|
||||
name: "OpenAI Test RR",
|
||||
apiKey: "sk-test",
|
||||
});
|
||||
|
||||
// 1. Populate lockout failure count = 4
|
||||
recordModelLockoutFailure(
|
||||
provider,
|
||||
dynamicConnId,
|
||||
"gpt-4",
|
||||
"rate_limit_exceeded",
|
||||
429,
|
||||
120_000,
|
||||
null,
|
||||
{ exactCooldownMs: 60_000 }
|
||||
);
|
||||
for (let i = 0; i < 3; i++) {
|
||||
recordModelLockoutFailure(
|
||||
provider,
|
||||
dynamicConnId,
|
||||
"gpt-4",
|
||||
"rate_limit_exceeded",
|
||||
429,
|
||||
120_000,
|
||||
null,
|
||||
{ exactCooldownMs: 60_000 }
|
||||
);
|
||||
}
|
||||
|
||||
// Verify initial failureCount is 4
|
||||
const initialLockout = getModelLockoutInfo(provider, dynamicConnId, "gpt-4");
|
||||
assert.equal(initialLockout?.failureCount, 4);
|
||||
|
||||
// 2. Record provider cooldown
|
||||
recordProviderCooldown(provider, dynamicConnId, settings);
|
||||
assert.ok(isProviderInCooldown(provider, dynamicConnId, settings));
|
||||
|
||||
// 3. Invoke handleComboChat with round-robin strategy
|
||||
const result = await handleComboChat({
|
||||
body: { stream: false },
|
||||
combo: {
|
||||
name: comboName,
|
||||
strategy: "round-robin",
|
||||
models: [modelStr],
|
||||
config: { maxRetries: 0, concurrencyPerModel: 1, queueTimeoutMs: 1000 },
|
||||
},
|
||||
handleSingleModel: async () => {
|
||||
return new Response(JSON.stringify({ choices: [{ message: { content: "ok" } }] }), {
|
||||
status: 200,
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
"X-OmniRoute-Selected-Connection-Id": dynamicConnId,
|
||||
},
|
||||
});
|
||||
},
|
||||
isModelAvailable: async () => true,
|
||||
log: createLog(),
|
||||
settings: null,
|
||||
allCombos: null,
|
||||
});
|
||||
|
||||
assert.equal(result.ok, true);
|
||||
|
||||
// 4. Assertions
|
||||
const decayCheck = decayModelFailureCount(provider, dynamicConnId, "gpt-4");
|
||||
assert.equal(
|
||||
decayCheck.newFailureCount,
|
||||
1,
|
||||
"failure count should decay from 4 to 2, and now to 1"
|
||||
);
|
||||
|
||||
assert.equal(
|
||||
isProviderInCooldown(provider, dynamicConnId, settings),
|
||||
false,
|
||||
"provider cooldown should be cleared on success"
|
||||
);
|
||||
|
||||
let persisted: any = null;
|
||||
for (let i = 0; i < 20; i++) {
|
||||
persisted = await settingsDb.getLKGP(comboName, comboName);
|
||||
if (persisted?.connectionId === dynamicConnId) {
|
||||
break;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
}
|
||||
assert.equal(persisted?.provider, provider);
|
||||
assert.equal(
|
||||
persisted?.connectionId,
|
||||
dynamicConnId,
|
||||
"LKGP connectionId must be the dynamic connection ID"
|
||||
);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user