From a4b4bca2ee92093891cd1be8b46483b34e293d13 Mon Sep 17 00:00:00 2001 From: mdigitalbh81 Date: Tue, 1 Sep 2026 00:47:29 -0300 Subject: [PATCH] fix(combo): stop retries when pinned Codex model is unavailable (#12240) --- ...ve-codex-turn-pin-model-scoped-fallback.md | 1 + open-sse/services/combo.ts | 52 +- open-sse/services/combo/nativeCodexTurnPin.ts | 129 +++++ .../unit/native-codex-turn-pin-10379.test.ts | 165 +++++- ...dex-turn-pin-model-scoped-fallback.test.ts | 533 ++++++++++++++++++ 5 files changed, 847 insertions(+), 33 deletions(-) create mode 100644 changelog.d/fixes/native-codex-turn-pin-model-scoped-fallback.md create mode 100644 tests/unit/native-codex-turn-pin-model-scoped-fallback.test.ts diff --git a/changelog.d/fixes/native-codex-turn-pin-model-scoped-fallback.md b/changelog.d/fixes/native-codex-turn-pin-model-scoped-fallback.md new file mode 100644 index 0000000000..28068a63fd --- /dev/null +++ b/changelog.d/fixes/native-codex-turn-pin-model-scoped-fallback.md @@ -0,0 +1 @@ +- **fix(combo):** return non-retryable HTTP 400 when all candidates for a pinned native Codex turn are unavailable due to model-scoped lockout, terminating the turn cleanly while preserving turn continuity and enabling standard Combo routing on subsequent turns diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index e184220e16..08e16e89f8 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -231,6 +231,8 @@ export { import { applyComboTargetExhaustion } from "./combo/targetExhaustion.ts"; import { applyNativeCodexTurnPin, + areAllPinnedTargetsModelScopedUnusable, + createPinnedModelUnavailableResponse, getNativeCodexTurnPin, pinNativeCodexTurn, } from "./combo/nativeCodexTurnPin.ts"; @@ -960,30 +962,49 @@ async function handleComboChatInner({ const { stickyWeightedLimit, getWeightedStepKeyForTarget, preScreenMap } = targetResolution; const _sticky = targetResolution.sticky; let orderedTargets = targetResolution.orderedTargets; + const quotaCutoffResetWindowConfig = resolveResetWindowConfig(config as Record); + if (activeNativeTurnPin) { - orderedTargets = applyNativeCodexTurnPin(orderedTargets, activeNativeTurnPin); - if (orderedTargets.length === 0) { - // #11371: quota-share ordering already reserved a winner slot; release it on - // this early exit (idempotent). + const pinnedTargets = applyNativeCodexTurnPin(orderedTargets, activeNativeTurnPin); + if (pinnedTargets.length === 0) { + //#11371: quota-share ordering reserved a winner slot; release on + //early exit (idempotent). targetResolution.quotaShareRelease?.(); - return errorResponse( - 409, - "The pinned native Codex turn target is no longer available; the turn cannot be moved to another provider" + log.warn( + "COMBO", + `Native Codex turn cannot continue: pinned model ${activeNativeTurnPin.modelStr} unavailable (target not in combo); preserving turn pin and terminating turn` + ); + return createPinnedModelUnavailableResponse(); + } + const allPinnedUnusable = await areAllPinnedTargetsModelScopedUnusable({ + pinnedTargets, + resilienceSettings, + quotaCutoffResetWindowConfig, + comboName: combo.name, + body: body as Record, + log, + isModelAvailable, + }); + if (allPinnedUnusable) { + targetResolution.quotaShareRelease?.(); + log.warn( + "COMBO", + `Native Codex turn cannot continue: pinned model ${activeNativeTurnPin.modelStr} is unavailable (model-scoped); preserving turn pin and terminating turn` + ); + return createPinnedModelUnavailableResponse(); + } else { + orderedTargets = pinnedTargets; + log.info( + "COMBO", + `Native Codex turn pinned to ${activeNativeTurnPin.modelStr} on connection ${activeNativeTurnPin.connectionId.slice(0, 8)}` ); } - log.info( - "COMBO", - `Native Codex turn pinned to ${activeNativeTurnPin.modelStr} connection ${activeNativeTurnPin.connectionId.slice(0, 8)}` - ); } // #5923 (Finding #4) — reset-window config for the shared per-target quota- // exhaustion cutoff below. The "auto" strategy already applies its own cutoff // via buildAutoCandidates/routableCandidates, so this only affects the other // 16 strategies (priority, weighted, etc.) that funnel through executeTarget. - const quotaCutoffResetWindowConfig = resolveResetWindowConfig(config as Record); - - // QA P0 diagnostics: record the order in which targets were actually attempted // (provider/model ids only) so a terminal combo failure can report the attempt // sequence alongside pool size + exhaustion reasons. Accumulates across set retries. const comboAttemptOrder: Array<{ provider: string; model: string }> = []; @@ -1253,7 +1274,8 @@ async function handleComboChatInner({ if ( resilienceSettings.providerCooldown.enabled && Boolean(provider && provider !== "unknown") && - isProviderInCooldown(provider, target.connectionId ?? undefined, resilienceSettings) + (isProviderInCooldown(provider, target.connectionId ?? undefined, resilienceSettings) || + isProviderInCooldown(provider, undefined, resilienceSettings)) ) { log.info("COMBO", `Skipping ${modelStr} — provider ${provider} in global cooldown`); recordComboDecision(traceInvocationId, { diff --git a/open-sse/services/combo/nativeCodexTurnPin.ts b/open-sse/services/combo/nativeCodexTurnPin.ts index 4fc175b933..9910d8d993 100644 --- a/open-sse/services/combo/nativeCodexTurnPin.ts +++ b/open-sse/services/combo/nativeCodexTurnPin.ts @@ -1,4 +1,15 @@ import { createHash } from "node:crypto"; +import { buildErrorBody } from "../../utils/error.ts"; +import { isModelLocked, hasPerModelQuota } from "../accountFallback.ts"; +import { isProviderInCooldown } from "../providerCooldownTracker.ts"; +import { getCircuitBreaker } from "../../../src/shared/utils/circuitBreaker.ts"; +import type { ResilienceSettings } from "../../../src/lib/resilience/settings"; +import { checkCredentialGate } from "../credentialGate.ts"; +import { canAffordRequest } from "../../../src/lib/quota/quotaScheduler.ts"; +import { resolveQuotaExhaustionCutoffForTarget } from "./quotaExhaustionCutoff.ts"; +import type { ResetWindowConfig } from "./quotaScoring.ts"; +import { parseModel } from "../model.ts"; +import type { ComboLogger, IsModelAvailable } from "./types.ts"; import type { ResolvedComboTarget } from "./types.ts"; @@ -150,6 +161,124 @@ export function revokeNativeCodexTurnPinsForConnection(connectionId: string): nu return revoked; } +export const NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE_CODE = "NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE"; +export const NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE_MESSAGE = + "The model handling this native Codex turn is no longer available. This turn cannot switch providers or models after output has been emitted. Start a new turn to allow Combo routing to select another model."; + +export function createPinnedModelUnavailableResponse(): Response { + const body = buildErrorBody(400, NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE_MESSAGE, undefined, { + code: NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE_CODE, + type: "invalid_request_error", + }); + return new Response(JSON.stringify(body), { + status: 400, + headers: { "Content-Type": "application/json" }, + }); +} + +export interface CheckPinnedTargetsModelScopedUnusableOptions { + pinnedTargets: ResolvedComboTarget[]; + resilienceSettings?: ResilienceSettings | null; + quotaCutoffResetWindowConfig?: ResetWindowConfig; + comboName: string; + body: Record; + log?: ComboLogger; + isModelAvailable?: IsModelAvailable; +} + +export async function isPinnedTargetModelScopedUnusable(args: { + target: ResolvedComboTarget; + resilienceSettings?: ResilienceSettings | null; + quotaCutoffResetWindowConfig?: ResetWindowConfig; + comboName: string; + body: Record; + log?: ComboLogger; + isModelAvailable?: IsModelAvailable; +}): Promise { + const { + target, + resilienceSettings, + quotaCutoffResetWindowConfig, + comboName, + body, + log, + isModelAvailable, + } = args; + const provider = target.provider; + const connectionId = target.connectionId || ""; + const rawModel = parseModel(target.modelStr).model || target.modelStr; + + if (provider && provider !== "unknown") { + const cb = getCircuitBreaker(provider); + if (cb.getStatus().state === "OPEN") return false; + if ( + resilienceSettings?.providerCooldown?.enabled && + (isProviderInCooldown(provider, connectionId || undefined, resilienceSettings) || + isProviderInCooldown(provider, undefined, resilienceSettings)) + ) { + return false; + } + } + + if ( + connectionId && + checkCredentialGate(connectionId, provider, target.modelStr).allowed === false + ) { + return false; + } + + if (provider && rawModel && isModelLocked(provider, connectionId, rawModel)) return true; + + if ( + process.env.OMNIROUTE_QUOTA_AWARE_ROUTING === "1" && + provider && + connectionId && + !canAffordRequest(connectionId, target.modelStr, body).affordable + ) { + return true; + } + + if (provider && connectionId && quotaCutoffResetWindowConfig) { + const cutoff = await resolveQuotaExhaustionCutoffForTarget( + provider, + connectionId, + resilienceSettings, + quotaCutoffResetWindowConfig, + comboName, + log ?? { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} } + ); + if (cutoff.blocked) return true; + } + + if (isModelAvailable) { + const available = await Promise.resolve(isModelAvailable(target.modelStr, target)).catch( + () => true + ); + if ( + !available && + provider && + rawModel && + (isModelLocked(provider, connectionId, rawModel) || hasPerModelQuota(provider, rawModel)) + ) { + return true; + } + } + + return false; +} + +export async function areAllPinnedTargetsModelScopedUnusable( + options: CheckPinnedTargetsModelScopedUnusableOptions +): Promise { + if (!options.pinnedTargets?.length) return false; + for (const target of options.pinnedTargets) { + if (!(await isPinnedTargetModelScopedUnusable({ target, ...options }))) { + return false; + } + } + return true; +} + export function clearNativeCodexTurnPinsForTests(): void { pins.clear(); } diff --git a/tests/unit/native-codex-turn-pin-10379.test.ts b/tests/unit/native-codex-turn-pin-10379.test.ts index 54a58bdb4d..df469648b7 100644 --- a/tests/unit/native-codex-turn-pin-10379.test.ts +++ b/tests/unit/native-codex-turn-pin-10379.test.ts @@ -1,14 +1,26 @@ -// #10379: Native Codex turn pin must allow fill-first failover across -// compatible connections for the same provider + model. -import { test } from "node:test"; +//#10379: Native Codex turn pin must allow fill-first failover across +//compatible connections for the same provider and model. +import test from "node:test"; import assert from "node:assert/strict"; const { applyNativeCodexTurnPin, pinNativeCodexTurn, getNativeCodexTurnPin, + createPinnedModelUnavailableResponse, + NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE_CODE, + NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE_MESSAGE, + isPinnedTargetModelScopedUnusable, + areAllPinnedTargetsModelScopedUnusable, clearNativeCodexTurnPinsForTests, } = await import("../../open-sse/services/combo/nativeCodexTurnPin.ts"); +const { lockExactModel, clearAllModelLockouts } = + await import("../../open-sse/services/accountFallback.ts"); +const { getCircuitBreaker, resetAllCircuitBreakers } = + await import("../../src/shared/utils/circuitBreaker.ts"); +const { recordProviderCooldown, clearCooldownState } = + await import("../../open-sse/services/providerCooldownTracker.ts"); +const { resolveResilienceSettings } = await import("../../src/lib/resilience/settings.ts"); const BODY = { client_metadata: { @@ -30,7 +42,7 @@ function makeTarget(connectionId: string, model = "gpt-5.6-sol", provider = "cod }; } -test("pinned connection is preferred but siblings are included as fallback", () => { +test("pinned connection is preferred and siblings are included as fallback", () => { clearNativeCodexTurnPinsForTests(); pinNativeCodexTurn({ body: BODY, @@ -39,12 +51,12 @@ test("pinned connection is preferred but siblings are included as fallback", () connectionId: "conn-1", }); - const pin = getNativeCodexTurnPin(BODY, "test-combo")!; + const pin = getNativeCodexTurnPin(BODY, "test-combo"); const targets = [makeTarget("conn-1"), makeTarget("conn-2"), makeTarget("conn-3")]; - const result = applyNativeCodexTurnPin(targets, pin); + const result = applyNativeCodexTurnPin(targets, pin!); - assert.equal(result.length, 3, "all compatible connections should be returned"); - assert.equal(result[0].connectionId, "conn-1", "pinned connection should be first"); + assert.equal(result.length, 3, "all compatible connections returned"); + assert.equal(result[0].connectionId, "conn-1", "pinned connection first"); assert.deepEqual(result[0].allowedConnectionIds, ["conn-1", "conn-2", "conn-3"]); }); @@ -57,9 +69,9 @@ test("fallback connections share the same allowedConnectionIds", () => { connectionId: "conn-2", }); - const pin = getNativeCodexTurnPin(BODY, "test-combo")!; + const pin = getNativeCodexTurnPin(BODY, "test-combo"); const targets = [makeTarget("conn-1"), makeTarget("conn-2"), makeTarget("conn-3")]; - const result = applyNativeCodexTurnPin(targets, pin); + const result = applyNativeCodexTurnPin(targets, pin!); assert.equal(result[0].connectionId, "conn-2"); assert.equal(result[1].connectionId, "conn-1"); @@ -78,15 +90,15 @@ test("incompatible targets (different provider/model) are excluded", () => { connectionId: "conn-1", }); - const pin = getNativeCodexTurnPin(BODY, "test-combo")!; + const pin = getNativeCodexTurnPin(BODY, "test-combo"); const targets = [ makeTarget("conn-1"), makeTarget("conn-2"), makeTarget("conn-other", "different-model", "other-provider"), ]; - const result = applyNativeCodexTurnPin(targets, pin); + const result = applyNativeCodexTurnPin(targets, pin!); - assert.equal(result.length, 2, "incompatible target should be excluded"); + assert.equal(result.length, 2, "incompatible target excluded"); assert.deepEqual(result[0].allowedConnectionIds, ["conn-1", "conn-2"]); }); @@ -99,14 +111,14 @@ test("empty result when no compatible targets exist", () => { connectionId: "conn-1", }); - const pin = getNativeCodexTurnPin(BODY, "test-combo")!; + const pin = getNativeCodexTurnPin(BODY, "test-combo"); const targets = [makeTarget("conn-x", "other-model", "other-provider")]; - const result = applyNativeCodexTurnPin(targets, pin); + const result = applyNativeCodexTurnPin(targets, pin!); assert.equal(result.length, 0); }); -test("pinNativeCodexTurn allows connectionId change for same provider+model", () => { +test("pinNativeCodexTurn allows connectionId change on same provider+model", () => { clearNativeCodexTurnPinsForTests(); pinNativeCodexTurn({ body: BODY, @@ -123,8 +135,8 @@ test("pinNativeCodexTurn allows connectionId change for same provider+model", () connectionId: "conn-2", }); - const pin = getNativeCodexTurnPin(BODY, "test-combo")!; - assert.equal(pin.connectionId, "conn-2", "pin should update to new connection"); + const pin = getNativeCodexTurnPin(BODY, "test-combo"); + assert.equal(pin?.connectionId, "conn-2", "pin updated to new connection"); }); test("pinNativeCodexTurn rejects provider/model change", () => { @@ -147,3 +159,120 @@ test("pinNativeCodexTurn rejects provider/model change", () => { /Native Codex turn target changed/ ); }); + +test("createPinnedModelUnavailableResponse constructs non-retryable HTTP 400 error response", async () => { + const response = createPinnedModelUnavailableResponse(); + assert.equal(response.status, 400); + const data = await response.json(); + assert.equal(data.error.code, NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE_CODE); + assert.equal(data.error.message, NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE_MESSAGE); + assert.equal(data.error.type, "invalid_request_error"); +}); + +test("isPinnedTargetModelScopedUnusable distinguishes model lockout from provider/connection outages", async () => { + clearAllModelLockouts(); + clearCooldownState(); + resetAllCircuitBreakers(); + + const resilienceSettings = resolveResilienceSettings({ + resilienceSettings: { + providerCooldown: { enabled: true, minRetryCooldownMs: 5000, maxRetryCooldownMs: 300000 }, + }, + }); + + const target = makeTarget("conn-1", "antigravity/claude-opus-4-6-thinking", "antigravity"); + + // 1. Healthy target -> false + assert.equal( + await isPinnedTargetModelScopedUnusable({ + target, + comboName: "test-combo", + body: BODY, + resilienceSettings, + }), + false + ); + + // 2. Exact model lockout -> true + lockExactModel("antigravity", "conn-1", "claude-opus-4-6-thinking", "quota_exhausted", 60_000); + assert.equal( + await isPinnedTargetModelScopedUnusable({ + target, + comboName: "test-combo", + body: BODY, + resilienceSettings, + }), + true + ); + + // 3. Provider circuit breaker OPEN -> false (not model-scoped) + const cb = getCircuitBreaker("antigravity", { failureThreshold: 1, resetTimeout: 60000 }); + try { + await cb.execute(async () => { + throw new Error("503"); + }); + } catch {} + assert.equal( + await isPinnedTargetModelScopedUnusable({ + target, + comboName: "test-combo", + body: BODY, + resilienceSettings, + }), + false + ); + + // Reset breaker, test provider cooldown + cb.reset(); + recordProviderCooldown("antigravity", undefined, resilienceSettings); + assert.equal( + await isPinnedTargetModelScopedUnusable({ + target, + comboName: "test-combo", + body: BODY, + resilienceSettings, + }), + false + ); +}); + +test("areAllPinnedTargetsModelScopedUnusable returns true only when all candidates are model-scoped unusable", async () => { + clearAllModelLockouts(); + clearCooldownState(); + resetAllCircuitBreakers(); + + const target1 = makeTarget("conn-1", "antigravity/claude-opus-4-6-thinking", "antigravity"); + const target2 = makeTarget("conn-2", "antigravity/claude-opus-4-6-thinking", "antigravity"); + + // Both healthy -> false + assert.equal( + await areAllPinnedTargetsModelScopedUnusable({ + pinnedTargets: [target1, target2], + comboName: "test-combo", + body: BODY, + }), + false + ); + + // Only target1 locked -> false + lockExactModel("antigravity", "conn-1", "claude-opus-4-6-thinking", "quota_exhausted", 60_000); + assert.equal( + await areAllPinnedTargetsModelScopedUnusable({ + pinnedTargets: [target1, target2], + comboName: "test-combo", + body: BODY, + }), + false + ); + + // Both locked -> true + lockExactModel("antigravity", "conn-2", "claude-opus-4-6-thinking", "quota_exhausted", 60_000); + assert.equal( + await areAllPinnedTargetsModelScopedUnusable({ + pinnedTargets: [target1, target2], + comboName: "test-combo", + body: BODY, + }), + true + ); +}); diff --git a/tests/unit/native-codex-turn-pin-model-scoped-fallback.test.ts b/tests/unit/native-codex-turn-pin-model-scoped-fallback.test.ts new file mode 100644 index 0000000000..314dd3efdb --- /dev/null +++ b/tests/unit/native-codex-turn-pin-model-scoped-fallback.test.ts @@ -0,0 +1,533 @@ +import test, { describe, beforeEach } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-turn-pin-repro-")); +const ORIGINAL_DATA_DIR = process.env.DATA_DIR; +process.env.DATA_DIR = TEST_DATA_DIR; + +const { handleComboChat } = await import("../../open-sse/services/combo.ts"); +const { lockExactModel, clearAllModelLockouts } = + await import("../../open-sse/services/accountFallback.ts"); +const { + getNativeCodexTurnPin, + clearNativeCodexTurnPinsForTests, + NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE_CODE, + NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE_MESSAGE, +} = await import("../../open-sse/services/combo/nativeCodexTurnPin.ts"); +const { recordProviderCooldown, isProviderInCooldown, clearCooldownState } = + await import("../../open-sse/services/providerCooldownTracker.ts"); +const { getCircuitBreaker, resetAllCircuitBreakers } = + await import("../../src/shared/utils/circuitBreaker.ts"); +const { resolveResilienceSettings } = await import("../../src/lib/resilience/settings.ts"); +const core = await import("../../src/lib/db/core.ts"); +const providersDb = await import("../../src/lib/db/providers.ts"); + +const testSettings = { + resilienceSettings: { + providerCooldown: { enabled: true, minRetryCooldownMs: 5000, maxRetryCooldownMs: 300000 }, + }, +}; + +const settings = resolveResilienceSettings(testSettings); + +function createLog(entries: Array<{ level: string; tag: string; msg: string }> = []) { + return { + info: (tag: string, msg: string) => { + entries.push({ level: "info", tag, msg }); + }, + warn: (tag: string, msg: string) => { + entries.push({ level: "warn", tag, msg }); + }, + error: (tag: string, msg: string) => { + entries.push({ level: "error", tag, msg }); + }, + debug: (tag: string, msg: string) => { + entries.push({ level: "debug", tag, msg }); + }, + entries, + }; +} + +async function cleanupTestDataDir() { + let lastError: unknown; + for (let attempt = 0; attempt < 5; attempt += 1) { + try { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + return; + } catch (error) { + lastError = error; + await new Promise((resolve) => setTimeout(resolve, 25)); + } + } + if (lastError) throw lastError; +} + +test.after(async () => { + await cleanupTestDataDir(); + process.env.DATA_DIR = ORIGINAL_DATA_DIR; +}); + +beforeEach(async () => { + clearAllModelLockouts(); + clearCooldownState(); + resetAllCircuitBreakers(); + clearNativeCodexTurnPinsForTests(); +}); + +describe("Native Codex Turn Pin model-scoped fallback", () => { + const comboName = "Codex"; + const opusModel = "antigravity/claude-opus-4-6-thinking"; + const geminiModel = "antigravity/gemini-3.7-flash-high"; + const codexModel = "codex/gpt-5.5-high"; + + const comboConfig = { + name: comboName, + strategy: "fill-first" as const, + models: [opusModel, geminiModel, codexModel], + config: { + maxRetries: 0, + concurrencyPerModel: 1, + queueTimeoutMs: 1000, + }, + }; + + const nativeTurnBody = { + stream: false, + client_metadata: { + "x-codex-turn-metadata": JSON.stringify({ + thread_id: "thread-prod-123", + turn_id: "turn-prod-456", + }), + }, + }; + + test("3-Phase Production Scenario: Phase 1 Opus pins -> Phase 2 terminal 400 preserving pin -> Phase 3 new turn routes Gemini", async () => { + const conn1 = await providersDb.createProviderConnection({ + provider: "antigravity", + authType: "oauth", + name: "Antigravity Account 1", + }); + const conn2 = await providersDb.createProviderConnection({ + provider: "antigravity", + authType: "oauth", + name: "Antigravity Account 2", + }); + await providersDb.createProviderConnection({ + provider: "codex", + authType: "apikey", + name: "Codex Key", + apiKey: "sk-codex-test", + }); + + const conn1Id = conn1.id; + const conn2Id = conn2.id; + + const attemptedModels: string[] = []; + + // Phase 1: Native turn-prod-456 on thread-prod-123 -> Opus succeeds, pin created + const phase1Result = await handleComboChat({ + body: nativeTurnBody, + combo: comboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async (_body, modelStr) => { + attemptedModels.push(modelStr); + return new Response( + JSON.stringify({ choices: [{ message: { content: "opus output" } }] }), + { + status: 200, + headers: { + "content-type": "application/json", + "x-omniroute-selected-connection-id": conn1Id, + }, + } + ); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: testSettings, + allCombos: null, + }); + + assert.equal(phase1Result.ok, true); + assert.deepEqual(attemptedModels, [opusModel]); + + const pin = getNativeCodexTurnPin(nativeTurnBody, comboName); + assert.ok(pin, "Turn pin created after phase 1"); + assert.equal(pin.modelStr, opusModel); + assert.equal(pin.provider, "antigravity"); + assert.equal(pin.connectionId, conn1Id); + + // Phase 2: SAME native turn (turn-prod-456) -> Opus becomes locked on all Antigravity accounts + lockExactModel("antigravity", conn1Id, "claude-opus-4-6-thinking", "quota_exhausted", 60_000); + lockExactModel("antigravity", conn2Id, "claude-opus-4-6-thinking", "quota_exhausted", 60_000); + lockExactModel("antigravity", "", "claude-opus-4-6-thinking", "quota_exhausted", 60_000); + + attemptedModels.length = 0; + const phase2LogEntries: Array<{ level: string; tag: string; msg: string }> = []; + + const phase2Result = await handleComboChat({ + body: nativeTurnBody, + combo: comboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async (_body, modelStr) => { + attemptedModels.push(modelStr); + return new Response(JSON.stringify({ error: "unexpected model dispatch" }), { + status: 500, + }); + }, + isModelAvailable: async () => true, + log: createLog(phase2LogEntries), + settings: testSettings, + allCombos: null, + }); + + // Phase 2 assertions: non-retryable 400 Bad Request terminates turn without reconnect storms + assert.equal(phase2Result.status, 400, "Phase 2 must return non-retryable 400 Bad Request"); + assert.equal(phase2Result.ok, false); + const phase2Body = await phase2Result.json(); + assert.equal(phase2Body.error.code, NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE_CODE); + assert.equal(phase2Body.error.message, NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE_MESSAGE); + assert.equal(phase2Body.error.type, "invalid_request_error"); + assert.deepEqual( + attemptedModels, + [], + "Zero models dispatched during phase 2 (no mid-turn switch to Gemini or Codex)" + ); + assert.equal( + isProviderInCooldown("antigravity", undefined, settings), + false, + "Antigravity provider must NOT be marked globally exhausted" + ); + + // Pinned turn is NOT released mid-turn + const pinAfterPhase2 = getNativeCodexTurnPin(nativeTurnBody, comboName); + assert.ok(pinAfterPhase2, "Turn pin must remain active for turn-prod-456"); + assert.equal(pinAfterPhase2.modelStr, opusModel, "Turn pin remains locked to Opus"); + + const terminalLog = phase2LogEntries.find( + (e) => + e.tag === "COMBO" && + e.msg.includes("Native Codex turn cannot continue") && + e.msg.includes("model-scoped") + ); + assert.ok(terminalLog, "Should log structured warning about model-scoped turn termination"); + + // Phase 3: NEW native turn (turn-prod-457) in same thread -> Opus still locked, normal Combo routing selects Gemini + const phase3TurnBody = { + stream: false, + client_metadata: { + "x-codex-turn-metadata": JSON.stringify({ + thread_id: "thread-prod-123", + turn_id: "turn-prod-457", + }), + }, + }; + + attemptedModels.length = 0; + const phase3Result = await handleComboChat({ + body: phase3TurnBody, + combo: comboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async (_body, modelStr) => { + attemptedModels.push(modelStr); + if (modelStr === geminiModel) { + return new Response( + JSON.stringify({ choices: [{ message: { content: "gemini output" } }] }), + { + status: 200, + headers: { + "content-type": "application/json", + "x-omniroute-selected-connection-id": conn1Id, + }, + } + ); + } + return new Response(JSON.stringify({ error: "unexpected model" }), { status: 500 }); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: testSettings, + allCombos: null, + }); + + assert.equal(phase3Result.ok, true, "Phase 3 must succeed with next healthy combo model"); + assert.deepEqual( + attemptedModels, + [geminiModel], + "Gemini attempted and succeeded; Codex GPT not called" + ); + + const pinPhase3 = getNativeCodexTurnPin(phase3TurnBody, comboName); + assert.ok(pinPhase3, "New turn pin created for phase 3"); + assert.equal(pinPhase3.modelStr, geminiModel, "Phase 3 pinned to Gemini"); + + const pinPhase2Check = getNativeCodexTurnPin(nativeTurnBody, comboName); + assert.equal(pinPhase2Check?.modelStr, opusModel, "Phase 2 turn pin still intact on Opus"); + }); + + test("Pinned connection fails over to sibling connection for same provider+model when sibling healthy", async () => { + const conn1Id = "conn-1"; + const conn2Id = "conn-2"; + + const explicitComboConfig = { + name: comboName, + strategy: "fill-first" as const, + models: [ + { id: "s1", kind: "model" as const, model: opusModel, connectionId: conn1Id, weight: 1 }, + { id: "s2", kind: "model" as const, model: opusModel, connectionId: conn2Id, weight: 1 }, + { id: "s3", kind: "model" as const, model: geminiModel, connectionId: conn1Id, weight: 1 }, + ], + config: { maxRetries: 0, concurrencyPerModel: 1, queueTimeoutMs: 1000 }, + }; + + // Phase 1: Opus succeeds on conn1 + await handleComboChat({ + body: nativeTurnBody, + combo: explicitComboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async () => + new Response(JSON.stringify({ choices: [{ message: { content: "opus conn1" } }] }), { + status: 200, + headers: { "x-omniroute-selected-connection-id": conn1Id }, + }), + isModelAvailable: async () => true, + log: createLog(), + settings: testSettings, + allCombos: null, + }); + + // Phase 2: Lock ONLY conn1 Opus, conn2 remains healthy + lockExactModel("antigravity", conn1Id, "claude-opus-4-6-thinking", "quota_exhausted", 60_000); + + const attempted: Array<{ modelStr: string; connectionId?: string }> = []; + const phase2Result = await handleComboChat({ + body: nativeTurnBody, + combo: explicitComboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async (_body, modelStr, target) => { + attempted.push({ modelStr, connectionId: target?.connectionId ?? undefined }); + return new Response(JSON.stringify({ choices: [{ message: { content: "opus conn2" } }] }), { + status: 200, + headers: { "x-omniroute-selected-connection-id": conn2Id }, + }); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: testSettings, + allCombos: null, + }); + + assert.equal(phase2Result.ok, true); + assert.equal(attempted.length, 1); + assert.equal(attempted[0].modelStr, opusModel, "Opus must remain pinned"); + assert.equal(attempted[0].connectionId, conn2Id, "Connection must fail over to conn2"); + }); + + test("Turn pin NOT released when provider circuit breaker is OPEN", async () => { + const conn1 = await providersDb.createProviderConnection({ + provider: "antigravity", + authType: "oauth", + name: "Antigravity Account 1", + }); + + await handleComboChat({ + body: nativeTurnBody, + combo: comboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async () => + new Response(JSON.stringify({ choices: [{ message: { content: "opus" } }] }), { + status: 200, + headers: { "x-omniroute-selected-connection-id": conn1.id }, + }), + isModelAvailable: async () => true, + log: createLog(), + settings: testSettings, + allCombos: null, + }); + + // Trip provider circuit breaker + const cb = getCircuitBreaker("antigravity", { failureThreshold: 1, resetTimeout: 60000 }); + try { + await cb.execute(async () => { + throw new Error("simulated 503"); + }); + } catch { + // expected + } + assert.equal(cb.getStatus().state, "OPEN"); + + const attempted: string[] = []; + const result = await handleComboChat({ + body: nativeTurnBody, + combo: comboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async (_body, modelStr) => { + attempted.push(modelStr); + return new Response(JSON.stringify({ choices: [{ message: { content: "ok" } }] }), { + status: 200, + }); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: testSettings, + allCombos: null, + }); + + assert.equal(result.ok, false, "Should fail due to provider circuit breaker OPEN"); + assert.equal(attempted.length, 0, "No targets should be attempted"); + }); + + test("Turn pin NOT released when provider in global cooldown", async () => { + const conn1 = await providersDb.createProviderConnection({ + provider: "antigravity", + authType: "oauth", + name: "Antigravity Account 1", + }); + + await handleComboChat({ + body: nativeTurnBody, + combo: comboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async () => + new Response(JSON.stringify({ choices: [{ message: { content: "opus" } }] }), { + status: 200, + headers: { "x-omniroute-selected-connection-id": conn1.id }, + }), + isModelAvailable: async () => true, + log: createLog(), + settings: testSettings, + allCombos: null, + }); + + recordProviderCooldown("antigravity", undefined, settings); + assert.equal(isProviderInCooldown("antigravity", undefined, settings), true); + + const attempted: string[] = []; + const result = await handleComboChat({ + body: nativeTurnBody, + combo: comboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async (_body, modelStr) => { + attempted.push(modelStr); + return new Response(JSON.stringify({ choices: [{ message: { content: "ok" } }] }), { + status: 200, + }); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: testSettings, + allCombos: null, + }); + + assert.equal(result.ok, false); + assert.equal(attempted.length, 0); + }); + + test("Request without active turn pin retains full Combo fallback when first model is locked", async () => { + const conn1 = await providersDb.createProviderConnection({ + provider: "antigravity", + authType: "oauth", + name: "Antigravity Account 1", + }); + const codexConn = await providersDb.createProviderConnection({ + provider: "codex", + authType: "apikey", + name: "Codex Key", + apiKey: "sk-codex-test", + }); + + const unpinnedBody = { stream: false }; + + // Lock Opus + lockExactModel("antigravity", conn1.id, "claude-opus-4-6-thinking", "quota_exhausted", 60_000); + lockExactModel("antigravity", "", "claude-opus-4-6-thinking", "quota_exhausted", 60_000); + + const attempted: string[] = []; + + // Gemini fails transiently (500), falls back to Codex GPT-5.5 in normal combo chain + const result = await handleComboChat({ + body: unpinnedBody, + combo: comboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async (_body, modelStr) => { + attempted.push(modelStr); + if (modelStr === geminiModel) { + return new Response(JSON.stringify({ error: { message: "gemini server error" } }), { + status: 500, + }); + } + if (modelStr === codexModel) { + return new Response( + JSON.stringify({ choices: [{ message: { content: "codex output" } }] }), + { + status: 200, + headers: { "x-omniroute-selected-connection-id": codexConn.id }, + } + ); + } + return new Response(JSON.stringify({ error: "unexpected model" }), { status: 500 }); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: testSettings, + allCombos: null, + }); + + assert.equal(result.ok, true); + assert.deepEqual( + attempted, + [geminiModel, codexModel], + "Should try Gemini, then Codex in combo order" + ); + }); + + test("Retrying failed Phase 2 turn repeatedly yields terminal 400 without mid-turn cross-model leak", async () => { + const conn1 = await providersDb.createProviderConnection({ + provider: "antigravity", + authType: "oauth", + name: "Antigravity Account 1", + }); + + // Phase 1: Opus succeeds + await handleComboChat({ + body: nativeTurnBody, + combo: comboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async () => + new Response(JSON.stringify({ choices: [{ message: { content: "opus" } }] }), { + status: 200, + headers: { "x-omniroute-selected-connection-id": conn1.id }, + }), + isModelAvailable: async () => true, + log: createLog(), + settings: testSettings, + allCombos: null, + }); + + lockExactModel("antigravity", conn1.id, "claude-opus-4-6-thinking", "quota_exhausted", 60_000); + lockExactModel("antigravity", "", "claude-opus-4-6-thinking", "quota_exhausted", 60_000); + + for (let retry = 0; retry < 3; retry += 1) { + const attempted: string[] = []; + const result = await handleComboChat({ + body: nativeTurnBody, + combo: comboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async (_b, m) => { + attempted.push(m); + return new Response(JSON.stringify({ ok: true }), { status: 200 }); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: testSettings, + allCombos: null, + }); + assert.equal(result.status, 400); + assert.equal(attempted.length, 0); + } + }); +});