From 65263a4fe9840e2ced3fac74694e7646f15f16aa Mon Sep 17 00:00:00 2001 From: opensource-elearning <159253500+opensource-elearning@users.noreply.github.com> Date: Fri, 18 Sep 2026 20:01:52 +0530 Subject: [PATCH] fix(cli): Codex long-session turn-pin fallback + codex-settings key resolution (#13564 #13563) (#13566) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(sse): release native Codex turn pin when pinned model is model-scoped unusable (#13564) Long-running Codex sessions die with 400 NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE whenever the model pinned to the current turn becomes model-scoped unusable mid-session (per-model quota lockout, connection cooldown, exhausted accounts). Claude Code has no equivalent pin and already falls back to the next healthy combo model; Codex now matches. Release the turn pin when all pinned provider+model targets are model-scoped unusable and fall through to full combo routing, re-pinning to whichever model succeeds. Preserve the pin on provider-wide outages (circuit breaker OPEN, provider cooldown) and when the pinned target is still healthy. Also prunes the stale ESLint suppression entry for combo.ts that this change orphaned (createPinnedModelUnavailableResponse import dropped; pre-existing getBootstrapLatencyMs remains the sole residual unused var). * fix(api): resolve codex-settings apiKey via canonical resolver instead of 400 (#13563) Applying Codex settings from /dashboard/cli-code/codex always failed with 400 "baseUrl, apiKey and model are required" when the dashboard sent an empty apiKey (cloud mode with no management key selected) — baseUrl and model are already Zod-gated, so that response could only ever fire on the empty key. The codex-settings route had diverged from the sibling CLI tools (cline/forge/ openclaw/grok-build/jcode): an inline if(!apiKey) 400 guard plus a hand-rolled getApiKeyById lookup, instead of the shared resolveApiKey(keyId, apiKey) helper which resolves by keyId, falls back to the submitted apiKey, then to sk_omniroute. This change makes codex-settings use the canonical resolver, so: - empty apiKey + valid keyId -> the real DB key is written to auth.json - empty apiKey + no keyId -> sk_omniroute default (config still applies) - explicit apiKey -> written verbatim (unchanged) * docs(changelog): add fragments for Codex turn-pin fallback and codex-settings apiKey resolution --- .../fixes/13566-codex-settings-apikey.md | 1 + .../fixes/13566-codex-turnpin-fallback.md | 1 + open-sse/services/combo.ts | 59 ++-- open-sse/services/combo/nativeCodexTurnPin.ts | 5 + src/app/api/cli-tools/codex-settings/route.ts | 24 +- .../codex-settings-api-key-resolution.test.ts | 115 ++++++++ ...dex-turn-pin-model-scoped-fallback.test.ts | 276 ++++++++++++++++-- 7 files changed, 404 insertions(+), 77 deletions(-) create mode 100644 changelog.d/fixes/13566-codex-settings-apikey.md create mode 100644 changelog.d/fixes/13566-codex-turnpin-fallback.md create mode 100644 tests/unit/codex-settings-api-key-resolution.test.ts diff --git a/changelog.d/fixes/13566-codex-settings-apikey.md b/changelog.d/fixes/13566-codex-settings-apikey.md new file mode 100644 index 0000000000..6c109c0428 --- /dev/null +++ b/changelog.d/fixes/13566-codex-settings-apikey.md @@ -0,0 +1 @@ +- fix(api): resolve the codex-settings `apiKey` through the canonical key resolver instead of an inline 400 guard, so the dashboard Apply flow no longer fails with `baseUrl, apiKey and model are required` in cloud mode when no management key is selected (#13563) diff --git a/changelog.d/fixes/13566-codex-turnpin-fallback.md b/changelog.d/fixes/13566-codex-turnpin-fallback.md new file mode 100644 index 0000000000..3264555f7b --- /dev/null +++ b/changelog.d/fixes/13566-codex-turnpin-fallback.md @@ -0,0 +1 @@ +- fix(sse): release the native Codex turn pin when the pinned model becomes model-scoped unusable, so a long-running Codex session falls back to the next healthy combo model instead of dying to a terminal `400 NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE` (#13564) diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index caded1d748..5dad34a275 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -90,8 +90,8 @@ export { import { applyNativeCodexTurnPin, areAllPinnedTargetsModelScopedUnusable, - createPinnedModelUnavailableResponse, getNativeCodexTurnPin, + releaseNativeCodexTurnPin, } from "./combo/nativeCodexTurnPin.ts"; import { pinIsDurablyUnhealthy, @@ -840,37 +840,40 @@ async function handleComboChatInner({ if (activeNativeTurnPin) { const pinnedTargets = applyNativeCodexTurnPin(orderedTargets, activeNativeTurnPin); if (pinnedTargets.length === 0) { - //#11371: quota-share ordering reserved a winner slot; release on - //early exit (idempotent). - targetResolution.quotaShareRelease?.(); + // Pinned model no longer exists in the combo — release pin and fall through + // to full combo routing so the turn can continue with a healthy model. + releaseNativeCodexTurnPin(body as Record, combo.name); log.warn( "COMBO", - `Native Codex turn cannot continue: pinned model ${activeNativeTurnPin.modelStr} unavailable (target not in combo); preserving turn pin and terminating turn` + `Native Codex turn pin released: pinned model ${activeNativeTurnPin.modelStr} no longer in combo; falling back to full combo routing` ); - return createPinnedModelUnavailableResponse(); - } - const allPinnedUnusable = await areAllPinnedTargetsModelScopedUnusable({ - pinnedTargets, - resilienceSettings, - quotaCutoffResetWindowConfig, - comboName: combo.name, - body: body as Record, - log, - isModelAvailable, - }); - if (allPinnedUnusable) { - targetResolution.quotaShareRelease?.(); - log.warn( - "COMBO", - `Native Codex turn cannot continue: pinned model ${activeNativeTurnPin.modelStr} is unavailable (model-scoped); preserving turn pin and terminating turn` - ); - return createPinnedModelUnavailableResponse(); } else { - orderedTargets = pinnedTargets; - log.info( - "COMBO", - `Native Codex turn pinned to ${activeNativeTurnPin.modelStr} on connection ${activeNativeTurnPin.connectionId.slice(0, 8)}` - ); + const allPinnedUnusable = await areAllPinnedTargetsModelScopedUnusable({ + pinnedTargets, + resilienceSettings, + quotaCutoffResetWindowConfig, + comboName: combo.name, + body: body as Record, + log, + isModelAvailable, + }); + if (allPinnedUnusable) { + // All pinned provider+model targets are model-scoped unusable — release + // the pin and fall through to full combo routing so the turn can try + // other models in the combo pool. This matches Claude Code's behavior + // where no turn pin allows natural multi-model fallback. + releaseNativeCodexTurnPin(body as Record, combo.name); + log.warn( + "COMBO", + `Native Codex turn pin released: pinned model ${activeNativeTurnPin.modelStr} model-scoped unavailable; falling back to full combo routing` + ); + } else { + orderedTargets = pinnedTargets; + log.info( + "COMBO", + `Native Codex turn pinned to ${activeNativeTurnPin.modelStr} on connection ${activeNativeTurnPin.connectionId.slice(0, 8)}` + ); + } } } diff --git a/open-sse/services/combo/nativeCodexTurnPin.ts b/open-sse/services/combo/nativeCodexTurnPin.ts index 2b6d94945f..b030d0a47e 100644 --- a/open-sse/services/combo/nativeCodexTurnPin.ts +++ b/open-sse/services/combo/nativeCodexTurnPin.ts @@ -280,6 +280,11 @@ export async function areAllPinnedTargetsModelScopedUnusable( return true; } +export function releaseNativeCodexTurnPin(body: Record, comboName: string): void { + const key = nativeCodexTurnKey(body, comboName); + if (key) pins.delete(key); +} + export function clearNativeCodexTurnPinsForTests(): void { pins.clear(); } diff --git a/src/app/api/cli-tools/codex-settings/route.ts b/src/app/api/cli-tools/codex-settings/route.ts index 40a094c840..6ebed75312 100644 --- a/src/app/api/cli-tools/codex-settings/route.ts +++ b/src/app/api/cli-tools/codex-settings/route.ts @@ -13,7 +13,7 @@ import { createMultiBackup } from "@/shared/services/backupService"; import { saveCliToolLastConfigured, deleteCliToolLastConfigured } from "@/lib/db/cliToolState"; import { cliModelConfigSchema } from "@/shared/validation/schemas"; import { isValidationFailure, validateBody } from "@/shared/validation/helpers"; -import { getApiKeyById } from "@/lib/db/apiKeys"; +import { resolveApiKey } from "@/shared/services/apiKeyResolver"; import { normalizeCodexBaseUrl } from "@/shared/utils/codexBaseUrl"; import { migrateCodexFeatureFlags } from "@/shared/utils/codexConfig"; @@ -214,25 +214,9 @@ export async function POST(request: Request) { return NextResponse.json({ error: validation.error }, { status: 400 }); } const { baseUrl, model, reasoningEffort, wireApi, modelMappings } = validation.data; - let { apiKey } = validation.data; - if (!apiKey) { - return NextResponse.json( - { error: "baseUrl, apiKey and model are required" }, - { status: 400 } - ); - } - - // Resolve real key from DB by ID - if (keyId) { - try { - const keyRecord = await getApiKeyById(keyId); - if (keyRecord?.key) { - apiKey = keyRecord.key as string; - } - } catch { - // Non-critical: fall back to whatever value was in apiKey - } - } + // Canonical key resolution (#13563): by keyId -> submitted apiKey -> sk_omniroute. + // Matches cline/forge/openclaw/grok-build/jcode-settings. + const apiKey = await resolveApiKey(keyId, validation.data.apiKey); const codexDir = getCodexDir(); const configPath = getCodexConfigPath(); diff --git a/tests/unit/codex-settings-api-key-resolution.test.ts b/tests/unit/codex-settings-api-key-resolution.test.ts new file mode 100644 index 0000000000..55b051a606 --- /dev/null +++ b/tests/unit/codex-settings-api-key-resolution.test.ts @@ -0,0 +1,115 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { SignJWT } from "jose"; + +/** + * Regression test for #13563. + * + * Applying Codex settings from /dashboard/cli-code/codex always failed with + * `400 baseUrl, apiKey and model are required` whenever the dashboard sent an + * empty apiKey — which it does in cloud mode (CLOUD_URL set) with no management + * key selected. `baseUrl` and `model` are already Zod-gated (min(1)), so this + * response could only ever fire on an empty apiKey, yet the error text points + * at all three fields. + * + * The codex-settings route had diverged from the canonical API-key resolution + * used by cline/forge/openclaw/grok-build/jcode-settings: an inline + * `if (!apiKey) return 400` guard plus a hand-rolled `getApiKeyById` lookup, + * instead of the shared `resolveApiKey(keyId, apiKey)`. That helper resolves by + * keyId first, then falls back to the submitted apiKey, then to `sk_omniroute`. + * + * This test drives the real POST handler end-to-end (real DB-backed API key, + * real JWT auth cookie) and asserts the value written into auth.json. + */ + +const TEST_ROOT = fs.mkdtempSync(path.join(os.tmpdir(), "omr-codex-settings-key-")); +const TEST_HOME = path.join(TEST_ROOT, "fake-home"); +fs.mkdirSync(TEST_HOME, { recursive: true }); + +const originalHome = os.homedir; +const originalDataDir = process.env.DATA_DIR; +const originalJwtSecret = process.env.JWT_SECRET; +const originalWriteFlag = process.env.CLI_ALLOW_CONFIG_WRITES; + +os.homedir = () => TEST_HOME; +process.env.DATA_DIR = path.join(TEST_ROOT, "data"); +process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "codex-settings-key-api-secret"; +process.env.CLI_ALLOW_CONFIG_WRITES = "true"; + +const core = await import("../../src/lib/db/core.ts"); +const apiKeysDb = await import("../../src/lib/db/apiKeys.ts"); +const route = await import("../../src/app/api/cli-tools/codex-settings/route.ts"); + +const AUTH_PATH = path.join(TEST_HOME, ".codex", "auth.json"); + +test.after(async () => { + core.resetDbInstance(); + os.homedir = originalHome; + if (originalDataDir === undefined) delete process.env.DATA_DIR; + else process.env.DATA_DIR = originalDataDir; + if (originalJwtSecret === undefined) delete process.env.JWT_SECRET; + else process.env.JWT_SECRET = originalJwtSecret; + if (originalWriteFlag === undefined) delete process.env.CLI_ALLOW_CONFIG_WRITES; + else process.env.CLI_ALLOW_CONFIG_WRITES = originalWriteFlag; + fs.rmSync(TEST_ROOT, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); +}); + +const authCookie = async (): Promise => { + process.env.JWT_SECRET = "codex-settings-key-api-jwt"; + const token = await new SignJWT({ authenticated: true, sub: "codex-settings-key-api-test" }) + .setProtectedHeader({ alg: "HS256" }) + .setExpirationTime("1h") + .sign(new TextEncoder().encode(process.env.JWT_SECRET)); + return `auth_token=${token}`; +}; + +const post = async (body: Record) => + route.POST( + new Request("http://localhost/api/cli-tools/codex-settings", { + method: "POST", + headers: { + cookie: await authCookie(), + "Content-Type": "application/json", + }, + body: JSON.stringify({ + baseUrl: "http://localhost:20128/api/v1", + model: "gpt-5.6-sol", + ...body, + }), + }) + ); + +const readWrittenApiKey = (): string | null => { + try { + return JSON.parse(fs.readFileSync(AUTH_PATH, "utf8")).OPENAI_API_KEY ?? null; + } catch { + return null; + } +}; + +test("#13563: POST codex-settings with empty apiKey resolves the real key from keyId instead of 400", async () => { + const created = await apiKeysDb.createApiKey("codex-settings-key-test", "codex-settings-machine"); + assert.ok(created.key && created.key.length > 0, "createApiKey must return a real plaintext key"); + + const response = await post({ apiKey: "", keyId: created.id }); + + assert.equal(response.status, 200); + assert.equal(readWrittenApiKey(), created.key); +}); + +test("#13563: POST codex-settings with empty apiKey and no keyId writes sk_omniroute instead of 400", async () => { + const response = await post({ apiKey: "" }); + + assert.equal(response.status, 200); + assert.equal(readWrittenApiKey(), "sk_omniroute"); +}); + +test("#13563: POST codex-settings with an explicit apiKey still writes it verbatim", async () => { + const response = await post({ apiKey: "sk-test-explicit" }); + + assert.equal(response.status, 200); + assert.equal(readWrittenApiKey(), "sk-test-explicit"); +}); diff --git a/tests/unit/native-codex-turn-pin-model-scoped-fallback.test.ts b/tests/unit/native-codex-turn-pin-model-scoped-fallback.test.ts index d37fb55cb8..18df842125 100644 --- a/tests/unit/native-codex-turn-pin-model-scoped-fallback.test.ts +++ b/tests/unit/native-codex-turn-pin-model-scoped-fallback.test.ts @@ -11,12 +11,8 @@ process.env.DATA_DIR = TEST_DATA_DIR; const { handleComboChat } = await import("../../open-sse/services/combo.ts"); const { lockExactModel, clearAllModelLockouts } = await import("../../open-sse/services/accountFallback.ts"); -const { - getNativeCodexTurnPin, - clearNativeCodexTurnPinsForTests, - NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE_CODE, - NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE_MESSAGE, -} = await import("../../open-sse/services/combo/nativeCodexTurnPin.ts"); +const { getNativeCodexTurnPin, clearNativeCodexTurnPinsForTests } = + await import("../../open-sse/services/combo/nativeCodexTurnPin.ts"); const { recordProviderCooldown, isProviderInCooldown, clearCooldownState } = await import("../../open-sse/services/providerCooldownTracker.ts"); const { PROVIDER_PROFILES } = await import("../../open-sse/config/constants.ts"); @@ -107,7 +103,7 @@ describe("Native Codex Turn Pin model-scoped fallback", () => { }, }; - test("3-Phase Production Scenario: Phase 1 Opus pins -> Phase 2 terminal 400 preserving pin -> Phase 3 new turn routes Gemini", async () => { + test("3-Phase Production Scenario: Phase 1 Opus pins -> Phase 2 pinned model unusable -> pin released, Gemini takes over -> Phase 3 new turn routes Gemini", async () => { const conn1 = await providersDb.createProviderConnection({ provider: "antigravity", authType: "oauth", @@ -163,7 +159,10 @@ describe("Native Codex Turn Pin model-scoped fallback", () => { assert.equal(pin.provider, "antigravity"); assert.equal(pin.connectionId, conn1Id); - // Phase 2: SAME native turn (turn-prod-456) -> Opus becomes locked on all Antigravity accounts + // Phase 2: SAME native turn (turn-prod-456) -> Opus becomes locked on all Antigravity + // accounts. The pinned model is model-scoped unusable, so the turn pin is released and the + // combo falls back to the next healthy model (Gemini) — matching Claude Code's natural + // multi-model fallback for long-running sessions. lockExactModel("antigravity", conn1Id, "claude-opus-4-6-thinking", "quota_exhausted", 60_000); lockExactModel("antigravity", conn2Id, "claude-opus-4-6-thinking", "quota_exhausted", 60_000); lockExactModel("antigravity", "", "claude-opus-4-6-thinking", "quota_exhausted", 60_000); @@ -177,6 +176,18 @@ describe("Native Codex Turn Pin model-scoped fallback", () => { clientManagedResponsesContext: true, handleSingleModel: async (_body, modelStr) => { attemptedModels.push(modelStr); + if (modelStr === geminiModel) { + return new Response( + JSON.stringify({ choices: [{ message: { content: "gemini continued" } }] }), + { + status: 200, + headers: { + "content-type": "application/json", + "x-omniroute-selected-connection-id": conn2Id, + }, + } + ); + } return new Response(JSON.stringify({ error: "unexpected model dispatch" }), { status: 500, }); @@ -187,17 +198,17 @@ describe("Native Codex Turn Pin model-scoped fallback", () => { allCombos: null, }); - // Phase 2 assertions: non-retryable 400 Bad Request terminates turn without reconnect storms - assert.equal(phase2Result.status, 400, "Phase 2 must return non-retryable 400 Bad Request"); - assert.equal(phase2Result.ok, false); - const phase2Body = await phase2Result.json(); - assert.equal(phase2Body.error.code, NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE_CODE); - assert.equal(phase2Body.error.message, NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE_MESSAGE); - assert.equal(phase2Body.error.type, "invalid_request_error"); + // Phase 2 assertions: pin released and the turn continues on Gemini + assert.equal( + phase2Result.ok, + true, + "Phase 2 must succeed after falling back off the locked model" + ); + assert.equal(phase2Result.status, 200); assert.deepEqual( attemptedModels, - [], - "Zero models dispatched during phase 2 (no mid-turn switch to Gemini or Codex)" + [geminiModel], + "Gemini (first healthy model) tried and succeeded; Opus skipped, Codex not called" ); assert.equal( isProviderInCooldown("antigravity", undefined, settings), @@ -205,18 +216,19 @@ describe("Native Codex Turn Pin model-scoped fallback", () => { "Antigravity provider must NOT be marked globally exhausted" ); - // Pinned turn is NOT released mid-turn + // Turn pin is released, then re-pinned to Gemini for the remainder of the turn const pinAfterPhase2 = getNativeCodexTurnPin(nativeTurnBody, comboName); - assert.ok(pinAfterPhase2, "Turn pin must remain active for turn-prod-456"); - assert.equal(pinAfterPhase2.modelStr, opusModel, "Turn pin remains locked to Opus"); + assert.ok(pinAfterPhase2, "New turn pin created after mid-turn fallback"); + assert.equal(pinAfterPhase2.modelStr, geminiModel, "Turn pin now locked to Gemini"); + assert.equal(pinAfterPhase2.provider, "antigravity"); - const terminalLog = phase2LogEntries.find( + const releaseLog = phase2LogEntries.find( (e) => e.tag === "COMBO" && - e.msg.includes("Native Codex turn cannot continue") && - e.msg.includes("model-scoped") + e.msg.includes("Native Codex turn pin released") && + e.msg.includes("model-scoped unavailable") ); - assert.ok(terminalLog, "Should log structured warning about model-scoped turn termination"); + assert.ok(releaseLog, "Should log structured warning about pin release and fallback"); // Phase 3: NEW native turn (turn-prod-457) in same thread -> Opus still locked, normal Combo routing selects Gemini const phase3TurnBody = { @@ -268,7 +280,7 @@ describe("Native Codex Turn Pin model-scoped fallback", () => { assert.equal(pinPhase3.modelStr, geminiModel, "Phase 3 pinned to Gemini"); const pinPhase2Check = getNativeCodexTurnPin(nativeTurnBody, comboName); - assert.equal(pinPhase2Check?.modelStr, opusModel, "Phase 2 turn pin still intact on Opus"); + assert.equal(pinPhase2Check?.modelStr, geminiModel, "Phase 2 turn pin remains on Gemini"); }); test("Pinned connection fails over to sibling connection for same provider+model when sibling healthy", async () => { @@ -491,7 +503,7 @@ describe("Native Codex Turn Pin model-scoped fallback", () => { ); }); - test("Retrying failed Phase 2 turn repeatedly yields terminal 400 without mid-turn cross-model leak", async () => { + test("Retrying failed Phase 2 turn after pinned model becomes unusable succeeds against healthy sibling, re-pinned without flapping", async () => { const conn1 = await providersDb.createProviderConnection({ provider: "antigravity", authType: "oauth", @@ -525,15 +537,221 @@ describe("Native Codex Turn Pin model-scoped fallback", () => { clientManagedResponsesContext: true, handleSingleModel: async (_b, m) => { attempted.push(m); - return new Response(JSON.stringify({ ok: true }), { status: 200 }); + return new Response( + JSON.stringify({ choices: [{ message: { content: "gemini retry" } }] }), + { status: 200, headers: { "x-omniroute-selected-connection-id": conn1.id } } + ); }, isModelAvailable: async () => true, log: createLog(), settings: testSettings, allCombos: null, }); - assert.equal(result.status, 400); - assert.equal(attempted.length, 0); + assert.equal(result.ok, true, `Retry ${retry} must succeed after pin release`); + assert.equal(result.status, 200); + assert.deepEqual( + attempted, + [geminiModel], + `Retry ${retry} must hit Gemini only (Opus skipped, no flapping)` + ); + const pin = getNativeCodexTurnPin(nativeTurnBody, comboName); + assert.equal(pin?.modelStr, geminiModel, `Retry ${retry} re-pins the turn to Gemini`); } }); + + test("Pinned model becomes model-scoped unusable mid-turn -> pin released and combo falls back to healthy models", async () => { + // This test reproduces the user's production failure: + // - Turn 1: Opus succeeds, pin created + // - Turn 2 (same turn_id): Opus locked on all accounts, but combo has Gemini and Codex as healthy fallbacks + // - Expected: pin released, Gemini tried and succeeds, new pin created for Gemini + // - This matches Claude Code's behavior where no turn pin allows natural fallback + + const conn1 = await providersDb.createProviderConnection({ + provider: "antigravity", + authType: "oauth", + name: "Antigravity Account 1", + }); + const conn2 = await providersDb.createProviderConnection({ + provider: "antigravity", + authType: "oauth", + name: "Antigravity Account 2", + }); + await providersDb.createProviderConnection({ + provider: "codex", + authType: "apikey", + name: "Codex Key", + apiKey: "sk-codex-test", + }); + + const conn1Id = conn1.id; + const conn2Id = conn2.id; + + const attemptedModels: string[] = []; + + // Phase 1: Native turn-prod-456 on thread-prod-123 -> Opus succeeds, pin created + const phase1Result = await handleComboChat({ + body: nativeTurnBody, + combo: comboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async (_body, modelStr) => { + attemptedModels.push(modelStr); + return new Response( + JSON.stringify({ choices: [{ message: { content: "opus output" } }] }), + { + status: 200, + headers: { + "content-type": "application/json", + "x-omniroute-selected-connection-id": conn1Id, + }, + } + ); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: testSettings, + allCombos: null, + }); + + assert.equal(phase1Result.ok, true); + assert.deepEqual(attemptedModels, [opusModel]); + + const pin = getNativeCodexTurnPin(nativeTurnBody, comboName); + assert.ok(pin, "Turn pin created after phase 1"); + assert.equal(pin.modelStr, opusModel); + assert.equal(pin.provider, "antigravity"); + assert.equal(pin.connectionId, conn1Id); + + // Phase 2: SAME native turn (turn-prod-456) -> Opus becomes locked on ALL Antigravity accounts + // BUT combo has healthy fallbacks (Gemini, Codex) that should be tried + lockExactModel("antigravity", conn1Id, "claude-opus-4-6-thinking", "quota_exhausted", 60_000); + lockExactModel("antigravity", conn2Id, "claude-opus-4-6-thinking", "quota_exhausted", 60_000); + lockExactModel("antigravity", "", "claude-opus-4-6-thinking", "quota_exhausted", 60_000); + + attemptedModels.length = 0; + const phase2LogEntries: Array<{ level: string; tag: string; msg: string }> = []; + + const phase2Result = await handleComboChat({ + body: nativeTurnBody, + combo: comboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async (_body, modelStr) => { + attemptedModels.push(modelStr); + if (modelStr === geminiModel) { + return new Response( + JSON.stringify({ choices: [{ message: { content: "gemini output" } }] }), + { + status: 200, + headers: { + "content-type": "application/json", + "x-omniroute-selected-connection-id": conn2Id, + }, + } + ); + } + if (modelStr === codexModel) { + return new Response( + JSON.stringify({ choices: [{ message: { content: "codex output" } }] }), + { + status: 200, + headers: { + "content-type": "application/json", + "x-omniroute-selected-connection-id": conn1Id, + }, + } + ); + } + return new Response(JSON.stringify({ error: "unexpected model" }), { status: 500 }); + }, + isModelAvailable: async () => true, + log: createLog(phase2LogEntries), + settings: testSettings, + allCombos: null, + }); + + // Phase 2 assertions: should succeed by falling back to Gemini + assert.equal(phase2Result.ok, true, "Phase 2 must succeed by falling back to healthy model"); + assert.equal(phase2Result.status, 200); + assert.deepEqual( + attemptedModels, + [geminiModel], + "Should try Gemini (first healthy fallback) and succeed; Opus skipped, Codex not called" + ); + + // Pin should be released and re-created for Gemini + const pinAfterPhase2 = getNativeCodexTurnPin(nativeTurnBody, comboName); + assert.ok(pinAfterPhase2, "New turn pin must be created for Gemini"); + assert.equal(pinAfterPhase2.modelStr, geminiModel, "Turn pin updated to Gemini"); + assert.equal(pinAfterPhase2.provider, "antigravity"); + + // Should log warning about pin release + const releaseLog = phase2LogEntries.find( + (e) => + e.tag === "COMBO" && + e.msg.includes("Native Codex turn pin released") && + e.msg.includes("model-scoped unavailable") + ); + assert.ok(releaseLog, "Should log structured warning about pin release and fallback"); + + // Phase 3: Same turn continues with Gemini pinned -> should use Gemini + attemptedModels.length = 0; + const phase3Result = await handleComboChat({ + body: nativeTurnBody, + combo: comboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async (_body, modelStr) => { + attemptedModels.push(modelStr); + return new Response( + JSON.stringify({ choices: [{ message: { content: "gemini continued" } }] }), + { + status: 200, + headers: { "x-omniroute-selected-connection-id": conn2Id }, + } + ); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: testSettings, + allCombos: null, + }); + + assert.equal(phase3Result.ok, true, "Phase 3 must continue with pinned Gemini"); + assert.deepEqual(attemptedModels, [geminiModel], "Should use pinned Gemini"); + + // Phase 4: NEW turn (turn-prod-457) -> normal combo routing, Opus still locked, Gemini selected + const phase4TurnBody = { + stream: false, + client_metadata: { + "x-codex-turn-metadata": JSON.stringify({ + thread_id: "thread-prod-123", + turn_id: "turn-prod-457", + }), + }, + }; + + attemptedModels.length = 0; + const phase4Result = await handleComboChat({ + body: phase4TurnBody, + combo: comboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async (_body, modelStr) => { + attemptedModels.push(modelStr); + return new Response( + JSON.stringify({ choices: [{ message: { content: "gemini new turn" } }] }), + { + status: 200, + headers: { "x-omniroute-selected-connection-id": conn2Id }, + } + ); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: testSettings, + allCombos: null, + }); + + assert.equal(phase4Result.ok, true); + assert.deepEqual(attemptedModels, [geminiModel]); + const pinPhase4 = getNativeCodexTurnPin(phase4TurnBody, comboName); + assert.equal(pinPhase4.modelStr, geminiModel, "New turn pinned to Gemini"); + }); });