fix(combo): stop retries when pinned Codex model is unavailable (#12240)

changelog.d/fixes/native-codex-turn-pin-model-scoped-fallback.md

@@ -0,0 +1 @@
+- **fix(combo):** return non-retryable HTTP 400 when all candidates for a pinned native Codex turn are unavailable due to model-scoped lockout, terminating the turn cleanly while preserving turn continuity and enabling standard Combo routing on subsequent turns

open-sse/services/combo.ts

@@ -231,6 +231,8 @@ export {
 import { applyComboTargetExhaustion } from "./combo/targetExhaustion.ts";
 import {
   applyNativeCodexTurnPin,
+  areAllPinnedTargetsModelScopedUnusable,
+  createPinnedModelUnavailableResponse,
   getNativeCodexTurnPin,
   pinNativeCodexTurn,
 } from "./combo/nativeCodexTurnPin.ts";
@@ -960,30 +962,49 @@ async function handleComboChatInner({
   const { stickyWeightedLimit, getWeightedStepKeyForTarget, preScreenMap } = targetResolution;
   const _sticky = targetResolution.sticky;
   let orderedTargets = targetResolution.orderedTargets;
+  const quotaCutoffResetWindowConfig = resolveResetWindowConfig(config as Record<string, unknown>);
+
   if (activeNativeTurnPin) {
-    orderedTargets = applyNativeCodexTurnPin(orderedTargets, activeNativeTurnPin);
-    if (orderedTargets.length === 0) {
-      // #11371: quota-share ordering already reserved a winner slot; release it on
-      // this early exit (idempotent).
+    const pinnedTargets = applyNativeCodexTurnPin(orderedTargets, activeNativeTurnPin);
+    if (pinnedTargets.length === 0) {
+      //#11371: quota-share ordering reserved a winner slot; release on
+      //early exit (idempotent).
       targetResolution.quotaShareRelease?.();
-      return errorResponse(
-        409,
-        "The pinned native Codex turn target is no longer available; the turn cannot be moved to another provider"
+      log.warn(
+        "COMBO",
+        `Native Codex turn cannot continue: pinned model ${activeNativeTurnPin.modelStr} unavailable (target not in combo); preserving turn pin and terminating turn`
+      );
+      return createPinnedModelUnavailableResponse();
+    }
+    const allPinnedUnusable = await areAllPinnedTargetsModelScopedUnusable({
+      pinnedTargets,
+      resilienceSettings,
+      quotaCutoffResetWindowConfig,
+      comboName: combo.name,
+      body: body as Record<string, unknown>,
+      log,
+      isModelAvailable,
+    });
+    if (allPinnedUnusable) {
+      targetResolution.quotaShareRelease?.();
+      log.warn(
+        "COMBO",
+        `Native Codex turn cannot continue: pinned model ${activeNativeTurnPin.modelStr} is unavailable (model-scoped); preserving turn pin and terminating turn`
+      );
+      return createPinnedModelUnavailableResponse();
+    } else {
+      orderedTargets = pinnedTargets;
+      log.info(
+        "COMBO",
+        `Native Codex turn pinned to ${activeNativeTurnPin.modelStr} on connection ${activeNativeTurnPin.connectionId.slice(0, 8)}`
       );
     }
-    log.info(
-      "COMBO",
-      `Native Codex turn pinned to ${activeNativeTurnPin.modelStr} connection ${activeNativeTurnPin.connectionId.slice(0, 8)}`
-    );
   }
 
   // #5923 (Finding #4) — reset-window config for the shared per-target quota-
   // exhaustion cutoff below. The "auto" strategy already applies its own cutoff
   // via buildAutoCandidates/routableCandidates, so this only affects the other
   // 16 strategies (priority, weighted, etc.) that funnel through executeTarget.
-  const quotaCutoffResetWindowConfig = resolveResetWindowConfig(config as Record<string, unknown>);
-
-  // QA P0 diagnostics: record the order in which targets were actually attempted
   // (provider/model ids only) so a terminal combo failure can report the attempt
   // sequence alongside pool size + exhaustion reasons. Accumulates across set retries.
   const comboAttemptOrder: Array<{ provider: string; model: string }> = [];
@@ -1253,7 +1274,8 @@ async function handleComboChatInner({
         if (
           resilienceSettings.providerCooldown.enabled &&
           Boolean(provider && provider !== "unknown") &&
-          isProviderInCooldown(provider, target.connectionId ?? undefined, resilienceSettings)
+          (isProviderInCooldown(provider, target.connectionId ?? undefined, resilienceSettings) ||
+            isProviderInCooldown(provider, undefined, resilienceSettings))
         ) {
           log.info("COMBO", `Skipping ${modelStr} — provider ${provider} in global cooldown`);
           recordComboDecision(traceInvocationId, {

open-sse/services/combo/nativeCodexTurnPin.ts

@@ -1,4 +1,15 @@
 import { createHash } from "node:crypto";
+import { buildErrorBody } from "../../utils/error.ts";
+import { isModelLocked, hasPerModelQuota } from "../accountFallback.ts";
+import { isProviderInCooldown } from "../providerCooldownTracker.ts";
+import { getCircuitBreaker } from "../../../src/shared/utils/circuitBreaker.ts";
+import type { ResilienceSettings } from "../../../src/lib/resilience/settings";
+import { checkCredentialGate } from "../credentialGate.ts";
+import { canAffordRequest } from "../../../src/lib/quota/quotaScheduler.ts";
+import { resolveQuotaExhaustionCutoffForTarget } from "./quotaExhaustionCutoff.ts";
+import type { ResetWindowConfig } from "./quotaScoring.ts";
+import { parseModel } from "../model.ts";
+import type { ComboLogger, IsModelAvailable } from "./types.ts";
 
 import type { ResolvedComboTarget } from "./types.ts";
 
@@ -150,6 +161,124 @@ export function revokeNativeCodexTurnPinsForConnection(connectionId: string): nu
   return revoked;
 }
 
+export const NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE_CODE = "NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE";
+export const NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE_MESSAGE =
+  "The model handling this native Codex turn is no longer available. This turn cannot switch providers or models after output has been emitted. Start a new turn to allow Combo routing to select another model.";
+
+export function createPinnedModelUnavailableResponse(): Response {
+  const body = buildErrorBody(400, NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE_MESSAGE, undefined, {
+    code: NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE_CODE,
+    type: "invalid_request_error",
+  });
+  return new Response(JSON.stringify(body), {
+    status: 400,
+    headers: { "Content-Type": "application/json" },
+  });
+}
+
+export interface CheckPinnedTargetsModelScopedUnusableOptions {
+  pinnedTargets: ResolvedComboTarget[];
+  resilienceSettings?: ResilienceSettings | null;
+  quotaCutoffResetWindowConfig?: ResetWindowConfig;
+  comboName: string;
+  body: Record<string, unknown>;
+  log?: ComboLogger;
+  isModelAvailable?: IsModelAvailable;
+}
+
+export async function isPinnedTargetModelScopedUnusable(args: {
+  target: ResolvedComboTarget;
+  resilienceSettings?: ResilienceSettings | null;
+  quotaCutoffResetWindowConfig?: ResetWindowConfig;
+  comboName: string;
+  body: Record<string, unknown>;
+  log?: ComboLogger;
+  isModelAvailable?: IsModelAvailable;
+}): Promise<boolean> {
+  const {
+    target,
+    resilienceSettings,
+    quotaCutoffResetWindowConfig,
+    comboName,
+    body,
+    log,
+    isModelAvailable,
+  } = args;
+  const provider = target.provider;
+  const connectionId = target.connectionId || "";
+  const rawModel = parseModel(target.modelStr).model || target.modelStr;
+
+  if (provider && provider !== "unknown") {
+    const cb = getCircuitBreaker(provider);
+    if (cb.getStatus().state === "OPEN") return false;
+    if (
+      resilienceSettings?.providerCooldown?.enabled &&
+      (isProviderInCooldown(provider, connectionId || undefined, resilienceSettings) ||
+        isProviderInCooldown(provider, undefined, resilienceSettings))
+    ) {
+      return false;
+    }
+  }
+
+  if (
+    connectionId &&
+    checkCredentialGate(connectionId, provider, target.modelStr).allowed === false
+  ) {
+    return false;
+  }
+
+  if (provider && rawModel && isModelLocked(provider, connectionId, rawModel)) return true;
+
+  if (
+    process.env.OMNIROUTE_QUOTA_AWARE_ROUTING === "1" &&
+    provider &&
+    connectionId &&
+    !canAffordRequest(connectionId, target.modelStr, body).affordable
+  ) {
+    return true;
+  }
+
+  if (provider && connectionId && quotaCutoffResetWindowConfig) {
+    const cutoff = await resolveQuotaExhaustionCutoffForTarget(
+      provider,
+      connectionId,
+      resilienceSettings,
+      quotaCutoffResetWindowConfig,
+      comboName,
+      log ?? { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} }
+    );
+    if (cutoff.blocked) return true;
+  }
+
+  if (isModelAvailable) {
+    const available = await Promise.resolve(isModelAvailable(target.modelStr, target)).catch(
+      () => true
+    );
+    if (
+      !available &&
+      provider &&
+      rawModel &&
+      (isModelLocked(provider, connectionId, rawModel) || hasPerModelQuota(provider, rawModel))
+    ) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+export async function areAllPinnedTargetsModelScopedUnusable(
+  options: CheckPinnedTargetsModelScopedUnusableOptions
+): Promise<boolean> {
+  if (!options.pinnedTargets?.length) return false;
+  for (const target of options.pinnedTargets) {
+    if (!(await isPinnedTargetModelScopedUnusable({ target, ...options }))) {
+      return false;
+    }
+  }
+  return true;
+}
+
 export function clearNativeCodexTurnPinsForTests(): void {
   pins.clear();
 }

tests/unit/native-codex-turn-pin-10379.test.ts

@@ -1,14 +1,26 @@
-// #10379: Native Codex turn pin must allow fill-first failover across
-// compatible connections for the same provider + model.
-import { test } from "node:test";
+//#10379: Native Codex turn pin must allow fill-first failover across
+//compatible connections for the same provider and model.
+import test from "node:test";
 import assert from "node:assert/strict";
 
 const {
   applyNativeCodexTurnPin,
   pinNativeCodexTurn,
   getNativeCodexTurnPin,
+  createPinnedModelUnavailableResponse,
+  NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE_CODE,
+  NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE_MESSAGE,
+  isPinnedTargetModelScopedUnusable,
+  areAllPinnedTargetsModelScopedUnusable,
   clearNativeCodexTurnPinsForTests,
 } = await import("../../open-sse/services/combo/nativeCodexTurnPin.ts");
+const { lockExactModel, clearAllModelLockouts } =
+  await import("../../open-sse/services/accountFallback.ts");
+const { getCircuitBreaker, resetAllCircuitBreakers } =
+  await import("../../src/shared/utils/circuitBreaker.ts");
+const { recordProviderCooldown, clearCooldownState } =
+  await import("../../open-sse/services/providerCooldownTracker.ts");
+const { resolveResilienceSettings } = await import("../../src/lib/resilience/settings.ts");
 
 const BODY = {
   client_metadata: {
@@ -30,7 +42,7 @@ function makeTarget(connectionId: string, model = "gpt-5.6-sol", provider = "cod
   };
 }
 
-test("pinned connection is preferred but siblings are included as fallback", () => {
+test("pinned connection is preferred and siblings are included as fallback", () => {
   clearNativeCodexTurnPinsForTests();
   pinNativeCodexTurn({
     body: BODY,
@@ -39,12 +51,12 @@ test("pinned connection is preferred but siblings are included as fallback", ()
     connectionId: "conn-1",
   });
 
-  const pin = getNativeCodexTurnPin(BODY, "test-combo")!;
+  const pin = getNativeCodexTurnPin(BODY, "test-combo");
   const targets = [makeTarget("conn-1"), makeTarget("conn-2"), makeTarget("conn-3")];
-  const result = applyNativeCodexTurnPin(targets, pin);
+  const result = applyNativeCodexTurnPin(targets, pin!);
 
-  assert.equal(result.length, 3, "all compatible connections should be returned");
-  assert.equal(result[0].connectionId, "conn-1", "pinned connection should be first");
+  assert.equal(result.length, 3, "all compatible connections returned");
+  assert.equal(result[0].connectionId, "conn-1", "pinned connection first");
   assert.deepEqual(result[0].allowedConnectionIds, ["conn-1", "conn-2", "conn-3"]);
 });
 
@@ -57,9 +69,9 @@ test("fallback connections share the same allowedConnectionIds", () => {
     connectionId: "conn-2",
   });
 
-  const pin = getNativeCodexTurnPin(BODY, "test-combo")!;
+  const pin = getNativeCodexTurnPin(BODY, "test-combo");
   const targets = [makeTarget("conn-1"), makeTarget("conn-2"), makeTarget("conn-3")];
-  const result = applyNativeCodexTurnPin(targets, pin);
+  const result = applyNativeCodexTurnPin(targets, pin!);
 
   assert.equal(result[0].connectionId, "conn-2");
   assert.equal(result[1].connectionId, "conn-1");
@@ -78,15 +90,15 @@ test("incompatible targets (different provider/model) are excluded", () => {
     connectionId: "conn-1",
   });
 
-  const pin = getNativeCodexTurnPin(BODY, "test-combo")!;
+  const pin = getNativeCodexTurnPin(BODY, "test-combo");
   const targets = [
     makeTarget("conn-1"),
     makeTarget("conn-2"),
     makeTarget("conn-other", "different-model", "other-provider"),
   ];
-  const result = applyNativeCodexTurnPin(targets, pin);
+  const result = applyNativeCodexTurnPin(targets, pin!);
 
-  assert.equal(result.length, 2, "incompatible target should be excluded");
+  assert.equal(result.length, 2, "incompatible target excluded");
   assert.deepEqual(result[0].allowedConnectionIds, ["conn-1", "conn-2"]);
 });
 
@@ -99,14 +111,14 @@ test("empty result when no compatible targets exist", () => {
     connectionId: "conn-1",
   });
 
-  const pin = getNativeCodexTurnPin(BODY, "test-combo")!;
+  const pin = getNativeCodexTurnPin(BODY, "test-combo");
   const targets = [makeTarget("conn-x", "other-model", "other-provider")];
-  const result = applyNativeCodexTurnPin(targets, pin);
+  const result = applyNativeCodexTurnPin(targets, pin!);
 
   assert.equal(result.length, 0);
 });
 
-test("pinNativeCodexTurn allows connectionId change for same provider+model", () => {
+test("pinNativeCodexTurn allows connectionId change on same provider+model", () => {
   clearNativeCodexTurnPinsForTests();
   pinNativeCodexTurn({
     body: BODY,
@@ -123,8 +135,8 @@ test("pinNativeCodexTurn allows connectionId change for same provider+model", ()
     connectionId: "conn-2",
   });
 
-  const pin = getNativeCodexTurnPin(BODY, "test-combo")!;
-  assert.equal(pin.connectionId, "conn-2", "pin should update to new connection");
+  const pin = getNativeCodexTurnPin(BODY, "test-combo");
+  assert.equal(pin?.connectionId, "conn-2", "pin updated to new connection");
 });
 
 test("pinNativeCodexTurn rejects provider/model change", () => {
@@ -147,3 +159,120 @@ test("pinNativeCodexTurn rejects provider/model change", () => {
     /Native Codex turn target changed/
   );
 });
+
+test("createPinnedModelUnavailableResponse constructs non-retryable HTTP 400 error response", async () => {
+  const response = createPinnedModelUnavailableResponse();
+  assert.equal(response.status, 400);
+  const data = await response.json();
+  assert.equal(data.error.code, NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE_CODE);
+  assert.equal(data.error.message, NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE_MESSAGE);
+  assert.equal(data.error.type, "invalid_request_error");
+});
+
+test("isPinnedTargetModelScopedUnusable distinguishes model lockout from provider/connection outages", async () => {
+  clearAllModelLockouts();
+  clearCooldownState();
+  resetAllCircuitBreakers();
+
+  const resilienceSettings = resolveResilienceSettings({
+    resilienceSettings: {
+      providerCooldown: { enabled: true, minRetryCooldownMs: 5000, maxRetryCooldownMs: 300000 },
+    },
+  });
+
+  const target = makeTarget("conn-1", "antigravity/claude-opus-4-6-thinking", "antigravity");
+
+  // 1. Healthy target -> false
+  assert.equal(
+    await isPinnedTargetModelScopedUnusable({
+      target,
+      comboName: "test-combo",
+      body: BODY,
+      resilienceSettings,
+    }),
+    false
+  );
+
+  // 2. Exact model lockout -> true
+  lockExactModel("antigravity", "conn-1", "claude-opus-4-6-thinking", "quota_exhausted", 60_000);
+  assert.equal(
+    await isPinnedTargetModelScopedUnusable({
+      target,
+      comboName: "test-combo",
+      body: BODY,
+      resilienceSettings,
+    }),
+    true
+  );
+
+  // 3. Provider circuit breaker OPEN -> false (not model-scoped)
+  const cb = getCircuitBreaker("antigravity", { failureThreshold: 1, resetTimeout: 60000 });
+  try {
+    await cb.execute(async () => {
+      throw new Error("503");
+    });
+  } catch {}
+  assert.equal(
+    await isPinnedTargetModelScopedUnusable({
+      target,
+      comboName: "test-combo",
+      body: BODY,
+      resilienceSettings,
+    }),
+    false
+  );
+
+  // Reset breaker, test provider cooldown
+  cb.reset();
+  recordProviderCooldown("antigravity", undefined, resilienceSettings);
+  assert.equal(
+    await isPinnedTargetModelScopedUnusable({
+      target,
+      comboName: "test-combo",
+      body: BODY,
+      resilienceSettings,
+    }),
+    false
+  );
+});
+
+test("areAllPinnedTargetsModelScopedUnusable returns true only when all candidates are model-scoped unusable", async () => {
+  clearAllModelLockouts();
+  clearCooldownState();
+  resetAllCircuitBreakers();
+
+  const target1 = makeTarget("conn-1", "antigravity/claude-opus-4-6-thinking", "antigravity");
+  const target2 = makeTarget("conn-2", "antigravity/claude-opus-4-6-thinking", "antigravity");
+
+  // Both healthy -> false
+  assert.equal(
+    await areAllPinnedTargetsModelScopedUnusable({
+      pinnedTargets: [target1, target2],
+      comboName: "test-combo",
+      body: BODY,
+    }),
+    false
+  );
+
+  // Only target1 locked -> false
+  lockExactModel("antigravity", "conn-1", "claude-opus-4-6-thinking", "quota_exhausted", 60_000);
+  assert.equal(
+    await areAllPinnedTargetsModelScopedUnusable({
+      pinnedTargets: [target1, target2],
+      comboName: "test-combo",
+      body: BODY,
+    }),
+    false
+  );
+
+  // Both locked -> true
+  lockExactModel("antigravity", "conn-2", "claude-opus-4-6-thinking", "quota_exhausted", 60_000);
+  assert.equal(
+    await areAllPinnedTargetsModelScopedUnusable({
+      pinnedTargets: [target1, target2],
+      comboName: "test-combo",
+      body: BODY,
+    }),
+    true
+  );
+});

tests/unit/native-codex-turn-pin-model-scoped-fallback.test.ts

@@ -0,0 +1,533 @@
+import test, { describe, beforeEach } from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-turn-pin-repro-"));
+const ORIGINAL_DATA_DIR = process.env.DATA_DIR;
+process.env.DATA_DIR = TEST_DATA_DIR;
+
+const { handleComboChat } = await import("../../open-sse/services/combo.ts");
+const { lockExactModel, clearAllModelLockouts } =
+  await import("../../open-sse/services/accountFallback.ts");
+const {
+  getNativeCodexTurnPin,
+  clearNativeCodexTurnPinsForTests,
+  NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE_CODE,
+  NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE_MESSAGE,
+} = await import("../../open-sse/services/combo/nativeCodexTurnPin.ts");
+const { recordProviderCooldown, isProviderInCooldown, clearCooldownState } =
+  await import("../../open-sse/services/providerCooldownTracker.ts");
+const { getCircuitBreaker, resetAllCircuitBreakers } =
+  await import("../../src/shared/utils/circuitBreaker.ts");
+const { resolveResilienceSettings } = await import("../../src/lib/resilience/settings.ts");
+const core = await import("../../src/lib/db/core.ts");
+const providersDb = await import("../../src/lib/db/providers.ts");
+
+const testSettings = {
+  resilienceSettings: {
+    providerCooldown: { enabled: true, minRetryCooldownMs: 5000, maxRetryCooldownMs: 300000 },
+  },
+};
+
+const settings = resolveResilienceSettings(testSettings);
+
+function createLog(entries: Array<{ level: string; tag: string; msg: string }> = []) {
+  return {
+    info: (tag: string, msg: string) => {
+      entries.push({ level: "info", tag, msg });
+    },
+    warn: (tag: string, msg: string) => {
+      entries.push({ level: "warn", tag, msg });
+    },
+    error: (tag: string, msg: string) => {
+      entries.push({ level: "error", tag, msg });
+    },
+    debug: (tag: string, msg: string) => {
+      entries.push({ level: "debug", tag, msg });
+    },
+    entries,
+  };
+}
+
+async function cleanupTestDataDir() {
+  let lastError: unknown;
+  for (let attempt = 0; attempt < 5; attempt += 1) {
+    try {
+      core.resetDbInstance();
+      fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+      return;
+    } catch (error) {
+      lastError = error;
+      await new Promise((resolve) => setTimeout(resolve, 25));
+    }
+  }
+  if (lastError) throw lastError;
+}
+
+test.after(async () => {
+  await cleanupTestDataDir();
+  process.env.DATA_DIR = ORIGINAL_DATA_DIR;
+});
+
+beforeEach(async () => {
+  clearAllModelLockouts();
+  clearCooldownState();
+  resetAllCircuitBreakers();
+  clearNativeCodexTurnPinsForTests();
+});
+
+describe("Native Codex Turn Pin model-scoped fallback", () => {
+  const comboName = "Codex";
+  const opusModel = "antigravity/claude-opus-4-6-thinking";
+  const geminiModel = "antigravity/gemini-3.7-flash-high";
+  const codexModel = "codex/gpt-5.5-high";
+
+  const comboConfig = {
+    name: comboName,
+    strategy: "fill-first" as const,
+    models: [opusModel, geminiModel, codexModel],
+    config: {
+      maxRetries: 0,
+      concurrencyPerModel: 1,
+      queueTimeoutMs: 1000,
+    },
+  };
+
+  const nativeTurnBody = {
+    stream: false,
+    client_metadata: {
+      "x-codex-turn-metadata": JSON.stringify({
+        thread_id: "thread-prod-123",
+        turn_id: "turn-prod-456",
+      }),
+    },
+  };
+
+  test("3-Phase Production Scenario: Phase 1 Opus pins -> Phase 2 terminal 400 preserving pin -> Phase 3 new turn routes Gemini", async () => {
+    const conn1 = await providersDb.createProviderConnection({
+      provider: "antigravity",
+      authType: "oauth",
+      name: "Antigravity Account 1",
+    });
+    const conn2 = await providersDb.createProviderConnection({
+      provider: "antigravity",
+      authType: "oauth",
+      name: "Antigravity Account 2",
+    });
+    await providersDb.createProviderConnection({
+      provider: "codex",
+      authType: "apikey",
+      name: "Codex Key",
+      apiKey: "sk-codex-test",
+    });
+
+    const conn1Id = conn1.id;
+    const conn2Id = conn2.id;
+
+    const attemptedModels: string[] = [];
+
+    // Phase 1: Native turn-prod-456 on thread-prod-123 -> Opus succeeds, pin created
+    const phase1Result = await handleComboChat({
+      body: nativeTurnBody,
+      combo: comboConfig,
+      clientManagedResponsesContext: true,
+      handleSingleModel: async (_body, modelStr) => {
+        attemptedModels.push(modelStr);
+        return new Response(
+          JSON.stringify({ choices: [{ message: { content: "opus output" } }] }),
+          {
+            status: 200,
+            headers: {
+              "content-type": "application/json",
+              "x-omniroute-selected-connection-id": conn1Id,
+            },
+          }
+        );
+      },
+      isModelAvailable: async () => true,
+      log: createLog(),
+      settings: testSettings,
+      allCombos: null,
+    });
+
+    assert.equal(phase1Result.ok, true);
+    assert.deepEqual(attemptedModels, [opusModel]);
+
+    const pin = getNativeCodexTurnPin(nativeTurnBody, comboName);
+    assert.ok(pin, "Turn pin created after phase 1");
+    assert.equal(pin.modelStr, opusModel);
+    assert.equal(pin.provider, "antigravity");
+    assert.equal(pin.connectionId, conn1Id);
+
+    // Phase 2: SAME native turn (turn-prod-456) -> Opus becomes locked on all Antigravity accounts
+    lockExactModel("antigravity", conn1Id, "claude-opus-4-6-thinking", "quota_exhausted", 60_000);
+    lockExactModel("antigravity", conn2Id, "claude-opus-4-6-thinking", "quota_exhausted", 60_000);
+    lockExactModel("antigravity", "", "claude-opus-4-6-thinking", "quota_exhausted", 60_000);
+
+    attemptedModels.length = 0;
+    const phase2LogEntries: Array<{ level: string; tag: string; msg: string }> = [];
+
+    const phase2Result = await handleComboChat({
+      body: nativeTurnBody,
+      combo: comboConfig,
+      clientManagedResponsesContext: true,
+      handleSingleModel: async (_body, modelStr) => {
+        attemptedModels.push(modelStr);
+        return new Response(JSON.stringify({ error: "unexpected model dispatch" }), {
+          status: 500,
+        });
+      },
+      isModelAvailable: async () => true,
+      log: createLog(phase2LogEntries),
+      settings: testSettings,
+      allCombos: null,
+    });
+
+    // Phase 2 assertions: non-retryable 400 Bad Request terminates turn without reconnect storms
+    assert.equal(phase2Result.status, 400, "Phase 2 must return non-retryable 400 Bad Request");
+    assert.equal(phase2Result.ok, false);
+    const phase2Body = await phase2Result.json();
+    assert.equal(phase2Body.error.code, NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE_CODE);
+    assert.equal(phase2Body.error.message, NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE_MESSAGE);
+    assert.equal(phase2Body.error.type, "invalid_request_error");
+    assert.deepEqual(
+      attemptedModels,
+      [],
+      "Zero models dispatched during phase 2 (no mid-turn switch to Gemini or Codex)"
+    );
+    assert.equal(
+      isProviderInCooldown("antigravity", undefined, settings),
+      false,
+      "Antigravity provider must NOT be marked globally exhausted"
+    );
+
+    // Pinned turn is NOT released mid-turn
+    const pinAfterPhase2 = getNativeCodexTurnPin(nativeTurnBody, comboName);
+    assert.ok(pinAfterPhase2, "Turn pin must remain active for turn-prod-456");
+    assert.equal(pinAfterPhase2.modelStr, opusModel, "Turn pin remains locked to Opus");
+
+    const terminalLog = phase2LogEntries.find(
+      (e) =>
+        e.tag === "COMBO" &&
+        e.msg.includes("Native Codex turn cannot continue") &&
+        e.msg.includes("model-scoped")
+    );
+    assert.ok(terminalLog, "Should log structured warning about model-scoped turn termination");
+
+    // Phase 3: NEW native turn (turn-prod-457) in same thread -> Opus still locked, normal Combo routing selects Gemini
+    const phase3TurnBody = {
+      stream: false,
+      client_metadata: {
+        "x-codex-turn-metadata": JSON.stringify({
+          thread_id: "thread-prod-123",
+          turn_id: "turn-prod-457",
+        }),
+      },
+    };
+
+    attemptedModels.length = 0;
+    const phase3Result = await handleComboChat({
+      body: phase3TurnBody,
+      combo: comboConfig,
+      clientManagedResponsesContext: true,
+      handleSingleModel: async (_body, modelStr) => {
+        attemptedModels.push(modelStr);
+        if (modelStr === geminiModel) {
+          return new Response(
+            JSON.stringify({ choices: [{ message: { content: "gemini output" } }] }),
+            {
+              status: 200,
+              headers: {
+                "content-type": "application/json",
+                "x-omniroute-selected-connection-id": conn1Id,
+              },
+            }
+          );
+        }
+        return new Response(JSON.stringify({ error: "unexpected model" }), { status: 500 });
+      },
+      isModelAvailable: async () => true,
+      log: createLog(),
+      settings: testSettings,
+      allCombos: null,
+    });
+
+    assert.equal(phase3Result.ok, true, "Phase 3 must succeed with next healthy combo model");
+    assert.deepEqual(
+      attemptedModels,
+      [geminiModel],
+      "Gemini attempted and succeeded; Codex GPT not called"
+    );
+
+    const pinPhase3 = getNativeCodexTurnPin(phase3TurnBody, comboName);
+    assert.ok(pinPhase3, "New turn pin created for phase 3");
+    assert.equal(pinPhase3.modelStr, geminiModel, "Phase 3 pinned to Gemini");
+
+    const pinPhase2Check = getNativeCodexTurnPin(nativeTurnBody, comboName);
+    assert.equal(pinPhase2Check?.modelStr, opusModel, "Phase 2 turn pin still intact on Opus");
+  });
+
+  test("Pinned connection fails over to sibling connection for same provider+model when sibling healthy", async () => {
+    const conn1Id = "conn-1";
+    const conn2Id = "conn-2";
+
+    const explicitComboConfig = {
+      name: comboName,
+      strategy: "fill-first" as const,
+      models: [
+        { id: "s1", kind: "model" as const, model: opusModel, connectionId: conn1Id, weight: 1 },
+        { id: "s2", kind: "model" as const, model: opusModel, connectionId: conn2Id, weight: 1 },
+        { id: "s3", kind: "model" as const, model: geminiModel, connectionId: conn1Id, weight: 1 },
+      ],
+      config: { maxRetries: 0, concurrencyPerModel: 1, queueTimeoutMs: 1000 },
+    };
+
+    // Phase 1: Opus succeeds on conn1
+    await handleComboChat({
+      body: nativeTurnBody,
+      combo: explicitComboConfig,
+      clientManagedResponsesContext: true,
+      handleSingleModel: async () =>
+        new Response(JSON.stringify({ choices: [{ message: { content: "opus conn1" } }] }), {
+          status: 200,
+          headers: { "x-omniroute-selected-connection-id": conn1Id },
+        }),
+      isModelAvailable: async () => true,
+      log: createLog(),
+      settings: testSettings,
+      allCombos: null,
+    });
+
+    // Phase 2: Lock ONLY conn1 Opus, conn2 remains healthy
+    lockExactModel("antigravity", conn1Id, "claude-opus-4-6-thinking", "quota_exhausted", 60_000);
+
+    const attempted: Array<{ modelStr: string; connectionId?: string }> = [];
+    const phase2Result = await handleComboChat({
+      body: nativeTurnBody,
+      combo: explicitComboConfig,
+      clientManagedResponsesContext: true,
+      handleSingleModel: async (_body, modelStr, target) => {
+        attempted.push({ modelStr, connectionId: target?.connectionId ?? undefined });
+        return new Response(JSON.stringify({ choices: [{ message: { content: "opus conn2" } }] }), {
+          status: 200,
+          headers: { "x-omniroute-selected-connection-id": conn2Id },
+        });
+      },
+      isModelAvailable: async () => true,
+      log: createLog(),
+      settings: testSettings,
+      allCombos: null,
+    });
+
+    assert.equal(phase2Result.ok, true);
+    assert.equal(attempted.length, 1);
+    assert.equal(attempted[0].modelStr, opusModel, "Opus must remain pinned");
+    assert.equal(attempted[0].connectionId, conn2Id, "Connection must fail over to conn2");
+  });
+
+  test("Turn pin NOT released when provider circuit breaker is OPEN", async () => {
+    const conn1 = await providersDb.createProviderConnection({
+      provider: "antigravity",
+      authType: "oauth",
+      name: "Antigravity Account 1",
+    });
+
+    await handleComboChat({
+      body: nativeTurnBody,
+      combo: comboConfig,
+      clientManagedResponsesContext: true,
+      handleSingleModel: async () =>
+        new Response(JSON.stringify({ choices: [{ message: { content: "opus" } }] }), {
+          status: 200,
+          headers: { "x-omniroute-selected-connection-id": conn1.id },
+        }),
+      isModelAvailable: async () => true,
+      log: createLog(),
+      settings: testSettings,
+      allCombos: null,
+    });
+
+    // Trip provider circuit breaker
+    const cb = getCircuitBreaker("antigravity", { failureThreshold: 1, resetTimeout: 60000 });
+    try {
+      await cb.execute(async () => {
+        throw new Error("simulated 503");
+      });
+    } catch {
+      // expected
+    }
+    assert.equal(cb.getStatus().state, "OPEN");
+
+    const attempted: string[] = [];
+    const result = await handleComboChat({
+      body: nativeTurnBody,
+      combo: comboConfig,
+      clientManagedResponsesContext: true,
+      handleSingleModel: async (_body, modelStr) => {
+        attempted.push(modelStr);
+        return new Response(JSON.stringify({ choices: [{ message: { content: "ok" } }] }), {
+          status: 200,
+        });
+      },
+      isModelAvailable: async () => true,
+      log: createLog(),
+      settings: testSettings,
+      allCombos: null,
+    });
+
+    assert.equal(result.ok, false, "Should fail due to provider circuit breaker OPEN");
+    assert.equal(attempted.length, 0, "No targets should be attempted");
+  });
+
+  test("Turn pin NOT released when provider in global cooldown", async () => {
+    const conn1 = await providersDb.createProviderConnection({
+      provider: "antigravity",
+      authType: "oauth",
+      name: "Antigravity Account 1",
+    });
+
+    await handleComboChat({
+      body: nativeTurnBody,
+      combo: comboConfig,
+      clientManagedResponsesContext: true,
+      handleSingleModel: async () =>
+        new Response(JSON.stringify({ choices: [{ message: { content: "opus" } }] }), {
+          status: 200,
+          headers: { "x-omniroute-selected-connection-id": conn1.id },
+        }),
+      isModelAvailable: async () => true,
+      log: createLog(),
+      settings: testSettings,
+      allCombos: null,
+    });
+
+    recordProviderCooldown("antigravity", undefined, settings);
+    assert.equal(isProviderInCooldown("antigravity", undefined, settings), true);
+
+    const attempted: string[] = [];
+    const result = await handleComboChat({
+      body: nativeTurnBody,
+      combo: comboConfig,
+      clientManagedResponsesContext: true,
+      handleSingleModel: async (_body, modelStr) => {
+        attempted.push(modelStr);
+        return new Response(JSON.stringify({ choices: [{ message: { content: "ok" } }] }), {
+          status: 200,
+        });
+      },
+      isModelAvailable: async () => true,
+      log: createLog(),
+      settings: testSettings,
+      allCombos: null,
+    });
+
+    assert.equal(result.ok, false);
+    assert.equal(attempted.length, 0);
+  });
+
+  test("Request without active turn pin retains full Combo fallback when first model is locked", async () => {
+    const conn1 = await providersDb.createProviderConnection({
+      provider: "antigravity",
+      authType: "oauth",
+      name: "Antigravity Account 1",
+    });
+    const codexConn = await providersDb.createProviderConnection({
+      provider: "codex",
+      authType: "apikey",
+      name: "Codex Key",
+      apiKey: "sk-codex-test",
+    });
+
+    const unpinnedBody = { stream: false };
+
+    // Lock Opus
+    lockExactModel("antigravity", conn1.id, "claude-opus-4-6-thinking", "quota_exhausted", 60_000);
+    lockExactModel("antigravity", "", "claude-opus-4-6-thinking", "quota_exhausted", 60_000);
+
+    const attempted: string[] = [];
+
+    // Gemini fails transiently (500), falls back to Codex GPT-5.5 in normal combo chain
+    const result = await handleComboChat({
+      body: unpinnedBody,
+      combo: comboConfig,
+      clientManagedResponsesContext: true,
+      handleSingleModel: async (_body, modelStr) => {
+        attempted.push(modelStr);
+        if (modelStr === geminiModel) {
+          return new Response(JSON.stringify({ error: { message: "gemini server error" } }), {
+            status: 500,
+          });
+        }
+        if (modelStr === codexModel) {
+          return new Response(
+            JSON.stringify({ choices: [{ message: { content: "codex output" } }] }),
+            {
+              status: 200,
+              headers: { "x-omniroute-selected-connection-id": codexConn.id },
+            }
+          );
+        }
+        return new Response(JSON.stringify({ error: "unexpected model" }), { status: 500 });
+      },
+      isModelAvailable: async () => true,
+      log: createLog(),
+      settings: testSettings,
+      allCombos: null,
+    });
+
+    assert.equal(result.ok, true);
+    assert.deepEqual(
+      attempted,
+      [geminiModel, codexModel],
+      "Should try Gemini, then Codex in combo order"
+    );
+  });
+
+  test("Retrying failed Phase 2 turn repeatedly yields terminal 400 without mid-turn cross-model leak", async () => {
+    const conn1 = await providersDb.createProviderConnection({
+      provider: "antigravity",
+      authType: "oauth",
+      name: "Antigravity Account 1",
+    });
+
+    // Phase 1: Opus succeeds
+    await handleComboChat({
+      body: nativeTurnBody,
+      combo: comboConfig,
+      clientManagedResponsesContext: true,
+      handleSingleModel: async () =>
+        new Response(JSON.stringify({ choices: [{ message: { content: "opus" } }] }), {
+          status: 200,
+          headers: { "x-omniroute-selected-connection-id": conn1.id },
+        }),
+      isModelAvailable: async () => true,
+      log: createLog(),
+      settings: testSettings,
+      allCombos: null,
+    });
+
+    lockExactModel("antigravity", conn1.id, "claude-opus-4-6-thinking", "quota_exhausted", 60_000);
+    lockExactModel("antigravity", "", "claude-opus-4-6-thinking", "quota_exhausted", 60_000);
+
+    for (let retry = 0; retry < 3; retry += 1) {
+      const attempted: string[] = [];
+      const result = await handleComboChat({
+        body: nativeTurnBody,
+        combo: comboConfig,
+        clientManagedResponsesContext: true,
+        handleSingleModel: async (_b, m) => {
+          attempted.push(m);
+          return new Response(JSON.stringify({ ok: true }), { status: 200 });
+        },
+        isModelAvailable: async () => true,
+        log: createLog(),
+        settings: testSettings,
+        allCombos: null,
+      });
+      assert.equal(result.status, 400);
+      assert.equal(attempted.length, 0);
+    }
+  });
+});