diff --git a/changelog.d/fixes/13180-native-codex-turn-auto-resume.md b/changelog.d/fixes/13180-native-codex-turn-auto-resume.md new file mode 100644 index 0000000000..e0618cc419 --- /dev/null +++ b/changelog.d/fixes/13180-native-codex-turn-auto-resume.md @@ -0,0 +1 @@ +- **fix(combo):** auto-resume a pinned native Codex turn on a healthy sibling connection or model when the pinned provider becomes unavailable for a model-scoped reason (quota, model lockout) instead of failing the turn outright — provider-wide circuit-breaker/cooldown state, pending tool calls, opaque continuation state, and partial streams still block resume, and at most one auto-resume happens per logical turn ([#13180](https://github.com/diegosouzapw/OmniRoute/pull/13180)) — thanks @mdigitalbh81 diff --git a/config/quality/file-size-baseline.json b/config/quality/file-size-baseline.json index d3691e7cde..ea136a9e16 100644 --- a/config/quality/file-size-baseline.json +++ b/config/quality/file-size-baseline.json @@ -481,7 +481,7 @@ "open-sse/services/accountFallback.ts": 2517, "open-sse/services/adobeFireflyBrowserLogin.ts": 1401, "open-sse/services/combo.ts": 4080, - "open-sse/services/combo/executeTargetAttempt.ts": 1258, + "open-sse/services/combo/executeTargetAttempt.ts": 1273, "open-sse/translator/response/openai-responses.ts": 1518, "open-sse/utils/cursorAgentProtobuf.ts": 1588, "open-sse/utils/proxyFetch.ts": 1276, @@ -719,5 +719,6 @@ "_rebaseline_2026_09_17_11742_log_boundary_hardening": "PR #11742 (rebase para release/v3.8.51): open-sse/handlers/chatCore.ts 6219->6287. O crescimento e a unica parte da PR que sobreviveu ao tip: endurecimento da fronteira de LOG (mais amplo que a Hard Rule #12, que cobre respostas). Sao 136 linhas adicionadas, das quais ~40 sao chamadas diretas de sanitizacao — sanitizeErrorMessage em erro de plugin (onError), em timeout de semaforo e na failureMessage antes de ela chegar ao console.log e ao call-log; sanitizeUpstreamDetails no log de resposta malformada; getSafeErrorMetadata + try/catch nos pontos onde metadata hostil (Proxy) podia lançar. O resto da PR foi descartado por ja estar no tip (#12506/#12945/#13635 error boundaries, #12429 wreq-js, #11754 aposentadoria do ChatGPT Web) — open-sse/utils/ difere do tip por UMA linha (registro do identificador publico lmarena_stream_error).", "_rebaseline_2026_09_17_13670_allow_auto_combos": "PR #13670 (@fouadSalkini): per-key allowAutoCombos para gatear os combos auto/* embutidos. src/app/api/v1/models/catalog.ts 2075->2117 e src/lib/db/apiKeys.ts 1625->1659. Crescimento e 100% proprio da PR, nao herdado: medido no tip puro, catalog.ts esta em 2074 (abaixo do teto 2075) e apiKeys.ts em 1620 (abaixo de 1625). O aumento e a propria feature — o campo de permissao por chave precisa ser lido, validado e propagado ate o filtro do catalogo, e cada ponto e chamada explicita, nao extraivel sem esconder o gate. Coberto pelos 25 testes da PR. As demais violacoes desta arvore (chatHelpers.ts, chatCore.ts e tests/unit/chatcore-translation-paths.test.ts) sao base-red herdado do tip e nao foram tocadas aqui.", "_rebaseline_2026_09_17c_chatcore_translation_paths_test": "tests/unit/chatcore-translation-paths.test.ts 3447->3449 (#13173, prefixos de cache de meio de conversa do Fable — as assercoes novas do caso). Ultimo teto remanescente da leva de merges de 2026-09-17; os outros dois (chatHelpers.ts e chatCore.ts) foram absorvidos pelos rebaselines das proprias PRs que mergearam depois. Medido no tip limpo.", - "_rebaseline_2026_09_18_13929_antigravity_account_lease_merge": "PR #13929 (Re-land of #10011, @Ardem2025 via @diegosouzapw): the Antigravity account lease, merged onto the current release/v3.8.51 tip (which had independently moved chat.ts to 2520 and auth.ts to 3557 via unrelated PRs). Combined ceiling after merge: src/sse/handlers/chat.ts->2541, src/sse/services/auth.ts->3577. The lease registry, its lifecycle glue and its selection glue were extracted into three NEW modules (src/sse/services/antigravityRoutingState.ts, antigravityLeaseLifecycle.ts, antigravityLeaseSelection.ts) precisely to keep this growth to the call sites; what remains in chat.ts/auth.ts is the wiring itself, which cannot be moved out of the selection loop and the dispatch path. Every added hunk is inert unless ANTIGRAVITY_ACCOUNT_LEASE_ENABLED (default false) is on. Covered by tests/unit/antigravity-routing-state.test.ts, antigravity-lease-lifecycle.test.ts and antigravity-account-lease-flag.test.ts. UPDATE (re-sync 2026-09-18 after trains 3b/4d moved the tip): auth.ts 3577->3582 (same +29 own growth over a tip now at 3552). open-sse/executors/base.ts 1753->1754 is NOT this PR's growth — it is release-tip drift from train 3b (#13002 +5 / #13705 -4 net +1, both merged without a baseline entry); absorbed here by the captain session under the owner-approved train-rebaseline policy so the tip stops failing check:file-size for every PR boarding after it." + "_rebaseline_2026_09_18_13929_antigravity_account_lease_merge": "PR #13929 (Re-land of #10011, @Ardem2025 via @diegosouzapw): the Antigravity account lease, merged onto the current release/v3.8.51 tip (which had independently moved chat.ts to 2520 and auth.ts to 3557 via unrelated PRs). Combined ceiling after merge: src/sse/handlers/chat.ts->2541, src/sse/services/auth.ts->3577. The lease registry, its lifecycle glue and its selection glue were extracted into three NEW modules (src/sse/services/antigravityRoutingState.ts, antigravityLeaseLifecycle.ts, antigravityLeaseSelection.ts) precisely to keep this growth to the call sites; what remains in chat.ts/auth.ts is the wiring itself, which cannot be moved out of the selection loop and the dispatch path. Every added hunk is inert unless ANTIGRAVITY_ACCOUNT_LEASE_ENABLED (default false) is on. Covered by tests/unit/antigravity-routing-state.test.ts, antigravity-lease-lifecycle.test.ts and antigravity-account-lease-flag.test.ts. UPDATE (re-sync 2026-09-18 after trains 3b/4d moved the tip): auth.ts 3577->3582 (same +29 own growth over a tip now at 3552). open-sse/executors/base.ts 1753->1754 is NOT this PR's growth — it is release-tip drift from train 3b (#13002 +5 / #13705 -4 net +1, both merged without a baseline entry); absorbed here by the captain session under the owner-approved train-rebaseline policy so the tip stops failing check:file-size for every PR boarding after it.", + "_rebaseline_2026_09_19_14162_native_codex_auto_resume": "PR #14162 (re-land of #13180, @mdigitalbh81 via @diegosouzapw): native Codex turn auto-resume. open-sse/services/combo/executeTargetAttempt.ts 1258->1273 (+15). Growth is 100% the PR's own, measured against the clean tip (1258 there, gate green): the pin step now advances the logical turn generation and logs the resumed provider/model when the attempt is an auto-resume dispatch, and the generation is passed into pinNativeCodexTurn — the branch has to sit at the pin site because that is the only place the winning target and effective connection are known. Covered by tests/unit/native-codex-auto-resume.test.ts + native-codex-auto-resume-guards.test.ts (15/15) and #13564's native-codex-turn-pin-model-scoped-fallback.test.ts (7/7)." } diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index 5dad34a275..1913b7e5c1 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -90,6 +90,8 @@ export { import { applyNativeCodexTurnPin, areAllPinnedTargetsModelScopedUnusable, + canAutoResumeNativeCodexTurn, + createPinnedModelUnavailableResponse, getNativeCodexTurnPin, releaseNativeCodexTurnPin, } from "./combo/nativeCodexTurnPin.ts"; @@ -131,6 +133,22 @@ import { executeTargetAttempt } from "./combo/executeTargetAttempt.ts"; import type { AttemptLoopDeps, AttemptLoopState } from "./combo/attemptLoopTypes.ts"; import { clearStaleLKGP } from "./combo/staleLkgpClear.ts"; +// Native Codex auto-resume (#13180) rejection reasons that mean the turn either carries +// state unsafe to hand to an untested alternate model (pending tool calls, opaque +// provider-specific continuation state) or has already used its one allowed resume for +// this logical turn. These must terminate the turn rather than fall through to #13564's +// plain "release pin and route naturally" fallback. Every other reason (e.g. the request +// does not use the Responses-API input/messages shape #13180's eligibility check needs) +// falls through unchanged so non-native-turn-shaped Codex requests keep working exactly +// as before #13180. +const NATIVE_CODEX_AUTO_RESUME_UNSAFE_REASONS = new Set([ + "pending_tool_call", + "unsafe_provider_state", + "no_alternate_target", + "no_healthy_alternate_target", + "max_resumes_exceeded", +]); + export { RESET_WINDOW_NAMES, QUOTA_SOFT_DEPRIORITIZE_FACTOR, setCandidateQuotaSoftPenalty }; export { scoreAutoTargets, expandAutoComboCandidatePool }; export type { SingleModelTarget, ResolvedComboTarget }; @@ -776,9 +794,10 @@ async function handleComboChatInner({ }); if (runtimeUnitDispatch) return runtimeUnitDispatch; - const activeNativeTurnPin = clientManagedResponsesContext + let activeNativeTurnPin = clientManagedResponsesContext ? getNativeCodexTurnPin(body, combo.name) : null; + let isAutoResuming = false; // Route new round-robin turns to the specialized handler. A native Codex // continuation with an established provider/account pin must use the common @@ -858,15 +877,60 @@ async function handleComboChatInner({ isModelAvailable, }); if (allPinnedUnusable) { - // All pinned provider+model targets are model-scoped unusable — release - // the pin and fall through to full combo routing so the turn can try - // other models in the combo pool. This matches Claude Code's behavior - // where no turn pin allows natural multi-model fallback. - releaseNativeCodexTurnPin(body as Record, combo.name); - log.warn( - "COMBO", - `Native Codex turn pin released: pinned model ${activeNativeTurnPin.modelStr} model-scoped unavailable; falling back to full combo routing` - ); + const autoResumeEligibility = await canAutoResumeNativeCodexTurn({ + body: body as Record, + comboName: combo.name, + activePin: activeNativeTurnPin, + allTargets: orderedTargets, + resilienceSettings, + quotaCutoffResetWindowConfig, + isModelAvailable, + log, + }); + + if (autoResumeEligibility.eligible === true) { + const selectedAlternate = autoResumeEligibility.selectedTarget; + log.info( + "COMBO", + `Native Codex auto-resume eligible: previous provider/model=${activeNativeTurnPin.provider}/${activeNativeTurnPin.modelStr}, previous logical turn generation=${autoResumeEligibility.previousPin.generation ?? 0}, reason=model_scoped_unavailable` + ); + log.info( + "COMBO", + `Native Codex auto-resume started: previous provider/model=${activeNativeTurnPin.provider}/${activeNativeTurnPin.modelStr}, target provider/model=${selectedAlternate.provider}/${selectedAlternate.modelStr}, target generation=${autoResumeEligibility.nextGeneration}` + ); + const alternateTargets = orderedTargets.filter( + (t) => + t.modelStr === selectedAlternate.modelStr && t.provider === selectedAlternate.provider + ); + orderedTargets = alternateTargets; + activeNativeTurnPin = null; + isAutoResuming = true; + } else if (NATIVE_CODEX_AUTO_RESUME_UNSAFE_REASONS.has(autoResumeEligibility.reason)) { + // These specific rejection reasons mean the turn carries state (pending + // tool calls, opaque provider-specific continuation state) or has + // already exhausted its resume budget, so handing it to an untested + // alternate model via natural combo routing (#13564's plain fallback) + // would be unsafe or would violate #13180's "at most one auto-resume + // per logical turn" bound. Terminate instead of falling through. + targetResolution.quotaShareRelease?.(); + log.warn( + "COMBO", + `Native Codex turn cannot continue: pinned model ${activeNativeTurnPin.modelStr} is unavailable (model-scoped); auto-resume rejected (${autoResumeEligibility.reason}); preserving turn pin and terminating turn` + ); + return createPinnedModelUnavailableResponse(); + } else { + // Every other rejection reason (e.g. the request body does not carry + // the Responses-API `input`/`messages` shape #13180's eligibility + // check needs) means auto-resume simply cannot be evaluated — it says + // nothing about the request being unsafe. Fall back to the plain + // release-and-route-naturally behavior (#13564) so non-native-turn or + // legacy-shaped Codex requests keep working exactly as before #13180. + releaseNativeCodexTurnPin(body as Record, combo.name); + log.warn( + "COMBO", + `Native Codex turn pin released: pinned model ${activeNativeTurnPin.modelStr} model-scoped unavailable; auto-resume not eligible (${autoResumeEligibility.reason}); falling back to full combo routing` + ); + } } else { orderedTargets = pinnedTargets; log.info( @@ -989,6 +1053,7 @@ async function handleComboChatInner({ releaseStickyPinOnFailure, clearStaleLKGP, clientManagedResponsesContext, + nativeCodexAutoResume: isAutoResuming, reasoningTokenBufferEnabled, stickyWeightedLimit, getWeightedStepKeyForTarget, diff --git a/open-sse/services/combo/attemptLoopTypes.ts b/open-sse/services/combo/attemptLoopTypes.ts index c60fa50c5e..d43ffd51c6 100644 --- a/open-sse/services/combo/attemptLoopTypes.ts +++ b/open-sse/services/combo/attemptLoopTypes.ts @@ -102,6 +102,7 @@ export type AttemptLoopDeps = { * gate tests keep compiling; attempt uses defaults when absent. */ clientManagedResponsesContext?: boolean; + nativeCodexAutoResume?: boolean; reasoningTokenBufferEnabled?: boolean; stickyWeightedLimit?: number; getWeightedStepKeyForTarget?: (target: ResolvedComboTarget) => string | null; diff --git a/open-sse/services/combo/executeTargetAttempt.ts b/open-sse/services/combo/executeTargetAttempt.ts index 2c0a261421..21252fb777 100644 --- a/open-sse/services/combo/executeTargetAttempt.ts +++ b/open-sse/services/combo/executeTargetAttempt.ts @@ -71,7 +71,7 @@ import { isModelScoped400, } from "./comboPredicates.ts"; import { applyComboTargetExhaustion } from "./targetExhaustion.ts"; -import { pinNativeCodexTurn } from "./nativeCodexTurnPin.ts"; +import { advanceNativeCodexTurnGeneration, pinNativeCodexTurn } from "./nativeCodexTurnPin.ts"; import { recordComboDecision } from "./decisionTrace.ts"; import { recordProviderCooldown } from "../providerCooldownTracker.ts"; import { @@ -469,12 +469,27 @@ export async function executeTargetAttempt(opts: { } if (Boolean(deps.clientManagedResponsesContext) && effectiveConnectionId) { - pinNativeCodexTurn({ - body: deps.body, - comboName: deps.combo.name, - target, - connectionId: effectiveConnectionId, - }); + if (deps.nativeCodexAutoResume) { + const nextGen = advanceNativeCodexTurnGeneration(deps.body, deps.combo.name); + deps.log.info( + "COMBO", + `Native Codex auto-resume routed: new provider/model=${target.modelStr} on connection ${effectiveConnectionId.slice(0, 8)} (logical turn generation ${nextGen})` + ); + pinNativeCodexTurn({ + body: deps.body, + comboName: deps.combo.name, + target, + connectionId: effectiveConnectionId, + generation: nextGen ?? undefined, + }); + } else { + pinNativeCodexTurn({ + body: deps.body, + comboName: deps.combo.name, + target, + connectionId: effectiveConnectionId, + }); + } } // Success decay: a healthy response walks the model's lockout failure diff --git a/open-sse/services/combo/nativeCodexTurnPin.ts b/open-sse/services/combo/nativeCodexTurnPin.ts index b030d0a47e..20567171f4 100644 --- a/open-sse/services/combo/nativeCodexTurnPin.ts +++ b/open-sse/services/combo/nativeCodexTurnPin.ts @@ -13,18 +13,30 @@ import type { ComboLogger, IsModelAvailable } from "./types.ts"; import type { ResolvedComboTarget } from "./types.ts"; -type NativeTurnPin = { +export type NativeTurnPin = { comboName: string; modelStr: string; provider: string; connectionId: string; createdAt: number; expiresAt: number; + generation?: number; }; +export interface NativeCodexTurnRecord { + comboName: string; + threadId: string; + turnId: string; + activeGeneration: number; + pins: Map; + createdAt: number; + expiresAt: number; +} + +export const MAX_AUTORESUMES_PER_TURN = 1; const TTL_MS = 45 * 60_000; const MAX_PINS = 1_000; -const pins = new Map(); +const turns = new Map(); function record(value: unknown): Record | undefined { return value && typeof value === "object" && !Array.isArray(value) @@ -57,21 +69,51 @@ export function nativeCodexTurnKey( } function prune(now = Date.now()): void { - for (const [key, pin] of pins) if (pin.expiresAt <= now) pins.delete(key); - while (pins.size > MAX_PINS) { - const oldest = pins.keys().next().value as string | undefined; + for (const [key, rec] of turns) if (rec.expiresAt <= now) turns.delete(key); + while (turns.size > MAX_PINS) { + const oldest = turns.keys().next().value as string | undefined; if (!oldest) break; - pins.delete(oldest); + turns.delete(oldest); } } export function getNativeCodexTurnPin( body: Record, - comboName: string + comboName: string, + generation?: number ): NativeTurnPin | null { prune(); const key = nativeCodexTurnKey(body, comboName); - return key ? (pins.get(key) ?? null) : null; + if (!key) return null; + const rec = turns.get(key); + if (!rec) return null; + const gen = generation !== undefined ? generation : rec.activeGeneration; + return rec.pins.get(gen) ?? null; +} + +export function getNativeCodexTurnActiveGeneration( + body: Record, + comboName: string +): number { + prune(); + const key = nativeCodexTurnKey(body, comboName); + if (!key) return 0; + return turns.get(key)?.activeGeneration ?? 0; +} + +export function advanceNativeCodexTurnGeneration( + body: Record, + comboName: string +): number | null { + prune(); + const key = nativeCodexTurnKey(body, comboName); + if (!key) return null; + const rec = turns.get(key); + if (!rec) return null; + rec.activeGeneration += 1; + const now = Date.now(); + rec.expiresAt = now + TTL_MS; + return rec.activeGeneration; } export function pinNativeCodexTurn(args: { @@ -79,10 +121,27 @@ export function pinNativeCodexTurn(args: { comboName: string; target: ResolvedComboTarget; connectionId: string; + generation?: number; }): void { const key = nativeCodexTurnKey(args.body, args.comboName); if (!key || !args.connectionId) return; - const existing = pins.get(key); + let rec = turns.get(key); + const now = Date.now(); + if (!rec) { + const metadata = turnMetadata(args.body); + rec = { + comboName: args.comboName, + threadId: typeof metadata?.thread_id === "string" ? metadata.thread_id : "", + turnId: typeof metadata?.turn_id === "string" ? metadata.turn_id : "", + activeGeneration: 0, + pins: new Map(), + createdAt: now, + expiresAt: now + TTL_MS, + }; + turns.set(key, rec); + } + const gen = args.generation !== undefined ? args.generation : rec.activeGeneration; + const existing = rec.pins.get(gen); if ( existing && (existing.modelStr !== args.target.modelStr || existing.provider !== args.target.provider) @@ -91,15 +150,16 @@ export function pinNativeCodexTurn(args: { } // ConnectionId changes are allowed (failover to sibling connection) // as long as provider + model stay the same. - const now = Date.now(); - pins.set(key, { + rec.pins.set(gen, { comboName: args.comboName, modelStr: args.target.modelStr, provider: args.target.provider, connectionId: args.connectionId, createdAt: existing?.createdAt ?? now, expiresAt: now + TTL_MS, + generation: gen, }); + rec.expiresAt = now + TTL_MS; prune(now); } @@ -153,10 +213,16 @@ export function applyNativeCodexTurnPin( export function revokeNativeCodexTurnPinsForConnection(connectionId: string): number { let revoked = 0; - for (const [key, pin] of pins) { - if (pin.connectionId !== connectionId) continue; - pins.delete(key); - revoked += 1; + for (const [key, rec] of turns) { + for (const [gen, pin] of rec.pins) { + if (pin.connectionId === connectionId) { + rec.pins.delete(gen); + revoked += 1; + } + } + if (rec.pins.size === 0) { + turns.delete(key); + } } return revoked; } @@ -282,9 +348,408 @@ export async function areAllPinnedTargetsModelScopedUnusable( export function releaseNativeCodexTurnPin(body: Record, comboName: string): void { const key = nativeCodexTurnKey(body, comboName); - if (key) pins.delete(key); + if (key) turns.delete(key); } export function clearNativeCodexTurnPinsForTests(): void { - pins.clear(); + turns.clear(); +} + +export function hasUnresolvedToolCalls(body: Record): boolean { + const input: unknown[] = Array.isArray(body.input) + ? body.input + : Array.isArray(body.messages) + ? body.messages + : []; + if (input.length === 0) return false; + + const callCounts = new Map(); + const outputCounts = new Map(); + + const processPart = (part: unknown): boolean => { + if (!part || typeof part !== "object") return true; + const rec = part as Record; + const type = typeof rec.type === "string" ? rec.type : ""; + + if (type === "function_call" || type === "custom_tool_call") { + const callId = + (typeof rec.call_id === "string" && rec.call_id.trim() ? rec.call_id.trim() : "") || + (typeof rec.id === "string" && rec.id.trim() ? rec.id.trim() : ""); + if (!callId) return false; + callCounts.set(callId, (callCounts.get(callId) ?? 0) + 1); + } else if (type === "function_call_output" || type === "custom_tool_call_output") { + const callId = + (typeof rec.call_id === "string" && rec.call_id.trim() ? rec.call_id.trim() : "") || + (typeof rec.id === "string" && rec.id.trim() ? rec.id.trim() : ""); + if (!callId) return false; + outputCounts.set(callId, (outputCounts.get(callId) ?? 0) + 1); + } else if (type === "tool_use") { + const callId = typeof rec.id === "string" && rec.id.trim() ? rec.id.trim() : ""; + if (!callId) return false; + callCounts.set(callId, (callCounts.get(callId) ?? 0) + 1); + } else if (type === "tool_result") { + const callId = + (typeof rec.tool_use_id === "string" && rec.tool_use_id.trim() + ? rec.tool_use_id.trim() + : "") || (typeof rec.id === "string" && rec.id.trim() ? rec.id.trim() : ""); + if (!callId) return false; + outputCounts.set(callId, (outputCounts.get(callId) ?? 0) + 1); + } + return true; + }; + + for (const item of input) { + if (!item || typeof item !== "object") continue; + const rec = item as Record; + const role = typeof rec.role === "string" ? rec.role : ""; + + if (rec.function_call && typeof rec.function_call === "object" && role === "assistant") { + return true; + } + if (role === "function") { + return true; + } + + if (!processPart(rec)) return true; + + if (role === "assistant" && Array.isArray(rec.tool_calls)) { + for (const tc of rec.tool_calls) { + if (!tc || typeof tc !== "object") return true; + const id = + typeof (tc as Record).id === "string" + ? ((tc as Record).id as string).trim() + : ""; + if (!id) return true; + callCounts.set(id, (callCounts.get(id) ?? 0) + 1); + } + } else if (role === "tool") { + const toolCallId = + (typeof rec.tool_call_id === "string" && rec.tool_call_id.trim() + ? rec.tool_call_id.trim() + : "") || + (typeof rec.call_id === "string" && rec.call_id.trim() ? rec.call_id.trim() : "") || + (typeof rec.id === "string" && rec.id.trim() ? rec.id.trim() : ""); + if (!toolCallId) return true; + outputCounts.set(toolCallId, (outputCounts.get(toolCallId) ?? 0) + 1); + } + + if (Array.isArray(rec.content)) { + for (const part of rec.content) { + if (!processPart(part)) return true; + } + } + if (Array.isArray(rec.output)) { + for (const part of rec.output) { + if (!processPart(part)) return true; + } + } + } + + if (callCounts.size === 0 && outputCounts.size === 0) return false; + if (callCounts.size !== outputCounts.size) return true; + for (const [id, count] of callCounts) { + if (count !== 1) return true; + if (outputCounts.get(id) !== 1) return true; + } + for (const [id, count] of outputCounts) { + if (count !== 1) return true; + if (!callCounts.has(id)) return true; + } + + return false; +} + +function isUnsafeItemOrPart(rec: Record): boolean { + const type = typeof rec.type === "string" ? rec.type : ""; + if (type === "item_reference" || type === "redacted_thinking") return true; + if (type === "encrypted_content") return true; + if (typeof rec.previous_response_id === "string" && rec.previous_response_id.trim() !== "") + return true; + if (typeof rec.previousResponseId === "string" && rec.previousResponseId.trim() !== "") + return true; + if (typeof rec.continuation_token === "string" && rec.continuation_token.trim() !== "") + return true; + if (typeof rec.continuationToken === "string" && rec.continuationToken.trim() !== "") return true; + if (typeof rec.encrypted_content === "string" && rec.encrypted_content.trim() !== "") return true; + if (typeof rec.encryptedContent === "string" && rec.encryptedContent.trim() !== "") return true; + if (typeof rec.encrypted_reasoning === "string" && rec.encrypted_reasoning.trim() !== "") + return true; + if (typeof rec.encryptedReasoning === "string" && rec.encryptedReasoning.trim() !== "") + return true; + if (typeof rec.thought_signature === "string" && rec.thought_signature.trim() !== "") return true; + if (typeof rec.thoughtSignature === "string" && rec.thoughtSignature.trim() !== "") return true; + if (typeof rec.signature === "string" && rec.signature.trim() !== "") return true; + if ( + rec.provider_metadata && + typeof rec.provider_metadata === "object" && + Object.keys(rec.provider_metadata as object).length > 0 + ) { + return true; + } + if ( + rec.providerMetadata && + typeof rec.providerMetadata === "object" && + Object.keys(rec.providerMetadata as object).length > 0 + ) { + return true; + } + if ( + rec.provider_data && + typeof rec.provider_data === "object" && + Object.keys(rec.provider_data as object).length > 0 + ) { + return true; + } + if ( + rec.providerData && + typeof rec.providerData === "object" && + Object.keys(rec.providerData as object).length > 0 + ) { + return true; + } + return false; +} + +export function hasProviderSpecificUnsafeContinuationState( + body: Record, + _activePin?: NativeTurnPin +): boolean { + if (typeof body.previous_response_id === "string" && body.previous_response_id.trim() !== "") { + return true; + } + if (typeof body.previousResponseId === "string" && body.previousResponseId.trim() !== "") { + return true; + } + if (typeof body.continuation_token === "string" && body.continuation_token.trim() !== "") { + return true; + } + if (typeof body.continuationToken === "string" && body.continuationToken.trim() !== "") { + return true; + } + if (typeof body.response_id === "string" && body.response_id.trim() !== "") { + return true; + } + if (typeof body.responseId === "string" && body.responseId.trim() !== "") { + return true; + } + if (typeof body.parent_response_id === "string" && body.parent_response_id.trim() !== "") { + return true; + } + if (typeof body.parentResponseId === "string" && body.parentResponseId.trim() !== "") { + return true; + } + if (typeof body.thought_signature === "string" && body.thought_signature.trim() !== "") { + return true; + } + if (typeof body.thoughtSignature === "string" && body.thoughtSignature.trim() !== "") { + return true; + } + if (typeof body.signature === "string" && body.signature.trim() !== "") { + return true; + } + if (typeof body.conversation_id === "string" && body.conversation_id.trim() !== "") { + return true; + } + if (typeof body.conversationId === "string" && body.conversationId.trim() !== "") { + return true; + } + if ( + body.conversation && + typeof body.conversation === "object" && + Object.keys(body.conversation as object).length > 0 + ) { + return true; + } + if ( + body.provider_metadata && + typeof body.provider_metadata === "object" && + Object.keys(body.provider_metadata as object).length > 0 + ) { + return true; + } + if ( + body.providerMetadata && + typeof body.providerMetadata === "object" && + Object.keys(body.providerMetadata as object).length > 0 + ) { + return true; + } + if ( + body.provider_data && + typeof body.provider_data === "object" && + Object.keys(body.provider_data as object).length > 0 + ) { + return true; + } + if ( + body.providerData && + typeof body.providerData === "object" && + Object.keys(body.providerData as object).length > 0 + ) { + return true; + } + + const input: unknown[] = Array.isArray(body.input) + ? body.input + : Array.isArray(body.messages) + ? body.messages + : []; + + for (const item of input) { + if (!item || typeof item !== "object") continue; + const rec = item as Record; + if (isUnsafeItemOrPart(rec)) return true; + + if (Array.isArray(rec.content)) { + for (const part of rec.content) { + if ( + part && + typeof part === "object" && + isUnsafeItemOrPart(part as Record) + ) { + return true; + } + } + } + if (Array.isArray(rec.output)) { + for (const outItem of rec.output) { + if ( + outItem && + typeof outItem === "object" && + isUnsafeItemOrPart(outItem as Record) + ) { + return true; + } + } + } + if (Array.isArray(rec.summary)) { + for (const sumItem of rec.summary) { + if ( + sumItem && + typeof sumItem === "object" && + isUnsafeItemOrPart(sumItem as Record) + ) { + return true; + } + } + } + if (Array.isArray(rec.tool_calls)) { + for (const tc of rec.tool_calls) { + if (tc && typeof tc === "object" && isUnsafeItemOrPart(tc as Record)) { + return true; + } + } + } + } + + return false; +} + +export interface CanAutoResumeNativeCodexTurnOptions { + body: Record; + comboName: string; + activePin: NativeTurnPin; + allTargets: ResolvedComboTarget[]; + resilienceSettings?: ResilienceSettings | null; + quotaCutoffResetWindowConfig?: ResetWindowConfig; + isModelAvailable?: IsModelAvailable; + log?: ComboLogger; +} + +export type AutoResumeDecision = + | { + eligible: true; + nextGeneration: number; + previousPin: NativeTurnPin; + selectedTarget: ResolvedComboTarget; + } + | { + eligible: false; + reason: string; + details?: Record; + }; + +export async function canAutoResumeNativeCodexTurn( + options: CanAutoResumeNativeCodexTurnOptions +): Promise { + const { + body, + comboName, + activePin, + allTargets, + resilienceSettings, + quotaCutoffResetWindowConfig, + isModelAvailable, + } = options; + + const key = nativeCodexTurnKey(body, comboName); + if (!key) return { eligible: false, reason: "invalid_turn_key" }; + + const rec = turns.get(key); + const currentGen = rec?.activeGeneration ?? 0; + if (currentGen >= MAX_AUTORESUMES_PER_TURN) { + return { eligible: false, reason: "max_resumes_exceeded" }; + } + + const inputList = Array.isArray(body.input) + ? body.input + : Array.isArray(body.messages) + ? body.messages + : null; + if (!inputList || inputList.length === 0) { + return { eligible: false, reason: "missing_or_empty_input" }; + } + + if (hasUnresolvedToolCalls(body)) { + return { eligible: false, reason: "pending_tool_call" }; + } + + if (hasProviderSpecificUnsafeContinuationState(body, activePin)) { + return { eligible: false, reason: "unsafe_provider_state" }; + } + + const alternateTargets = allTargets.filter( + (t) => t.modelStr !== activePin.modelStr || t.provider !== activePin.provider + ); + if (alternateTargets.length === 0) { + return { eligible: false, reason: "no_alternate_target" }; + } + + let selectedTarget: ResolvedComboTarget | null = null; + for (const alt of alternateTargets) { + if (alt.provider && alt.provider !== "unknown") { + const cb = getCircuitBreaker(alt.provider); + if (cb.getStatus().state === "OPEN") continue; + } + if ( + resilienceSettings?.providerCooldown?.enabled && + (isProviderInCooldown(alt.provider, alt.connectionId || undefined, resilienceSettings) || + isProviderInCooldown(alt.provider, undefined, resilienceSettings)) + ) { + continue; + } + const unusable = await isPinnedTargetModelScopedUnusable({ + target: alt, + resilienceSettings, + quotaCutoffResetWindowConfig, + comboName, + body, + isModelAvailable, + }); + if (!unusable) { + selectedTarget = alt; + break; + } + } + + if (!selectedTarget) { + return { eligible: false, reason: "no_healthy_alternate_target" }; + } + + return { + eligible: true, + nextGeneration: currentGen + 1, + previousPin: activePin, + selectedTarget, + }; } diff --git a/tests/unit/native-codex-auto-resume-guards.test.ts b/tests/unit/native-codex-auto-resume-guards.test.ts new file mode 100644 index 0000000000..16d7d89eb5 --- /dev/null +++ b/tests/unit/native-codex-auto-resume-guards.test.ts @@ -0,0 +1,416 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +// Pure guard predicates of the native Codex auto-resume path (#13180). Split out of +// tests/unit/native-codex-auto-resume.test.ts (which exercises the full combo flow +// against a scratch DB) so each file stays under the 1200-line test cap; these three +// cases need no DB, no combo config and no fixtures. +const { + hasUnresolvedToolCalls, + hasProviderSpecificUnsafeContinuationState, + MAX_AUTORESUMES_PER_TURN, +} = await import("../../open-sse/services/combo/nativeCodexTurnPin.ts"); + +test("MAX_AUTORESUMES_PER_TURN constant is 1", () => { + assert.equal(MAX_AUTORESUMES_PER_TURN, 1); +}); + +test("hasUnresolvedToolCalls correctly validates 1:1 call-output pairs and rejects duplicates/orphans/nested", () => { + // Empty input: no tool calls + assert.equal(hasUnresolvedToolCalls({}), false); + assert.equal(hasUnresolvedToolCalls({ input: [] }), false); + + // Nested output array with unresolved tool_use is unsafe (true) + assert.equal( + hasUnresolvedToolCalls({ + input: [ + { + type: "function_call_output", + call_id: "c1", + output: [{ type: "tool_use", id: "tu-nested", name: "bash" }], + }, + ], + }), + true + ); + + // Two calls with same call_id and two outputs with same call_id (count=2 != 1) is unsafe (true) + assert.equal( + hasUnresolvedToolCalls({ + input: [ + { type: "function_call", call_id: "c-dup2", name: "cat", arguments: "{}" }, + { type: "function_call", call_id: "c-dup2", name: "cat", arguments: "{}" }, + { type: "function_call_output", call_id: "c-dup2", output: "out1" }, + { type: "function_call_output", call_id: "c-dup2", output: "out2" }, + ], + }), + true + ); + + // Two distinct calls with two distinct matching outputs is safe (false) + assert.equal( + hasUnresolvedToolCalls({ + input: [ + { type: "function_call", call_id: "c-1", name: "cat", arguments: "{}" }, + { type: "function_call", call_id: "c-2", name: "ls", arguments: "{}" }, + { type: "function_call_output", call_id: "c-1", output: "out1" }, + { type: "function_call_output", call_id: "c-2", output: "out2" }, + ], + }), + false + ); + + // Resolved function call (1 call, 1 matching output) + assert.equal( + hasUnresolvedToolCalls({ + input: [ + { type: "message", role: "user", content: "read file" }, + { type: "function_call", call_id: "call-1", name: "cat", arguments: "{}" }, + { type: "function_call_output", call_id: "call-1", output: "hello world" }, + ], + }), + false + ); + + // Unresolved function call (call with no output) + assert.equal( + hasUnresolvedToolCalls({ + input: [ + { type: "message", role: "user", content: "read file" }, + { type: "function_call", call_id: "call-1", name: "cat", arguments: "{}" }, + ], + }), + true + ); + + // Resolved custom tool call + assert.equal( + hasUnresolvedToolCalls({ + input: [ + { type: "message", role: "user", content: "patch file" }, + { type: "custom_tool_call", call_id: "call-2", name: "apply_patch", input: "diff" }, + { type: "custom_tool_call_output", call_id: "call-2", output: "ok" }, + ], + }), + false + ); + + // Unresolved custom tool call + assert.equal( + hasUnresolvedToolCalls({ + input: [ + { type: "message", role: "user", content: "patch file" }, + { type: "custom_tool_call", call_id: "call-2", name: "apply_patch", input: "diff" }, + ], + }), + true + ); + + // Anthropic tool_use and tool_result in content array (resolved) + assert.equal( + hasUnresolvedToolCalls({ + messages: [ + { + role: "assistant", + content: [{ type: "tool_use", id: "tu-1", name: "bash", input: {} }], + }, + { + role: "user", + content: [{ type: "tool_result", tool_use_id: "tu-1", content: "done" }], + }, + ], + }), + false + ); + + // Anthropic tool_use in content array (unresolved) + assert.equal( + hasUnresolvedToolCalls({ + messages: [ + { + role: "assistant", + content: [{ type: "tool_use", id: "tu-1", name: "bash", input: {} }], + }, + ], + }), + true + ); + + // Assistant message tool_calls format (resolved) + assert.equal( + hasUnresolvedToolCalls({ + input: [ + { + type: "message", + role: "assistant", + tool_calls: [{ id: "call-3", type: "function", function: { name: "shell" } }], + }, + { type: "message", role: "tool", tool_call_id: "call-3", content: "done" }, + ], + }), + false + ); + + // Assistant message tool_calls format (unresolved) + assert.equal( + hasUnresolvedToolCalls({ + input: [ + { + type: "message", + role: "assistant", + tool_calls: [{ id: "call-3", type: "function", function: { name: "shell" } }], + }, + ], + }), + true + ); + + // Duplicate tool call ID: two calls with same ID, one output -> unsafe (true) + assert.equal( + hasUnresolvedToolCalls({ + input: [ + { type: "function_call", call_id: "call-dup", name: "cat", arguments: "{}" }, + { type: "function_call", call_id: "call-dup", name: "cat", arguments: "{}" }, + { type: "function_call_output", call_id: "call-dup", output: "res" }, + ], + }), + true + ); + + // Duplicate tool output ID: one call, two outputs with same ID -> unsafe (true) + assert.equal( + hasUnresolvedToolCalls({ + input: [ + { type: "function_call", call_id: "call-dup-out", name: "cat", arguments: "{}" }, + { type: "function_call_output", call_id: "call-dup-out", output: "res1" }, + { type: "function_call_output", call_id: "call-dup-out", output: "res2" }, + ], + }), + true + ); + + // Orphaned tool output: output without matching call -> unsafe (true) + assert.equal( + hasUnresolvedToolCalls({ + input: [{ type: "function_call_output", call_id: "orphan-call", output: "res" }], + }), + true + ); + + // Malformed tool call with empty call_id -> unsafe (true) + assert.equal( + hasUnresolvedToolCalls({ + input: [{ type: "function_call", call_id: "", name: "cat", arguments: "{}" }], + }), + true + ); + + // Legacy unidentifiable function_call -> unsafe (true) + assert.equal( + hasUnresolvedToolCalls({ + input: [{ role: "assistant", function_call: { name: "test", arguments: "{}" } }], + }), + true + ); +}); + +test("hasProviderSpecificUnsafeContinuationState detects opaque provider state at all levels", () => { + // Clean input: safe + assert.equal( + hasProviderSpecificUnsafeContinuationState({ + input: [{ type: "message", role: "user", content: "hello" }], + }), + false + ); + + // conversation_id at root: unsafe + assert.equal( + hasProviderSpecificUnsafeContinuationState({ + conversation_id: "conv_12345", + input: [{ type: "message", role: "user", content: "hello" }], + }), + true + ); + + // conversation object at root: unsafe + assert.equal( + hasProviderSpecificUnsafeContinuationState({ + conversation: { id: "conv_67890" }, + input: [{ type: "message", role: "user", content: "hello" }], + }), + true + ); + + // Item with item-level previous_response_id: unsafe + assert.equal( + hasProviderSpecificUnsafeContinuationState({ + input: [ + { + type: "message", + role: "assistant", + previous_response_id: "resp_nested_prev", + content: "hello", + }, + ], + }), + true + ); + + // Item with item-level continuation_token: unsafe + assert.equal( + hasProviderSpecificUnsafeContinuationState({ + input: [ + { + type: "message", + role: "assistant", + continuation_token: "tok_nested_cont", + content: "hello", + }, + ], + }), + true + ); + + // previous_response_id: unsafe (binds to upstream response store) + assert.equal( + hasProviderSpecificUnsafeContinuationState({ + previous_response_id: "resp_12345_upstream", + input: [{ type: "message", role: "user", content: "hello" }], + }), + true + ); + + // continuation_token: unsafe + assert.equal( + hasProviderSpecificUnsafeContinuationState({ + continuation_token: "tok_opaque_blob", + input: [{ type: "message", role: "user", content: "hello" }], + }), + true + ); + + // response_id: unsafe + assert.equal( + hasProviderSpecificUnsafeContinuationState({ + response_id: "resp_999", + input: [{ type: "message", role: "user", content: "hello" }], + }), + true + ); + + // provider_metadata at root: unsafe + assert.equal( + hasProviderSpecificUnsafeContinuationState({ + provider_metadata: { openai: { message_id: "m1" } }, + input: [{ type: "message", role: "user", content: "hello" }], + }), + true + ); + + // item_reference: unsafe (server-side item ID) + assert.equal( + hasProviderSpecificUnsafeContinuationState({ + input: [{ type: "item_reference", id: "item_abc123" }], + }), + true + ); + + // reasoning item with encrypted_content: unsafe + assert.equal( + hasProviderSpecificUnsafeContinuationState({ + input: [ + { type: "reasoning", encrypted_content: "enc_blob_xyz" }, + { type: "message", role: "user", content: "hello" }, + ], + }), + true + ); + + // thinking item with thought_signature: unsafe + assert.equal( + hasProviderSpecificUnsafeContinuationState({ + input: [ + { type: "thinking", thought_signature: "sig_gemini_blob" }, + { type: "message", role: "user", content: "hello" }, + ], + }), + true + ); + + // redacted_thinking item: unsafe + assert.equal( + hasProviderSpecificUnsafeContinuationState({ + input: [{ type: "redacted_thinking", data: "redacted" }], + }), + true + ); + + // Nested thinking part inside content array with signature: unsafe + assert.equal( + hasProviderSpecificUnsafeContinuationState({ + messages: [ + { + role: "assistant", + content: [ + { type: "thinking", thinking: "deep thought", signature: "sig-xyz" }, + { type: "text", text: "hello" }, + ], + }, + ], + }), + true + ); + + // encrypted_content item: unsafe + assert.equal( + hasProviderSpecificUnsafeContinuationState({ + input: [{ type: "encrypted_content", encrypted_content: "enc_123" }], + }), + true + ); + + // Root body thoughtSignature (camelCase): unsafe + assert.equal( + hasProviderSpecificUnsafeContinuationState({ + thoughtSignature: "sig_camel_case", + input: [{ type: "message", role: "user", content: "hello" }], + }), + true + ); + + // Root body provider_data: unsafe + assert.equal( + hasProviderSpecificUnsafeContinuationState({ + provider_data: { gemini: { candidate_token_count: 50 } }, + input: [{ type: "message", role: "user", content: "hello" }], + }), + true + ); + + // Nested output array with encrypted_content: unsafe + assert.equal( + hasProviderSpecificUnsafeContinuationState({ + input: [ + { + type: "function_call_output", + call_id: "c1", + output: [{ type: "encrypted_content", encrypted_content: "enc_blob" }], + }, + ], + }), + true + ); + + // Nested summary array with thought_signature: unsafe + assert.equal( + hasProviderSpecificUnsafeContinuationState({ + input: [ + { + type: "reasoning", + summary: [{ type: "summary_text", text: "...", thought_signature: "sig" }], + }, + ], + }), + true + ); +}); diff --git a/tests/unit/native-codex-auto-resume.test.ts b/tests/unit/native-codex-auto-resume.test.ts new file mode 100644 index 0000000000..dac462bff4 --- /dev/null +++ b/tests/unit/native-codex-auto-resume.test.ts @@ -0,0 +1,1131 @@ +import test, { describe, beforeEach } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-autoresume-test-")); +const ORIGINAL_DATA_DIR = process.env.DATA_DIR; +process.env.DATA_DIR = TEST_DATA_DIR; + +const { handleComboChat } = await import("../../open-sse/services/combo.ts"); +const { lockExactModel, clearAllModelLockouts } = + await import("../../open-sse/services/accountFallback.ts"); +const { + pinNativeCodexTurn, + advanceNativeCodexTurnGeneration, + getNativeCodexTurnPin, + getNativeCodexTurnActiveGeneration, + clearNativeCodexTurnPinsForTests, + revokeNativeCodexTurnPinsForConnection, + NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE_CODE, +} = await import("../../open-sse/services/combo/nativeCodexTurnPin.ts"); +const { recordProviderCooldown, isProviderInCooldown, clearCooldownState } = + await import("../../open-sse/services/providerCooldownTracker.ts"); +const { PROVIDER_PROFILES } = await import("../../open-sse/config/constants.ts"); +const { getCircuitBreaker, resetAllCircuitBreakers } = + await import("../../src/shared/utils/circuitBreaker.ts"); +const { resolveResilienceSettings } = await import("../../src/lib/resilience/settings.ts"); +const core = await import("../../src/lib/db/core.ts"); +const providersDb = await import("../../src/lib/db/providers.ts"); + +const testSettings = { + resilienceSettings: { + providerCooldown: { + enabled: true, + minRetryCooldownMs: 5000, + maxRetryCooldownMs: 300000, + }, + comboCooldownWait: { enabled: false }, + }, +}; +const settings = resolveResilienceSettings(testSettings); + +function createLog(entries: Array<{ level: string; tag: string; msg: string }> = []) { + return { + info: (tag: string, msg: string) => entries.push({ level: "info", tag, msg }), + warn: (tag: string, msg: string) => entries.push({ level: "warn", tag, msg }), + error: (tag: string, msg: string) => entries.push({ level: "error", tag, msg }), + debug: (tag: string, msg: string) => entries.push({ level: "debug", tag, msg }), + entries, + }; +} + +async function cleanupTestDataDir() { + let lastError: unknown; + for (let attempt = 0; attempt < 5; attempt += 1) { + try { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + return; + } catch (error) { + lastError = error; + await new Promise((resolve) => setTimeout(resolve, 25)); + } + } + if (lastError) throw lastError; +} + +test.after(async () => { + await cleanupTestDataDir(); + process.env.DATA_DIR = ORIGINAL_DATA_DIR; +}); + +beforeEach(async () => { + clearAllModelLockouts(); + clearCooldownState(); + resetAllCircuitBreakers(); + clearNativeCodexTurnPinsForTests(); +}); + +describe("Native Codex Safe Auto-Resume", () => { + const comboName = "Codex"; + const opusModel = "antigravity/claude-opus-4-6-thinking"; + const geminiModel = "antigravity/gemini-3.7-flash-high"; + const codexModel = "codex/gpt-5.5-high"; + + const comboConfig = { + name: comboName, + strategy: "fill-first" as const, + models: [opusModel, geminiModel, codexModel], + config: { + maxRetries: 0, + concurrencyPerModel: 1, + queueTimeoutMs: 1000, + }, + }; + + test("5-Phase Production Scenario: Opus 429 Model Lockout -> Safe Auto-Resume to Gemini -> Subsequent Pinned to Gemini", async () => { + const conn1 = await providersDb.createProviderConnection({ + provider: "antigravity", + authType: "oauth", + name: "Antigravity Account 1", + }); + const conn2 = await providersDb.createProviderConnection({ + provider: "antigravity", + authType: "oauth", + name: "Antigravity Account 2", + }); + await providersDb.createProviderConnection({ + provider: "codex", + authType: "apikey", + name: "Codex Key", + apiKey: "sk-codex-test", + }); + const conn1Id = conn1.id; + const conn2Id = conn2.id; + const attemptedModels: string[] = []; + + const baseTurnMetadata = { + thread_id: "thread-autoresume-123", + turn_id: "turn-autoresume-456", + }; + + // PHASE 1: Opus succeeds for turn-autoresume-456, pin created for generation 0 + const phase1Body = { + stream: false, + client_metadata: { + "x-codex-turn-metadata": JSON.stringify(baseTurnMetadata), + }, + input: [{ type: "message", role: "user", content: "list files then edit" }], + }; + + const phase1Result = await handleComboChat({ + body: phase1Body, + combo: comboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async (_body, modelStr) => { + attemptedModels.push(modelStr); + return new Response( + JSON.stringify({ choices: [{ message: { content: "opus output" } }] }), + { + status: 200, + headers: { + "content-type": "application/json", + "x-omniroute-selected-connection-id": conn1Id, + }, + } + ); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: testSettings, + allCombos: null, + }); + + assert.equal(phase1Result.ok, true); + assert.deepEqual(attemptedModels, [opusModel]); + + const pinGen0 = getNativeCodexTurnPin(phase1Body, comboName, 0); + assert.ok(pinGen0, "Generation 0 pin created after Phase 1"); + assert.equal(pinGen0.modelStr, opusModel); + assert.equal(pinGen0.provider, "antigravity"); + assert.equal(pinGen0.connectionId, conn1Id); + assert.equal(getNativeCodexTurnActiveGeneration(phase1Body, comboName), 0); + + // PHASE 2 & 3: Tool output sent for SAME turn. Opus receives 429 lockout across all connections. + // Safe automatic resume triggers to Gemini. + lockExactModel("antigravity", conn1Id, "claude-opus-4-6-thinking", "quota_exhausted", 60_000); + lockExactModel("antigravity", conn2Id, "claude-opus-4-6-thinking", "quota_exhausted", 60_000); + lockExactModel("antigravity", "", "claude-opus-4-6-thinking", "quota_exhausted", 60_000); + + attemptedModels.length = 0; + const phase2LogEntries: Array<{ level: string; tag: string; msg: string }> = []; + const phase2Body = { + stream: false, + client_metadata: { + "x-codex-turn-metadata": JSON.stringify(baseTurnMetadata), + }, + input: [ + { type: "message", role: "user", content: "list files then edit" }, + { type: "function_call", call_id: "call-1", name: "ls", arguments: "{}" }, + { type: "function_call_output", call_id: "call-1", output: "main.ts\npackage.json" }, + ], + }; + + const phase2Result = await handleComboChat({ + body: phase2Body, + combo: comboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async (_body, modelStr) => { + attemptedModels.push(modelStr); + if (modelStr === geminiModel) { + return new Response( + JSON.stringify({ choices: [{ message: { content: "gemini resumed output" } }] }), + { + status: 200, + headers: { + "content-type": "application/json", + "x-omniroute-selected-connection-id": conn1Id, + }, + } + ); + } + return new Response(JSON.stringify({ error: "unexpected model" }), { status: 500 }); + }, + isModelAvailable: async () => true, + log: createLog(phase2LogEntries), + settings: testSettings, + allCombos: null, + }); + + assert.equal(phase2Result.ok, true, "Phase 2 must succeed automatically on Gemini"); + assert.deepEqual( + attemptedModels, + [geminiModel], + "Only Gemini dispatched (Opus skipped due to lockout)" + ); + + // Verify telemetry logs for auto-resume + const eligibleLog = phase2LogEntries.find((e) => + e.msg.includes("Native Codex auto-resume eligible") + ); + const startedLog = phase2LogEntries.find((e) => + e.msg.includes("Native Codex auto-resume started") + ); + const routedLog = phase2LogEntries.find((e) => + e.msg.includes("Native Codex auto-resume routed") + ); + assert.ok(eligibleLog, "Should log auto-resume eligible"); + assert.ok(startedLog, "Should log auto-resume started"); + assert.ok(routedLog, "Should log auto-resume routed to Gemini"); + + // PHASE 4: Verify generation isolation: Opus gen 0 pin NOT mutated, Gemini is gen 1 pin + const activeGen = getNativeCodexTurnActiveGeneration(phase2Body, comboName); + assert.equal(activeGen, 1, "Active generation is now 1"); + + const gen0PinCheck = getNativeCodexTurnPin(phase2Body, comboName, 0); + assert.ok(gen0PinCheck); + assert.equal(gen0PinCheck.modelStr, opusModel, "Generation 0 pin remains Opus (not mutated)"); + + const gen1PinCheck = getNativeCodexTurnPin(phase2Body, comboName, 1); + assert.ok(gen1PinCheck); + assert.equal(gen1PinCheck.modelStr, geminiModel, "Generation 1 pin is Gemini"); + + // Active pin query without generation returns current active (Gemini) + const currentActivePin = getNativeCodexTurnPin(phase2Body, comboName); + assert.equal(currentActivePin?.modelStr, geminiModel); + + // PHASE 5: Subsequent tool output for SAME turn stays pinned to Gemini + attemptedModels.length = 0; + const phase3Body = { + stream: false, + client_metadata: { + "x-codex-turn-metadata": JSON.stringify(baseTurnMetadata), + }, + input: [ + { type: "message", role: "user", content: "list files then edit" }, + { type: "function_call", call_id: "call-1", name: "ls", arguments: "{}" }, + { type: "function_call_output", call_id: "call-1", output: "main.ts\npackage.json" }, + { type: "function_call", call_id: "call-2", name: "cat", arguments: '{"file":"main.ts"}' }, + { type: "function_call_output", call_id: "call-2", output: "console.log('hi')" }, + ], + }; + + const phase3Result = await handleComboChat({ + body: phase3Body, + combo: comboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async (_body, modelStr) => { + attemptedModels.push(modelStr); + return new Response( + JSON.stringify({ choices: [{ message: { content: "gemini step 2 output" } }] }), + { + status: 200, + headers: { + "content-type": "application/json", + "x-omniroute-selected-connection-id": conn1Id, + }, + } + ); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: testSettings, + allCombos: null, + }); + + assert.equal(phase3Result.ok, true); + assert.deepEqual(attemptedModels, [geminiModel], "Subsequent request stayed pinned to Gemini"); + }); + + test("Opaque continuation state (previous_response_id) rejects auto-resume and returns HTTP 400", async () => { + const conn1 = await providersDb.createProviderConnection({ + provider: "antigravity", + authType: "oauth", + name: "Antigravity Account 1", + }); + + const baseTurnMetadata = { + thread_id: "thread-unsafe-state", + turn_id: "turn-unsafe-state", + }; + + // Phase 1: Opus succeeds + const phase1Body = { + stream: false, + client_metadata: { "x-codex-turn-metadata": JSON.stringify(baseTurnMetadata) }, + input: [{ type: "message", role: "user", content: "hello" }], + }; + + await handleComboChat({ + body: phase1Body, + combo: comboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async () => + new Response(JSON.stringify({ choices: [{ message: { content: "opus" } }] }), { + status: 200, + headers: { "x-omniroute-selected-connection-id": conn1.id }, + }), + isModelAvailable: async () => true, + log: createLog(), + settings: testSettings, + allCombos: null, + }); + + // Lock Opus + lockExactModel("antigravity", conn1.id, "claude-opus-4-6-thinking", "quota_exhausted", 60_000); + lockExactModel("antigravity", "", "claude-opus-4-6-thinking", "quota_exhausted", 60_000); + + // Request with previous_response_id + const unsafeBody = { + stream: false, + previous_response_id: "resp_opus_pinned_upstream", + client_metadata: { "x-codex-turn-metadata": JSON.stringify(baseTurnMetadata) }, + input: [ + { type: "message", role: "user", content: "hello" }, + { type: "function_call", call_id: "c1", name: "ls", arguments: "{}" }, + { type: "function_call_output", call_id: "c1", output: "ok" }, + ], + }; + + const attempted: string[] = []; + const logs: Array<{ level: string; tag: string; msg: string }> = []; + const result = await handleComboChat({ + body: unsafeBody, + combo: comboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async (_b, m) => { + attempted.push(m); + return new Response(JSON.stringify({ ok: true }), { status: 200 }); + }, + isModelAvailable: async () => true, + log: createLog(logs), + settings: testSettings, + allCombos: null, + }); + + assert.equal(result.status, 400); + const data = await result.json(); + assert.equal(data.error.code, NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE_CODE); + assert.equal(attempted.length, 0, "No model dispatched"); + + const rejectLog = logs.find( + (e) => e.msg.includes("auto-resume rejected") && e.msg.includes("unsafe_provider_state") + ); + assert.ok(rejectLog, "Should log rejection reason unsafe_provider_state"); + }); + + test("Pending tool call prevents auto-resume and returns HTTP 400", async () => { + const conn1 = await providersDb.createProviderConnection({ + provider: "antigravity", + authType: "oauth", + name: "Antigravity Account 1", + }); + + const baseTurnMetadata = { + thread_id: "thread-pending-123", + turn_id: "turn-pending-456", + }; + + // Phase 1: Opus succeeds + const phase1Body = { + stream: false, + client_metadata: { "x-codex-turn-metadata": JSON.stringify(baseTurnMetadata) }, + input: [{ type: "message", role: "user", content: "hello" }], + }; + + await handleComboChat({ + body: phase1Body, + combo: comboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async () => + new Response(JSON.stringify({ choices: [{ message: { content: "opus" } }] }), { + status: 200, + headers: { "x-omniroute-selected-connection-id": conn1.id }, + }), + isModelAvailable: async () => true, + log: createLog(), + settings: testSettings, + allCombos: null, + }); + + // Lock Opus + lockExactModel("antigravity", conn1.id, "claude-opus-4-6-thinking", "quota_exhausted", 60_000); + lockExactModel("antigravity", "", "claude-opus-4-6-thinking", "quota_exhausted", 60_000); + + // Request with UNRESOLVED tool call (missing tool output) + const pendingToolBody = { + stream: false, + client_metadata: { "x-codex-turn-metadata": JSON.stringify(baseTurnMetadata) }, + input: [ + { type: "message", role: "user", content: "hello" }, + { type: "function_call", call_id: "call-unresolved", name: "shell", arguments: "{}" }, + ], + }; + + const attempted: string[] = []; + const logs: Array<{ level: string; tag: string; msg: string }> = []; + const result = await handleComboChat({ + body: pendingToolBody, + combo: comboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async (_b, m) => { + attempted.push(m); + return new Response(JSON.stringify({ ok: true }), { status: 200 }); + }, + isModelAvailable: async () => true, + log: createLog(logs), + settings: testSettings, + allCombos: null, + }); + + assert.equal(result.status, 400, "Must return HTTP 400 when tool call unresolved"); + const data = await result.json(); + assert.equal(data.error.code, NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE_CODE); + assert.equal(attempted.length, 0, "No model dispatched"); + + const rejectLog = logs.find( + (e) => e.msg.includes("auto-resume rejected") && e.msg.includes("pending_tool_call") + ); + assert.ok(rejectLog, "Should log rejection reason pending_tool_call"); + }); + + test("Partial stream safety: Opus emits partial SSE stream chunks then fails -> Gemini is NOT dispatched mid-stream", async () => { + const conn1 = await providersDb.createProviderConnection({ + provider: "antigravity", + authType: "oauth", + name: "Antigravity Account 1", + }); + + const baseTurnMetadata = { + thread_id: "thread-partial-stream-safety", + turn_id: "turn-partial-stream-safety", + }; + + // Phase 1: Opus succeeds + const phase1Body = { + stream: true, + client_metadata: { "x-codex-turn-metadata": JSON.stringify(baseTurnMetadata) }, + input: [{ type: "message", role: "user", content: "hello" }], + }; + + await handleComboChat({ + body: phase1Body, + combo: comboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async () => + new Response(JSON.stringify({ choices: [{ message: { content: "opus" } }] }), { + status: 200, + headers: { "x-omniroute-selected-connection-id": conn1.id }, + }), + isModelAvailable: async () => true, + log: createLog(), + settings: testSettings, + allCombos: null, + }); + + // Phase 2: Request is sent. Opus is NOT locked before dispatch. + // Opus returns a stream that emits partial bytes and then aborts/fails. + // Invariant: Gemini MUST NOT be dispatched during this request. + const phase2Body = { + stream: true, + client_metadata: { "x-codex-turn-metadata": JSON.stringify(baseTurnMetadata) }, + input: [ + { type: "message", role: "user", content: "hello" }, + { type: "function_call", call_id: "c1", name: "ls", arguments: "{}" }, + { type: "function_call_output", call_id: "c1", output: "ok" }, + ], + }; + + const attempted: string[] = []; + const stream = new ReadableStream({ + start(controller) { + controller.enqueue( + new TextEncoder().encode('data: {"choices":[{"delta":{"content":"partial output"}}]}\n\n') + ); + controller.error(new Error("Mid-stream connection reset")); + }, + }); + + const _result = await handleComboChat({ + body: phase2Body, + combo: comboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async (_b, m) => { + attempted.push(m); + return new Response(stream, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: testSettings, + allCombos: null, + }); + + // Pinned turn with maxRetries=0 dispatches strictly Opus; Gemini must NOT be called + assert.deepEqual(attempted, [opusModel], "Only pinned Opus dispatched; Gemini never called"); + assert.equal( + getNativeCodexTurnActiveGeneration(phase2Body, comboName), + 0, + "Generation remains 0 on runtime failure" + ); + }); + + test("Sibling connection is preferred over auto-resume", async () => { + const conn1Id = "conn-sib-1"; + const conn2Id = "conn-sib-2"; + + const explicitComboConfig = { + name: comboName, + strategy: "fill-first" as const, + models: [ + { id: "s1", kind: "model" as const, model: opusModel, connectionId: conn1Id, weight: 1 }, + { id: "s2", kind: "model" as const, model: opusModel, connectionId: conn2Id, weight: 1 }, + { id: "s3", kind: "model" as const, model: geminiModel, connectionId: conn1Id, weight: 1 }, + ], + config: { maxRetries: 0, concurrencyPerModel: 1, queueTimeoutMs: 1000 }, + }; + + const turnBody = { + stream: false, + client_metadata: { + "x-codex-turn-metadata": JSON.stringify({ + thread_id: "thread-sib", + turn_id: "turn-sib", + }), + }, + input: [{ type: "message", role: "user", content: "test" }], + }; + + // Phase 1: Opus succeeds on conn1 + await handleComboChat({ + body: turnBody, + combo: explicitComboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async () => + new Response(JSON.stringify({ choices: [{ message: { content: "opus conn1" } }] }), { + status: 200, + headers: { "x-omniroute-selected-connection-id": conn1Id }, + }), + isModelAvailable: async () => true, + log: createLog(), + settings: testSettings, + allCombos: null, + }); + + // Lock ONLY conn1 Opus; conn2 remains healthy + lockExactModel("antigravity", conn1Id, "claude-opus-4-6-thinking", "quota_exhausted", 60_000); + + const attempted: Array<{ modelStr: string; connectionId?: string }> = []; + const result = await handleComboChat({ + body: turnBody, + combo: explicitComboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async (_body, modelStr, target) => { + attempted.push({ modelStr, connectionId: target?.connectionId || undefined }); + return new Response(JSON.stringify({ choices: [{ message: { content: "opus conn2" } }] }), { + status: 200, + headers: { "x-omniroute-selected-connection-id": conn2Id }, + }); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: testSettings, + allCombos: null, + }); + + assert.equal(result.ok, true); + assert.equal(attempted.length, 1); + assert.equal(attempted[0].modelStr, opusModel, "Opus remains pinned to sibling connection"); + assert.equal(attempted[0].connectionId, conn2Id, "Connection failed over to conn2"); + assert.equal( + getNativeCodexTurnActiveGeneration(turnBody, comboName), + 0, + "No generation advance on sibling failover" + ); + }); + + test("No healthy alternate model in combo returns HTTP 400 NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE and does NOT advance generation", async () => { + const conn1 = await providersDb.createProviderConnection({ + provider: "antigravity", + authType: "oauth", + name: "Antigravity Account 1", + }); + + const singleModelComboConfig = { + name: "OpusOnly", + strategy: "fill-first" as const, + models: [opusModel], + config: { maxRetries: 0, concurrencyPerModel: 1, queueTimeoutMs: 1000 }, + }; + + const turnBody = { + stream: false, + client_metadata: { + "x-codex-turn-metadata": JSON.stringify({ + thread_id: "thread-single", + turn_id: "turn-single", + }), + }, + input: [{ type: "message", role: "user", content: "test" }], + }; + + // Phase 1: Opus succeeds + await handleComboChat({ + body: turnBody, + combo: singleModelComboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async () => + new Response(JSON.stringify({ choices: [{ message: { content: "opus" } }] }), { + status: 200, + headers: { "x-omniroute-selected-connection-id": conn1.id }, + }), + isModelAvailable: async () => true, + log: createLog(), + settings: testSettings, + allCombos: null, + }); + + // Lock Opus + lockExactModel("antigravity", conn1.id, "claude-opus-4-6-thinking", "quota_exhausted", 60_000); + lockExactModel("antigravity", "", "claude-opus-4-6-thinking", "quota_exhausted", 60_000); + + const attempted: string[] = []; + const result = await handleComboChat({ + body: turnBody, + combo: singleModelComboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async (_b, m) => { + attempted.push(m); + return new Response(JSON.stringify({ ok: true }), { status: 200 }); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: testSettings, + allCombos: null, + }); + + assert.equal(result.status, 400); + const data = await result.json(); + assert.equal(data.error.code, NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE_CODE); + assert.equal(attempted.length, 0); + assert.equal( + getNativeCodexTurnActiveGeneration(turnBody, "OpusOnly"), + 0, + "Generation must not advance when no alternate target exists" + ); + }); + + test("Provider circuit breaker OPEN does NOT trigger auto-resume", async () => { + const conn1 = await providersDb.createProviderConnection({ + provider: "antigravity", + authType: "oauth", + name: "Antigravity Account 1", + }); + + const turnBody = { + stream: false, + client_metadata: { + "x-codex-turn-metadata": JSON.stringify({ + thread_id: "thread-cb", + turn_id: "turn-cb", + }), + }, + input: [ + { type: "message", role: "user", content: "cmd" }, + { type: "function_call", call_id: "c1", name: "ls", arguments: "{}" }, + { type: "function_call_output", call_id: "c1", output: "ok" }, + ], + }; + + // Phase 1: Opus succeeds + await handleComboChat({ + body: turnBody, + combo: comboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async () => + new Response(JSON.stringify({ choices: [{ message: { content: "opus" } }] }), { + status: 200, + headers: { "x-omniroute-selected-connection-id": conn1.id }, + }), + isModelAvailable: async () => true, + log: createLog(), + settings: testSettings, + allCombos: null, + }); + + // Trip provider circuit breaker (provider-wide failure) + const cb = getCircuitBreaker("antigravity", { failureThreshold: 1, resetTimeout: 60000 }); + try { + await cb.execute(async () => { + throw new Error("simulated 503"); + }); + } catch { + // expected + } + assert.equal(cb.getStatus().state, "OPEN"); + + const attempted: string[] = []; + const result = await handleComboChat({ + body: turnBody, + combo: comboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async (_b, m) => { + attempted.push(m); + return new Response(JSON.stringify({ ok: true }), { status: 200 }); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: testSettings, + allCombos: null, + }); + + assert.equal(result.ok, false, "Should fail due to provider circuit breaker OPEN"); + assert.equal(attempted.length, 0, "No targets attempted"); + assert.equal( + getNativeCodexTurnActiveGeneration(turnBody, comboName), + 0, + "Circuit breaker does not advance generation" + ); + }); + + test("Provider global cooldown does NOT trigger auto-resume", async () => { + const conn1 = await providersDb.createProviderConnection({ + provider: "antigravity", + authType: "oauth", + name: "Antigravity Account 1", + }); + + const turnBody = { + stream: false, + client_metadata: { + "x-codex-turn-metadata": JSON.stringify({ + thread_id: "thread-cd", + turn_id: "turn-cd", + }), + }, + input: [ + { type: "message", role: "user", content: "cmd" }, + { type: "function_call", call_id: "c1", name: "ls", arguments: "{}" }, + { type: "function_call_output", call_id: "c1", output: "ok" }, + ], + }; + + // Phase 1: Opus succeeds + await handleComboChat({ + body: turnBody, + combo: comboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async () => + new Response(JSON.stringify({ choices: [{ message: { content: "opus" } }] }), { + status: 200, + headers: { "x-omniroute-selected-connection-id": conn1.id }, + }), + isModelAvailable: async () => true, + log: createLog(), + settings: testSettings, + allCombos: null, + }); + + // Trigger provider global cooldown. + // Current upstream requires providerFailureThreshold failures before + // the whole provider is considered cooling. + for (let i = 0; i < PROVIDER_PROFILES.oauth.providerFailureThreshold; i += 1) { + recordProviderCooldown("antigravity", undefined, settings); + } + assert.equal(isProviderInCooldown("antigravity", undefined, settings), true); + + const attempted: string[] = []; + const result = await handleComboChat({ + body: turnBody, + combo: comboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async (_b, m) => { + attempted.push(m); + return new Response(JSON.stringify({ ok: true }), { status: 200 }); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: testSettings, + allCombos: null, + }); + + assert.equal(result.ok, false); + assert.equal(attempted.length, 0); + assert.equal(getNativeCodexTurnActiveGeneration(turnBody, comboName), 0); + }); + + test("MAX_AUTORESUMES_PER_TURN = 1 stops cascading: Opus -> Gemini succeeds, but second failure in same turn returns terminal 400", async () => { + const conn1 = await providersDb.createProviderConnection({ + provider: "antigravity", + authType: "oauth", + name: "Antigravity Account 1", + }); + await providersDb.createProviderConnection({ + provider: "codex", + authType: "apikey", + name: "Codex Key", + apiKey: "sk-codex-test", + }); + + const baseTurnMetadata = { + thread_id: "thread-max-cascade-1", + turn_id: "turn-max-cascade-1", + }; + + const turnBody = { + stream: false, + client_metadata: { + "x-codex-turn-metadata": JSON.stringify(baseTurnMetadata), + }, + input: [ + { type: "message", role: "user", content: "cascade test" }, + { type: "function_call", call_id: "c1", name: "ls", arguments: "{}" }, + { type: "function_call_output", call_id: "c1", output: "ok" }, + ], + }; + + // Gen 0: Opus succeeds + await handleComboChat({ + body: turnBody, + combo: comboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async () => + new Response(JSON.stringify({ choices: [{ message: { content: "opus" } }] }), { + status: 200, + headers: { "x-omniroute-selected-connection-id": conn1.id }, + }), + isModelAvailable: async () => true, + log: createLog(), + settings: testSettings, + allCombos: null, + }); + assert.equal(getNativeCodexTurnActiveGeneration(turnBody, comboName), 0); + + // Lock Opus -> 1st auto-resume to Gemini (Gen 1) SUCCEEDS + lockExactModel("antigravity", conn1.id, "claude-opus-4-6-thinking", "quota_exhausted", 60_000); + lockExactModel("antigravity", "", "claude-opus-4-6-thinking", "quota_exhausted", 60_000); + + const resGen1 = await handleComboChat({ + body: turnBody, + combo: comboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async (_b, m) => { + if (m === geminiModel) { + return new Response(JSON.stringify({ choices: [{ message: { content: "gemini" } }] }), { + status: 200, + headers: { "x-omniroute-selected-connection-id": conn1.id }, + }); + } + return new Response(JSON.stringify({ error: "fail" }), { status: 500 }); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: testSettings, + allCombos: null, + }); + + assert.equal(resGen1.ok, true); + assert.equal(getNativeCodexTurnActiveGeneration(turnBody, comboName), 1); + + // Now lock Gemini as well in the SAME turn: 2nd auto-resume MUST BE REJECTED (policy = 1) + lockExactModel("antigravity", conn1.id, "gemini-3.7-flash-high", "quota_exhausted", 60_000); + lockExactModel("antigravity", "", "gemini-3.7-flash-high", "quota_exhausted", 60_000); + + const attemptedGen2: string[] = []; + const logsGen2: Array<{ level: string; tag: string; msg: string }> = []; + const resGen2 = await handleComboChat({ + body: turnBody, + combo: comboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async (_b, m) => { + attemptedGen2.push(m); + return new Response(JSON.stringify({ choices: [{ message: { content: "codex" } }] }), { + status: 200, + }); + }, + isModelAvailable: async () => true, + log: createLog(logsGen2), + settings: testSettings, + allCombos: null, + }); + + assert.equal(resGen2.status, 400, "Must return HTTP 400: max resumes exceeded"); + const dataGen2 = await resGen2.json(); + assert.equal(dataGen2.error.code, NATIVE_CODEX_PINNED_MODEL_UNAVAILABLE_CODE); + assert.equal(attemptedGen2.length, 0, "Codex must NOT be dispatched on 2nd cascade"); + assert.equal( + getNativeCodexTurnActiveGeneration(turnBody, comboName), + 1, + "Generation remains 1" + ); + + const maxLog = logsGen2.find( + (e) => e.msg.includes("auto-resume rejected") && e.msg.includes("max_resumes_exceeded") + ); + assert.ok(maxLog, "Should log max_resumes_exceeded rejection"); + }); + + test("Pin immutability, multi-generation revocation, and TTL expiry cleanup", () => { + const mockBody = { + client_metadata: { + "x-codex-turn-metadata": JSON.stringify({ + thread_id: "thread-immutability", + turn_id: "turn-immutability", + }), + }, + }; + + // Pin Generation 0 on conn-A + pinNativeCodexTurn({ + body: mockBody, + comboName, + target: { + kind: "model", + stepId: "s1", + executionKey: "ek1", + modelStr: opusModel, + provider: "antigravity", + providerId: null, + connectionId: "conn-A", + weight: 1, + label: null, + }, + connectionId: "conn-A", + }); + + const gen0Pin = getNativeCodexTurnPin(mockBody, comboName, 0); + assert.equal(gen0Pin?.modelStr, opusModel); + assert.equal(gen0Pin?.connectionId, "conn-A"); + + // Advance to Gen 1 and pin on conn-B + advanceNativeCodexTurnGeneration(mockBody, comboName); + pinNativeCodexTurn({ + body: mockBody, + comboName, + target: { + kind: "model", + stepId: "s2", + executionKey: "ek2", + modelStr: geminiModel, + provider: "antigravity", + providerId: null, + connectionId: "conn-B", + weight: 1, + label: null, + }, + connectionId: "conn-B", + }); + + // Verify Gen 0 is still Opus on conn-A (immutability check) + const gen0Check = getNativeCodexTurnPin(mockBody, comboName, 0); + assert.equal(gen0Check?.modelStr, opusModel); + assert.equal(gen0Check?.connectionId, "conn-A"); + + // Verify Gen 1 is Gemini on conn-B + const gen1Check = getNativeCodexTurnPin(mockBody, comboName, 1); + assert.equal(gen1Check?.modelStr, geminiModel); + assert.equal(gen1Check?.connectionId, "conn-B"); + + // Revoke pins for conn-A only: Gen 0 is deleted, Gen 1 is intact + const revokedConnA = revokeNativeCodexTurnPinsForConnection("conn-A"); + assert.equal(revokedConnA, 1); + assert.equal(getNativeCodexTurnPin(mockBody, comboName, 0), null); + assert.equal(getNativeCodexTurnPin(mockBody, comboName, 1)?.modelStr, geminiModel); + + // Revoke pins for conn-B: Gen 1 is deleted, turn record is fully removed + const revokedConnB = revokeNativeCodexTurnPinsForConnection("conn-B"); + assert.equal(revokedConnB, 1); + assert.equal(getNativeCodexTurnPin(mockBody, comboName, 1), null); + assert.equal(getNativeCodexTurnActiveGeneration(mockBody, comboName), 0); + }); + + test("Auto-resume dispatch failure does not advance generation or cascade to 3rd model", async () => { + const conn1 = await providersDb.createProviderConnection({ + provider: "antigravity", + authType: "oauth", + name: "Antigravity Account 1", + }); + await providersDb.createProviderConnection({ + provider: "codex", + authType: "apikey", + name: "Codex Key", + apiKey: "sk-codex-test", + }); + + const baseTurnMetadata = { + thread_id: "thread-fail-no-cascade", + turn_id: "turn-fail-no-cascade", + }; + + const turnBody = { + stream: false, + client_metadata: { + "x-codex-turn-metadata": JSON.stringify(baseTurnMetadata), + }, + input: [ + { type: "message", role: "user", content: "cmd" }, + { type: "function_call", call_id: "c1", name: "ls", arguments: "{}" }, + { type: "function_call_output", call_id: "c1", output: "ok" }, + ], + }; + + // Phase 1: Opus succeeds (Gen 0) + await handleComboChat({ + body: turnBody, + combo: comboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async () => + new Response(JSON.stringify({ choices: [{ message: { content: "opus" } }] }), { + status: 200, + headers: { "x-omniroute-selected-connection-id": conn1.id }, + }), + isModelAvailable: async () => true, + log: createLog(), + settings: testSettings, + allCombos: null, + }); + assert.equal(getNativeCodexTurnActiveGeneration(turnBody, comboName), 0); + + // Lock Opus + lockExactModel("antigravity", conn1.id, "claude-opus-4-6-thinking", "quota_exhausted", 60_000); + lockExactModel("antigravity", "", "claude-opus-4-6-thinking", "quota_exhausted", 60_000); + + // Phase 2: Auto-resume routes to Gemini, but Gemini upstream fails (500) + // Invariant: Codex (3rd model) MUST NOT be dispatched in this same request! + const attempted: string[] = []; + const res = await handleComboChat({ + body: turnBody, + combo: comboConfig, + clientManagedResponsesContext: true, + handleSingleModel: async (_b, m) => { + attempted.push(m); + if (m === geminiModel) { + return new Response(JSON.stringify({ error: "gemini temporary 500" }), { status: 500 }); + } + return new Response( + JSON.stringify({ choices: [{ message: { content: "codex leaked" } }] }), + { + status: 200, + } + ); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: testSettings, + allCombos: null, + }); + + assert.equal(res.ok, false); + assert.deepEqual(attempted, [geminiModel], "Only Gemini attempted; no cascade to Codex"); + // Because Gemini failed, active generation was NOT committed to 1 + assert.equal( + getNativeCodexTurnActiveGeneration(turnBody, comboName), + 0, + "Generation remains 0 on dispatch failure" + ); + assert.equal( + getNativeCodexTurnPin(turnBody, comboName, 0)?.modelStr, + opusModel, + "Gen 0 pin remains Opus" + ); + }); + + test("TTL expiry cleans up turn record and prevents memory leak", () => { + const mockBody = { + client_metadata: { + "x-codex-turn-metadata": JSON.stringify({ + thread_id: "thread-ttl-test", + turn_id: "turn-ttl-test", + }), + }, + }; + + pinNativeCodexTurn({ + body: mockBody, + comboName, + target: { + kind: "model", + stepId: "s1", + executionKey: "ek1", + modelStr: opusModel, + provider: "antigravity", + providerId: null, + connectionId: "conn-ttl", + weight: 1, + label: null, + }, + connectionId: "conn-ttl", + }); + + assert.ok(getNativeCodexTurnPin(mockBody, comboName)); + + // Advance Date.now past TTL_MS (45 minutes = 2_700_000 ms) + const origDateNow = Date.now; + try { + Date.now = () => origDateNow() + 46 * 60 * 1000; + // Prune is triggered on read + assert.equal(getNativeCodexTurnPin(mockBody, comboName), null, "Expired pin pruned"); + assert.equal( + getNativeCodexTurnActiveGeneration(mockBody, comboName), + 0, + "Expired turn record pruned" + ); + } finally { + Date.now = origDateNow; + } + }); +});