From e5b240479c48e3e807620c52a47eab0a90c1742f Mon Sep 17 00:00:00 2001 From: Xiangzhe <32761048+xz-dev@users.noreply.github.com> Date: Sun, 19 Jul 2026 08:17:49 +0800 Subject: [PATCH] fix(codex): preserve GPT-5.6 reasoning contract (#7012) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(codex): preserve GPT-5.6 reasoning contract * fix(vscode): expose Responses text models * fix(codex): keep GPT-5.6 limits through discovery * fix(ci): extract isUsableChatModel helpers to satisfy complexity ratchet Splitting the supported_endpoints/output_modalities guard clauses into excludesChatAndResponsesEndpoints() / excludesTextOutputModality() drops isUsableChatModel's cyclomatic complexity from 16 to under the ratchet's max of 15 (complexity-ratchets gate: 2057 -> 2056, back at baseline). Behavior is unchanged; existing vscode/codex route tests cover it. Co-authored-by: diegosouzapw * fix(codex): merge capacity limits conservatively (smaller of live vs pinned wins) Resolve the #7012 catalog-merge policy collision: instead of the pinned GPT-5.6 contract always winning for a fixed set of model ids, capacity limits (inputTokenLimit/outputTokenLimit) now merge via mergeCapacityLimitConservatively — Math.min(pinned, live) when both are present, so OmniRoute never promises more context than the account can actually serve. All other overlapping fields still take the live value unconditionally. Guard tests cover both directions (pinned smaller wins / pinned larger loses) at the route level and via an isolated helper-level unit test. Co-authored-by: diegosouzapw --------- Co-authored-by: Xiangzhe Co-authored-by: diegosouzapw --- open-sse/config/providers/shared.ts | 6 +- .../request/openai-responses/helpers.ts | 16 ++- .../request/openai-responses/toResponses.ts | 2 +- .../providers/[id]/models/discovery/codex.ts | 58 +++++++++- src/app/api/v1/vscode/[token]/models/route.ts | 55 ++++++--- src/lib/vscode/reasoningMetadata.ts | 22 +++- src/shared/reasoning/effortStandardization.ts | 6 +- tests/unit/codex-capacity-limit-merge.test.ts | 75 +++++++++++++ tests/unit/codex-gpt56-catalog.test.ts | 2 +- tests/unit/executor-codex-gpt56.test.ts | 31 +++++ .../provider-models-discovery-split.test.ts | 27 ++++- .../unit/provider-models-route-codex.test.ts | 50 ++++++++- tests/unit/vscode-responses-models.test.ts | 106 ++++++++++++++++++ tests/unit/vscode-shared-metadata.test.ts | 18 ++- tests/unit/vscode-token-routes-gpt56.test.ts | 8 +- tests/unit/vscode-token-routes.test.ts | 50 ++++----- 16 files changed, 460 insertions(+), 72 deletions(-) create mode 100644 tests/unit/codex-capacity-limit-merge.test.ts create mode 100644 tests/unit/vscode-responses-models.test.ts diff --git a/open-sse/config/providers/shared.ts b/open-sse/config/providers/shared.ts index fd9d809381..405c13ea32 100644 --- a/open-sse/config/providers/shared.ts +++ b/open-sse/config/providers/shared.ts @@ -312,15 +312,15 @@ export const GPT_5_6_API_CAPABILITIES = { maxOutputTokens: 128000, } as const; -// Codex's live catalog reports a 372K usable input budget for GPT-5.6. -// Keep the reserved 128K output budget explicit, matching the GPT-5.5 catalog contract. +// Codex's live catalog reports a 372K context window for GPT-5.6. +// Keep the input and output limits explicit for catalog consumers that expose them separately. export const GPT_5_6_CODEX_CAPABILITIES = { targetFormat: "openai-responses", toolCalling: true, supportsReasoning: true, supportsVision: true, supportsXHighEffort: true, - contextLength: 500000, + contextLength: 372000, maxInputTokens: 372000, maxOutputTokens: 128000, } as const; diff --git a/open-sse/translator/request/openai-responses/helpers.ts b/open-sse/translator/request/openai-responses/helpers.ts index f50f5132ce..a65a7fe86b 100644 --- a/open-sse/translator/request/openai-responses/helpers.ts +++ b/open-sse/translator/request/openai-responses/helpers.ts @@ -49,9 +49,21 @@ export function imageUrlToText(value: unknown): string { return toString(record.url); } -export function normalizeResponsesReasoningEffort(value: unknown): string { +const CODEX_GPT_5_6_MODEL_PATTERN = + /^gpt-5\.6-(?:sol|terra|luna)(?:-(?:none|low|medium|high|xhigh|max|ultra))?$/; + +function supportsNativeMaxReasoningEffort(model: unknown): boolean { + const normalizedModel = toString(model) + .trim() + .toLowerCase() + .replace(/^(?:codex|cx)\//, ""); + return CODEX_GPT_5_6_MODEL_PATTERN.test(normalizedModel); +} + +export function normalizeResponsesReasoningEffort(value: unknown, model?: unknown): string { const effort = toString(value).toLowerCase(); - return effort === "max" ? "xhigh" : effort; + if (effort !== "max") return effort; + return supportsNativeMaxReasoningEffort(model) ? "max" : "xhigh"; } export function shouldRequestClaudeSummarizedThinking(value: unknown): boolean { diff --git a/open-sse/translator/request/openai-responses/toResponses.ts b/open-sse/translator/request/openai-responses/toResponses.ts index 48eaea8533..859f2d60d7 100644 --- a/open-sse/translator/request/openai-responses/toResponses.ts +++ b/open-sse/translator/request/openai-responses/toResponses.ts @@ -380,7 +380,7 @@ export function openaiToOpenAIResponsesRequest( if (root.reasoning !== undefined) { result.reasoning = root.reasoning; } else if (root.reasoning_effort !== undefined) { - const effort = normalizeResponsesReasoningEffort(root.reasoning_effort); + const effort = normalizeResponsesReasoningEffort(root.reasoning_effort, model ?? root.model); if (effort && effort !== "none") { // Effort-only chat request: default a reasoning summary so the upstream // streams thinking back (see the constant's note above). diff --git a/src/app/api/providers/[id]/models/discovery/codex.ts b/src/app/api/providers/[id]/models/discovery/codex.ts index 7992d36b67..6e71592a3f 100644 --- a/src/app/api/providers/[id]/models/discovery/codex.ts +++ b/src/app/api/providers/[id]/models/discovery/codex.ts @@ -292,9 +292,62 @@ function localCatalogModelToCodexDiscoveryModel( }; } +/** + * Capacity limits (input/output token caps) merge CONSERVATIVELY: the smaller + * of the pinned local-catalog value and the live-discovery value wins, never + * the larger. Overpromising context lets a request run past what the account + * can actually serve — the upstream truncates mid-conversation and can burn a + * combo fallback. Underpromising only leaves capacity on the table, which is + * a performance loss, not a broken request. When only one side has a value, + * that value passes through unchanged (nothing to reconcile). + * + * This is the ONE deliberate exception to "live wins on overlapping fields" + * below — name/apiFormat/supportedEndpoints/supportsThinking/supportsVision + * etc. still take the live value unconditionally. Don't extend this + * conservative rule to other fields without updating the policy comment on + * mergeCodexLiveModelsWithLocalCatalog (#7012). + */ +function mergeCapacityLimitConservatively( + pinnedValue: number | undefined, + liveValue: number | undefined +): number | undefined { + if (typeof pinnedValue === "number" && typeof liveValue === "number") { + return Math.min(pinnedValue, liveValue); + } + return typeof liveValue === "number" ? liveValue : pinnedValue; +} + +function mergeLiveAndLocalCodexModel( + liveModel: CodexDiscoveryModel, + localModel: CodexDiscoveryModel +): CodexDiscoveryModel { + const merged: CodexDiscoveryModel = { ...localModel, ...liveModel }; + const inputTokenLimit = mergeCapacityLimitConservatively( + localModel.inputTokenLimit, + liveModel.inputTokenLimit + ); + const outputTokenLimit = mergeCapacityLimitConservatively( + localModel.outputTokenLimit, + liveModel.outputTokenLimit + ); + if (typeof inputTokenLimit === "number") { + merged.inputTokenLimit = inputTokenLimit; + } else { + delete merged.inputTokenLimit; + } + if (typeof outputTokenLimit === "number") { + merged.outputTokenLimit = outputTokenLimit; + } else { + delete merged.outputTokenLimit; + } + return merged; +} + /** * Live/GitHub discovery is the source of truth for "what exists". * Explicit filters (denylist / predicates) are the policy layer for "what we show". + * Live wins on overlapping fields, EXCEPT capacity limits (input/output token + * caps) — those merge conservatively, see mergeCapacityLimitConservatively. * Do NOT reintroduce curated-only allowlisting as the default path (#6862 / #6859). */ export function mergeCodexLiveModelsWithLocalCatalog( @@ -312,7 +365,10 @@ export function mergeCodexLiveModelsWithLocalCatalog( if (!localModel.id) continue; const normalizedLocal = localCatalogModelToCodexDiscoveryModel(localModel); const existing = merged.get(localModel.id); - merged.set(localModel.id, existing ? { ...normalizedLocal, ...existing } : normalizedLocal); + merged.set( + localModel.id, + existing ? mergeLiveAndLocalCodexModel(existing, normalizedLocal) : normalizedLocal + ); } return Array.from(merged.values()); diff --git a/src/app/api/v1/vscode/[token]/models/route.ts b/src/app/api/v1/vscode/[token]/models/route.ts index 946a8cc866..5d0e39fad5 100644 --- a/src/app/api/v1/vscode/[token]/models/route.ts +++ b/src/app/api/v1/vscode/[token]/models/route.ts @@ -72,29 +72,51 @@ type EnrichModelForVscodeOptions = { preserveNativeId?: boolean; }; +const TEXT_GENERATION_API_FORMATS = new Set([ + "chat-completions", + "responses", + "openai-responses", +]); + +function usesResponsesApi(model: CatalogModelEntry) { + return ( + model.api_format === "responses" || + model.api_format === "openai-responses" || + model.supported_endpoints?.includes("responses") === true + ); +} + +function excludesChatAndResponsesEndpoints(model: CatalogModelEntry) { + return ( + Array.isArray(model.supported_endpoints) && + model.supported_endpoints.length > 0 && + !model.supported_endpoints.includes("chat") && + !model.supported_endpoints.includes("responses") + ); +} + +function excludesTextOutputModality(model: CatalogModelEntry) { + return ( + Array.isArray(model.output_modalities) && + model.output_modalities.length > 0 && + !model.output_modalities.includes("text") + ); +} + function isUsableChatModel(model: CatalogModelEntry) { if (typeof model.owned_by === "string" && model.owned_by.trim().toLowerCase() === "combo") { return false; } if (typeof model.parent === "string" && model.parent.length > 0) return false; if (typeof model.type === "string" && model.type !== "chat") return false; - if (typeof model.api_format === "string" && model.api_format !== "chat-completions") { - return false; - } if ( - Array.isArray(model.supported_endpoints) && - model.supported_endpoints.length > 0 && - !model.supported_endpoints.includes("chat") - ) { - return false; - } - if ( - Array.isArray(model.output_modalities) && - model.output_modalities.length > 0 && - !model.output_modalities.includes("text") + typeof model.api_format === "string" && + !TEXT_GENERATION_API_FORMATS.has(model.api_format) ) { return false; } + if (excludesChatAndResponsesEndpoints(model)) return false; + if (excludesTextOutputModality(model)) return false; return true; } @@ -230,9 +252,10 @@ export function enrichModelForVscode( name: options.preserveNativeId ? getVscodeRawModelDisplayName(presentationModel) : getVscodeModelDisplayName(presentationModel), - url: reasoningEffortValues - ? `${tokenBaseUrl}/responses#models.ai.azure.com` - : `${tokenBaseUrl}/chat/completions#models.ai.azure.com`, + url: + reasoningEffortValues || usesResponsesApi(model) + ? `${tokenBaseUrl}/responses#models.ai.azure.com` + : `${tokenBaseUrl}/chat/completions#models.ai.azure.com`, toolCalling: resolvedCapabilities.toolCalling === true, vision: resolvedCapabilities.supportsVision === true, maxInputTokens: diff --git a/src/lib/vscode/reasoningMetadata.ts b/src/lib/vscode/reasoningMetadata.ts index 03780176d5..52e7262207 100644 --- a/src/lib/vscode/reasoningMetadata.ts +++ b/src/lib/vscode/reasoningMetadata.ts @@ -177,8 +177,28 @@ export function getReasoningVariantBaseModelId(modelId: string) { return matchReasoningEffortSuffix(modelId)?.baseModelId || modelId; } +function getCodexGpt56DefaultReasoningEffort(model: VscodeCatalogModel) { + const modelId = getCatalogModelName(model); + const parsed = parseModel(modelId, ""); + const providerId = (parsed.provider || model.owned_by || "").trim().toLowerCase(); + if (providerId !== "codex" && providerId !== "cx") return undefined; + + const providerModelId = (parsed.model || model.root || modelId.split("/").pop() || modelId) + .trim() + .toLowerCase(); + const match = providerModelId.match( + /^gpt-5\.6-(sol|terra|luna)(?:-(?:none|low|medium|high|xhigh|max|ultra))?$/ + ); + if (!match) return undefined; + return match[1] === "sol" ? "low" : "medium"; +} + export function getDefaultReasoningEffort(model: VscodeCatalogModel, supportedValues?: string[]) { - return inferSelectedReasoningEffort(model, supportedValues) || DEFAULT_REASONING_EFFORT; + return ( + inferSelectedReasoningEffort(model, supportedValues) || + getCodexGpt56DefaultReasoningEffort(model) || + DEFAULT_REASONING_EFFORT + ); } export function buildReasoningConfigSchema( diff --git a/src/shared/reasoning/effortStandardization.ts b/src/shared/reasoning/effortStandardization.ts index 611433b8ff..f6924f4cc1 100644 --- a/src/shared/reasoning/effortStandardization.ts +++ b/src/shared/reasoning/effortStandardization.ts @@ -18,7 +18,7 @@ export const CANONICAL_EFFORT_VALUES = ["none", "low", "medium", "high", "xhigh" export type CanonicalEffort = (typeof CANONICAL_EFFORT_VALUES)[number]; -/** Add provider-native GPT-5.6 effort levels without widening the global request vocabulary. */ +/** Use provider-native GPT-5.6 effort levels without widening the global request vocabulary. */ export function extendCodexGpt56EffortValues( provider: string | null | undefined, model: string | null | undefined, @@ -39,8 +39,8 @@ export function extendCodexGpt56EffortValues( ); if (!match) return values; - const additions = match[1] === "luna" ? ["max"] : ["max", "ultra"]; - return [...new Set([...values, ...additions])]; + const nativeValues = ["low", "medium", "high", "xhigh", "max"]; + return match[1] === "luna" ? nativeValues : [...nativeValues, "ultra"]; } /** diff --git a/tests/unit/codex-capacity-limit-merge.test.ts b/tests/unit/codex-capacity-limit-merge.test.ts new file mode 100644 index 0000000000..ec60a7580d --- /dev/null +++ b/tests/unit/codex-capacity-limit-merge.test.ts @@ -0,0 +1,75 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +// Isolated unit coverage for the capacity-limit merge policy, independent of +// the /api/providers/[id]/models route. See +// src/app/api/providers/[id]/models/discovery/codex.ts::mergeCapacityLimitConservatively +// and the guard test in tests/unit/provider-models-route-codex.test.ts (#7012). +const codexDiscovery = await import( + "../../src/app/api/providers/[id]/models/discovery/codex.ts" +); + +function liveModel(overrides: Record) { + return { + id: "gpt-5.6-sol", + name: "GPT 5.6 Sol Live", + owned_by: "codex" as const, + apiFormat: "responses" as const, + supportedEndpoints: ["responses"] as ["responses"], + ...overrides, + }; +} + +test("mergeCodexLiveModelsWithLocalCatalog: pinned (local) limit wins when it is SMALLER than live", () => { + const merged = codexDiscovery.mergeCodexLiveModelsWithLocalCatalog( + [liveModel({ inputTokenLimit: 999999, outputTokenLimit: 999999 })], + [ + { + id: "gpt-5.6-sol", + name: "GPT 5.6 Sol", + maxInputTokens: 372000, + maxOutputTokens: 128000, + }, + ] + ); + const model = merged.find((m) => m.id === "gpt-5.6-sol"); + // Pinned (372000/128000) is smaller than live (999999/999999) — the smaller, + // safer value must win so requests never overrun the account's real budget. + assert.equal(model?.inputTokenLimit, 372000); + assert.equal(model?.outputTokenLimit, 128000); + // Non-capacity fields are unaffected — live still wins on those. + assert.equal(model?.name, "GPT 5.6 Sol Live"); +}); + +test("mergeCodexLiveModelsWithLocalCatalog: live limit wins when the pinned (local) value is LARGER", () => { + const merged = codexDiscovery.mergeCodexLiveModelsWithLocalCatalog( + [liveModel({ inputTokenLimit: 100000, outputTokenLimit: 50000 })], + [ + { + id: "gpt-5.6-sol", + name: "GPT 5.6 Sol", + maxInputTokens: 372000, + maxOutputTokens: 128000, + }, + ] + ); + const model = merged.find((m) => m.id === "gpt-5.6-sol"); + // Live (100000/50000) is smaller than pinned (372000/128000) here — the + // smaller live value must win, not the larger pinned contract. + assert.equal(model?.inputTokenLimit, 100000); + assert.equal(model?.outputTokenLimit, 50000); +}); + +test("mergeCodexLiveModelsWithLocalCatalog: uses whichever side has a value when only one side defines it", () => { + const liveOnly = codexDiscovery.mergeCodexLiveModelsWithLocalCatalog( + [liveModel({ inputTokenLimit: 200000 })], + [{ id: "gpt-5.6-sol", name: "GPT 5.6 Sol" }] + ); + assert.equal(liveOnly.find((m) => m.id === "gpt-5.6-sol")?.inputTokenLimit, 200000); + + const pinnedOnly = codexDiscovery.mergeCodexLiveModelsWithLocalCatalog( + [liveModel({ inputTokenLimit: undefined })], + [{ id: "gpt-5.6-sol", name: "GPT 5.6 Sol", maxInputTokens: 372000 }] + ); + assert.equal(pinnedOnly.find((m) => m.id === "gpt-5.6-sol")?.inputTokenLimit, 372000); +}); diff --git a/tests/unit/codex-gpt56-catalog.test.ts b/tests/unit/codex-gpt56-catalog.test.ts index 4c5d3c6d30..a24c307a27 100644 --- a/tests/unit/codex-gpt56-catalog.test.ts +++ b/tests/unit/codex-gpt56-catalog.test.ts @@ -36,7 +36,7 @@ test("Codex catalog exposes the GPT-5.6 lineup in configured priority order", () for (const modelId of expectedIds) { const model = models.find((entry) => entry.id === modelId); assert.ok(model, `codex must expose ${modelId}`); - assert.equal(model.contextLength, 500000); + assert.equal(model.contextLength, 372000); assert.equal(model.maxInputTokens, 372000); assert.equal(model.maxOutputTokens, 128000); assert.equal(model.targetFormat, "openai-responses"); diff --git a/tests/unit/executor-codex-gpt56.test.ts b/tests/unit/executor-codex-gpt56.test.ts index 3e5d2b9506..4c2fd01a1e 100644 --- a/tests/unit/executor-codex-gpt56.test.ts +++ b/tests/unit/executor-codex-gpt56.test.ts @@ -2,6 +2,37 @@ import test from "node:test"; import assert from "node:assert/strict"; import { CodexExecutor } from "../../open-sse/executors/codex.ts"; +import { openaiToOpenAIResponsesRequest } from "../../open-sse/translator/request/openai-responses/toResponses.ts"; + +test("Chat-to-Codex translation preserves max only for GPT-5.6", () => { + const executor = new CodexExecutor(); + const cases = [ + { model: "gpt-5.6-sol", expectedEffort: "max" }, + { model: "gpt-5.6-terra", expectedEffort: "max" }, + { model: "gpt-5.6-luna", expectedEffort: "max" }, + { model: "gpt-5.5", expectedEffort: "xhigh" }, + ]; + + for (const { model, expectedEffort } of cases) { + const translated = openaiToOpenAIResponsesRequest( + model, + { + model, + messages: [{ role: "user", content: "test" }], + reasoning_effort: "max", + }, + true, + {} + ); + const result = executor.transformRequest(model, translated, true, { + requestEndpointPath: "/chat/completions", + }); + + assert.equal(result.model, model); + assert.equal(result.reasoning.effort, expectedEffort, model); + assert.equal(result.reasoning_effort, undefined); + } +}); test("CodexExecutor.transformRequest preserves max effort for GPT-5.6", () => { const executor = new CodexExecutor(); diff --git a/tests/unit/provider-models-discovery-split.test.ts b/tests/unit/provider-models-discovery-split.test.ts index 9dd3502575..bdb9d2d871 100644 --- a/tests/unit/provider-models-discovery-split.test.ts +++ b/tests/unit/provider-models-discovery-split.test.ts @@ -304,7 +304,7 @@ test("codex.enrichCodexModelsFromGithubCatalog keeps live entitlement list autho assert.equal(enriched[0]?.supportsVision, true); }); -test("codex.mergeCodexLiveModelsWithLocalCatalog auto-includes remote-only models", () => { +test("codex.mergeCodexLiveModelsWithLocalCatalog merges capacity limits conservatively (smaller wins)", () => { const merged = mergeCodexLiveModelsWithLocalCatalog( [ { @@ -320,19 +320,25 @@ test("codex.mergeCodexLiveModelsWithLocalCatalog auto-includes remote-only model owned_by: "codex", apiFormat: "responses", supportedEndpoints: ["responses"], - inputTokenLimit: 999999, + inputTokenLimit: 272000, supportsVision: true, }, + { + id: "gpt-5.5", + name: "Live GPT 5.5", + inputTokenLimit: 300000, + }, ], [ { id: "gpt-5.6-sol", name: "GPT 5.6 Sol", - contextLength: 500000, + contextLength: 372000, maxInputTokens: 372000, maxOutputTokens: 128000, }, - { id: "gpt-5.6-sol-low", name: "GPT 5.6 Sol (Low)", contextLength: 500000 }, + { id: "gpt-5.6-sol-low", name: "GPT 5.6 Sol (Low)", contextLength: 372000 }, + { id: "gpt-5.5", name: "GPT 5.5", maxInputTokens: 272000 }, ] ); @@ -341,10 +347,19 @@ test("codex.mergeCodexLiveModelsWithLocalCatalog auto-includes remote-only model assert.ok(ids.includes("gpt-5.6-sol")); assert.ok(ids.includes("gpt-5.6-sol-low")); const sol = merged.find((model) => model.id === "gpt-5.6-sol"); - // Local catalog enriches known IDs; live fields win on overlap via merge order. - assert.equal(sol?.inputTokenLimit, 999999); + assert.equal(sol?.name, "Live Sol"); + // Live (272000) is SMALLER than the pinned contract (372000) here — the + // smaller value wins so OmniRoute never promises more context than the + // live account can actually serve (#7012). + assert.equal(sol?.inputTokenLimit, 272000); assert.equal(sol?.supportsVision, true); + // Output limit is pinned-only (live has none) — passes through unchanged. assert.equal(sol?.outputTokenLimit, 128000); + assert.equal( + merged.find((model) => model.id === "gpt-5.5")?.inputTokenLimit, + 272000, + "capacity limits merge conservatively for all Codex models, not only the pinned GPT-5.6 ids — the smaller of live (300000) vs. pinned (272000) wins" + ); }); test("codex discovery filters drop the GPT-5.4 family but keep other remote models", () => { diff --git a/tests/unit/provider-models-route-codex.test.ts b/tests/unit/provider-models-route-codex.test.ts index c9b3824793..2457c62727 100644 --- a/tests/unit/provider-models-route-codex.test.ts +++ b/tests/unit/provider-models-route-codex.test.ts @@ -176,10 +176,16 @@ test("provider models route merges live Codex models with the local catalog then assert.ok(modelIds.has("gpt-5.6-sol")); assert.ok(modelIds.has("gpt-5.6-sol-ultra")); assert.ok(modelIds.has("gpt-5.6-sol-max")); - // Live payload wins on overlapping fields; local catalog supplies local-only variants. + // Live payload wins on overlapping fields; local catalog supplies local-only + // variants. EXCEPTION: capacity limits (inputTokenLimit/outputTokenLimit) + // merge conservatively — the smaller of live vs. pinned wins, never the + // larger, so a stale/inflated live number can never make OmniRoute promise + // more context than the account can actually serve (#7012). Here the pinned + // GPT-5.6 Codex contract (372000/128000, see GPT_5_6_CODEX_CAPABILITIES) is + // smaller than the live payload's 999999/999999, so the pinned value wins. assert.equal(liveModel?.name, "GPT 5.6 Sol Live"); - assert.equal(liveModel?.inputTokenLimit, 999999); - assert.equal(liveModel?.outputTokenLimit, 999999); + assert.equal(liveModel?.inputTokenLimit, 372000); + assert.equal(liveModel?.outputTokenLimit, 128000); assert.equal(liveModel?.apiFormat, "responses"); assert.deepEqual(liveModel?.supportedEndpoints, ["responses"]); assert.equal(liveModel?.supportsThinking, true); @@ -200,6 +206,44 @@ test("provider models route merges live Codex models with the local catalog then assert.equal(syncedIds.has("stale-codex-model"), false); }); +test("provider models route: live token limit wins when it is SMALLER than the pinned local catalog value", async () => { + const connection = await seedCodexConnection({ + accessToken: "codex-access-token", + providerSpecificData: { chatgptAccountId: "account-123" }, + }); + + globalThis.fetch = async (url) => { + const requestUrl = String(url); + if (requestUrl.includes("raw.githubusercontent.com/openai/codex")) { + return Response.json({ models: [] }); + } + // Live reports a SMALLER budget than the pinned GPT-5.6 Codex contract + // (372000/128000, GPT_5_6_CODEX_CAPABILITIES) — e.g. a temporary + // account-level cap. The conservative merge must take the smaller live + // value here, not the larger pinned one (#7012). + return Response.json({ + models: [ + { + slug: "gpt-5.6-sol", + display_name: "GPT 5.6 Sol Live", + visibility: "list", + supported_in_api: true, + max_input_tokens: 100000, + max_output_tokens: 50000, + }, + ], + }); + }; + + const response = await callRoute(connection.id, "?refresh=true"); + const body = (await response.json()) as RouteBody; + const liveModel = body.models?.find((model) => model.id === "gpt-5.6-sol"); + + assert.equal(response.status, 200); + assert.equal(liveModel?.inputTokenLimit, 100000); + assert.equal(liveModel?.outputTokenLimit, 50000); +}); + test("provider models route uses the GitHub Codex catalog when live discovery fails", async () => { const connection = await seedCodexConnection({ accessToken: "codex-access-token" }); const seenUrls: string[] = []; diff --git a/tests/unit/vscode-responses-models.test.ts b/tests/unit/vscode-responses-models.test.ts new file mode 100644 index 0000000000..71cce125cd --- /dev/null +++ b/tests/unit/vscode-responses-models.test.ts @@ -0,0 +1,106 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-vscode-responses-")); +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "vscode-responses-models-secret"; + +const core = await import("../../src/lib/db/core.ts"); +const providersDb = await import("../../src/lib/db/providers.ts"); +const settingsDb = await import("../../src/lib/db/settings.ts"); +const apiKeysDb = await import("../../src/lib/db/apiKeys.ts"); +const modelsDb = await import("../../src/lib/db/models.ts"); +const vscodeModelsRoute = await import("../../src/app/api/v1/vscode/[token]/models/route.ts"); +const vscodeRawModelsRoute = + await import("../../src/app/api/v1/vscode/raw/[token]/models/route.ts"); + +type MetadataModel = { + id?: string; + root?: string; + url?: string; + api_format?: string; + supported_endpoints?: string[]; +}; + +async function resetStorage() { + core.resetDbInstance(); + apiKeysDb.resetApiKeyState(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +} + +test.beforeEach(async () => { + await resetStorage(); +}); + +test.after(() => { + core.resetDbInstance(); + apiKeysDb.resetApiKeyState(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +test("vscode model metadata routes keep Responses text-generation models", async () => { + await settingsDb.updateSettings({ + requireLogin: true, + password: "hashed-password", + requireAuthForModels: true, + }); + const connection = await providersDb.createProviderConnection({ + provider: "codex", + authType: "oauth", + name: "codex-vscode-responses-model", + accessToken: "codex-test-token", + isActive: true, + testStatus: "active", + providerSpecificData: {}, + }); + await modelsDb.replaceSyncedAvailableModelsForConnection("codex", connection.id, [ + { + id: "future-codex-responses", + name: "Future Codex Responses", + source: "imported", + apiFormat: "responses", + supportedEndpoints: ["responses"], + inputTokenLimit: 372000, + outputTokenLimit: 128000, + }, + ]); + const key = await apiKeysDb.createApiKey( + "vscode-responses-model", + "machine-vscode-responses-model" + ); + const params = { params: { token: key.key } }; + + const [rawResponse, groupedResponse] = await Promise.all([ + vscodeRawModelsRoute.GET( + new Request(`http://localhost/api/v1/vscode/raw/${encodeURIComponent(key.key)}/models`), + params + ), + vscodeModelsRoute.GET( + new Request(`http://localhost/api/v1/vscode/${encodeURIComponent(key.key)}/models`), + params + ), + ]); + const rawBody = (await rawResponse.json()) as { data?: MetadataModel[] }; + const groupedBody = (await groupedResponse.json()) as { data?: MetadataModel[] }; + const rawModel = (rawBody.data || []).find( + (entry) => entry.id === "cx/future-codex-responses" + ); + const groupedModel = (groupedBody.data || []).find( + (entry) => entry.root === "future-codex-responses" + ); + + assert.equal(rawResponse.status, 200); + assert.equal(groupedResponse.status, 200); + assert.ok(rawModel, "raw metadata route dropped a Responses text-generation model"); + assert.ok(groupedModel, "grouped metadata route dropped a Responses text-generation model"); + assert.equal(rawModel.api_format, "responses"); + assert.deepEqual(rawModel.supported_endpoints, ["responses"]); + assert.equal( + groupedModel.url, + `http://localhost/api/v1/vscode/${encodeURIComponent(key.key)}/responses#models.ai.azure.com` + ); +}); diff --git a/tests/unit/vscode-shared-metadata.test.ts b/tests/unit/vscode-shared-metadata.test.ts index 4ad453793d..635e7aa399 100644 --- a/tests/unit/vscode-shared-metadata.test.ts +++ b/tests/unit/vscode-shared-metadata.test.ts @@ -73,6 +73,11 @@ test("vscode reasoning metadata supports GPT-5.6 Max and Ultra without splitting owned_by: "codex", capabilities: { reasoning: true }, }; + const terra = { + id: "cx/gpt-5.6-terra", + owned_by: "codex", + capabilities: { reasoning: true }, + }; const luna = { id: "cx/gpt-5.6-luna", owned_by: "codex", @@ -80,7 +85,14 @@ test("vscode reasoning metadata supports GPT-5.6 Max and Ultra without splitting }; assert.deepEqual(reasoningMetadata.getReasoningEffortValues(sol), [ - "none", + "low", + "medium", + "high", + "xhigh", + "max", + "ultra", + ]); + assert.deepEqual(reasoningMetadata.getReasoningEffortValues(terra), [ "low", "medium", "high", @@ -89,13 +101,15 @@ test("vscode reasoning metadata supports GPT-5.6 Max and Ultra without splitting "ultra", ]); assert.deepEqual(reasoningMetadata.getReasoningEffortValues(luna), [ - "none", "low", "medium", "high", "xhigh", "max", ]); + assert.equal(reasoningMetadata.getDefaultReasoningEffort(sol), "low"); + assert.equal(reasoningMetadata.getDefaultReasoningEffort(terra), "medium"); + assert.equal(reasoningMetadata.getDefaultReasoningEffort(luna), "medium"); assert.equal( reasoningMetadata.inferSelectedReasoningEffort( { ...sol, id: "cx/gpt-5.6-sol-ultra" }, diff --git a/tests/unit/vscode-token-routes-gpt56.test.ts b/tests/unit/vscode-token-routes-gpt56.test.ts index 293e312a7c..f136199739 100644 --- a/tests/unit/vscode-token-routes-gpt56.test.ts +++ b/tests/unit/vscode-token-routes-gpt56.test.ts @@ -82,16 +82,16 @@ test("vscode raw models route exposes native GPT-5.6 IDs and effort tiers", asyn assert.equal(typeof defaultModel.created, "number"); assert.equal(defaultModel.owned_by, "codex"); assert.equal(defaultModel.name, "Codex GPT 5.6 Sol"); - assert.equal(typeof defaultModel.context_length, "number"); - assert.equal(typeof defaultModel.max_output_tokens, "number"); - assert.equal(typeof defaultModel.max_input_tokens, "number"); + assert.equal(defaultModel.context_length, 372000); + assert.equal(defaultModel.max_output_tokens, 128000); + assert.equal(defaultModel.max_input_tokens, 372000); assert.deepEqual(defaultModel.capabilities, { vision: true, tool_calling: true, reasoning: true, thinking: true, supportsThinking: true, - effort_tiers: ["none", "low", "medium", "high", "xhigh", "max", "ultra"], + effort_tiers: ["low", "medium", "high", "xhigh", "max", "ultra"], }); for (const field of [ "url", diff --git a/tests/unit/vscode-token-routes.test.ts b/tests/unit/vscode-token-routes.test.ts index 24dfebb4fe..59d9ad044d 100644 --- a/tests/unit/vscode-token-routes.test.ts +++ b/tests/unit/vscode-token-routes.test.ts @@ -255,7 +255,7 @@ test("vscode combos route resolves combo names through Ollama api/show", async ( assert.equal(body.model, "show-combo"); assert.equal(body.modelfile, "FROM show-combo"); assert.equal(body.details.family, "show-combo"); - assert.equal(body.model_info.context_length, 500000); + assert.equal(body.model_info.context_length, 372000); assert.deepEqual(body.supportsReasoningEffort, ["none", "low", "medium", "high", "xhigh"]); assert.equal(body.model_info.capabilities.reasoning, true); }); @@ -377,7 +377,6 @@ test("vscode tokenized models route keeps xhigh for codex models that advertise assert.equal(model.toolCalling, true); assert.equal(model.vision, true); assert.deepEqual(model.supportsReasoningEffort, [ - "none", "low", "medium", "high", @@ -386,7 +385,6 @@ test("vscode tokenized models route keeps xhigh for codex models that advertise "ultra", ]); assert.deepEqual(model.supportedReasoningEfforts, [ - "none", "low", "medium", "high", @@ -394,9 +392,8 @@ test("vscode tokenized models route keeps xhigh for codex models that advertise "max", "ultra", ]); - assert.equal(model.defaultReasoningEffort, "none"); + assert.equal(model.defaultReasoningEffort, "low"); assert.deepEqual(model.configSchema?.properties?.reasoningEffort?.enum, [ - "none", "low", "medium", "high", @@ -404,7 +401,7 @@ test("vscode tokenized models route keeps xhigh for codex models that advertise "max", "ultra", ]); - assert.equal(model.configSchema?.properties?.reasoningEffort?.default, "none"); + assert.equal(model.configSchema?.properties?.reasoningEffort?.default, "low"); const importedIds = new Set((body.data || []).map((entry: any) => entry.id)); assert.ok(!importedIds.has("cx/gpt-5.6-sol")); assert.ok(!importedIds.has("cx/gpt-5.6-sol__tier_priority")); @@ -651,7 +648,6 @@ test("vscode tokenized tags route exposes reasoning metadata for codex models", assert.equal(response.status, 200); assert.ok(model, "missing gpt-5.6-sol__provider_cx in tokenized VS Code tags route"); assert.deepEqual(model.supportsReasoningEffort, [ - "none", "low", "medium", "high", @@ -660,7 +656,6 @@ test("vscode tokenized tags route exposes reasoning metadata for codex models", "ultra", ]); assert.deepEqual(model.supports_reasoning_effort, [ - "none", "low", "medium", "high", @@ -669,7 +664,6 @@ test("vscode tokenized tags route exposes reasoning metadata for codex models", "ultra", ]); assert.deepEqual(model.supportedReasoningEfforts, [ - "none", "low", "medium", "high", @@ -677,12 +671,11 @@ test("vscode tokenized tags route exposes reasoning metadata for codex models", "max", "ultra", ]); - assert.equal(model.defaultReasoningEffort, "none"); + assert.equal(model.defaultReasoningEffort, "low"); assert.equal(model.selectedReasoningEffort, "none"); assert.equal(model.selected_reasoning_effort, "none"); assert.equal(model.details.family, "gpt-5.6-sol"); assert.deepEqual(model.configurationSchema?.properties?.reasoningEffort?.enum, [ - "none", "low", "medium", "high", @@ -690,9 +683,8 @@ test("vscode tokenized tags route exposes reasoning metadata for codex models", "max", "ultra", ]); - assert.equal(model.configurationSchema?.properties?.reasoningEffort?.default, "none"); + assert.equal(model.configurationSchema?.properties?.reasoningEffort?.default, "low"); assert.deepEqual(model.details.configurationSchema?.properties?.reasoningEffort?.enum, [ - "none", "low", "medium", "high", @@ -701,7 +693,6 @@ test("vscode tokenized tags route exposes reasoning metadata for codex models", "ultra", ]); assert.deepEqual(model.details.supports_reasoning_effort, [ - "none", "low", "medium", "high", @@ -775,13 +766,17 @@ test("vscode tokenized tags route only exposes usable canonical chat models", as `tag ${tagModel.name} should be chat-capable` ); assert.ok( - !catalogModel.api_format || catalogModel.api_format === "chat-completions", - `tag ${tagModel.name} should use chat-completions` + !catalogModel.api_format || + ["chat-completions", "responses", "openai-responses"].includes( + catalogModel.api_format + ), + `tag ${tagModel.name} should use a text-generation API format` ); assert.ok( !Array.isArray(catalogModel.supported_endpoints) || - catalogModel.supported_endpoints.includes("chat"), - `tag ${tagModel.name} should support chat` + catalogModel.supported_endpoints.includes("chat") || + catalogModel.supported_endpoints.includes("responses"), + `tag ${tagModel.name} should support text generation` ); assert.ok( !Array.isArray(catalogModel.output_modalities) || @@ -794,8 +789,11 @@ test("vscode tokenized tags route only exposes usable canonical chat models", as (model: any) => model.parent || (typeof model.type === "string" && model.type !== "chat") || - (typeof model.api_format === "string" && model.api_format !== "chat-completions") || - (Array.isArray(model.supported_endpoints) && !model.supported_endpoints.includes("chat")) || + (typeof model.api_format === "string" && + !["chat-completions", "responses", "openai-responses"].includes(model.api_format)) || + (Array.isArray(model.supported_endpoints) && + !model.supported_endpoints.includes("chat") && + !model.supported_endpoints.includes("responses")) || (Array.isArray(model.output_modalities) && !model.output_modalities.includes("text")) ); const tagNames = new Set((tagsBody.models || []).map((model: any) => model.name)); @@ -1040,7 +1038,6 @@ test("vscode tokenized api/show route exposes explicit reasoning effort metadata assert.equal(body.remote_model, "Codex GPT 5.6 Sol (Default)"); assert.equal(body.details.family, "gpt-5.6-sol"); assert.deepEqual(body.supportsReasoningEffort, [ - "none", "low", "medium", "high", @@ -1049,7 +1046,6 @@ test("vscode tokenized api/show route exposes explicit reasoning effort metadata "ultra", ]); assert.deepEqual(body.supports_reasoning_effort, [ - "none", "low", "medium", "high", @@ -1058,7 +1054,6 @@ test("vscode tokenized api/show route exposes explicit reasoning effort metadata "ultra", ]); assert.deepEqual(body.supportedReasoningEfforts, [ - "none", "low", "medium", "high", @@ -1066,11 +1061,10 @@ test("vscode tokenized api/show route exposes explicit reasoning effort metadata "max", "ultra", ]); - assert.equal(body.defaultReasoningEffort, "none"); + assert.equal(body.defaultReasoningEffort, "low"); assert.equal(body.selectedReasoningEffort, "none"); assert.equal(body.selected_reasoning_effort, "none"); assert.deepEqual(body.configurationSchema?.properties?.reasoningEffort?.enum, [ - "none", "low", "medium", "high", @@ -1078,12 +1072,11 @@ test("vscode tokenized api/show route exposes explicit reasoning effort metadata "max", "ultra", ]); - assert.equal(body.configurationSchema?.properties?.reasoningEffort?.default, "none"); + assert.equal(body.configurationSchema?.properties?.reasoningEffort?.default, "low"); assert.equal(body.model_info["general.basename"], "Codex GPT 5.6 Sol (Default)"); assert.equal(body.model_info["general.architecture"], "codex"); - assert.equal(body.model_info["codex.context_length"], 500000); + assert.equal(body.model_info["codex.context_length"], 372000); assert.deepEqual(body.model_info.supports_reasoning_effort, [ - "none", "low", "medium", "high", @@ -1093,7 +1086,6 @@ test("vscode tokenized api/show route exposes explicit reasoning effort metadata ]); assert.equal(body.model_info.selected_reasoning_effort, "none"); assert.deepEqual(body.model_info.capabilities.supports_reasoning_effort, [ - "none", "low", "medium", "high",