mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-07-26 09:52:11 +03:00
fix(codex): preserve GPT-5.6 reasoning contract (#7012)
* fix(codex): preserve GPT-5.6 reasoning contract * fix(vscode): expose Responses text models * fix(codex): keep GPT-5.6 limits through discovery * fix(ci): extract isUsableChatModel helpers to satisfy complexity ratchet Splitting the supported_endpoints/output_modalities guard clauses into excludesChatAndResponsesEndpoints() / excludesTextOutputModality() drops isUsableChatModel's cyclomatic complexity from 16 to under the ratchet's max of 15 (complexity-ratchets gate: 2057 -> 2056, back at baseline). Behavior is unchanged; existing vscode/codex route tests cover it. Co-authored-by: diegosouzapw <diegosouza.pw@gmail.com> * fix(codex): merge capacity limits conservatively (smaller of live vs pinned wins) Resolve the #7012 catalog-merge policy collision: instead of the pinned GPT-5.6 contract always winning for a fixed set of model ids, capacity limits (inputTokenLimit/outputTokenLimit) now merge via mergeCapacityLimitConservatively — Math.min(pinned, live) when both are present, so OmniRoute never promises more context than the account can actually serve. All other overlapping fields still take the live value unconditionally. Guard tests cover both directions (pinned smaller wins / pinned larger loses) at the route level and via an isolated helper-level unit test. Co-authored-by: diegosouzapw <diegosouza.pw@gmail.com> --------- Co-authored-by: Xiangzhe <xz-dev@users.noreply.github.com> Co-authored-by: diegosouzapw <diegosouza.pw@gmail.com>
This commit is contained in:
@@ -312,15 +312,15 @@ export const GPT_5_6_API_CAPABILITIES = {
|
||||
maxOutputTokens: 128000,
|
||||
} as const;
|
||||
|
||||
// Codex's live catalog reports a 372K usable input budget for GPT-5.6.
|
||||
// Keep the reserved 128K output budget explicit, matching the GPT-5.5 catalog contract.
|
||||
// Codex's live catalog reports a 372K context window for GPT-5.6.
|
||||
// Keep the input and output limits explicit for catalog consumers that expose them separately.
|
||||
export const GPT_5_6_CODEX_CAPABILITIES = {
|
||||
targetFormat: "openai-responses",
|
||||
toolCalling: true,
|
||||
supportsReasoning: true,
|
||||
supportsVision: true,
|
||||
supportsXHighEffort: true,
|
||||
contextLength: 500000,
|
||||
contextLength: 372000,
|
||||
maxInputTokens: 372000,
|
||||
maxOutputTokens: 128000,
|
||||
} as const;
|
||||
|
||||
@@ -49,9 +49,21 @@ export function imageUrlToText(value: unknown): string {
|
||||
return toString(record.url);
|
||||
}
|
||||
|
||||
export function normalizeResponsesReasoningEffort(value: unknown): string {
|
||||
const CODEX_GPT_5_6_MODEL_PATTERN =
|
||||
/^gpt-5\.6-(?:sol|terra|luna)(?:-(?:none|low|medium|high|xhigh|max|ultra))?$/;
|
||||
|
||||
function supportsNativeMaxReasoningEffort(model: unknown): boolean {
|
||||
const normalizedModel = toString(model)
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/^(?:codex|cx)\//, "");
|
||||
return CODEX_GPT_5_6_MODEL_PATTERN.test(normalizedModel);
|
||||
}
|
||||
|
||||
export function normalizeResponsesReasoningEffort(value: unknown, model?: unknown): string {
|
||||
const effort = toString(value).toLowerCase();
|
||||
return effort === "max" ? "xhigh" : effort;
|
||||
if (effort !== "max") return effort;
|
||||
return supportsNativeMaxReasoningEffort(model) ? "max" : "xhigh";
|
||||
}
|
||||
|
||||
export function shouldRequestClaudeSummarizedThinking(value: unknown): boolean {
|
||||
|
||||
@@ -380,7 +380,7 @@ export function openaiToOpenAIResponsesRequest(
|
||||
if (root.reasoning !== undefined) {
|
||||
result.reasoning = root.reasoning;
|
||||
} else if (root.reasoning_effort !== undefined) {
|
||||
const effort = normalizeResponsesReasoningEffort(root.reasoning_effort);
|
||||
const effort = normalizeResponsesReasoningEffort(root.reasoning_effort, model ?? root.model);
|
||||
if (effort && effort !== "none") {
|
||||
// Effort-only chat request: default a reasoning summary so the upstream
|
||||
// streams thinking back (see the constant's note above).
|
||||
|
||||
@@ -292,9 +292,62 @@ function localCatalogModelToCodexDiscoveryModel(
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Capacity limits (input/output token caps) merge CONSERVATIVELY: the smaller
|
||||
* of the pinned local-catalog value and the live-discovery value wins, never
|
||||
* the larger. Overpromising context lets a request run past what the account
|
||||
* can actually serve — the upstream truncates mid-conversation and can burn a
|
||||
* combo fallback. Underpromising only leaves capacity on the table, which is
|
||||
* a performance loss, not a broken request. When only one side has a value,
|
||||
* that value passes through unchanged (nothing to reconcile).
|
||||
*
|
||||
* This is the ONE deliberate exception to "live wins on overlapping fields"
|
||||
* below — name/apiFormat/supportedEndpoints/supportsThinking/supportsVision
|
||||
* etc. still take the live value unconditionally. Don't extend this
|
||||
* conservative rule to other fields without updating the policy comment on
|
||||
* mergeCodexLiveModelsWithLocalCatalog (#7012).
|
||||
*/
|
||||
function mergeCapacityLimitConservatively(
|
||||
pinnedValue: number | undefined,
|
||||
liveValue: number | undefined
|
||||
): number | undefined {
|
||||
if (typeof pinnedValue === "number" && typeof liveValue === "number") {
|
||||
return Math.min(pinnedValue, liveValue);
|
||||
}
|
||||
return typeof liveValue === "number" ? liveValue : pinnedValue;
|
||||
}
|
||||
|
||||
function mergeLiveAndLocalCodexModel(
|
||||
liveModel: CodexDiscoveryModel,
|
||||
localModel: CodexDiscoveryModel
|
||||
): CodexDiscoveryModel {
|
||||
const merged: CodexDiscoveryModel = { ...localModel, ...liveModel };
|
||||
const inputTokenLimit = mergeCapacityLimitConservatively(
|
||||
localModel.inputTokenLimit,
|
||||
liveModel.inputTokenLimit
|
||||
);
|
||||
const outputTokenLimit = mergeCapacityLimitConservatively(
|
||||
localModel.outputTokenLimit,
|
||||
liveModel.outputTokenLimit
|
||||
);
|
||||
if (typeof inputTokenLimit === "number") {
|
||||
merged.inputTokenLimit = inputTokenLimit;
|
||||
} else {
|
||||
delete merged.inputTokenLimit;
|
||||
}
|
||||
if (typeof outputTokenLimit === "number") {
|
||||
merged.outputTokenLimit = outputTokenLimit;
|
||||
} else {
|
||||
delete merged.outputTokenLimit;
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
|
||||
/**
|
||||
* Live/GitHub discovery is the source of truth for "what exists".
|
||||
* Explicit filters (denylist / predicates) are the policy layer for "what we show".
|
||||
* Live wins on overlapping fields, EXCEPT capacity limits (input/output token
|
||||
* caps) — those merge conservatively, see mergeCapacityLimitConservatively.
|
||||
* Do NOT reintroduce curated-only allowlisting as the default path (#6862 / #6859).
|
||||
*/
|
||||
export function mergeCodexLiveModelsWithLocalCatalog(
|
||||
@@ -312,7 +365,10 @@ export function mergeCodexLiveModelsWithLocalCatalog(
|
||||
if (!localModel.id) continue;
|
||||
const normalizedLocal = localCatalogModelToCodexDiscoveryModel(localModel);
|
||||
const existing = merged.get(localModel.id);
|
||||
merged.set(localModel.id, existing ? { ...normalizedLocal, ...existing } : normalizedLocal);
|
||||
merged.set(
|
||||
localModel.id,
|
||||
existing ? mergeLiveAndLocalCodexModel(existing, normalizedLocal) : normalizedLocal
|
||||
);
|
||||
}
|
||||
|
||||
return Array.from(merged.values());
|
||||
|
||||
@@ -72,29 +72,51 @@ type EnrichModelForVscodeOptions = {
|
||||
preserveNativeId?: boolean;
|
||||
};
|
||||
|
||||
const TEXT_GENERATION_API_FORMATS = new Set([
|
||||
"chat-completions",
|
||||
"responses",
|
||||
"openai-responses",
|
||||
]);
|
||||
|
||||
function usesResponsesApi(model: CatalogModelEntry) {
|
||||
return (
|
||||
model.api_format === "responses" ||
|
||||
model.api_format === "openai-responses" ||
|
||||
model.supported_endpoints?.includes("responses") === true
|
||||
);
|
||||
}
|
||||
|
||||
function excludesChatAndResponsesEndpoints(model: CatalogModelEntry) {
|
||||
return (
|
||||
Array.isArray(model.supported_endpoints) &&
|
||||
model.supported_endpoints.length > 0 &&
|
||||
!model.supported_endpoints.includes("chat") &&
|
||||
!model.supported_endpoints.includes("responses")
|
||||
);
|
||||
}
|
||||
|
||||
function excludesTextOutputModality(model: CatalogModelEntry) {
|
||||
return (
|
||||
Array.isArray(model.output_modalities) &&
|
||||
model.output_modalities.length > 0 &&
|
||||
!model.output_modalities.includes("text")
|
||||
);
|
||||
}
|
||||
|
||||
function isUsableChatModel(model: CatalogModelEntry) {
|
||||
if (typeof model.owned_by === "string" && model.owned_by.trim().toLowerCase() === "combo") {
|
||||
return false;
|
||||
}
|
||||
if (typeof model.parent === "string" && model.parent.length > 0) return false;
|
||||
if (typeof model.type === "string" && model.type !== "chat") return false;
|
||||
if (typeof model.api_format === "string" && model.api_format !== "chat-completions") {
|
||||
return false;
|
||||
}
|
||||
if (
|
||||
Array.isArray(model.supported_endpoints) &&
|
||||
model.supported_endpoints.length > 0 &&
|
||||
!model.supported_endpoints.includes("chat")
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
if (
|
||||
Array.isArray(model.output_modalities) &&
|
||||
model.output_modalities.length > 0 &&
|
||||
!model.output_modalities.includes("text")
|
||||
typeof model.api_format === "string" &&
|
||||
!TEXT_GENERATION_API_FORMATS.has(model.api_format)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
if (excludesChatAndResponsesEndpoints(model)) return false;
|
||||
if (excludesTextOutputModality(model)) return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
@@ -230,9 +252,10 @@ export function enrichModelForVscode(
|
||||
name: options.preserveNativeId
|
||||
? getVscodeRawModelDisplayName(presentationModel)
|
||||
: getVscodeModelDisplayName(presentationModel),
|
||||
url: reasoningEffortValues
|
||||
? `${tokenBaseUrl}/responses#models.ai.azure.com`
|
||||
: `${tokenBaseUrl}/chat/completions#models.ai.azure.com`,
|
||||
url:
|
||||
reasoningEffortValues || usesResponsesApi(model)
|
||||
? `${tokenBaseUrl}/responses#models.ai.azure.com`
|
||||
: `${tokenBaseUrl}/chat/completions#models.ai.azure.com`,
|
||||
toolCalling: resolvedCapabilities.toolCalling === true,
|
||||
vision: resolvedCapabilities.supportsVision === true,
|
||||
maxInputTokens:
|
||||
|
||||
@@ -177,8 +177,28 @@ export function getReasoningVariantBaseModelId(modelId: string) {
|
||||
return matchReasoningEffortSuffix(modelId)?.baseModelId || modelId;
|
||||
}
|
||||
|
||||
function getCodexGpt56DefaultReasoningEffort(model: VscodeCatalogModel) {
|
||||
const modelId = getCatalogModelName(model);
|
||||
const parsed = parseModel(modelId, "");
|
||||
const providerId = (parsed.provider || model.owned_by || "").trim().toLowerCase();
|
||||
if (providerId !== "codex" && providerId !== "cx") return undefined;
|
||||
|
||||
const providerModelId = (parsed.model || model.root || modelId.split("/").pop() || modelId)
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
const match = providerModelId.match(
|
||||
/^gpt-5\.6-(sol|terra|luna)(?:-(?:none|low|medium|high|xhigh|max|ultra))?$/
|
||||
);
|
||||
if (!match) return undefined;
|
||||
return match[1] === "sol" ? "low" : "medium";
|
||||
}
|
||||
|
||||
export function getDefaultReasoningEffort(model: VscodeCatalogModel, supportedValues?: string[]) {
|
||||
return inferSelectedReasoningEffort(model, supportedValues) || DEFAULT_REASONING_EFFORT;
|
||||
return (
|
||||
inferSelectedReasoningEffort(model, supportedValues) ||
|
||||
getCodexGpt56DefaultReasoningEffort(model) ||
|
||||
DEFAULT_REASONING_EFFORT
|
||||
);
|
||||
}
|
||||
|
||||
export function buildReasoningConfigSchema(
|
||||
|
||||
@@ -18,7 +18,7 @@ export const CANONICAL_EFFORT_VALUES = ["none", "low", "medium", "high", "xhigh"
|
||||
|
||||
export type CanonicalEffort = (typeof CANONICAL_EFFORT_VALUES)[number];
|
||||
|
||||
/** Add provider-native GPT-5.6 effort levels without widening the global request vocabulary. */
|
||||
/** Use provider-native GPT-5.6 effort levels without widening the global request vocabulary. */
|
||||
export function extendCodexGpt56EffortValues(
|
||||
provider: string | null | undefined,
|
||||
model: string | null | undefined,
|
||||
@@ -39,8 +39,8 @@ export function extendCodexGpt56EffortValues(
|
||||
);
|
||||
if (!match) return values;
|
||||
|
||||
const additions = match[1] === "luna" ? ["max"] : ["max", "ultra"];
|
||||
return [...new Set([...values, ...additions])];
|
||||
const nativeValues = ["low", "medium", "high", "xhigh", "max"];
|
||||
return match[1] === "luna" ? nativeValues : [...nativeValues, "ultra"];
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
75
tests/unit/codex-capacity-limit-merge.test.ts
Normal file
75
tests/unit/codex-capacity-limit-merge.test.ts
Normal file
@@ -0,0 +1,75 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
// Isolated unit coverage for the capacity-limit merge policy, independent of
|
||||
// the /api/providers/[id]/models route. See
|
||||
// src/app/api/providers/[id]/models/discovery/codex.ts::mergeCapacityLimitConservatively
|
||||
// and the guard test in tests/unit/provider-models-route-codex.test.ts (#7012).
|
||||
const codexDiscovery = await import(
|
||||
"../../src/app/api/providers/[id]/models/discovery/codex.ts"
|
||||
);
|
||||
|
||||
function liveModel(overrides: Record<string, unknown>) {
|
||||
return {
|
||||
id: "gpt-5.6-sol",
|
||||
name: "GPT 5.6 Sol Live",
|
||||
owned_by: "codex" as const,
|
||||
apiFormat: "responses" as const,
|
||||
supportedEndpoints: ["responses"] as ["responses"],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test("mergeCodexLiveModelsWithLocalCatalog: pinned (local) limit wins when it is SMALLER than live", () => {
|
||||
const merged = codexDiscovery.mergeCodexLiveModelsWithLocalCatalog(
|
||||
[liveModel({ inputTokenLimit: 999999, outputTokenLimit: 999999 })],
|
||||
[
|
||||
{
|
||||
id: "gpt-5.6-sol",
|
||||
name: "GPT 5.6 Sol",
|
||||
maxInputTokens: 372000,
|
||||
maxOutputTokens: 128000,
|
||||
},
|
||||
]
|
||||
);
|
||||
const model = merged.find((m) => m.id === "gpt-5.6-sol");
|
||||
// Pinned (372000/128000) is smaller than live (999999/999999) — the smaller,
|
||||
// safer value must win so requests never overrun the account's real budget.
|
||||
assert.equal(model?.inputTokenLimit, 372000);
|
||||
assert.equal(model?.outputTokenLimit, 128000);
|
||||
// Non-capacity fields are unaffected — live still wins on those.
|
||||
assert.equal(model?.name, "GPT 5.6 Sol Live");
|
||||
});
|
||||
|
||||
test("mergeCodexLiveModelsWithLocalCatalog: live limit wins when the pinned (local) value is LARGER", () => {
|
||||
const merged = codexDiscovery.mergeCodexLiveModelsWithLocalCatalog(
|
||||
[liveModel({ inputTokenLimit: 100000, outputTokenLimit: 50000 })],
|
||||
[
|
||||
{
|
||||
id: "gpt-5.6-sol",
|
||||
name: "GPT 5.6 Sol",
|
||||
maxInputTokens: 372000,
|
||||
maxOutputTokens: 128000,
|
||||
},
|
||||
]
|
||||
);
|
||||
const model = merged.find((m) => m.id === "gpt-5.6-sol");
|
||||
// Live (100000/50000) is smaller than pinned (372000/128000) here — the
|
||||
// smaller live value must win, not the larger pinned contract.
|
||||
assert.equal(model?.inputTokenLimit, 100000);
|
||||
assert.equal(model?.outputTokenLimit, 50000);
|
||||
});
|
||||
|
||||
test("mergeCodexLiveModelsWithLocalCatalog: uses whichever side has a value when only one side defines it", () => {
|
||||
const liveOnly = codexDiscovery.mergeCodexLiveModelsWithLocalCatalog(
|
||||
[liveModel({ inputTokenLimit: 200000 })],
|
||||
[{ id: "gpt-5.6-sol", name: "GPT 5.6 Sol" }]
|
||||
);
|
||||
assert.equal(liveOnly.find((m) => m.id === "gpt-5.6-sol")?.inputTokenLimit, 200000);
|
||||
|
||||
const pinnedOnly = codexDiscovery.mergeCodexLiveModelsWithLocalCatalog(
|
||||
[liveModel({ inputTokenLimit: undefined })],
|
||||
[{ id: "gpt-5.6-sol", name: "GPT 5.6 Sol", maxInputTokens: 372000 }]
|
||||
);
|
||||
assert.equal(pinnedOnly.find((m) => m.id === "gpt-5.6-sol")?.inputTokenLimit, 372000);
|
||||
});
|
||||
@@ -36,7 +36,7 @@ test("Codex catalog exposes the GPT-5.6 lineup in configured priority order", ()
|
||||
for (const modelId of expectedIds) {
|
||||
const model = models.find((entry) => entry.id === modelId);
|
||||
assert.ok(model, `codex must expose ${modelId}`);
|
||||
assert.equal(model.contextLength, 500000);
|
||||
assert.equal(model.contextLength, 372000);
|
||||
assert.equal(model.maxInputTokens, 372000);
|
||||
assert.equal(model.maxOutputTokens, 128000);
|
||||
assert.equal(model.targetFormat, "openai-responses");
|
||||
|
||||
@@ -2,6 +2,37 @@ import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import { CodexExecutor } from "../../open-sse/executors/codex.ts";
|
||||
import { openaiToOpenAIResponsesRequest } from "../../open-sse/translator/request/openai-responses/toResponses.ts";
|
||||
|
||||
test("Chat-to-Codex translation preserves max only for GPT-5.6", () => {
|
||||
const executor = new CodexExecutor();
|
||||
const cases = [
|
||||
{ model: "gpt-5.6-sol", expectedEffort: "max" },
|
||||
{ model: "gpt-5.6-terra", expectedEffort: "max" },
|
||||
{ model: "gpt-5.6-luna", expectedEffort: "max" },
|
||||
{ model: "gpt-5.5", expectedEffort: "xhigh" },
|
||||
];
|
||||
|
||||
for (const { model, expectedEffort } of cases) {
|
||||
const translated = openaiToOpenAIResponsesRequest(
|
||||
model,
|
||||
{
|
||||
model,
|
||||
messages: [{ role: "user", content: "test" }],
|
||||
reasoning_effort: "max",
|
||||
},
|
||||
true,
|
||||
{}
|
||||
);
|
||||
const result = executor.transformRequest(model, translated, true, {
|
||||
requestEndpointPath: "/chat/completions",
|
||||
});
|
||||
|
||||
assert.equal(result.model, model);
|
||||
assert.equal(result.reasoning.effort, expectedEffort, model);
|
||||
assert.equal(result.reasoning_effort, undefined);
|
||||
}
|
||||
});
|
||||
|
||||
test("CodexExecutor.transformRequest preserves max effort for GPT-5.6", () => {
|
||||
const executor = new CodexExecutor();
|
||||
|
||||
@@ -304,7 +304,7 @@ test("codex.enrichCodexModelsFromGithubCatalog keeps live entitlement list autho
|
||||
assert.equal(enriched[0]?.supportsVision, true);
|
||||
});
|
||||
|
||||
test("codex.mergeCodexLiveModelsWithLocalCatalog auto-includes remote-only models", () => {
|
||||
test("codex.mergeCodexLiveModelsWithLocalCatalog merges capacity limits conservatively (smaller wins)", () => {
|
||||
const merged = mergeCodexLiveModelsWithLocalCatalog(
|
||||
[
|
||||
{
|
||||
@@ -320,19 +320,25 @@ test("codex.mergeCodexLiveModelsWithLocalCatalog auto-includes remote-only model
|
||||
owned_by: "codex",
|
||||
apiFormat: "responses",
|
||||
supportedEndpoints: ["responses"],
|
||||
inputTokenLimit: 999999,
|
||||
inputTokenLimit: 272000,
|
||||
supportsVision: true,
|
||||
},
|
||||
{
|
||||
id: "gpt-5.5",
|
||||
name: "Live GPT 5.5",
|
||||
inputTokenLimit: 300000,
|
||||
},
|
||||
],
|
||||
[
|
||||
{
|
||||
id: "gpt-5.6-sol",
|
||||
name: "GPT 5.6 Sol",
|
||||
contextLength: 500000,
|
||||
contextLength: 372000,
|
||||
maxInputTokens: 372000,
|
||||
maxOutputTokens: 128000,
|
||||
},
|
||||
{ id: "gpt-5.6-sol-low", name: "GPT 5.6 Sol (Low)", contextLength: 500000 },
|
||||
{ id: "gpt-5.6-sol-low", name: "GPT 5.6 Sol (Low)", contextLength: 372000 },
|
||||
{ id: "gpt-5.5", name: "GPT 5.5", maxInputTokens: 272000 },
|
||||
]
|
||||
);
|
||||
|
||||
@@ -341,10 +347,19 @@ test("codex.mergeCodexLiveModelsWithLocalCatalog auto-includes remote-only model
|
||||
assert.ok(ids.includes("gpt-5.6-sol"));
|
||||
assert.ok(ids.includes("gpt-5.6-sol-low"));
|
||||
const sol = merged.find((model) => model.id === "gpt-5.6-sol");
|
||||
// Local catalog enriches known IDs; live fields win on overlap via merge order.
|
||||
assert.equal(sol?.inputTokenLimit, 999999);
|
||||
assert.equal(sol?.name, "Live Sol");
|
||||
// Live (272000) is SMALLER than the pinned contract (372000) here — the
|
||||
// smaller value wins so OmniRoute never promises more context than the
|
||||
// live account can actually serve (#7012).
|
||||
assert.equal(sol?.inputTokenLimit, 272000);
|
||||
assert.equal(sol?.supportsVision, true);
|
||||
// Output limit is pinned-only (live has none) — passes through unchanged.
|
||||
assert.equal(sol?.outputTokenLimit, 128000);
|
||||
assert.equal(
|
||||
merged.find((model) => model.id === "gpt-5.5")?.inputTokenLimit,
|
||||
272000,
|
||||
"capacity limits merge conservatively for all Codex models, not only the pinned GPT-5.6 ids — the smaller of live (300000) vs. pinned (272000) wins"
|
||||
);
|
||||
});
|
||||
|
||||
test("codex discovery filters drop the GPT-5.4 family but keep other remote models", () => {
|
||||
|
||||
@@ -176,10 +176,16 @@ test("provider models route merges live Codex models with the local catalog then
|
||||
assert.ok(modelIds.has("gpt-5.6-sol"));
|
||||
assert.ok(modelIds.has("gpt-5.6-sol-ultra"));
|
||||
assert.ok(modelIds.has("gpt-5.6-sol-max"));
|
||||
// Live payload wins on overlapping fields; local catalog supplies local-only variants.
|
||||
// Live payload wins on overlapping fields; local catalog supplies local-only
|
||||
// variants. EXCEPTION: capacity limits (inputTokenLimit/outputTokenLimit)
|
||||
// merge conservatively — the smaller of live vs. pinned wins, never the
|
||||
// larger, so a stale/inflated live number can never make OmniRoute promise
|
||||
// more context than the account can actually serve (#7012). Here the pinned
|
||||
// GPT-5.6 Codex contract (372000/128000, see GPT_5_6_CODEX_CAPABILITIES) is
|
||||
// smaller than the live payload's 999999/999999, so the pinned value wins.
|
||||
assert.equal(liveModel?.name, "GPT 5.6 Sol Live");
|
||||
assert.equal(liveModel?.inputTokenLimit, 999999);
|
||||
assert.equal(liveModel?.outputTokenLimit, 999999);
|
||||
assert.equal(liveModel?.inputTokenLimit, 372000);
|
||||
assert.equal(liveModel?.outputTokenLimit, 128000);
|
||||
assert.equal(liveModel?.apiFormat, "responses");
|
||||
assert.deepEqual(liveModel?.supportedEndpoints, ["responses"]);
|
||||
assert.equal(liveModel?.supportsThinking, true);
|
||||
@@ -200,6 +206,44 @@ test("provider models route merges live Codex models with the local catalog then
|
||||
assert.equal(syncedIds.has("stale-codex-model"), false);
|
||||
});
|
||||
|
||||
test("provider models route: live token limit wins when it is SMALLER than the pinned local catalog value", async () => {
|
||||
const connection = await seedCodexConnection({
|
||||
accessToken: "codex-access-token",
|
||||
providerSpecificData: { chatgptAccountId: "account-123" },
|
||||
});
|
||||
|
||||
globalThis.fetch = async (url) => {
|
||||
const requestUrl = String(url);
|
||||
if (requestUrl.includes("raw.githubusercontent.com/openai/codex")) {
|
||||
return Response.json({ models: [] });
|
||||
}
|
||||
// Live reports a SMALLER budget than the pinned GPT-5.6 Codex contract
|
||||
// (372000/128000, GPT_5_6_CODEX_CAPABILITIES) — e.g. a temporary
|
||||
// account-level cap. The conservative merge must take the smaller live
|
||||
// value here, not the larger pinned one (#7012).
|
||||
return Response.json({
|
||||
models: [
|
||||
{
|
||||
slug: "gpt-5.6-sol",
|
||||
display_name: "GPT 5.6 Sol Live",
|
||||
visibility: "list",
|
||||
supported_in_api: true,
|
||||
max_input_tokens: 100000,
|
||||
max_output_tokens: 50000,
|
||||
},
|
||||
],
|
||||
});
|
||||
};
|
||||
|
||||
const response = await callRoute(connection.id, "?refresh=true");
|
||||
const body = (await response.json()) as RouteBody;
|
||||
const liveModel = body.models?.find((model) => model.id === "gpt-5.6-sol");
|
||||
|
||||
assert.equal(response.status, 200);
|
||||
assert.equal(liveModel?.inputTokenLimit, 100000);
|
||||
assert.equal(liveModel?.outputTokenLimit, 50000);
|
||||
});
|
||||
|
||||
test("provider models route uses the GitHub Codex catalog when live discovery fails", async () => {
|
||||
const connection = await seedCodexConnection({ accessToken: "codex-access-token" });
|
||||
const seenUrls: string[] = [];
|
||||
|
||||
106
tests/unit/vscode-responses-models.test.ts
Normal file
106
tests/unit/vscode-responses-models.test.ts
Normal file
@@ -0,0 +1,106 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-vscode-responses-"));
|
||||
process.env.DATA_DIR = TEST_DATA_DIR;
|
||||
process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "vscode-responses-models-secret";
|
||||
|
||||
const core = await import("../../src/lib/db/core.ts");
|
||||
const providersDb = await import("../../src/lib/db/providers.ts");
|
||||
const settingsDb = await import("../../src/lib/db/settings.ts");
|
||||
const apiKeysDb = await import("../../src/lib/db/apiKeys.ts");
|
||||
const modelsDb = await import("../../src/lib/db/models.ts");
|
||||
const vscodeModelsRoute = await import("../../src/app/api/v1/vscode/[token]/models/route.ts");
|
||||
const vscodeRawModelsRoute =
|
||||
await import("../../src/app/api/v1/vscode/raw/[token]/models/route.ts");
|
||||
|
||||
type MetadataModel = {
|
||||
id?: string;
|
||||
root?: string;
|
||||
url?: string;
|
||||
api_format?: string;
|
||||
supported_endpoints?: string[];
|
||||
};
|
||||
|
||||
async function resetStorage() {
|
||||
core.resetDbInstance();
|
||||
apiKeysDb.resetApiKeyState();
|
||||
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
|
||||
fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
|
||||
}
|
||||
|
||||
test.beforeEach(async () => {
|
||||
await resetStorage();
|
||||
});
|
||||
|
||||
test.after(() => {
|
||||
core.resetDbInstance();
|
||||
apiKeysDb.resetApiKeyState();
|
||||
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test("vscode model metadata routes keep Responses text-generation models", async () => {
|
||||
await settingsDb.updateSettings({
|
||||
requireLogin: true,
|
||||
password: "hashed-password",
|
||||
requireAuthForModels: true,
|
||||
});
|
||||
const connection = await providersDb.createProviderConnection({
|
||||
provider: "codex",
|
||||
authType: "oauth",
|
||||
name: "codex-vscode-responses-model",
|
||||
accessToken: "codex-test-token",
|
||||
isActive: true,
|
||||
testStatus: "active",
|
||||
providerSpecificData: {},
|
||||
});
|
||||
await modelsDb.replaceSyncedAvailableModelsForConnection("codex", connection.id, [
|
||||
{
|
||||
id: "future-codex-responses",
|
||||
name: "Future Codex Responses",
|
||||
source: "imported",
|
||||
apiFormat: "responses",
|
||||
supportedEndpoints: ["responses"],
|
||||
inputTokenLimit: 372000,
|
||||
outputTokenLimit: 128000,
|
||||
},
|
||||
]);
|
||||
const key = await apiKeysDb.createApiKey(
|
||||
"vscode-responses-model",
|
||||
"machine-vscode-responses-model"
|
||||
);
|
||||
const params = { params: { token: key.key } };
|
||||
|
||||
const [rawResponse, groupedResponse] = await Promise.all([
|
||||
vscodeRawModelsRoute.GET(
|
||||
new Request(`http://localhost/api/v1/vscode/raw/${encodeURIComponent(key.key)}/models`),
|
||||
params
|
||||
),
|
||||
vscodeModelsRoute.GET(
|
||||
new Request(`http://localhost/api/v1/vscode/${encodeURIComponent(key.key)}/models`),
|
||||
params
|
||||
),
|
||||
]);
|
||||
const rawBody = (await rawResponse.json()) as { data?: MetadataModel[] };
|
||||
const groupedBody = (await groupedResponse.json()) as { data?: MetadataModel[] };
|
||||
const rawModel = (rawBody.data || []).find(
|
||||
(entry) => entry.id === "cx/future-codex-responses"
|
||||
);
|
||||
const groupedModel = (groupedBody.data || []).find(
|
||||
(entry) => entry.root === "future-codex-responses"
|
||||
);
|
||||
|
||||
assert.equal(rawResponse.status, 200);
|
||||
assert.equal(groupedResponse.status, 200);
|
||||
assert.ok(rawModel, "raw metadata route dropped a Responses text-generation model");
|
||||
assert.ok(groupedModel, "grouped metadata route dropped a Responses text-generation model");
|
||||
assert.equal(rawModel.api_format, "responses");
|
||||
assert.deepEqual(rawModel.supported_endpoints, ["responses"]);
|
||||
assert.equal(
|
||||
groupedModel.url,
|
||||
`http://localhost/api/v1/vscode/${encodeURIComponent(key.key)}/responses#models.ai.azure.com`
|
||||
);
|
||||
});
|
||||
@@ -73,6 +73,11 @@ test("vscode reasoning metadata supports GPT-5.6 Max and Ultra without splitting
|
||||
owned_by: "codex",
|
||||
capabilities: { reasoning: true },
|
||||
};
|
||||
const terra = {
|
||||
id: "cx/gpt-5.6-terra",
|
||||
owned_by: "codex",
|
||||
capabilities: { reasoning: true },
|
||||
};
|
||||
const luna = {
|
||||
id: "cx/gpt-5.6-luna",
|
||||
owned_by: "codex",
|
||||
@@ -80,7 +85,14 @@ test("vscode reasoning metadata supports GPT-5.6 Max and Ultra without splitting
|
||||
};
|
||||
|
||||
assert.deepEqual(reasoningMetadata.getReasoningEffortValues(sol), [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh",
|
||||
"max",
|
||||
"ultra",
|
||||
]);
|
||||
assert.deepEqual(reasoningMetadata.getReasoningEffortValues(terra), [
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
@@ -89,13 +101,15 @@ test("vscode reasoning metadata supports GPT-5.6 Max and Ultra without splitting
|
||||
"ultra",
|
||||
]);
|
||||
assert.deepEqual(reasoningMetadata.getReasoningEffortValues(luna), [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh",
|
||||
"max",
|
||||
]);
|
||||
assert.equal(reasoningMetadata.getDefaultReasoningEffort(sol), "low");
|
||||
assert.equal(reasoningMetadata.getDefaultReasoningEffort(terra), "medium");
|
||||
assert.equal(reasoningMetadata.getDefaultReasoningEffort(luna), "medium");
|
||||
assert.equal(
|
||||
reasoningMetadata.inferSelectedReasoningEffort(
|
||||
{ ...sol, id: "cx/gpt-5.6-sol-ultra" },
|
||||
|
||||
@@ -82,16 +82,16 @@ test("vscode raw models route exposes native GPT-5.6 IDs and effort tiers", asyn
|
||||
assert.equal(typeof defaultModel.created, "number");
|
||||
assert.equal(defaultModel.owned_by, "codex");
|
||||
assert.equal(defaultModel.name, "Codex GPT 5.6 Sol");
|
||||
assert.equal(typeof defaultModel.context_length, "number");
|
||||
assert.equal(typeof defaultModel.max_output_tokens, "number");
|
||||
assert.equal(typeof defaultModel.max_input_tokens, "number");
|
||||
assert.equal(defaultModel.context_length, 372000);
|
||||
assert.equal(defaultModel.max_output_tokens, 128000);
|
||||
assert.equal(defaultModel.max_input_tokens, 372000);
|
||||
assert.deepEqual(defaultModel.capabilities, {
|
||||
vision: true,
|
||||
tool_calling: true,
|
||||
reasoning: true,
|
||||
thinking: true,
|
||||
supportsThinking: true,
|
||||
effort_tiers: ["none", "low", "medium", "high", "xhigh", "max", "ultra"],
|
||||
effort_tiers: ["low", "medium", "high", "xhigh", "max", "ultra"],
|
||||
});
|
||||
for (const field of [
|
||||
"url",
|
||||
|
||||
@@ -255,7 +255,7 @@ test("vscode combos route resolves combo names through Ollama api/show", async (
|
||||
assert.equal(body.model, "show-combo");
|
||||
assert.equal(body.modelfile, "FROM show-combo");
|
||||
assert.equal(body.details.family, "show-combo");
|
||||
assert.equal(body.model_info.context_length, 500000);
|
||||
assert.equal(body.model_info.context_length, 372000);
|
||||
assert.deepEqual(body.supportsReasoningEffort, ["none", "low", "medium", "high", "xhigh"]);
|
||||
assert.equal(body.model_info.capabilities.reasoning, true);
|
||||
});
|
||||
@@ -377,7 +377,6 @@ test("vscode tokenized models route keeps xhigh for codex models that advertise
|
||||
assert.equal(model.toolCalling, true);
|
||||
assert.equal(model.vision, true);
|
||||
assert.deepEqual(model.supportsReasoningEffort, [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
@@ -386,7 +385,6 @@ test("vscode tokenized models route keeps xhigh for codex models that advertise
|
||||
"ultra",
|
||||
]);
|
||||
assert.deepEqual(model.supportedReasoningEfforts, [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
@@ -394,9 +392,8 @@ test("vscode tokenized models route keeps xhigh for codex models that advertise
|
||||
"max",
|
||||
"ultra",
|
||||
]);
|
||||
assert.equal(model.defaultReasoningEffort, "none");
|
||||
assert.equal(model.defaultReasoningEffort, "low");
|
||||
assert.deepEqual(model.configSchema?.properties?.reasoningEffort?.enum, [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
@@ -404,7 +401,7 @@ test("vscode tokenized models route keeps xhigh for codex models that advertise
|
||||
"max",
|
||||
"ultra",
|
||||
]);
|
||||
assert.equal(model.configSchema?.properties?.reasoningEffort?.default, "none");
|
||||
assert.equal(model.configSchema?.properties?.reasoningEffort?.default, "low");
|
||||
const importedIds = new Set((body.data || []).map((entry: any) => entry.id));
|
||||
assert.ok(!importedIds.has("cx/gpt-5.6-sol"));
|
||||
assert.ok(!importedIds.has("cx/gpt-5.6-sol__tier_priority"));
|
||||
@@ -651,7 +648,6 @@ test("vscode tokenized tags route exposes reasoning metadata for codex models",
|
||||
assert.equal(response.status, 200);
|
||||
assert.ok(model, "missing gpt-5.6-sol__provider_cx in tokenized VS Code tags route");
|
||||
assert.deepEqual(model.supportsReasoningEffort, [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
@@ -660,7 +656,6 @@ test("vscode tokenized tags route exposes reasoning metadata for codex models",
|
||||
"ultra",
|
||||
]);
|
||||
assert.deepEqual(model.supports_reasoning_effort, [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
@@ -669,7 +664,6 @@ test("vscode tokenized tags route exposes reasoning metadata for codex models",
|
||||
"ultra",
|
||||
]);
|
||||
assert.deepEqual(model.supportedReasoningEfforts, [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
@@ -677,12 +671,11 @@ test("vscode tokenized tags route exposes reasoning metadata for codex models",
|
||||
"max",
|
||||
"ultra",
|
||||
]);
|
||||
assert.equal(model.defaultReasoningEffort, "none");
|
||||
assert.equal(model.defaultReasoningEffort, "low");
|
||||
assert.equal(model.selectedReasoningEffort, "none");
|
||||
assert.equal(model.selected_reasoning_effort, "none");
|
||||
assert.equal(model.details.family, "gpt-5.6-sol");
|
||||
assert.deepEqual(model.configurationSchema?.properties?.reasoningEffort?.enum, [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
@@ -690,9 +683,8 @@ test("vscode tokenized tags route exposes reasoning metadata for codex models",
|
||||
"max",
|
||||
"ultra",
|
||||
]);
|
||||
assert.equal(model.configurationSchema?.properties?.reasoningEffort?.default, "none");
|
||||
assert.equal(model.configurationSchema?.properties?.reasoningEffort?.default, "low");
|
||||
assert.deepEqual(model.details.configurationSchema?.properties?.reasoningEffort?.enum, [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
@@ -701,7 +693,6 @@ test("vscode tokenized tags route exposes reasoning metadata for codex models",
|
||||
"ultra",
|
||||
]);
|
||||
assert.deepEqual(model.details.supports_reasoning_effort, [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
@@ -775,13 +766,17 @@ test("vscode tokenized tags route only exposes usable canonical chat models", as
|
||||
`tag ${tagModel.name} should be chat-capable`
|
||||
);
|
||||
assert.ok(
|
||||
!catalogModel.api_format || catalogModel.api_format === "chat-completions",
|
||||
`tag ${tagModel.name} should use chat-completions`
|
||||
!catalogModel.api_format ||
|
||||
["chat-completions", "responses", "openai-responses"].includes(
|
||||
catalogModel.api_format
|
||||
),
|
||||
`tag ${tagModel.name} should use a text-generation API format`
|
||||
);
|
||||
assert.ok(
|
||||
!Array.isArray(catalogModel.supported_endpoints) ||
|
||||
catalogModel.supported_endpoints.includes("chat"),
|
||||
`tag ${tagModel.name} should support chat`
|
||||
catalogModel.supported_endpoints.includes("chat") ||
|
||||
catalogModel.supported_endpoints.includes("responses"),
|
||||
`tag ${tagModel.name} should support text generation`
|
||||
);
|
||||
assert.ok(
|
||||
!Array.isArray(catalogModel.output_modalities) ||
|
||||
@@ -794,8 +789,11 @@ test("vscode tokenized tags route only exposes usable canonical chat models", as
|
||||
(model: any) =>
|
||||
model.parent ||
|
||||
(typeof model.type === "string" && model.type !== "chat") ||
|
||||
(typeof model.api_format === "string" && model.api_format !== "chat-completions") ||
|
||||
(Array.isArray(model.supported_endpoints) && !model.supported_endpoints.includes("chat")) ||
|
||||
(typeof model.api_format === "string" &&
|
||||
!["chat-completions", "responses", "openai-responses"].includes(model.api_format)) ||
|
||||
(Array.isArray(model.supported_endpoints) &&
|
||||
!model.supported_endpoints.includes("chat") &&
|
||||
!model.supported_endpoints.includes("responses")) ||
|
||||
(Array.isArray(model.output_modalities) && !model.output_modalities.includes("text"))
|
||||
);
|
||||
const tagNames = new Set((tagsBody.models || []).map((model: any) => model.name));
|
||||
@@ -1040,7 +1038,6 @@ test("vscode tokenized api/show route exposes explicit reasoning effort metadata
|
||||
assert.equal(body.remote_model, "Codex GPT 5.6 Sol (Default)");
|
||||
assert.equal(body.details.family, "gpt-5.6-sol");
|
||||
assert.deepEqual(body.supportsReasoningEffort, [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
@@ -1049,7 +1046,6 @@ test("vscode tokenized api/show route exposes explicit reasoning effort metadata
|
||||
"ultra",
|
||||
]);
|
||||
assert.deepEqual(body.supports_reasoning_effort, [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
@@ -1058,7 +1054,6 @@ test("vscode tokenized api/show route exposes explicit reasoning effort metadata
|
||||
"ultra",
|
||||
]);
|
||||
assert.deepEqual(body.supportedReasoningEfforts, [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
@@ -1066,11 +1061,10 @@ test("vscode tokenized api/show route exposes explicit reasoning effort metadata
|
||||
"max",
|
||||
"ultra",
|
||||
]);
|
||||
assert.equal(body.defaultReasoningEffort, "none");
|
||||
assert.equal(body.defaultReasoningEffort, "low");
|
||||
assert.equal(body.selectedReasoningEffort, "none");
|
||||
assert.equal(body.selected_reasoning_effort, "none");
|
||||
assert.deepEqual(body.configurationSchema?.properties?.reasoningEffort?.enum, [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
@@ -1078,12 +1072,11 @@ test("vscode tokenized api/show route exposes explicit reasoning effort metadata
|
||||
"max",
|
||||
"ultra",
|
||||
]);
|
||||
assert.equal(body.configurationSchema?.properties?.reasoningEffort?.default, "none");
|
||||
assert.equal(body.configurationSchema?.properties?.reasoningEffort?.default, "low");
|
||||
assert.equal(body.model_info["general.basename"], "Codex GPT 5.6 Sol (Default)");
|
||||
assert.equal(body.model_info["general.architecture"], "codex");
|
||||
assert.equal(body.model_info["codex.context_length"], 500000);
|
||||
assert.equal(body.model_info["codex.context_length"], 372000);
|
||||
assert.deepEqual(body.model_info.supports_reasoning_effort, [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
@@ -1093,7 +1086,6 @@ test("vscode tokenized api/show route exposes explicit reasoning effort metadata
|
||||
]);
|
||||
assert.equal(body.model_info.selected_reasoning_effort, "none");
|
||||
assert.deepEqual(body.model_info.capabilities.supports_reasoning_effort, [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
|
||||
Reference in New Issue
Block a user