mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-19 13:42:09 +03:00
fix(sse): clamp Azure gpt-4o-mini completion tokens to its 16384 ceiling
Azure gpt-4o-mini deployments accept at most 16384 completion tokens and 400 on anything larger: max_tokens is too large: 32000. This model supports at most 16384 completion tokens, whereas you provided 32000. The 32000 is OmniRoute's own doing: adjustMaxTokens raises any smaller max_tokens to DEFAULT_MIN_TOKENS (32000) whenever tools are present, to avoid truncated tool arguments. That floor has no upper bound, so an agentic client asking for far less still trips the model ceiling on its first turn. Add scoped maxOutputCap rules in paramSupport.ts for both Azure wire paths. PROVIDER_MAX_TOKENS is the wrong lever here - it is provider-wide, and the same Azure resource also serves GPT-5 deployments with a much higher ceiling. Regression guard: tests/unit/azure-max-output-clamp.test.ts, which also pins that the clamp does not leak to gpt-5.1 or to gpt-4o-mini on other providers.
This commit is contained in:
committed by
diegosouzapw
parent
37129db6af
commit
06541b1036
@@ -63,7 +63,12 @@ const STRIP_RULES: StripRule[] = [
|
||||
// MoonshotAI/kimi-cli#1124), and by upstream decolua/9router#2460. Scoped to
|
||||
// OmniRoute's actual volcengine Kimi id (not a broad /kimi/i regex) so it
|
||||
// never clamps an unrelated future Kimi listing whose Ark cap may differ.
|
||||
{ provider: "volcengine", match: /^kimi-k2-5-260127$/, maxOutputCap: 32768, clampToModelMaxOutput: true },
|
||||
{
|
||||
provider: "volcengine",
|
||||
match: /^kimi-k2-5-260127$/,
|
||||
maxOutputCap: 32768,
|
||||
clampToModelMaxOutput: true,
|
||||
},
|
||||
// #7364: Z.AI's glm-4.6v vision endpoint enforces a 32768 max_tokens ceiling
|
||||
// server-side and 400s when a client sends a larger explicit max_tokens (e.g. a
|
||||
// client defaulting to 65536). Scoped to both wire paths that can reach this
|
||||
@@ -75,6 +80,19 @@ const STRIP_RULES: StripRule[] = [
|
||||
// glmProvider.ts, maxOutputTokens: 32768, so clampToModelMaxOutput suffices).
|
||||
{ provider: "zai", match: /^glm-4\.6v$/i, maxOutputCap: 32768 },
|
||||
{ provider: "glm", match: /^glm-4\.6v$/i, clampToModelMaxOutput: true },
|
||||
// Azure gpt-4o-mini deployments cap completion tokens at 16384 and 400 on
|
||||
// anything larger: "max_tokens is too large: 32000. This model supports at
|
||||
// most 16384 completion tokens". OmniRoute's own tool-calling floor
|
||||
// (DEFAULT_MIN_TOKENS = 32000, applied by adjustMaxTokens) raises even a tiny
|
||||
// explicit max_tokens to 32000 whenever tools are present, so every agentic
|
||||
// client trips this on its first turn. PROVIDER_MAX_TOKENS is not the right
|
||||
// lever here: it is provider-wide, and the same Azure resource also serves
|
||||
// GPT-5 deployments whose ceiling is far higher. Azure deployment names are
|
||||
// operator-chosen, hence a prefix match rather than an exact id, and the
|
||||
// models are passthrough (no catalog maxOutputTokens for clampToModelMaxOutput
|
||||
// to read), hence the fixed cap.
|
||||
{ provider: "azure-openai", match: /^gpt-4o-mini/i, maxOutputCap: 16384 },
|
||||
{ provider: "azure-ai", match: /^gpt-4o-mini/i, maxOutputCap: 16384 },
|
||||
];
|
||||
|
||||
function matches(rule: StripRule, model: string): boolean {
|
||||
|
||||
58
tests/unit/azure-max-output-clamp.test.ts
Normal file
58
tests/unit/azure-max-output-clamp.test.ts
Normal file
@@ -0,0 +1,58 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import { stripUnsupportedParams } from "../../open-sse/translator/paramSupport.ts";
|
||||
|
||||
/**
|
||||
* Regression guard for the Azure gpt-4o-mini completion-token ceiling.
|
||||
*
|
||||
* Observed against a live Azure deployment:
|
||||
* azure-openai/gpt-4o-mini-dz
|
||||
* -> 400 "max_tokens is too large: 32000. This model supports at most
|
||||
* 16384 completion tokens, whereas you provided 32000."
|
||||
*
|
||||
* The 32000 is OmniRoute's own doing: `adjustMaxTokens` raises any smaller
|
||||
* max_tokens to DEFAULT_MIN_TOKENS (32000) whenever tools are present, so an
|
||||
* agentic client trips this on its first turn even when it asked for far less.
|
||||
*/
|
||||
|
||||
test("azure gpt-4o-mini clamps max_tokens to the 16384 ceiling", () => {
|
||||
const out = stripUnsupportedParams("azure-openai", "gpt-4o-mini-dz", {
|
||||
max_tokens: 32000,
|
||||
messages: [],
|
||||
}) as Record<string, unknown>;
|
||||
|
||||
assert.equal(out.max_tokens, 16384);
|
||||
});
|
||||
|
||||
test("the clamp applies on the azure-ai wire path too", () => {
|
||||
const out = stripUnsupportedParams("azure-ai", "gpt-4o-mini", {
|
||||
max_completion_tokens: 32000,
|
||||
}) as Record<string, unknown>;
|
||||
|
||||
assert.equal(out.max_completion_tokens, 16384);
|
||||
});
|
||||
|
||||
test("a value already under the ceiling is left alone", () => {
|
||||
const out = stripUnsupportedParams("azure-openai", "gpt-4o-mini", {
|
||||
max_tokens: 800,
|
||||
}) as Record<string, unknown>;
|
||||
|
||||
assert.equal(out.max_tokens, 800);
|
||||
});
|
||||
|
||||
test("the clamp is scoped — larger Azure deployments keep their budget", () => {
|
||||
const out = stripUnsupportedParams("azure-ai", "gpt-5.1", {
|
||||
max_tokens: 32000,
|
||||
}) as Record<string, unknown>;
|
||||
|
||||
assert.equal(out.max_tokens, 32000);
|
||||
});
|
||||
|
||||
test("the clamp does not leak to gpt-4o-mini on other providers", () => {
|
||||
const out = stripUnsupportedParams("openai", "gpt-4o-mini", {
|
||||
max_tokens: 32000,
|
||||
}) as Record<string, unknown>;
|
||||
|
||||
assert.equal(out.max_tokens, 32000);
|
||||
});
|
||||
Reference in New Issue
Block a user