diff --git a/open-sse/services/learnedReasoningEffortCaps.ts b/open-sse/services/learnedReasoningEffortCaps.ts index 5ed852765c..568b2e3798 100644 --- a/open-sse/services/learnedReasoningEffortCaps.ts +++ b/open-sse/services/learnedReasoningEffortCaps.ts @@ -12,7 +12,21 @@ * `sanitizeReasoningEffortForProvider` in `executors/base/reasoningEffort.ts`) * so the 4xx→retry round-trip is paid at most once per process per provider+model. * - * `clampToLearned` implements downgrade-only clamping: greatest accepted <= demand. + * `clampToLearned` implements nearest-tier clamping: smallest accepted >= demand, + * falling back to the greatest accepted when demand exceeds every accepted value. + * (#11295 — unified with the static "declared" clamp in + * `executors/base/reasoningEffort.ts`, which already used nearest-tier semantics. + * Before #11295, this learned clamp was downgrade-only — greatest accepted <= + * demand — so the SAME accepted set {low,high,max} produced medium→low here but + * medium→high via the declared path: identical inputs, opposite outputs, + * depending only on whether the model had a static registry entry. #11274's + * DeepSeek native mapping is the precedent for nearest-tier. This also fixes a + * standalone bug: a request BELOW the learned floor (e.g. none/minimal on a + * model that only ever advertised {low,high,max}) used to return null — no + * clamp — so the too-low value passed straight through to the upstream, which + * 400'd again on every subsequent request without ever learning a lower floor. + * Nearest-tier naturally fixes this too: the smallest accepted value is always + * >= any demand below the floor, so it is returned instead of null. * * In-memory only (same operator-accepted tradeoff as the thinking-budget cache): * restart resets, the first request after a restart may re-learn at the cost of @@ -132,25 +146,39 @@ export function recordLearnedReasoningEffort( } /** - * Return the greatest accepted value <= effortStr (downgrade only), or null - * if effortStr is already accepted, below the minimum, or not in ORDER. + * Return the nearest-tier accepted value for effortStr: the smallest accepted + * value with rank >= effortStr's rank, or — when effortStr's rank exceeds every + * accepted value (demand above the learned ceiling) — the greatest accepted + * value. Returns null only when effortStr is already accepted (no clamp + * needed), empty, or not a recognized member of REASONING_EFFORT_ORDER. + * + * Mirrors the declared-capability clamp in `executors/base/reasoningEffort.ts` + * (#11295): both now use nearest-tier semantics so the same accepted set + * produces the same mapping regardless of whether the model has a static + * registry entry or was only learned reactively from an upstream 4xx. */ export function clampToLearned(effortStr: string, accepted: Set): string | null { if (!effortStr || accepted.has(effortStr)) return null; const rank = rankOf(effortStr); if (rank === -1) return null; - const minRank = Math.min(...[...accepted].map((v) => rankOf(v))); - if (rank < minRank) return null; - let best: string | null = null; - let bestRank = -1; + + let nearestAbove: string | null = null; + let nearestAboveRank = Infinity; + let highest: string | null = null; + let highestRank = -1; for (const v of accepted) { const r = rankOf(v); - if (r <= rank && r > bestRank) { - bestRank = r; - best = v; + if (r < 0) continue; + if (r >= rank && r < nearestAboveRank) { + nearestAboveRank = r; + nearestAbove = v; + } + if (r > highestRank) { + highestRank = r; + highest = v; } } - return best; + return nearestAbove ?? highest; } // Matches prose shapes: OVH's "@ai-sdk/openai-compatible" deserializer diff --git a/tests/unit/learned-reasoning-effort-caps.test.ts b/tests/unit/learned-reasoning-effort-caps.test.ts index c057351814..b656f34bbb 100644 --- a/tests/unit/learned-reasoning-effort-caps.test.ts +++ b/tests/unit/learned-reasoning-effort-caps.test.ts @@ -130,13 +130,18 @@ test("a later, lower accepted-list does ratchet the cap down", () => { assert.equal((getLearnedReasoningEffort("acme", "model-x") as unknown as Set).size, 2); }); -test("clampToLearned medium→low when accepted is low,high,max", async () => { +// #11295: nearest-tier semantics (smallest accepted >= demand) — unified with +// the declared/static clamp. Was downgrade-only (greatest accepted <= demand, +// medium→low) before #11295. +test("clampToLearned medium→high when accepted is low,high,max (nearest-tier, #11295)", async () => { const { clampToLearned } = await import("../../open-sse/services/learnedReasoningEffortCaps.ts"); - assert.equal(clampToLearned("medium", new Set(["low", "high", "max"])), "low"); + assert.equal(clampToLearned("medium", new Set(["low", "high", "max"])), "high"); }); -test("clampToLearned xhigh→high when accepted is low,high,max", async () => { +// #11295: xhigh(rank 5) has no accepted tier >= it among {low,high,max} +// (max=6 IS >= 5, so nearest-tier picks max) — was downgrade-only high before. +test("clampToLearned xhigh→max when accepted is low,high,max (nearest-tier, #11295)", async () => { const { clampToLearned } = await import("../../open-sse/services/learnedReasoningEffortCaps.ts"); - assert.equal(clampToLearned("xhigh", new Set(["low", "high", "max"])), "high"); + assert.equal(clampToLearned("xhigh", new Set(["low", "high", "max"])), "max"); }); test("clampToLearned ultra→max when accepted is low,high,max", async () => { const { clampToLearned } = await import("../../open-sse/services/learnedReasoningEffortCaps.ts"); @@ -154,17 +159,25 @@ test("clampToLearned returns null when already accepted", async () => { const { clampToLearned } = await import("../../open-sse/services/learnedReasoningEffortCaps.ts"); assert.equal(clampToLearned("low", new Set(["low", "high", "max"])), null); }); -test("clampToLearned returns null when effort < min (no upgrade)", async () => { +// #11295: a sub-floor demand (below every accepted value) now maps to the +// accepted floor instead of returning null. Pre-#11295 this returned null — +// no clamp — so the too-low value passed straight through to the upstream, +// which 400'd again on every subsequent request without ever learning a +// lower floor. +test("clampToLearned maps sub-floor demand to the accepted floor instead of null (#11295)", async () => { const { clampToLearned } = await import("../../open-sse/services/learnedReasoningEffortCaps.ts"); - assert.equal(clampToLearned("low", new Set(["high", "max"])), null); + assert.equal(clampToLearned("low", new Set(["high", "max"])), "high"); }); test("clampToLearned returns null for turbo (not in ORDER)", async () => { const { clampToLearned } = await import("../../open-sse/services/learnedReasoningEffortCaps.ts"); assert.equal(clampToLearned("turbo", new Set(["low", "high", "max"])), null); }); -test("clampToLearned returns null when effort is none but accepted is low,high,max", async () => { +// #11295: none is below the learned floor {low,high,max} — nearest-tier maps +// it to the floor (low) instead of returning null (no clamp, upstream 400s +// again with no chance to ever learn a lower floor). +test("clampToLearned maps none to the floor (low) when accepted is low,high,max (#11295)", async () => { const { clampToLearned } = await import("../../open-sse/services/learnedReasoningEffortCaps.ts"); - assert.equal(clampToLearned("none", new Set(["low", "high", "max"])), null); + assert.equal(clampToLearned("none", new Set(["low", "high", "max"])), "low"); }); test("recordLearned stores Set and getLearned returns Set", () => { const s = recordLearnedReasoningEffort("acme", "m1", ["low", "high", "max"]); diff --git a/tests/unit/reasoning-effort-clamp-and-retry.test.ts b/tests/unit/reasoning-effort-clamp-and-retry.test.ts index fa925216c6..b67451e6b4 100644 --- a/tests/unit/reasoning-effort-clamp-and-retry.test.ts +++ b/tests/unit/reasoning-effort-clamp-and-retry.test.ts @@ -108,7 +108,7 @@ test("a second request for the same provider+model sends the learned value on th } }); -test("400 please use low, high, or max clamps and retries once", async () => { +test("400 please use low, high, or max clamps and retries once (nearest-tier: medium -> high, #11295)", async () => { const executor = new SimpleExecutor(); const originalFetch = globalThis.fetch; const capturedBodies: Record[] = []; @@ -140,7 +140,10 @@ test("400 please use low, high, or max clamps and retries once", async () => { }); assert.equal(capturedBodies.length, 2); assert.equal(capturedBodies[0].reasoning_effort, "medium"); - assert.equal(capturedBodies[1].reasoning_effort, "low"); + // #11295: nearest-tier — smallest accepted >= demand — maps medium(3) to + // high(4), the smallest accepted rank at or above it (was "low" under the + // old downgrade-only direction). + assert.equal(capturedBodies[1].reasoning_effort, "high"); const learned = getLearnedReasoningEffort("openai-compatible-chat-eaff6869", "x-preview-f-free") as unknown as Set; assert.ok(learned instanceof Set); assert.ok(learned.has("low")); @@ -190,7 +193,7 @@ test("400 please use low, medium with ultra retries to medium", async () => { } }); -test("no-op clamp does not retry: learned {high,max} with low request stays single-fetch", async () => { +test("sub-floor clamp now retries: learned {high,max} with low request clamps up to high (#11295)", async () => { const executor = new SimpleExecutor(); const originalFetch = globalThis.fetch; const capturedBodies: Record[] = []; @@ -214,17 +217,20 @@ test("no-op clamp does not retry: learned {high,max} with low request stays sing }; try { - // low is below the learned minimum {high,max}: downgrade-only passthrough, - // sanitizer leaves the body unchanged -> no identical-body retry. + // #11295: low is below the learned minimum {high,max}. Pre-#11295 this was + // a downgrade-only passthrough (no clamp, no retry, upstream stayed 400 + // forever). Nearest-tier now clamps up to the accepted floor (high) and + // retries once, succeeding. const result = await executor.execute({ model: "x-preview-f-free-3", body: { reasoning_effort: "low" }, stream: false, credentials: {}, }); - assert.equal(capturedBodies.length, 1); + assert.equal(capturedBodies.length, 2); assert.equal(capturedBodies[0].reasoning_effort, "low"); - assert.equal(result.response.status, 400); + assert.equal(capturedBodies[1].reasoning_effort, "high"); + assert.equal(result.response.status, 200); } finally { globalThis.fetch = originalFetch; } diff --git a/tests/unit/reasoning-effort-clamp-direction-consistency.test.ts b/tests/unit/reasoning-effort-clamp-direction-consistency.test.ts new file mode 100644 index 0000000000..7d09967ec9 --- /dev/null +++ b/tests/unit/reasoning-effort-clamp-direction-consistency.test.ts @@ -0,0 +1,79 @@ +// #11295 — the learned clamp (reactive, from upstream 4xx) and the declared +// clamp (static registry `supportedThinkingEfforts`) used to disagree on +// direction for the identical accepted set {low,high,max}: the learned path +// was downgrade-only (medium -> low) while the declared path was already +// nearest-tier (medium -> high). Same inputs, opposite outputs, depending only +// on whether the model happened to have a static registry entry. This test +// proves the two paths now agree, and that a request below the learned floor +// (previously silently passed through unmapped, returning null from +// clampToLearned) is now mapped up to the nearest accepted tier instead. +import { test, after, beforeEach } from "node:test"; +import assert from "node:assert/strict"; +import { clampToLearned } from "../../open-sse/services/learnedReasoningEffortCaps.ts"; +import { sanitizeReasoningEffortForProvider } from "../../open-sse/executors/base/reasoningEffort.ts"; +import { + recordLearnedReasoningEffort, + __test_resetLearnedReasoningEffortCaps, +} from "../../open-sse/services/learnedReasoningEffortCaps.ts"; + +beforeEach(() => { + __test_resetLearnedReasoningEffortCaps(); +}); + +after(() => { + __test_resetLearnedReasoningEffortCaps(); +}); + +test("clampToLearned: nearest-tier medium -> high when accepted is {low,high,max} (was low pre-#11295)", () => { + assert.equal(clampToLearned("medium", new Set(["low", "high", "max"])), "high"); +}); + +test("sanitizeReasoningEffortForProvider maps medium identically for a LEARNED-only model and a DECLARED model with the same {low,high,max} accepted set", () => { + // Learned side: a custom OpenAI-compatible connection that has no static + // registry entry — the only source of truth is the reactively-learned set. + recordLearnedReasoningEffort("acme-oai-compatible", "custom-reasoner", [ + "low", + "high", + "max", + ]); + const learnedResult = sanitizeReasoningEffortForProvider( + { reasoning_effort: "medium" }, + "acme-oai-compatible", + "custom-reasoner" + ) as Record; + + // Declared side: opencode-go/ox-alpha-free, whose registry entry declares + // supportedThinkingEfforts: ["low", "high", "max"] (see reasoningEffort.ts + // comment referencing the Console Go 400 case). + const declaredResult = sanitizeReasoningEffortForProvider( + { reasoning_effort: "medium" }, + "opencode-go", + "ox-alpha-free" + ) as Record; + + assert.equal(learnedResult.reasoning_effort, "high"); + assert.equal(declaredResult.reasoning_effort, "high"); + assert.equal(learnedResult.reasoning_effort, declaredResult.reasoning_effort); +}); + +test("sub-floor request (none) on a learned-only model with floor {low,high,max} maps to low, not a pass-through null-clamp", () => { + recordLearnedReasoningEffort("acme-oai-compatible", "custom-reasoner-2", [ + "low", + "high", + "max", + ]); + const result = sanitizeReasoningEffortForProvider( + { reasoning_effort: "none" }, + "acme-oai-compatible", + "custom-reasoner-2" + ) as Record; + assert.equal(result.reasoning_effort, "low"); +}); + +test("clampToLearned: sub-floor demand (none) below accepted {low,high,max} maps to the accepted floor (low), not null", () => { + assert.equal(clampToLearned("none", new Set(["low", "high", "max"])), "low"); +}); + +test("clampToLearned: sub-floor demand (low) below accepted {high,max} maps to the accepted floor (high), not null", () => { + assert.equal(clampToLearned("low", new Set(["high", "max"])), "high"); +}); diff --git a/tests/unit/reasoning-effort-learned-capability.test.ts b/tests/unit/reasoning-effort-learned-capability.test.ts index 9dab460d8f..36ac5d13d8 100644 --- a/tests/unit/reasoning-effort-learned-capability.test.ts +++ b/tests/unit/reasoning-effort-learned-capability.test.ts @@ -90,23 +90,25 @@ test("deepseek's non-ordinal max<->xhigh translation is untouched by the learned assert.equal(result.reasoning_effort, "max"); }); -test("proactive clamp: medium→low for learned {low,high,max}", () => { +// #11295: nearest-tier — smallest accepted >= demand — replaces the old +// downgrade-only (greatest accepted <= demand) direction. +test("proactive clamp: medium→high for learned {low,high,max} (nearest-tier, #11295)", () => { recordLearnedReasoningEffort("opencode-zen-direct", "x-preview-f-free", ["low", "high", "max"]); const out = sanitizeReasoningEffortForProvider( { reasoning_effort: "medium", model: "x-preview-f-free" }, "opencode-zen-direct", "x-preview-f-free" ) as { reasoning_effort: string }; - assert.equal(out.reasoning_effort, "low"); + assert.equal(out.reasoning_effort, "high"); }); -test("proactive clamp: xhigh→high for learned {low,high,max}", () => { +test("proactive clamp: xhigh→max for learned {low,high,max} (nearest-tier, #11295)", () => { recordLearnedReasoningEffort("opencode-zen-direct", "x-preview-f-free-2", ["low", "high", "max"]); const out = sanitizeReasoningEffortForProvider( { reasoning_effort: "xhigh", model: "x-preview-f-free-2" }, "opencode-zen-direct", "x-preview-f-free-2" ) as { reasoning_effort: string }; - assert.equal(out.reasoning_effort, "high"); + assert.equal(out.reasoning_effort, "max"); }); test("proactive clamp: ultra→max for learned {low,high,max}", () => { recordLearnedReasoningEffort("opencode-zen-direct", "x-preview-f-free-3", ["low", "high", "max"]); @@ -135,14 +137,16 @@ test("proactive clamp: high→medium for learned {low,medium}", () => { ) as { reasoning_effort: string }; assert.equal(out.reasoning_effort, "medium"); }); -test("no upgrade: low stays low for learned {high,max}", () => { +// #11295: sub-floor demand (low, below the learned floor {high,max}) now +// clamps up to the floor instead of passing through unchanged. +test("sub-floor clamp: low→high for learned {high,max} (#11295)", () => { recordLearnedReasoningEffort("acme", "m3", ["high", "max"]); const out = sanitizeReasoningEffortForProvider( { reasoning_effort: "low", model: "m3" }, "acme", "m3" ) as { reasoning_effort: string }; - assert.equal(out.reasoning_effort, "low"); + assert.equal(out.reasoning_effort, "high"); }); test("custom model ultra→medium for learned {low,medium}", () => { recordLearnedReasoningEffort("openai-compatible-chat-eaff6869", "qwen3-coder-30b-a3b-instruct-2", ["low", "medium"]);