mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-23 07:32:20 +03:00
fix(providers): learn reasoning_effort capability from upstream 4xx instead of a hardcoded/opt-out default (#11116)
Validated on the combined batch board over release/v3.8.50 tip d91238b7: static gates clean, typecheck:core clean, focused tests green.
Learned reasoning_effort caps mirror the merged learnedThinkingCaps mechanism: parse the upstream 4xx enum, clamp, retry once, consult proactively — covers custom openai-compatible connections the static registry can't. 22 new test cases + full regression list green. Fixes #11111. Thank you @maxmad64bis!
This commit is contained in:
@@ -0,0 +1 @@
|
||||
- **fix(providers):** `reasoning_effort` now learns the accepted values from a provider's own 400/422 response and clamps to the highest one instead of forwarding an unsupported `xhigh`/`max` (or a hardcoded `"high"` fallback) — fixes custom OpenAI-compatible connections and registered providers with no reasoning metadata ([#11116](https://github.com/diegosouzapw/OmniRoute/pull/11116)) — thanks @maxmad64bis
|
||||
@@ -171,6 +171,7 @@ export const HTTP_STATUS = {
|
||||
FORBIDDEN: 403,
|
||||
NOT_FOUND: 404,
|
||||
NOT_ACCEPTABLE: 406,
|
||||
UNPROCESSABLE_ENTITY: 422,
|
||||
REQUEST_TIMEOUT: 408,
|
||||
GONE: 410,
|
||||
RATE_LIMITED: 429,
|
||||
@@ -263,11 +264,17 @@ export const PROVIDER_PROFILES = {
|
||||
circuitBreakerReset: envInt("OMNIROUTE_CIRCUIT_BREAKER_API_KEY_RESET_MS", 30000),
|
||||
// Provider-level circuit breaker (entire provider cooldown after repeated failures)
|
||||
providerFailureThreshold: envInt("OMNIROUTE_PROVIDER_BREAKER_API_KEY_FAILURE_THRESHOLD", 15), // Scaled for 500+ connections (was 5)
|
||||
providerFailureWindowMs: envInt("OMNIROUTE_PROVIDER_BREAKER_API_KEY_FAILURE_WINDOW_MS", 1800000), // 30min window (was 20min)
|
||||
providerFailureWindowMs: envInt(
|
||||
"OMNIROUTE_PROVIDER_BREAKER_API_KEY_FAILURE_WINDOW_MS",
|
||||
1800000
|
||||
), // 30min window (was 20min)
|
||||
providerCooldownMs: envInt("OMNIROUTE_PROVIDER_BREAKER_API_KEY_COOLDOWN_MS", 600000), // 10min cooldown when threshold reached
|
||||
degradationThreshold: envInt("OMNIROUTE_PROVIDER_BREAKER_API_KEY_DEGRADATION_THRESHOLD", 7),
|
||||
maxBackoffMultiplier: envInt("OMNIROUTE_PROVIDER_BREAKER_API_KEY_MAX_BACKOFF_MULTIPLIER", 4),
|
||||
backoffEscalationCount: envInt("OMNIROUTE_PROVIDER_BREAKER_API_KEY_BACKOFF_ESCALATION_COUNT", 3),
|
||||
backoffEscalationCount: envInt(
|
||||
"OMNIROUTE_PROVIDER_BREAKER_API_KEY_BACKOFF_ESCALATION_COUNT",
|
||||
3
|
||||
),
|
||||
},
|
||||
// Local providers (localhost inference backends like Ollama, LM Studio, oMLX).
|
||||
// Not yet wired into getProviderProfile() — will be used when local provider_nodes
|
||||
|
||||
@@ -20,6 +20,10 @@ import {
|
||||
recordLearnedThinkingCap,
|
||||
parseThinkingBudgetMax,
|
||||
} from "../services/learnedThinkingCaps.ts";
|
||||
import {
|
||||
recordLearnedReasoningEffort,
|
||||
parseReasoningEffortEnum,
|
||||
} from "../services/learnedReasoningEffortCaps.ts";
|
||||
import {
|
||||
getParamFilterConfig,
|
||||
addParamToBlocklist,
|
||||
@@ -826,6 +830,9 @@ export class BaseExecutor {
|
||||
// loop. The learned cap is also recorded process-wide via
|
||||
// recordLearnedThinkingCap so future requests skip the 400 entirely.
|
||||
let thinkingBudgetClampedMax: number | null = null;
|
||||
// Set by the reasoning_effort 4xx clamp-and-retry below — guards the same
|
||||
// "fires at most once per URL" invariant as thinkingBudgetClampedMax above.
|
||||
let reasoningEffortClamped = false;
|
||||
|
||||
for (let urlIndex = 0; urlIndex < fallbackCount; urlIndex++) {
|
||||
const requestCredentials = withForcedResponsesUpstream(
|
||||
@@ -1529,6 +1536,49 @@ export class BaseExecutor {
|
||||
}
|
||||
}
|
||||
|
||||
// Reasoning-effort enum 4xx clamp-and-retry (any provider/model without a
|
||||
// declared reasoning_effort capability — custom OpenAI-compatible
|
||||
// connections, or a registered provider the registry hasn't caught up
|
||||
// with). Mirrors the thinking_budget clamp-and-retry above: parse the
|
||||
// upstream-advertised accepted values, record them process-wide (so
|
||||
// FUTURE requests clamp proactively via sanitizeReasoningEffortForProvider
|
||||
// → getLearnedReasoningEffort), clamp the live transformedBody by
|
||||
// re-running the sanitizer, and retry the same URL once.
|
||||
if (
|
||||
(response.status === HTTP_STATUS.BAD_REQUEST ||
|
||||
response.status === HTTP_STATUS.UNPROCESSABLE_ENTITY) &&
|
||||
!reasoningEffortClamped &&
|
||||
transformedBody &&
|
||||
typeof transformedBody === "object"
|
||||
) {
|
||||
const errText = await response
|
||||
.clone()
|
||||
.text()
|
||||
.catch(() => "");
|
||||
const acceptedValues = parseReasoningEffortEnum(errText);
|
||||
if (acceptedValues) {
|
||||
reasoningEffortClamped = true;
|
||||
const learned = recordLearnedReasoningEffort(this.provider, model, acceptedValues);
|
||||
if (learned) {
|
||||
transformedBody = sanitizeReasoningEffortForProvider(
|
||||
transformedBody,
|
||||
this.provider,
|
||||
model,
|
||||
log
|
||||
);
|
||||
let retryBody = JSON.stringify(transformedBody);
|
||||
if (usesClaudeCodeProtocol || this.provider === "claude") {
|
||||
retryBody = await signRequestBody(retryBody);
|
||||
}
|
||||
log?.info?.(
|
||||
"REASONING_SANITIZE",
|
||||
`Upstream ${response.status} rejected reasoning_effort on ${url} — clamped to ${learned} and retrying (learned for ${this.provider}/${model})`
|
||||
);
|
||||
response = await fetchWithStartTimeout(url, { ...fetchOptions, body: retryBody });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Generic reactive 400 field-downgrade; each field is stripped at most once.
|
||||
if (
|
||||
response.status === HTTP_STATUS.BAD_REQUEST &&
|
||||
|
||||
@@ -8,6 +8,10 @@ import {
|
||||
getProviderModel,
|
||||
getProviderModels,
|
||||
} from "../../config/providerModels.ts";
|
||||
import {
|
||||
getLearnedReasoningEffort,
|
||||
REASONING_EFFORT_ORDER,
|
||||
} from "../../services/learnedReasoningEffortCaps.ts";
|
||||
|
||||
/**
|
||||
* Sanitize reasoning_effort for providers that don't accept all values.
|
||||
@@ -338,10 +342,24 @@ export function sanitizeReasoningEffortForProvider(
|
||||
|
||||
const supportsXHigh = supportsXHighEffort(provider, modelStr);
|
||||
const supportsMax = supportsMaxEffortForProvider(provider, modelStr);
|
||||
// Highest value we've actually seen this provider+model accept in a real
|
||||
// upstream 4xx (learnedReasoningEffortCaps.ts) — takes priority over the
|
||||
// static registry (which defaults to "supports everything" when there's no
|
||||
// entry, e.g. custom OpenAI-compatible connections) and over the hardcoded
|
||||
// "high" fallback below (which isn't always valid either).
|
||||
const learnedCap = getLearnedReasoningEffort(provider, modelStr);
|
||||
const learnedRank = learnedCap ? REASONING_EFFORT_ORDER.indexOf(learnedCap) : -1;
|
||||
|
||||
// ── xhigh handling ──────────────────────────────────────────────────────
|
||||
// xhigh is OmniRoute-internal. Map it to the best effort the model accepts.
|
||||
if (effortStr === "xhigh") {
|
||||
if (learnedCap && learnedRank < REASONING_EFFORT_ORDER.indexOf("xhigh")) {
|
||||
log?.info?.(
|
||||
"REASONING_SANITIZE",
|
||||
`${provider}/${modelStr}: clamped reasoning_effort xhigh → ${learnedCap} (learned)`
|
||||
);
|
||||
return writeEffortValue(b, learnedCap, c);
|
||||
}
|
||||
if (supportsXHigh) return body; // model accepts xhigh natively
|
||||
if (supportsMax) {
|
||||
log?.info?.(
|
||||
@@ -366,6 +384,13 @@ export function sanitizeReasoningEffortForProvider(
|
||||
// upstream, and if it 400s the user gets a clear signal. This prevents
|
||||
// new models from being unusable for weeks until they're whitelisted (#8057).
|
||||
if (effortStr === "max") {
|
||||
if (learnedCap && learnedRank < REASONING_EFFORT_ORDER.indexOf("max")) {
|
||||
log?.info?.(
|
||||
"REASONING_SANITIZE",
|
||||
`${provider}/${modelStr}: clamped reasoning_effort max → ${learnedCap} (learned)`
|
||||
);
|
||||
return writeEffortValue(b, learnedCap, c);
|
||||
}
|
||||
if (supportsMax) return body; // explicitly known to accept max
|
||||
|
||||
// A model that explicitly advertises its accepted tiers is safe to normalize.
|
||||
|
||||
126
open-sse/services/learnedReasoningEffortCaps.ts
Normal file
126
open-sse/services/learnedReasoningEffortCaps.ts
Normal file
@@ -0,0 +1,126 @@
|
||||
/**
|
||||
* Learned Reasoning-Effort Caps — reactive capability memory for providers/models
|
||||
* OmniRoute has no static registry entry for (custom OpenAI-compatible connections,
|
||||
* or any registered provider whose registry entry carries no reasoning metadata).
|
||||
*
|
||||
* Same shape as `learnedThinkingCaps.ts` (thinking_budget), generalized from a
|
||||
* numeric budget to an ordinal reasoning_effort scale: on a 4xx whose body
|
||||
* enumerates the accepted values, `base.ts`'s executor calls
|
||||
* `recordLearnedReasoningEffort`, which stores the highest recognized value in a
|
||||
* module-level Map keyed "provider:model" (lowercased). Subsequent requests for
|
||||
* the same provider+model read the cap via `getLearnedReasoningEffort` (consulted
|
||||
* by `sanitizeReasoningEffortForProvider` in `executors/base/reasoningEffort.ts`)
|
||||
* so the 4xx→retry round-trip is paid at most once per process per provider+model.
|
||||
*
|
||||
* In-memory only (same operator-accepted tradeoff as the thinking-budget cache):
|
||||
* restart resets, the first request after a restart may re-learn at the cost of
|
||||
* one upstream 4xx.
|
||||
*/
|
||||
|
||||
export const REASONING_EFFORT_ORDER: readonly string[] = [
|
||||
"none",
|
||||
"minimal",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh",
|
||||
"max",
|
||||
];
|
||||
|
||||
// key: `${provider}:${model}` lowercased → highest value known to be accepted.
|
||||
const learnedCaps = new Map<string, string>();
|
||||
|
||||
function buildKey(provider: string | null | undefined, model: string | null | undefined): string {
|
||||
const p = typeof provider === "string" ? provider.trim().toLowerCase() : "";
|
||||
const m = typeof model === "string" ? model.trim().toLowerCase() : "";
|
||||
if (!p || !m) return "";
|
||||
return `${p}:${m}`;
|
||||
}
|
||||
|
||||
function rankOf(value: string): number {
|
||||
return REASONING_EFFORT_ORDER.indexOf(value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the learned cap for provider+model, or null when nothing has been
|
||||
* learned yet (no upstream 4xx recorded). Keyed case-insensitively.
|
||||
*/
|
||||
export function getLearnedReasoningEffort(
|
||||
provider: string | null | undefined,
|
||||
model: string | null | undefined
|
||||
): string | null {
|
||||
const key = buildKey(provider, model);
|
||||
if (!key) return null;
|
||||
return learnedCaps.get(key) ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Record that `acceptedValues` is the enum the upstream advertised for
|
||||
* provider+model, and store the highest recognized value as the learned cap.
|
||||
* Returns the stored value, or null when `acceptedValues` contained no token
|
||||
* from `REASONING_EFFORT_ORDER` (nothing usable to learn) or the key is unusable.
|
||||
*
|
||||
* Always monotonically decreases: if a cap already stored ranks lower than the
|
||||
* newly computed highest, the stored (lower) value wins and is returned
|
||||
* unchanged. This keeps a later, laxer-looking response (or a race between
|
||||
* concurrent requests) from ratcheting the cap back up.
|
||||
*/
|
||||
export function recordLearnedReasoningEffort(
|
||||
provider: string | null | undefined,
|
||||
model: string | null | undefined,
|
||||
acceptedValues: string[]
|
||||
): string | null {
|
||||
const key = buildKey(provider, model);
|
||||
if (!key) return null;
|
||||
|
||||
let best: string | null = null;
|
||||
let bestRank = -1;
|
||||
for (const raw of acceptedValues) {
|
||||
const rank = rankOf(raw);
|
||||
if (rank > bestRank) {
|
||||
bestRank = rank;
|
||||
best = raw;
|
||||
}
|
||||
}
|
||||
if (best === null) return null;
|
||||
|
||||
const existing = learnedCaps.get(key);
|
||||
if (existing !== undefined && rankOf(existing) <= bestRank) {
|
||||
return existing; // already learned an equal-or-lower cap; keep it
|
||||
}
|
||||
learnedCaps.set(key, best);
|
||||
return best;
|
||||
}
|
||||
|
||||
// Matches both prose shapes observed: OVH's `@ai-sdk/openai-compatible`
|
||||
// deserializer ("expected one of `a`, `b`") and a generic vendor prose form
|
||||
// ("Supported types are a, b, and c").
|
||||
const LIST_INTRO = /(?:expected one of|supported (?:types|values) are)[:\s]*([^.]+)/i;
|
||||
|
||||
/**
|
||||
* Extract the upstream-advertised accepted reasoning_effort values from a 4xx
|
||||
* error body. Returns only tokens present in REASONING_EFFORT_ORDER (unknown
|
||||
* tokens are dropped defensively) in the order they appeared, or null when the
|
||||
* text names no recognized enum member.
|
||||
*/
|
||||
export function parseReasoningEffortEnum(errText: unknown): string[] | null {
|
||||
if (typeof errText !== "string" || !errText) return null;
|
||||
const match = LIST_INTRO.exec(errText);
|
||||
if (!match) return null;
|
||||
const tokens = match[1]
|
||||
.split(/,|\band\b|&/i)
|
||||
.map((t) =>
|
||||
t
|
||||
.replace(/`/g, "")
|
||||
.replace(/\([^)]*\)/g, "")
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
)
|
||||
.filter((t) => t.length > 0 && REASONING_EFFORT_ORDER.includes(t));
|
||||
return tokens.length > 0 ? tokens : null;
|
||||
}
|
||||
|
||||
/** Test-only: clear the learned-cap Map between tests. */
|
||||
export function __test_resetLearnedReasoningEffortCaps(): void {
|
||||
learnedCaps.clear();
|
||||
}
|
||||
7
tests/unit/http-status-unprocessable-entity.test.ts
Normal file
7
tests/unit/http-status-unprocessable-entity.test.ts
Normal file
@@ -0,0 +1,7 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { HTTP_STATUS } from "../../open-sse/config/constants.ts";
|
||||
|
||||
test("HTTP_STATUS declares UNPROCESSABLE_ENTITY as 422", () => {
|
||||
assert.equal(HTTP_STATUS.UNPROCESSABLE_ENTITY, 422);
|
||||
});
|
||||
126
tests/unit/learned-reasoning-effort-caps.test.ts
Normal file
126
tests/unit/learned-reasoning-effort-caps.test.ts
Normal file
@@ -0,0 +1,126 @@
|
||||
import { test, after, beforeEach } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
REASONING_EFFORT_ORDER,
|
||||
parseReasoningEffortEnum,
|
||||
recordLearnedReasoningEffort,
|
||||
getLearnedReasoningEffort,
|
||||
__test_resetLearnedReasoningEffortCaps,
|
||||
} from "../../open-sse/services/learnedReasoningEffortCaps.ts";
|
||||
|
||||
beforeEach(() => {
|
||||
__test_resetLearnedReasoningEffortCaps();
|
||||
});
|
||||
|
||||
after(() => {
|
||||
__test_resetLearnedReasoningEffortCaps();
|
||||
});
|
||||
|
||||
// ── REASONING_EFFORT_ORDER ──────────────────────────────────────────────────
|
||||
|
||||
test("REASONING_EFFORT_ORDER is none < minimal < low < medium < high < xhigh < max", () => {
|
||||
assert.deepEqual(REASONING_EFFORT_ORDER, [
|
||||
"none",
|
||||
"minimal",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh",
|
||||
"max",
|
||||
]);
|
||||
});
|
||||
|
||||
// ── parseReasoningEffortEnum ────────────────────────────────────────────────
|
||||
|
||||
test("parseReasoningEffortEnum extracts the real OVH 422 enum (backtick-quoted)", () => {
|
||||
const err =
|
||||
"Failed to deserialize the JSON body into the target type: reasoning_effort: " +
|
||||
"unknown variant `xhigh`, expected one of `none`, `high`, `medium`, `low`, `minimal`";
|
||||
assert.deepEqual(parseReasoningEffortEnum(err), ["none", "high", "medium", "low", "minimal"]);
|
||||
});
|
||||
|
||||
test("parseReasoningEffortEnum extracts a bare comma/and-joined enum with annotations", () => {
|
||||
const err =
|
||||
"Unexpected reasoning effort high. Supported types are xhigh (default), medium, and low.";
|
||||
assert.deepEqual(parseReasoningEffortEnum(err), ["xhigh", "medium", "low"]);
|
||||
});
|
||||
|
||||
test("parseReasoningEffortEnum drops unrecognized tokens", () => {
|
||||
const err = "expected one of `none`, `turbo`, `high`";
|
||||
assert.deepEqual(parseReasoningEffortEnum(err), ["none", "high"]);
|
||||
});
|
||||
|
||||
test("parseReasoningEffortEnum returns null for unrelated error text", () => {
|
||||
assert.equal(parseReasoningEffortEnum("connection refused"), null);
|
||||
assert.equal(parseReasoningEffortEnum(""), null);
|
||||
assert.equal(parseReasoningEffortEnum(null), null);
|
||||
assert.equal(parseReasoningEffortEnum(undefined), null);
|
||||
});
|
||||
|
||||
test("parseReasoningEffortEnum returns null when the list has no recognized token", () => {
|
||||
assert.equal(parseReasoningEffortEnum("expected one of `foo`, `bar`"), null);
|
||||
});
|
||||
|
||||
// ── recordLearnedReasoningEffort / getLearnedReasoningEffort ───────────────
|
||||
|
||||
test("records the highest recognized value from the accepted list", () => {
|
||||
const learned = recordLearnedReasoningEffort("ovh", "qwen3-coder-30b-a3b-instruct", [
|
||||
"none",
|
||||
"high",
|
||||
"medium",
|
||||
"low",
|
||||
"minimal",
|
||||
]);
|
||||
assert.equal(learned, "high");
|
||||
assert.equal(getLearnedReasoningEffort("ovh", "qwen3-coder-30b-a3b-instruct"), "high");
|
||||
});
|
||||
|
||||
test("returns null and stores nothing when acceptedValues has no recognized token", () => {
|
||||
const learned = recordLearnedReasoningEffort("acme", "model-x", ["foo", "bar"]);
|
||||
assert.equal(learned, null);
|
||||
assert.equal(getLearnedReasoningEffort("acme", "model-x"), null);
|
||||
});
|
||||
|
||||
test("monotonic decrease: a later, higher accepted-list never ratchets the cap back up", () => {
|
||||
recordLearnedReasoningEffort("acme", "model-x", ["none", "low", "medium"]);
|
||||
const learned = recordLearnedReasoningEffort("acme", "model-x", [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh",
|
||||
]);
|
||||
assert.equal(learned, "medium");
|
||||
assert.equal(getLearnedReasoningEffort("acme", "model-x"), "medium");
|
||||
});
|
||||
|
||||
test("a later, lower accepted-list does ratchet the cap down", () => {
|
||||
recordLearnedReasoningEffort("acme", "model-x", ["none", "low", "medium", "high"]);
|
||||
const learned = recordLearnedReasoningEffort("acme", "model-x", ["none", "low"]);
|
||||
assert.equal(learned, "low");
|
||||
assert.equal(getLearnedReasoningEffort("acme", "model-x"), "low");
|
||||
});
|
||||
|
||||
test("getLearnedReasoningEffort returns null for unknown provider+model", () => {
|
||||
assert.equal(getLearnedReasoningEffort("acme", "unknown-model"), null);
|
||||
});
|
||||
|
||||
test("getLearnedReasoningEffort is keyed case-insensitively on provider+model", () => {
|
||||
recordLearnedReasoningEffort("OVH", "Qwen3-Coder-30B", ["none", "high"]);
|
||||
assert.equal(getLearnedReasoningEffort("ovh", "qwen3-coder-30b"), "high");
|
||||
assert.equal(getLearnedReasoningEffort("OVH", "QWEN3-CODER-30B"), "high");
|
||||
});
|
||||
|
||||
test("different providers for the same model id have independent caps", () => {
|
||||
recordLearnedReasoningEffort("ovh", "shared-model", ["none", "high"]);
|
||||
assert.equal(getLearnedReasoningEffort("openrouter", "shared-model"), null);
|
||||
});
|
||||
|
||||
test("handles empty/null provider or model gracefully", () => {
|
||||
assert.equal(getLearnedReasoningEffort("", "m"), null);
|
||||
assert.equal(getLearnedReasoningEffort("p", ""), null);
|
||||
assert.equal(getLearnedReasoningEffort(null, "m"), null);
|
||||
assert.equal(getLearnedReasoningEffort("p", null), null);
|
||||
assert.equal(recordLearnedReasoningEffort("", "m", ["high"]), null);
|
||||
assert.equal(recordLearnedReasoningEffort("p", "", ["high"]), null);
|
||||
});
|
||||
110
tests/unit/reasoning-effort-clamp-and-retry.test.ts
Normal file
110
tests/unit/reasoning-effort-clamp-and-retry.test.ts
Normal file
@@ -0,0 +1,110 @@
|
||||
import { test, after, beforeEach } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { BaseExecutor } from "../../open-sse/executors/base.ts";
|
||||
import {
|
||||
getLearnedReasoningEffort,
|
||||
recordLearnedReasoningEffort,
|
||||
__test_resetLearnedReasoningEffortCaps,
|
||||
} from "../../open-sse/services/learnedReasoningEffortCaps.ts";
|
||||
|
||||
const OVH_422_BODY = JSON.stringify({
|
||||
error: {
|
||||
message:
|
||||
"Failed to deserialize the JSON body into the target type: reasoning_effort: " +
|
||||
"unknown variant `xhigh`, expected one of `none`, `high`, `medium`, `low`, `minimal`",
|
||||
},
|
||||
});
|
||||
|
||||
// Passthrough executor: returns the body unchanged so we assert on exactly what
|
||||
// base.ts sends upstream.
|
||||
class SimpleExecutor extends BaseExecutor {
|
||||
constructor() {
|
||||
super("openai-compatible-chat-eaff6869", {
|
||||
baseUrls: ["https://oai.endpoints.kepler.ai.cloud.ovh.net/v1/chat/completions"],
|
||||
});
|
||||
}
|
||||
async transformRequest(_model: string, body: Record<string, unknown>) {
|
||||
return { ...body };
|
||||
}
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
__test_resetLearnedReasoningEffortCaps();
|
||||
});
|
||||
|
||||
after(() => {
|
||||
__test_resetLearnedReasoningEffortCaps();
|
||||
});
|
||||
|
||||
test("422 'unknown variant xhigh, expected one of ...' clamps reasoning_effort and retries once", async () => {
|
||||
const executor = new SimpleExecutor();
|
||||
const originalFetch = globalThis.fetch;
|
||||
const capturedBodies: Record<string, unknown>[] = [];
|
||||
|
||||
globalThis.fetch = async (_url: string | URL | Request, init: RequestInit = {}) => {
|
||||
const body = JSON.parse(String(init.body));
|
||||
capturedBodies.push(body);
|
||||
if (capturedBodies.length === 1) {
|
||||
return new Response(OVH_422_BODY, {
|
||||
status: 422,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
}
|
||||
return new Response(JSON.stringify({ ok: true }), {
|
||||
status: 200,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await executor.execute({
|
||||
model: "qwen3-coder-30b-a3b-instruct",
|
||||
body: { reasoning_effort: "xhigh" },
|
||||
stream: false,
|
||||
credentials: {},
|
||||
});
|
||||
assert.equal(capturedBodies.length, 2);
|
||||
assert.equal(capturedBodies[0].reasoning_effort, "xhigh");
|
||||
assert.equal(capturedBodies[1].reasoning_effort, "high");
|
||||
assert.equal(
|
||||
getLearnedReasoningEffort("openai-compatible-chat-eaff6869", "qwen3-coder-30b-a3b-instruct"),
|
||||
"high"
|
||||
);
|
||||
assert.equal(result.response.status, 200);
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test("a second request for the same provider+model sends the learned value on the first try", async () => {
|
||||
const executor = new SimpleExecutor();
|
||||
const originalFetch = globalThis.fetch;
|
||||
const capturedBodies: Record<string, unknown>[] = [];
|
||||
|
||||
globalThis.fetch = async (_url: string | URL | Request, init: RequestInit = {}) => {
|
||||
const body = JSON.parse(String(init.body));
|
||||
capturedBodies.push(body);
|
||||
return new Response(JSON.stringify({ ok: true }), {
|
||||
status: 200,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
};
|
||||
|
||||
try {
|
||||
recordLearnedReasoningEffort(
|
||||
"openai-compatible-chat-eaff6869",
|
||||
"qwen3-coder-30b-a3b-instruct",
|
||||
["none", "high", "medium", "low", "minimal"]
|
||||
);
|
||||
await executor.execute({
|
||||
model: "qwen3-coder-30b-a3b-instruct",
|
||||
body: { reasoning_effort: "xhigh" },
|
||||
stream: false,
|
||||
credentials: {},
|
||||
});
|
||||
assert.equal(capturedBodies.length, 1);
|
||||
assert.equal(capturedBodies[0].reasoning_effort, "high");
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
91
tests/unit/reasoning-effort-learned-capability.test.ts
Normal file
91
tests/unit/reasoning-effort-learned-capability.test.ts
Normal file
@@ -0,0 +1,91 @@
|
||||
import { test, after, beforeEach } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { sanitizeReasoningEffortForProvider } from "../../open-sse/executors/base/reasoningEffort.ts";
|
||||
import {
|
||||
recordLearnedReasoningEffort,
|
||||
__test_resetLearnedReasoningEffortCaps,
|
||||
} from "../../open-sse/services/learnedReasoningEffortCaps.ts";
|
||||
|
||||
beforeEach(() => {
|
||||
__test_resetLearnedReasoningEffortCaps();
|
||||
});
|
||||
|
||||
after(() => {
|
||||
__test_resetLearnedReasoningEffortCaps();
|
||||
});
|
||||
|
||||
test("unregistered/custom provider+model: no learned cap yet sends xhigh unchanged", () => {
|
||||
const body = { reasoning_effort: "xhigh" };
|
||||
const result = sanitizeReasoningEffortForProvider(
|
||||
body,
|
||||
"openai-compatible-chat-eaff6869",
|
||||
"qwen3-coder-30b-a3b-instruct"
|
||||
) as { reasoning_effort: string };
|
||||
assert.equal(result.reasoning_effort, "xhigh");
|
||||
});
|
||||
|
||||
test("unregistered/custom provider+model: a learned cap clamps xhigh down to it", () => {
|
||||
recordLearnedReasoningEffort("openai-compatible-chat-eaff6869", "qwen3-coder-30b-a3b-instruct", [
|
||||
"none",
|
||||
"high",
|
||||
"medium",
|
||||
"low",
|
||||
"minimal",
|
||||
]);
|
||||
const body = { reasoning_effort: "xhigh" };
|
||||
const result = sanitizeReasoningEffortForProvider(
|
||||
body,
|
||||
"openai-compatible-chat-eaff6869",
|
||||
"qwen3-coder-30b-a3b-instruct"
|
||||
) as { reasoning_effort: string };
|
||||
assert.equal(result.reasoning_effort, "high");
|
||||
});
|
||||
|
||||
test("learned cap only clamps when the requested effort is above it", () => {
|
||||
recordLearnedReasoningEffort("acme", "model-x", ["none", "low", "medium"]);
|
||||
const body = { reasoning_effort: "low" };
|
||||
const result = sanitizeReasoningEffortForProvider(body, "acme", "model-x") as {
|
||||
reasoning_effort: string;
|
||||
};
|
||||
assert.equal(result.reasoning_effort, "low");
|
||||
});
|
||||
|
||||
test("registry says supportsXHighEffort:false (and no supportsMax path) with a learned cap below 'high': uses the learned cap, not the hardcoded 'high'", () => {
|
||||
// claude-haiku-4-5 is registered with supportsXHighEffort:false
|
||||
// (open-sse/config/providers/registry/claude/index.ts) and its family is
|
||||
// excluded from supportsClaudeMaxEffort (CLAUDE_MAX_EFFORT_UNSUPPORTED_FAMILY_PATTERNS
|
||||
// in providerModels.ts), so it reaches the hardcoded-"high" line today —
|
||||
// a real registry-covered case. Teach a lower cap and confirm it wins.
|
||||
recordLearnedReasoningEffort("claude", "claude-haiku-4-5-20251001", ["none", "low", "medium"]);
|
||||
const body = { reasoning_effort: "xhigh" };
|
||||
const result = sanitizeReasoningEffortForProvider(
|
||||
body,
|
||||
"claude",
|
||||
"claude-haiku-4-5-20251001"
|
||||
) as {
|
||||
reasoning_effort: string;
|
||||
};
|
||||
assert.equal(result.reasoning_effort, "medium");
|
||||
});
|
||||
|
||||
test("registry says supportsXHighEffort:false with no learned cap: falls back to hardcoded 'high' (unchanged behavior)", () => {
|
||||
const body = { reasoning_effort: "xhigh" };
|
||||
const result = sanitizeReasoningEffortForProvider(
|
||||
body,
|
||||
"claude",
|
||||
"claude-haiku-4-5-20251001"
|
||||
) as {
|
||||
reasoning_effort: string;
|
||||
};
|
||||
assert.equal(result.reasoning_effort, "high");
|
||||
});
|
||||
|
||||
test("deepseek's non-ordinal max<->xhigh translation is untouched by the learned-cap catch-all", () => {
|
||||
recordLearnedReasoningEffort("deepseek", "deepseek-v4", ["none", "low"]);
|
||||
const body = { reasoning_effort: "xhigh" };
|
||||
const result = sanitizeReasoningEffortForProvider(body, "deepseek", "deepseek-v4") as {
|
||||
reasoning_effort: string;
|
||||
};
|
||||
// deepseek's special case returns early — xhigh -> max, never reaches the catch-all.
|
||||
assert.equal(result.reasoning_effort, "max");
|
||||
});
|
||||
Reference in New Issue
Block a user