mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-25 08:32:11 +03:00
fix(sse): learn accepted reasoning_effort sets and clamp downgrade-only (#11232)
Validated on the combined 12-PR batch board (worktree off origin/release/v3.8.50 @ 8f390eff): focused node:test suites 183/184 (the single red was a load-induced pollUntil flake — 11/11 when re-run isolated, this file included), typecheck:core clean, file-size/changelog/complexity/cognitive gates all within baseline, mutation-coverage gate no-drift. Learned effort sets now cost at most one 400 per provider+model. Thank you @maxmad64bis!
This commit is contained in:
Dizzle
@@ -1559,22 +1559,31 @@ export class BaseExecutor {
|
||||
if (acceptedValues) {
|
||||
reasoningEffortClamped = true;
|
||||
const learned = recordLearnedReasoningEffort(this.provider, model, acceptedValues);
|
||||
if (learned) {
|
||||
if (learned && learned.size > 0) {
|
||||
const beforeRetry = JSON.stringify(transformedBody);
|
||||
transformedBody = sanitizeReasoningEffortForProvider(
|
||||
transformedBody,
|
||||
this.provider,
|
||||
model,
|
||||
log
|
||||
);
|
||||
let retryBody = JSON.stringify(transformedBody);
|
||||
if (usesClaudeCodeProtocol || this.provider === "claude") {
|
||||
retryBody = await signRequestBody(retryBody);
|
||||
const afterRetry = JSON.stringify(transformedBody);
|
||||
if (beforeRetry === afterRetry) {
|
||||
log?.info?.(
|
||||
"REASONING_SANITIZE",
|
||||
`Upstream ${response.status} rejected reasoning_effort on ${url} — learned ${[...learned].join(",")} but clamp was no-op for ${this.provider}/${model}, not retrying`
|
||||
);
|
||||
} else {
|
||||
let retryBody = JSON.stringify(transformedBody);
|
||||
if (usesClaudeCodeProtocol || this.provider === "claude") {
|
||||
retryBody = await signRequestBody(retryBody);
|
||||
}
|
||||
log?.info?.(
|
||||
"REASONING_SANITIZE",
|
||||
`Upstream ${response.status} rejected reasoning_effort on ${url} — clamped to ${[...learned].join(",")} and retrying (learned for ${this.provider}/${model})`
|
||||
);
|
||||
response = await fetchWithStartTimeout(url, { ...fetchOptions, body: retryBody });
|
||||
}
|
||||
log?.info?.(
|
||||
"REASONING_SANITIZE",
|
||||
`Upstream ${response.status} rejected reasoning_effort on ${url} — clamped to ${learned} and retrying (learned for ${this.provider}/${model})`
|
||||
);
|
||||
response = await fetchWithStartTimeout(url, { ...fetchOptions, body: retryBody });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,7 +10,7 @@ import {
|
||||
} from "../../config/providerModels.ts";
|
||||
import {
|
||||
getLearnedReasoningEffort,
|
||||
REASONING_EFFORT_ORDER,
|
||||
clampToLearned,
|
||||
} from "../../services/learnedReasoningEffortCaps.ts";
|
||||
|
||||
/**
|
||||
@@ -340,26 +340,29 @@ export function sanitizeReasoningEffortForProvider(
|
||||
return body;
|
||||
}
|
||||
|
||||
// Generic learned clamp (downgrade-only: greatest accepted <= demand).
|
||||
// Sits AFTER the per-provider early returns by design: deepseek/command-code/
|
||||
// ollama-cloud have deliberate static translations that take precedence; the
|
||||
// learned set governs every other provider and all effort values, before the
|
||||
// xhigh/max static fallbacks below.
|
||||
const learnedSet = getLearnedReasoningEffort(provider, modelStr);
|
||||
if (learnedSet && learnedSet.size > 0 && !learnedSet.has(effortStr)) {
|
||||
const clamped = clampToLearned(effortStr, learnedSet);
|
||||
if (clamped && clamped !== effortStr) {
|
||||
log?.info?.(
|
||||
"REASONING_SANITIZE",
|
||||
`${provider}/${modelStr}: clamped reasoning_effort ${effortStr} → ${clamped} (learned)`
|
||||
);
|
||||
return writeEffortValue(b, clamped, c);
|
||||
}
|
||||
}
|
||||
|
||||
const supportsXHigh = supportsXHighEffort(provider, modelStr);
|
||||
const supportsMax = supportsMaxEffortForProvider(provider, modelStr);
|
||||
// Highest value we've actually seen this provider+model accept in a real
|
||||
// upstream 4xx (learnedReasoningEffortCaps.ts) — takes priority over the
|
||||
// static registry (which defaults to "supports everything" when there's no
|
||||
// entry, e.g. custom OpenAI-compatible connections) and over the hardcoded
|
||||
// "high" fallback below (which isn't always valid either).
|
||||
const learnedCap = getLearnedReasoningEffort(provider, modelStr);
|
||||
const learnedRank = learnedCap ? REASONING_EFFORT_ORDER.indexOf(learnedCap) : -1;
|
||||
|
||||
// ── xhigh handling ──────────────────────────────────────────────────────
|
||||
// xhigh is OmniRoute-internal. Map it to the best effort the model accepts.
|
||||
if (effortStr === "xhigh") {
|
||||
if (learnedCap && learnedRank < REASONING_EFFORT_ORDER.indexOf("xhigh")) {
|
||||
log?.info?.(
|
||||
"REASONING_SANITIZE",
|
||||
`${provider}/${modelStr}: clamped reasoning_effort xhigh → ${learnedCap} (learned)`
|
||||
);
|
||||
return writeEffortValue(b, learnedCap, c);
|
||||
}
|
||||
if (supportsXHigh) return body; // model accepts xhigh natively
|
||||
if (supportsMax) {
|
||||
log?.info?.(
|
||||
@@ -384,13 +387,6 @@ export function sanitizeReasoningEffortForProvider(
|
||||
// upstream, and if it 400s the user gets a clear signal. This prevents
|
||||
// new models from being unusable for weeks until they're whitelisted (#8057).
|
||||
if (effortStr === "max") {
|
||||
if (learnedCap && learnedRank < REASONING_EFFORT_ORDER.indexOf("max")) {
|
||||
log?.info?.(
|
||||
"REASONING_SANITIZE",
|
||||
`${provider}/${modelStr}: clamped reasoning_effort max → ${learnedCap} (learned)`
|
||||
);
|
||||
return writeEffortValue(b, learnedCap, c);
|
||||
}
|
||||
if (supportsMax) return body; // explicitly known to accept max
|
||||
|
||||
// A model that explicitly advertises its accepted tiers is safe to normalize.
|
||||
@@ -407,7 +403,7 @@ export function sanitizeReasoningEffortForProvider(
|
||||
)?.supportedThinkingEfforts;
|
||||
const maxFallback =
|
||||
Array.isArray(explicitEfforts) && !explicitEfforts.includes("max")
|
||||
? ["xhigh", "high", "medium", "low"].find((tier) => explicitEfforts.includes(tier))
|
||||
? ["ultra", "xhigh", "high", "medium", "low"].find((tier) => explicitEfforts.includes(tier))
|
||||
: undefined;
|
||||
if (maxFallback) {
|
||||
log?.info?.(
|
||||
|
||||
@@ -6,12 +6,14 @@
|
||||
* Same shape as `learnedThinkingCaps.ts` (thinking_budget), generalized from a
|
||||
* numeric budget to an ordinal reasoning_effort scale: on a 4xx whose body
|
||||
* enumerates the accepted values, `base.ts`'s executor calls
|
||||
* `recordLearnedReasoningEffort`, which stores the highest recognized value in a
|
||||
* module-level Map keyed "provider:model" (lowercased). Subsequent requests for
|
||||
* the same provider+model read the cap via `getLearnedReasoningEffort` (consulted
|
||||
* by `sanitizeReasoningEffortForProvider` in `executors/base/reasoningEffort.ts`)
|
||||
* `recordLearnedReasoningEffort`, which stores the accepted set in a module-level
|
||||
* Map keyed "provider:model" (lowercased). Subsequent requests for the same
|
||||
* provider+model read the set via `getLearnedReasoningEffort` (consulted by
|
||||
* `sanitizeReasoningEffortForProvider` in `executors/base/reasoningEffort.ts`)
|
||||
* so the 4xx→retry round-trip is paid at most once per process per provider+model.
|
||||
*
|
||||
* `clampToLearned` implements downgrade-only clamping: greatest accepted <= demand.
|
||||
*
|
||||
* In-memory only (same operator-accepted tradeoff as the thinking-budget cache):
|
||||
* restart resets, the first request after a restart may re-learn at the cost of
|
||||
* one upstream 4xx.
|
||||
@@ -25,10 +27,11 @@ export const REASONING_EFFORT_ORDER: readonly string[] = [
|
||||
"high",
|
||||
"xhigh",
|
||||
"max",
|
||||
"ultra",
|
||||
];
|
||||
|
||||
// key: `${provider}:${model}` lowercased → highest value known to be accepted.
|
||||
const learnedCaps = new Map<string, string>();
|
||||
// key: `${provider}:${model}` lowercased → accepted set.
|
||||
const learnedCaps = new Map<string, Set<string>>();
|
||||
|
||||
function buildKey(provider: string | null | undefined, model: string | null | undefined): string {
|
||||
const p = typeof provider === "string" ? provider.trim().toLowerCase() : "";
|
||||
@@ -41,61 +44,89 @@ function rankOf(value: string): number {
|
||||
return REASONING_EFFORT_ORDER.indexOf(value);
|
||||
}
|
||||
|
||||
function isSubset(a: Set<string>, b: Set<string>): boolean {
|
||||
for (const v of a) if (!b.has(v)) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the learned cap for provider+model, or null when nothing has been
|
||||
* learned yet (no upstream 4xx recorded). Keyed case-insensitively.
|
||||
* Return the learned accepted set for provider+model, or null when nothing has
|
||||
* been learned yet (no upstream 4xx recorded). Keyed case-insensitively.
|
||||
*/
|
||||
export function getLearnedReasoningEffort(
|
||||
provider: string | null | undefined,
|
||||
model: string | null | undefined
|
||||
): string | null {
|
||||
): Set<string> | null {
|
||||
const key = buildKey(provider, model);
|
||||
if (!key) return null;
|
||||
return learnedCaps.get(key) ?? null;
|
||||
const v = learnedCaps.get(key);
|
||||
return v ? new Set(v) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Record that `acceptedValues` is the enum the upstream advertised for
|
||||
* provider+model, and store the highest recognized value as the learned cap.
|
||||
* Returns the stored value, or null when `acceptedValues` contained no token
|
||||
* from `REASONING_EFFORT_ORDER` (nothing usable to learn) or the key is unusable.
|
||||
* provider+model, and store the accepted set. Returns the stored set, or null
|
||||
* when `acceptedValues` contained no token from `REASONING_EFFORT_ORDER`.
|
||||
*
|
||||
* Always monotonically decreases: if a cap already stored ranks lower than the
|
||||
* newly computed highest, the stored (lower) value wins and is returned
|
||||
* unchanged. This keeps a later, laxer-looking response (or a race between
|
||||
* concurrent requests) from ratcheting the cap back up.
|
||||
* Monotonically non-expanding: if existing ⊆ newSet, keep existing (never
|
||||
* re-expand); if newSet ⊂ existing, replace (more restrictive); if neither
|
||||
* subset, keep existing.
|
||||
*/
|
||||
export function recordLearnedReasoningEffort(
|
||||
provider: string | null | undefined,
|
||||
model: string | null | undefined,
|
||||
acceptedValues: string[]
|
||||
): string | null {
|
||||
): Set<string> | null {
|
||||
const key = buildKey(provider, model);
|
||||
if (!key) return null;
|
||||
|
||||
let best: string | null = null;
|
||||
let bestRank = -1;
|
||||
const newSet = new Set<string>();
|
||||
for (const raw of acceptedValues) {
|
||||
const rank = rankOf(raw);
|
||||
if (rank > bestRank) {
|
||||
bestRank = rank;
|
||||
best = raw;
|
||||
}
|
||||
const lowered = typeof raw === "string" ? raw.trim().toLowerCase() : "";
|
||||
if (lowered && REASONING_EFFORT_ORDER.includes(lowered)) newSet.add(lowered);
|
||||
}
|
||||
if (best === null) return null;
|
||||
if (newSet.size === 0) return null;
|
||||
|
||||
const existing = learnedCaps.get(key);
|
||||
if (existing !== undefined && rankOf(existing) <= bestRank) {
|
||||
return existing; // already learned an equal-or-lower cap; keep it
|
||||
if (existing !== undefined) {
|
||||
// Defensive copies: never hand out the live cached Set.
|
||||
if (isSubset(existing, newSet)) return new Set(existing);
|
||||
if (isSubset(newSet, existing)) {
|
||||
learnedCaps.set(key, newSet);
|
||||
return new Set(newSet);
|
||||
}
|
||||
return new Set(existing);
|
||||
}
|
||||
learnedCaps.set(key, newSet);
|
||||
return new Set(newSet);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the greatest accepted value <= effortStr (downgrade only), or null
|
||||
* if effortStr is already accepted, below the minimum, or not in ORDER.
|
||||
*/
|
||||
export function clampToLearned(effortStr: string, accepted: Set<string>): string | null {
|
||||
if (!effortStr || accepted.has(effortStr)) return null;
|
||||
const rank = rankOf(effortStr);
|
||||
if (rank === -1) return null;
|
||||
const minRank = Math.min(...[...accepted].map((v) => rankOf(v)));
|
||||
if (rank < minRank) return null;
|
||||
let best: string | null = null;
|
||||
let bestRank = -1;
|
||||
for (const v of accepted) {
|
||||
const r = rankOf(v);
|
||||
if (r <= rank && r > bestRank) {
|
||||
bestRank = r;
|
||||
best = v;
|
||||
}
|
||||
}
|
||||
learnedCaps.set(key, best);
|
||||
return best;
|
||||
}
|
||||
|
||||
// Matches both prose shapes observed: OVH's `@ai-sdk/openai-compatible`
|
||||
// deserializer ("expected one of `a`, `b`") and a generic vendor prose form
|
||||
// ("Supported types are a, b, and c").
|
||||
const LIST_INTRO = /(?:expected one of|supported (?:types|values) are)[:\s]*([^.]+)/i;
|
||||
// Matches prose shapes: OVH's "@ai-sdk/openai-compatible" deserializer
|
||||
// ("expected one of `a`, `b`"), generic ("Supported types are a, b, and c"),
|
||||
// and "please use a, b, or c".
|
||||
const LIST_INTRO = /(?:expected one of|supported (?:types|values) are|please use)[:\s]*([^.]+)/i;
|
||||
|
||||
/**
|
||||
* Extract the upstream-advertised accepted reasoning_effort values from a 4xx
|
||||
@@ -108,13 +139,14 @@ export function parseReasoningEffortEnum(errText: unknown): string[] | null {
|
||||
const match = LIST_INTRO.exec(errText);
|
||||
if (!match) return null;
|
||||
const tokens = match[1]
|
||||
.split(/,|\band\b|&/i)
|
||||
.split(/,|\b(?:and|or)\b|&/i)
|
||||
.map((t) =>
|
||||
t
|
||||
.replace(/`/g, "")
|
||||
.replace(/\([^)]*\)/g, "")
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/^[^a-z]+|[^a-z]+$/g, "")
|
||||
)
|
||||
.filter((t) => t.length > 0 && REASONING_EFFORT_ORDER.includes(t));
|
||||
return tokens.length > 0 ? tokens : null;
|
||||
|
||||
@@ -18,7 +18,7 @@ after(() => {
|
||||
|
||||
// ── REASONING_EFFORT_ORDER ──────────────────────────────────────────────────
|
||||
|
||||
test("REASONING_EFFORT_ORDER is none < minimal < low < medium < high < xhigh < max", () => {
|
||||
test("REASONING_EFFORT_ORDER is none < minimal < low < medium < high < xhigh < max < ultra", () => {
|
||||
assert.deepEqual(REASONING_EFFORT_ORDER, [
|
||||
"none",
|
||||
"minimal",
|
||||
@@ -27,6 +27,24 @@ test("REASONING_EFFORT_ORDER is none < minimal < low < medium < high < xhigh < m
|
||||
"high",
|
||||
"xhigh",
|
||||
"max",
|
||||
"ultra",
|
||||
]);
|
||||
});
|
||||
|
||||
test("REASONING_EFFORT_ORDER ends with ultra", () => {
|
||||
assert.equal(REASONING_EFFORT_ORDER.at(-1), "ultra");
|
||||
});
|
||||
test("parseReasoningEffortEnum extracts please use low, high, or max", () => {
|
||||
const err =
|
||||
"This model always engages in thinking and cannot be disabled; please use low, high, or max";
|
||||
assert.deepEqual(parseReasoningEffortEnum(err), ["low", "high", "max"]);
|
||||
});
|
||||
test("parseReasoningEffortEnum extracts please use with ultra", () => {
|
||||
assert.deepEqual(parseReasoningEffortEnum("please use low, high, max, ultra"), [
|
||||
"low",
|
||||
"high",
|
||||
"max",
|
||||
"ultra",
|
||||
]);
|
||||
});
|
||||
|
||||
@@ -70,9 +88,10 @@ test("records the highest recognized value from the accepted list", () => {
|
||||
"medium",
|
||||
"low",
|
||||
"minimal",
|
||||
]);
|
||||
assert.equal(learned, "high");
|
||||
assert.equal(getLearnedReasoningEffort("ovh", "qwen3-coder-30b-a3b-instruct"), "high");
|
||||
]) as unknown as Set<string>;
|
||||
assert.ok(learned instanceof Set);
|
||||
assert.ok(learned.has("high"));
|
||||
assert.equal((getLearnedReasoningEffort("ovh", "qwen3-coder-30b-a3b-instruct") as unknown as Set<string>).has("high"), true);
|
||||
});
|
||||
|
||||
test("returns null and stores nothing when acceptedValues has no recognized token", () => {
|
||||
@@ -89,26 +108,77 @@ test("monotonic decrease: a later, higher accepted-list never ratchets the cap b
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh",
|
||||
]);
|
||||
assert.equal(learned, "medium");
|
||||
assert.equal(getLearnedReasoningEffort("acme", "model-x"), "medium");
|
||||
]) as unknown as Set<string>;
|
||||
assert.equal(learned.size, 3);
|
||||
assert.ok(learned.has("medium"));
|
||||
assert.equal((getLearnedReasoningEffort("acme", "model-x") as unknown as Set<string>).size, 3);
|
||||
});
|
||||
|
||||
test("a later, lower accepted-list does ratchet the cap down", () => {
|
||||
recordLearnedReasoningEffort("acme", "model-x", ["none", "low", "medium", "high"]);
|
||||
const learned = recordLearnedReasoningEffort("acme", "model-x", ["none", "low"]);
|
||||
assert.equal(learned, "low");
|
||||
assert.equal(getLearnedReasoningEffort("acme", "model-x"), "low");
|
||||
const learned = recordLearnedReasoningEffort("acme", "model-x", ["none", "low"]) as unknown as Set<string>;
|
||||
assert.equal(learned.size, 2);
|
||||
assert.ok(learned.has("low"));
|
||||
assert.equal((getLearnedReasoningEffort("acme", "model-x") as unknown as Set<string>).size, 2);
|
||||
});
|
||||
|
||||
test("clampToLearned medium→low when accepted is low,high,max", async () => {
|
||||
const { clampToLearned } = await import("../../open-sse/services/learnedReasoningEffortCaps.ts");
|
||||
assert.equal(clampToLearned("medium", new Set(["low", "high", "max"])), "low");
|
||||
});
|
||||
test("clampToLearned xhigh→high when accepted is low,high,max", async () => {
|
||||
const { clampToLearned } = await import("../../open-sse/services/learnedReasoningEffortCaps.ts");
|
||||
assert.equal(clampToLearned("xhigh", new Set(["low", "high", "max"])), "high");
|
||||
});
|
||||
test("clampToLearned ultra→max when accepted is low,high,max", async () => {
|
||||
const { clampToLearned } = await import("../../open-sse/services/learnedReasoningEffortCaps.ts");
|
||||
assert.equal(clampToLearned("ultra", new Set(["low", "high", "max"])), "max");
|
||||
});
|
||||
test("clampToLearned ultra→medium when accepted is low,medium", async () => {
|
||||
const { clampToLearned } = await import("../../open-sse/services/learnedReasoningEffortCaps.ts");
|
||||
assert.equal(clampToLearned("ultra", new Set(["low", "medium"])), "medium");
|
||||
});
|
||||
test("clampToLearned high→medium when accepted is low,medium", async () => {
|
||||
const { clampToLearned } = await import("../../open-sse/services/learnedReasoningEffortCaps.ts");
|
||||
assert.equal(clampToLearned("high", new Set(["low", "medium"])), "medium");
|
||||
});
|
||||
test("clampToLearned returns null when already accepted", async () => {
|
||||
const { clampToLearned } = await import("../../open-sse/services/learnedReasoningEffortCaps.ts");
|
||||
assert.equal(clampToLearned("low", new Set(["low", "high", "max"])), null);
|
||||
});
|
||||
test("clampToLearned returns null when effort < min (no upgrade)", async () => {
|
||||
const { clampToLearned } = await import("../../open-sse/services/learnedReasoningEffortCaps.ts");
|
||||
assert.equal(clampToLearned("low", new Set(["high", "max"])), null);
|
||||
});
|
||||
test("clampToLearned returns null for turbo (not in ORDER)", async () => {
|
||||
const { clampToLearned } = await import("../../open-sse/services/learnedReasoningEffortCaps.ts");
|
||||
assert.equal(clampToLearned("turbo", new Set(["low", "high", "max"])), null);
|
||||
});
|
||||
test("clampToLearned returns null when effort is none but accepted is low,high,max", async () => {
|
||||
const { clampToLearned } = await import("../../open-sse/services/learnedReasoningEffortCaps.ts");
|
||||
assert.equal(clampToLearned("none", new Set(["low", "high", "max"])), null);
|
||||
});
|
||||
test("recordLearned stores Set and getLearned returns Set", () => {
|
||||
const s = recordLearnedReasoningEffort("acme", "m1", ["low", "high", "max"]);
|
||||
assert.ok(s instanceof Set);
|
||||
assert.deepEqual([...(s as unknown as Set<string>)].sort(), ["high", "low", "max"]);
|
||||
const g = getLearnedReasoningEffort("acme", "m1");
|
||||
assert.ok(g instanceof Set);
|
||||
});
|
||||
test("monotonicity incomparable: keep existing when neither subset", () => {
|
||||
recordLearnedReasoningEffort("acme", "m4", ["low", "high", "max"]);
|
||||
const s4 = recordLearnedReasoningEffort("acme", "m4", ["low", "medium"]);
|
||||
assert.equal((s4 as unknown as Set<string>).size, 3);
|
||||
assert.ok((s4 as unknown as Set<string>).has("high"));
|
||||
});
|
||||
test("getLearnedReasoningEffort returns null for unknown provider+model", () => {
|
||||
assert.equal(getLearnedReasoningEffort("acme", "unknown-model"), null);
|
||||
});
|
||||
|
||||
test("getLearnedReasoningEffort is keyed case-insensitively on provider+model", () => {
|
||||
recordLearnedReasoningEffort("OVH", "Qwen3-Coder-30B", ["none", "high"]);
|
||||
assert.equal(getLearnedReasoningEffort("ovh", "qwen3-coder-30b"), "high");
|
||||
assert.equal(getLearnedReasoningEffort("OVH", "QWEN3-CODER-30B"), "high");
|
||||
assert.ok((getLearnedReasoningEffort("ovh", "qwen3-coder-30b") as unknown as Set<string>).has("high"));
|
||||
assert.ok((getLearnedReasoningEffort("OVH", "QWEN3-CODER-30B") as unknown as Set<string>).has("high"));
|
||||
});
|
||||
|
||||
test("different providers for the same model id have independent caps", () => {
|
||||
|
||||
@@ -66,9 +66,8 @@ test("422 'unknown variant xhigh, expected one of ...' clamps reasoning_effort a
|
||||
assert.equal(capturedBodies.length, 2);
|
||||
assert.equal(capturedBodies[0].reasoning_effort, "xhigh");
|
||||
assert.equal(capturedBodies[1].reasoning_effort, "high");
|
||||
assert.equal(
|
||||
getLearnedReasoningEffort("openai-compatible-chat-eaff6869", "qwen3-coder-30b-a3b-instruct"),
|
||||
"high"
|
||||
assert.ok(
|
||||
(getLearnedReasoningEffort("openai-compatible-chat-eaff6869", "qwen3-coder-30b-a3b-instruct") as unknown as Set<string>).has("high")
|
||||
);
|
||||
assert.equal(result.response.status, 200);
|
||||
} finally {
|
||||
@@ -108,3 +107,125 @@ test("a second request for the same provider+model sends the learned value on th
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test("400 please use low, high, or max clamps and retries once", async () => {
|
||||
const executor = new SimpleExecutor();
|
||||
const originalFetch = globalThis.fetch;
|
||||
const capturedBodies: Record<string, unknown>[] = [];
|
||||
const BODY_400_PLEASE_USE = JSON.stringify({
|
||||
error: { message: "This model always engages in thinking and cannot be disabled; please use low, high, or max" },
|
||||
});
|
||||
|
||||
globalThis.fetch = async (_url: string | URL | Request, init: RequestInit = {}) => {
|
||||
const body = JSON.parse(String(init.body));
|
||||
capturedBodies.push(body);
|
||||
if (capturedBodies.length === 1) {
|
||||
return new Response(BODY_400_PLEASE_USE, {
|
||||
status: 400,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
}
|
||||
return new Response(JSON.stringify({ ok: true }), {
|
||||
status: 200,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await executor.execute({
|
||||
model: "x-preview-f-free",
|
||||
body: { reasoning_effort: "medium" },
|
||||
stream: false,
|
||||
credentials: {},
|
||||
});
|
||||
assert.equal(capturedBodies.length, 2);
|
||||
assert.equal(capturedBodies[0].reasoning_effort, "medium");
|
||||
assert.equal(capturedBodies[1].reasoning_effort, "low");
|
||||
const learned = getLearnedReasoningEffort("openai-compatible-chat-eaff6869", "x-preview-f-free") as unknown as Set<string>;
|
||||
assert.ok(learned instanceof Set);
|
||||
assert.ok(learned.has("low"));
|
||||
assert.ok(learned.has("high"));
|
||||
assert.equal(result.response.status, 200);
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test("400 please use low, medium with ultra retries to medium", async () => {
|
||||
const executor = new SimpleExecutor();
|
||||
const originalFetch = globalThis.fetch;
|
||||
const capturedBodies: Record<string, unknown>[] = [];
|
||||
const BODY_400_ULTRA = JSON.stringify({
|
||||
error: { message: "please use low, medium" },
|
||||
});
|
||||
|
||||
globalThis.fetch = async (_url: string | URL | Request, init: RequestInit = {}) => {
|
||||
const body = JSON.parse(String(init.body));
|
||||
capturedBodies.push(body);
|
||||
if (capturedBodies.length === 1) {
|
||||
return new Response(BODY_400_ULTRA, {
|
||||
status: 400,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
}
|
||||
return new Response(JSON.stringify({ ok: true }), {
|
||||
status: 200,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await executor.execute({
|
||||
model: "x-preview-f-free-2",
|
||||
body: { reasoning_effort: "ultra" },
|
||||
stream: false,
|
||||
credentials: {},
|
||||
});
|
||||
assert.equal(capturedBodies.length, 2);
|
||||
assert.equal(capturedBodies[0].reasoning_effort, "ultra");
|
||||
assert.equal(capturedBodies[1].reasoning_effort, "medium");
|
||||
assert.equal(result.response.status, 200);
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test("no-op clamp does not retry: learned {high,max} with low request stays single-fetch", async () => {
|
||||
const executor = new SimpleExecutor();
|
||||
const originalFetch = globalThis.fetch;
|
||||
const capturedBodies: Record<string, unknown>[] = [];
|
||||
const BODY_400_HIGH_MAX = JSON.stringify({
|
||||
error: { message: "please use high, or max" },
|
||||
});
|
||||
|
||||
globalThis.fetch = async (_url: string | URL | Request, init: RequestInit = {}) => {
|
||||
const body = JSON.parse(String(init.body));
|
||||
capturedBodies.push(body);
|
||||
if (capturedBodies.length === 1) {
|
||||
return new Response(BODY_400_HIGH_MAX, {
|
||||
status: 400,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
}
|
||||
return new Response(JSON.stringify({ ok: true }), {
|
||||
status: 200,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
};
|
||||
|
||||
try {
|
||||
// low is below the learned minimum {high,max}: downgrade-only passthrough,
|
||||
// sanitizer leaves the body unchanged -> no identical-body retry.
|
||||
const result = await executor.execute({
|
||||
model: "x-preview-f-free-3",
|
||||
body: { reasoning_effort: "low" },
|
||||
stream: false,
|
||||
credentials: {},
|
||||
});
|
||||
assert.equal(capturedBodies.length, 1);
|
||||
assert.equal(capturedBodies[0].reasoning_effort, "low");
|
||||
assert.equal(result.response.status, 400);
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
@@ -89,3 +89,67 @@ test("deepseek's non-ordinal max<->xhigh translation is untouched by the learned
|
||||
// deepseek's special case returns early — xhigh -> max, never reaches the catch-all.
|
||||
assert.equal(result.reasoning_effort, "max");
|
||||
});
|
||||
|
||||
test("proactive clamp: medium→low for learned {low,high,max}", () => {
|
||||
recordLearnedReasoningEffort("opencode-zen-direct", "x-preview-f-free", ["low", "high", "max"]);
|
||||
const out = sanitizeReasoningEffortForProvider(
|
||||
{ reasoning_effort: "medium", model: "x-preview-f-free" },
|
||||
"opencode-zen-direct",
|
||||
"x-preview-f-free"
|
||||
) as { reasoning_effort: string };
|
||||
assert.equal(out.reasoning_effort, "low");
|
||||
});
|
||||
test("proactive clamp: xhigh→high for learned {low,high,max}", () => {
|
||||
recordLearnedReasoningEffort("opencode-zen-direct", "x-preview-f-free-2", ["low", "high", "max"]);
|
||||
const out = sanitizeReasoningEffortForProvider(
|
||||
{ reasoning_effort: "xhigh", model: "x-preview-f-free-2" },
|
||||
"opencode-zen-direct",
|
||||
"x-preview-f-free-2"
|
||||
) as { reasoning_effort: string };
|
||||
assert.equal(out.reasoning_effort, "high");
|
||||
});
|
||||
test("proactive clamp: ultra→max for learned {low,high,max}", () => {
|
||||
recordLearnedReasoningEffort("opencode-zen-direct", "x-preview-f-free-3", ["low", "high", "max"]);
|
||||
const out = sanitizeReasoningEffortForProvider(
|
||||
{ reasoning_effort: "ultra", model: "x-preview-f-free-3" },
|
||||
"opencode-zen-direct",
|
||||
"x-preview-f-free-3"
|
||||
) as { reasoning_effort: string };
|
||||
assert.equal(out.reasoning_effort, "max");
|
||||
});
|
||||
test("proactive clamp: ultra→medium for learned {low,medium}", () => {
|
||||
recordLearnedReasoningEffort("acme", "m", ["low", "medium"]);
|
||||
const out = sanitizeReasoningEffortForProvider(
|
||||
{ reasoning_effort: "ultra", model: "m" },
|
||||
"acme",
|
||||
"m"
|
||||
) as { reasoning_effort: string };
|
||||
assert.equal(out.reasoning_effort, "medium");
|
||||
});
|
||||
test("proactive clamp: high→medium for learned {low,medium}", () => {
|
||||
recordLearnedReasoningEffort("acme", "m2", ["low", "medium"]);
|
||||
const out = sanitizeReasoningEffortForProvider(
|
||||
{ reasoning_effort: "high", model: "m2" },
|
||||
"acme",
|
||||
"m2"
|
||||
) as { reasoning_effort: string };
|
||||
assert.equal(out.reasoning_effort, "medium");
|
||||
});
|
||||
test("no upgrade: low stays low for learned {high,max}", () => {
|
||||
recordLearnedReasoningEffort("acme", "m3", ["high", "max"]);
|
||||
const out = sanitizeReasoningEffortForProvider(
|
||||
{ reasoning_effort: "low", model: "m3" },
|
||||
"acme",
|
||||
"m3"
|
||||
) as { reasoning_effort: string };
|
||||
assert.equal(out.reasoning_effort, "low");
|
||||
});
|
||||
test("custom model ultra→medium for learned {low,medium}", () => {
|
||||
recordLearnedReasoningEffort("openai-compatible-chat-eaff6869", "qwen3-coder-30b-a3b-instruct-2", ["low", "medium"]);
|
||||
const out = sanitizeReasoningEffortForProvider(
|
||||
{ reasoning_effort: "ultra", model: "qwen3-coder-30b-a3b-instruct-2" },
|
||||
"openai-compatible-chat-eaff6869",
|
||||
"qwen3-coder-30b-a3b-instruct-2"
|
||||
) as { reasoning_effort: string };
|
||||
assert.equal(out.reasoning_effort, "medium");
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user