mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-07-26 09:52:11 +03:00
fix(combos): default reasoning token buffer off
This commit is contained in:
@@ -2431,7 +2431,7 @@ async function handleRoundRobinCombo({
|
|||||||
const maxRetries = config.maxRetries ?? 1;
|
const maxRetries = config.maxRetries ?? 1;
|
||||||
const retryDelayMs = resolveDelayMs(config.retryDelayMs, 2000);
|
const retryDelayMs = resolveDelayMs(config.retryDelayMs, 2000);
|
||||||
const fallbackDelayMs = resolveDelayMs(config.fallbackDelayMs, 0);
|
const fallbackDelayMs = resolveDelayMs(config.fallbackDelayMs, 0);
|
||||||
const reasoningTokenBufferEnabled = config.reasoningTokenBufferEnabled !== false;
|
const reasoningTokenBufferEnabled = config.reasoningTokenBufferEnabled === true;
|
||||||
|
|
||||||
const resilienceSettings: ResilienceSettings = settings
|
const resilienceSettings: ResilienceSettings = settings
|
||||||
? resolveResilienceSettings(settings)
|
? resolveResilienceSettings(settings)
|
||||||
@@ -3032,7 +3032,10 @@ async function handleRoundRobinCombo({
|
|||||||
resilienceSettings.providerCooldown.enabled &&
|
resilienceSettings.providerCooldown.enabled &&
|
||||||
provider &&
|
provider &&
|
||||||
provider !== "unknown" &&
|
provider !== "unknown" &&
|
||||||
!(result.status === 500 && hasPerModelQuota(provider, parseModel(modelStr).model || modelStr))
|
!(
|
||||||
|
result.status === 500 &&
|
||||||
|
hasPerModelQuota(provider, parseModel(modelStr).model || modelStr)
|
||||||
|
)
|
||||||
) {
|
) {
|
||||||
recordProviderCooldown(
|
recordProviderCooldown(
|
||||||
provider,
|
provider,
|
||||||
|
|||||||
@@ -90,9 +90,7 @@ export function phaseComboSetup(ctx: ComboContext): ComboSetup {
|
|||||||
|
|
||||||
const universalHandoffConfig = resolveUniversalHandoffConfig(
|
const universalHandoffConfig = resolveUniversalHandoffConfig(
|
||||||
(combo.universal_handoff || combo.universalHandoff) as
|
(combo.universal_handoff || combo.universalHandoff) as
|
||||||
| Record<string, unknown>
|
Record<string, unknown> | null | undefined,
|
||||||
| null
|
|
||||||
| undefined,
|
|
||||||
relayOptions?.universalHandoffConfig as Record<string, unknown> | null | undefined
|
relayOptions?.universalHandoffConfig as Record<string, unknown> | null | undefined
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -118,7 +116,7 @@ export function phaseComboSetup(ctx: ComboContext): ComboSetup {
|
|||||||
FETCH_TIMEOUT_MS,
|
FETCH_TIMEOUT_MS,
|
||||||
DEFAULT_COMBO_TARGET_TIMEOUT_MS
|
DEFAULT_COMBO_TARGET_TIMEOUT_MS
|
||||||
);
|
);
|
||||||
const reasoningTokenBufferEnabled = config.reasoningTokenBufferEnabled !== false;
|
const reasoningTokenBufferEnabled = config.reasoningTokenBufferEnabled === true;
|
||||||
|
|
||||||
return {
|
return {
|
||||||
strategy,
|
strategy,
|
||||||
|
|||||||
@@ -53,7 +53,7 @@ const DEFAULT_COMBO_CONFIG = {
|
|||||||
maxComboDepth: 3,
|
maxComboDepth: 3,
|
||||||
nestedComboMode: "flatten",
|
nestedComboMode: "flatten",
|
||||||
trackMetrics: true,
|
trackMetrics: true,
|
||||||
reasoningTokenBufferEnabled: true,
|
reasoningTokenBufferEnabled: false,
|
||||||
manifestRouting: false,
|
manifestRouting: false,
|
||||||
// Complexity-aware auto routing (2026): when on, the auto router scores
|
// Complexity-aware auto routing (2026): when on, the auto router scores
|
||||||
// candidates by how well their tier matches the request's classified
|
// candidates by how well their tier matches the request's classified
|
||||||
|
|||||||
@@ -93,7 +93,7 @@ export default function ComboDefaultsTab() {
|
|||||||
retryDelayMs: 2000,
|
retryDelayMs: 2000,
|
||||||
maxComboDepth: 3,
|
maxComboDepth: 3,
|
||||||
trackMetrics: true,
|
trackMetrics: true,
|
||||||
reasoningTokenBufferEnabled: true,
|
reasoningTokenBufferEnabled: false,
|
||||||
handoffThreshold: 0.85,
|
handoffThreshold: 0.85,
|
||||||
handoffModel: "",
|
handoffModel: "",
|
||||||
maxMessagesForSummary: 30,
|
maxMessagesForSummary: 30,
|
||||||
@@ -292,8 +292,7 @@ export default function ComboDefaultsTab() {
|
|||||||
|
|
||||||
// Filtered provider list — excludes already-added ones, filtered by search query
|
// Filtered provider list — excludes already-added ones, filtered by search query
|
||||||
const filteredProviders = availableProviders.filter(
|
const filteredProviders = availableProviders.filter(
|
||||||
(p) =>
|
(p) => !providerOverrides[p.provider] && matchesSearch(p.provider, searchQuery)
|
||||||
!providerOverrides[p.provider] && matchesSearch(p.provider, searchQuery)
|
|
||||||
);
|
);
|
||||||
|
|
||||||
const handleDropdownKeyDown = (e: React.KeyboardEvent) => {
|
const handleDropdownKeyDown = (e: React.KeyboardEvent) => {
|
||||||
@@ -690,11 +689,11 @@ export default function ComboDefaultsTab() {
|
|||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
<Toggle
|
<Toggle
|
||||||
checked={comboDefaults.reasoningTokenBufferEnabled !== false}
|
checked={comboDefaults.reasoningTokenBufferEnabled === true}
|
||||||
onChange={() =>
|
onChange={() =>
|
||||||
setComboDefaults((prev) => ({
|
setComboDefaults((prev) => ({
|
||||||
...prev,
|
...prev,
|
||||||
reasoningTokenBufferEnabled: prev.reasoningTokenBufferEnabled === false,
|
reasoningTokenBufferEnabled: prev.reasoningTokenBufferEnabled !== true,
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
/>
|
/>
|
||||||
|
|||||||
@@ -55,7 +55,7 @@ export async function GET(request: Request) {
|
|||||||
maxMessagesForSummary: 30,
|
maxMessagesForSummary: 30,
|
||||||
maxComboDepth: 3,
|
maxComboDepth: 3,
|
||||||
trackMetrics: true,
|
trackMetrics: true,
|
||||||
reasoningTokenBufferEnabled: true,
|
reasoningTokenBufferEnabled: false,
|
||||||
zeroLatencyOptimizationsEnabled: false,
|
zeroLatencyOptimizationsEnabled: false,
|
||||||
},
|
},
|
||||||
providerOverrides,
|
providerOverrides,
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ test("getDefaultComboConfig returns a fresh copy of the defaults", () => {
|
|||||||
assert.equal(first.failoverBeforeRetry, true);
|
assert.equal(first.failoverBeforeRetry, true);
|
||||||
assert.equal(first.maxSetRetries, 0);
|
assert.equal(first.maxSetRetries, 0);
|
||||||
assert.equal(first.setRetryDelayMs, 2000);
|
assert.equal(first.setRetryDelayMs, 2000);
|
||||||
assert.equal(first.reasoningTokenBufferEnabled, true);
|
assert.equal(first.reasoningTokenBufferEnabled, false);
|
||||||
assert.equal(first.zeroLatencyOptimizationsEnabled, false);
|
assert.equal(first.zeroLatencyOptimizationsEnabled, false);
|
||||||
assert.equal(first.hedging, false);
|
assert.equal(first.hedging, false);
|
||||||
assert.equal(first.fallbackCompressionMode, "lite");
|
assert.equal(first.fallbackCompressionMode, "lite");
|
||||||
|
|||||||
@@ -2869,7 +2869,7 @@ test("handleComboChat aborts combo when 503 response does NOT contain the unavai
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("#3587 reasoning model gets max_tokens buffer applied", async () => {
|
test("#3587 reasoning model gets max_tokens buffer applied when explicitly enabled", async () => {
|
||||||
saveModelsDevCapabilities({
|
saveModelsDevCapabilities({
|
||||||
openai: {
|
openai: {
|
||||||
"gpt-4o-reasoning": capabilityEntry(12000, { reasoning: true, limit_output: 12000 }),
|
"gpt-4o-reasoning": capabilityEntry(12000, { reasoning: true, limit_output: 12000 }),
|
||||||
@@ -2882,6 +2882,7 @@ test("#3587 reasoning model gets max_tokens buffer applied", async () => {
|
|||||||
combo: {
|
combo: {
|
||||||
name: "reasoning-buffer",
|
name: "reasoning-buffer",
|
||||||
models: ["openai/gpt-4o-reasoning"],
|
models: ["openai/gpt-4o-reasoning"],
|
||||||
|
config: { reasoningTokenBufferEnabled: true },
|
||||||
},
|
},
|
||||||
handleSingleModel: async (body: Record<string, unknown>) => {
|
handleSingleModel: async (body: Record<string, unknown>) => {
|
||||||
bodies.push(body);
|
bodies.push(body);
|
||||||
@@ -2934,6 +2935,7 @@ test("#3587 reasoning buffer preserves max_tokens when the full buffer exceeds m
|
|||||||
combo: {
|
combo: {
|
||||||
name: "reasoning-buffer-near-cap",
|
name: "reasoning-buffer-near-cap",
|
||||||
models: ["openai/gemini-high-cap"],
|
models: ["openai/gemini-high-cap"],
|
||||||
|
config: { reasoningTokenBufferEnabled: true },
|
||||||
},
|
},
|
||||||
handleSingleModel: async (body: Record<string, unknown>) => {
|
handleSingleModel: async (body: Record<string, unknown>) => {
|
||||||
bodies.push(body);
|
bodies.push(body);
|
||||||
@@ -2983,7 +2985,7 @@ test("#3587 reasoning buffer is disabled without explicit model capability data"
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("#3588 reasoning token buffer feature flag preserves client max_tokens", async () => {
|
test("#3588 reasoning token buffer is disabled by default and preserves client max_tokens", async () => {
|
||||||
saveModelsDevCapabilities({
|
saveModelsDevCapabilities({
|
||||||
openai: {
|
openai: {
|
||||||
"flagged-reasoning": capabilityEntry(12000, { reasoning: true, limit_output: 12000 }),
|
"flagged-reasoning": capabilityEntry(12000, { reasoning: true, limit_output: 12000 }),
|
||||||
@@ -3009,11 +3011,7 @@ test("#3588 reasoning token buffer feature flag preserves client max_tokens", as
|
|||||||
},
|
},
|
||||||
isModelAvailable: async () => true,
|
isModelAvailable: async () => true,
|
||||||
log: createLog(),
|
log: createLog(),
|
||||||
settings: {
|
settings: null,
|
||||||
comboDefaults: {
|
|
||||||
reasoningTokenBufferEnabled: false,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
relayOptions: null,
|
relayOptions: null,
|
||||||
allCombos: null,
|
allCombos: null,
|
||||||
});
|
});
|
||||||
@@ -3078,6 +3076,7 @@ test("#3587 round-robin buffer does NOT compound across reasoning models", async
|
|||||||
name: "rr-reasoning-no-compound",
|
name: "rr-reasoning-no-compound",
|
||||||
strategy: "round-robin",
|
strategy: "round-robin",
|
||||||
models: ["openai/rr-reasoning-a", "openai/rr-reasoning-b"],
|
models: ["openai/rr-reasoning-a", "openai/rr-reasoning-b"],
|
||||||
|
config: { reasoningTokenBufferEnabled: true },
|
||||||
},
|
},
|
||||||
handleSingleModel: async (body: Record<string, unknown>, modelStr: string) => {
|
handleSingleModel: async (body: Record<string, unknown>, modelStr: string) => {
|
||||||
seen.push({ model: modelStr, maxTokens: body.max_tokens });
|
seen.push({ model: modelStr, maxTokens: body.max_tokens });
|
||||||
|
|||||||
@@ -15,7 +15,11 @@ test("createComboContext carries inputs and the body BY REFERENCE", () => {
|
|||||||
const body = { model: "auto", messages: [], stream: true };
|
const body = { model: "auto", messages: [], stream: true };
|
||||||
const combo = { name: "c1", models: ["a", "b"] };
|
const combo = { name: "c1", models: ["a", "b"] };
|
||||||
const ctx = createComboContext({ body, combo, log });
|
const ctx = createComboContext({ body, combo, log });
|
||||||
assert.equal(ctx.body, body, "body must be the same reference (not copied) for byte-identical pinning");
|
assert.equal(
|
||||||
|
ctx.body,
|
||||||
|
body,
|
||||||
|
"body must be the same reference (not copied) for byte-identical pinning"
|
||||||
|
);
|
||||||
assert.equal(ctx.combo, combo);
|
assert.equal(ctx.combo, combo);
|
||||||
assert.equal(ctx.settings, null);
|
assert.equal(ctx.settings, null);
|
||||||
assert.equal(ctx.relayOptions, null);
|
assert.equal(ctx.relayOptions, null);
|
||||||
@@ -34,7 +38,11 @@ test("phaseComboSetup resolves strategy/config/stream with pinning OFF (pure, no
|
|||||||
assert.equal(setup.effectiveSessionId, null);
|
assert.equal(setup.effectiveSessionId, null);
|
||||||
assert.equal(setup.clientRequestedStream, true, "body.stream === true");
|
assert.equal(setup.clientRequestedStream, true, "body.stream === true");
|
||||||
assert.equal(typeof setup.comboTargetTimeoutMs, "number");
|
assert.equal(typeof setup.comboTargetTimeoutMs, "number");
|
||||||
assert.equal(typeof setup.reasoningTokenBufferEnabled, "boolean");
|
assert.equal(
|
||||||
|
setup.reasoningTokenBufferEnabled,
|
||||||
|
false,
|
||||||
|
"reasoning token buffering is opt-in by default"
|
||||||
|
);
|
||||||
assert.ok(setup.config && typeof setup.config === "object", "config cascade resolved");
|
assert.ok(setup.config && typeof setup.config === "object", "config cascade resolved");
|
||||||
assert.ok(
|
assert.ok(
|
||||||
setup.resilienceSettings && typeof setup.resilienceSettings === "object",
|
setup.resilienceSettings && typeof setup.resilienceSettings === "object",
|
||||||
|
|||||||
Reference in New Issue
Block a user