From ce98c30cfb082b9c5f485851911c14981d507a61 Mon Sep 17 00:00:00 2001 From: "Bob.Hou" Date: Wed, 16 Sep 2026 00:47:32 -0400 Subject: [PATCH] fix(combos): stop Gemini thinking from failing dashboard combo tests (#13560) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two related fixes: the combo health probe sends `reasoning_effort: "none"` for Gemini-family models so the probe budget is not spent on thinking, and `detectMalformedNonStream` stops classifying a response with `finish_reason` `length`/`tool_calls`/`content_filter` and empty content as `empty_choices`. The second half is the important one: it brings the post-translation check in line with `isEmptyContentResponse` (`open-sse/services/errorClassifier.ts`, `LEGIT_EMPTY_OPENAI_FINISH`), which already treated those finish reasons as legitimate. Until now a response could pass the pre-translation check and still be rewritten into a synthetic 502 afterwards — for every non-streaming completion, not just combo probes. Validated as a combined board first (this PR merged with the 11 siblings of the same batch on the release tip): eslint on every changed file with the suppressions file, typecheck:core, check:open-sse-typecheck, complexity, cognitive-complexity, changelog-integrity, i18n new-key coverage, docs-sync, migration-numbering, provider-consistency and a duplicate-identifier audit all green, plus 275 passing / 0 failing focused node:test cases across the 28 test files the batch touches. Then re-validated alone on the fresh tip before this merge: conflicts re-resolved, file sizes rebaselined for this PR's own growth, eslint and this PR's focused tests re-run. Thanks @HouMinXi! --- changelog.d/fixes/combo-probe-no-thinking.md | 1 + open-sse/utils/diagnostics.ts | 21 ++++++++++++++++++-- src/lib/combos/testHealth.ts | 19 +++++++++++++++++- tests/unit/combo-test-health.test.ts | 11 +++++++++- tests/unit/combo-test-route.test.ts | 1 + tests/unit/diagnostics.test.ts | 8 ++++++++ 6 files changed, 57 insertions(+), 4 deletions(-) create mode 100644 changelog.d/fixes/combo-probe-no-thinking.md diff --git a/changelog.d/fixes/combo-probe-no-thinking.md b/changelog.d/fixes/combo-probe-no-thinking.md new file mode 100644 index 0000000000..8246088861 --- /dev/null +++ b/changelog.d/fixes/combo-probe-no-thinking.md @@ -0,0 +1 @@ +- **fix(combos):** Gemini combo probes send `reasoning_effort: none` so thinking does not eat the health-check budget; truncated `finish_reason: length` responses are no longer rewritten as empty-content 502s diff --git a/open-sse/utils/diagnostics.ts b/open-sse/utils/diagnostics.ts index ebee72e1e8..bb84d02461 100644 --- a/open-sse/utils/diagnostics.ts +++ b/open-sse/utils/diagnostics.ts @@ -313,7 +313,23 @@ export function detectMalformedNonStream(resp: unknown): MalformedReason | null return false; }); - if (!anyHasOutput) return "empty_choices"; + if (!anyHasOutput) { + // Same terminal stops isEmptyContentResponse already accepts as + // successful truncation, not a silent fake-success. Gemini 3.8 + // health probes that spend max_tokens on thinking come back as + // content:"" + finish_reason:"length". Treating that as empty_choices + // rewrites a valid 200 into 502 and fails dashboard Test all. + const truncatedAtLimit = choices.some((choice) => { + const c = choice as Record; + return ( + c?.finish_reason === "length" || + c?.finish_reason === "tool_calls" || + c?.finish_reason === "content_filter" + ); + }); + if (truncatedAtLimit) return null; + return "empty_choices"; + } return null; } @@ -323,7 +339,8 @@ export function describeMalformedNonStream( ): { message: string; code: string; type: string } { const body = resp && typeof resp === "object" ? (resp as Record) : null; if (body?.object === "response" && body.status === "failed") { - const err = body.error && typeof body.error === "object" ? (body.error as Record) : null; + const err = + body.error && typeof body.error === "object" ? (body.error as Record) : null; const rawMessage = typeof err?.message === "string" && err.message.trim().length > 0 ? err.message.trim() : null; return { diff --git a/src/lib/combos/testHealth.ts b/src/lib/combos/testHealth.ts index 140c4fa33e..9a9a793a6e 100644 --- a/src/lib/combos/testHealth.ts +++ b/src/lib/combos/testHealth.ts @@ -111,6 +111,10 @@ export function buildComboTestPrompt() { return COMBO_TEST_PROMPT; } +function isGeminiComboProbe(modelStr: string) { + return /(?:^|\/)gemini(?:-|$)/i.test(modelStr); +} + export function buildComboTestRequestBody( modelStr: string, isEmbedding: boolean = false, @@ -123,7 +127,13 @@ export function buildComboTestRequestBody( }; } - return { + const body: { + model: string; + messages: { role: string; content: string }[]; + max_tokens: number; + stream: boolean; + reasoning_effort?: "none"; + } = { model: modelStr, messages: [{ role: "user", content: buildComboTestPrompt() }], // Keep the smoke probe short so reasoning-heavy models do not burn the @@ -133,6 +143,13 @@ export function buildComboTestRequestBody( (options.stream ? STREAMING_MODEL_TEST_MAX_TOKENS : COMBO_TEST_MAX_TOKENS), stream: options.stream ?? false, }; + // Gemini 3.8 flash-high injects thinkingLevel=high unless the documented + // off-switch is set. Other providers must not see this field: some + // OpenAI-compatible endpoints 400 unknown parameters. + if (isGeminiComboProbe(modelStr)) { + body.reasoning_effort = "none"; + } + return body; } export type ComboTestStreamResult = { diff --git a/tests/unit/combo-test-health.test.ts b/tests/unit/combo-test-health.test.ts index c0e9ecf4c8..85dd4311c4 100644 --- a/tests/unit/combo-test-health.test.ts +++ b/tests/unit/combo-test-health.test.ts @@ -8,7 +8,7 @@ const { extractComboTestStreamText, } = await import("../../src/lib/combos/testHealth.ts"); -test("combo test helper builds a short smoke payload", () => { +test("combo test helper builds short smoke payload", () => { const body = buildComboTestRequestBody("openrouter/openai/gpt-5.4"); assert.equal(body.model, "openrouter/openai/gpt-5.4"); @@ -16,6 +16,15 @@ test("combo test helper builds a short smoke payload", () => { assert.equal(body.max_tokens, 64); assert.equal("temperature" in body, false); assert.equal(body.stream, false); + assert.equal("reasoning_effort" in body, false); +}); + +test("combo test helper turns off thinking for Gemini 3.8 flash-high probes", () => { + const body = buildComboTestRequestBody("agy/gemini-3.8-flash-high"); + + assert.equal(body.messages[0].content, "Reply with exactly: pong"); + assert.equal(body.max_tokens, 64); + assert.equal(body.reasoning_effort, "none"); }); test("combo test helper builds a small streaming model probe", () => { diff --git a/tests/unit/combo-test-route.test.ts b/tests/unit/combo-test-route.test.ts index 0053330f23..661b5a3066 100644 --- a/tests/unit/combo-test-route.test.ts +++ b/tests/unit/combo-test-route.test.ts @@ -148,6 +148,7 @@ test("combo test route marks a model healthy only when it returns assistant text assert.equal(forwardedBody.model, "openrouter/openai/gpt-5.4"); assert.equal(forwardedBody.messages[0].content, "Reply with exactly: pong"); assert.equal(forwardedBody.max_tokens, 64); + assert.equal("reasoning_effort" in forwardedBody, false); assert.equal("temperature" in forwardedBody, false); assert.equal(body.resolvedBy, "openrouter/openai/gpt-5.4"); assert.equal(body.results[0].status, "ok"); diff --git a/tests/unit/diagnostics.test.ts b/tests/unit/diagnostics.test.ts index 62da56fc0c..f4fd6438f9 100644 --- a/tests/unit/diagnostics.test.ts +++ b/tests/unit/diagnostics.test.ts @@ -128,6 +128,14 @@ test("detectMalformedNonStream returns 'empty_choices' when choice message has n assert.equal(detectMalformedNonStream(body), "empty_choices"); }); +test("detectMalformedNonStream returns null when empty content stopped at token limit", () => { + const body = { + choices: [{ index: 0, message: { role: "assistant", content: "" }, finish_reason: "length" }], + usage: { reasoning_tokens: 28 }, + }; + assert.equal(detectMalformedNonStream(body), null); +}); + test("detectMalformedNonStream returns null for valid chat completion", () => { const body = { choices: [{ message: { content: "Hello!", tool_calls: null }, finish_reason: "stop" }],