fix(sse): trust finish_reason over reasoning-ratio heuristic in response quality validation (#12262)

* fix(sse): trust finish_reason:length/max_tokens over the reasoning-ratio heuristic in response quality validation

A truncated response with empty content and reasoning_content present was
only rejected by validateResponseQuality() when reasoning consumed >=90%
of completion_tokens. A response truncated at a lower ratio (e.g. 63%)
passed through as "valid" even though the caller received no usable
content and finish_reason was explicitly "length" (or the alternate
"max_tokens" naming some providers use) -- an unambiguous truncation
signal the validator wasn't reading. Reproduced live against
nvidia/nemotron-3-super-120b-a12b: content:null, finish_reason:length,
reasoning_tokens 645/1024 (63%).

Trust finish_reason directly when it's reported, falling back to the
existing token-ratio heuristic only when it isn't. Does not affect the
deliberate-tiny-probe case (e.g. max_tokens:1 connectivity pings) --
those never produce reasoning_content, so the branch this change is in
doesn't run for them.

* docs(changelog): add fragment for #12262

---------

Co-authored-by: brick30llc-ctrl <admin@brick30.com>
This commit is contained in:
brick30llc-ctrl
2026-09-01 09:02:05 -05:00
committed by GitHub
parent f301d34ce5
commit 0ff164701d
3 changed files with 84 additions and 0 deletions

View File

@@ -199,6 +199,71 @@ test("#3587 reasoning via completion_tokens_details.reasoning_tokens → invalid
assert.match(out.reason ?? "", /reasoning consumed/i);
});
test("client-audit-2026-09-01: finish_reason:length + reasoning <90% of tokens → invalid (direct truncation signal beats the ratio heuristic)", async () => {
// Reproduces a live CT124 response: nvidia/nemotron truncated by max_tokens
// with content:null, finish_reason:"length", and reasoning at only 63% of
// completion_tokens (645/1024) — below the old 90% threshold, so it used to
// pass the validator as "valid" even though the caller got nothing usable.
const res = makeResponse({
choices: [
{
message: {
content: null,
reasoning: "Step-by-step analysis that never reached a final answer...",
},
finish_reason: "length",
},
],
usage: {
completion_tokens: 1024,
completion_tokens_details: { reasoning_tokens: 645 },
},
});
const out = await validateResponseQuality(res, false, silentLog);
assert.equal(out.valid, false, "should be invalid: finish_reason:length with empty content");
assert.match(out.reason ?? "", /truncated at token limit/i);
});
test("client-audit-2026-09-01: finish_reason:max_tokens (Anthropic-shape naming) + empty content → invalid", async () => {
const res = makeResponse({
choices: [
{
message: { content: null, reasoning_content: "Partial reasoning trace" },
finish_reason: "max_tokens",
},
],
usage: { completion_tokens: 512, reasoning_tokens: 100 },
});
const out = await validateResponseQuality(res, false, silentLog);
assert.equal(out.valid, false, "should be invalid: max_tokens finish_reason with empty content");
});
test("client-audit-2026-09-01: no finish_reason + reasoning <90% of tokens → still valid (regression guard, #3587 behavior preserved)", async () => {
// Same low ratio as the case above, but no finish_reason reported at all —
// the direct-truncation-signal branch must not fire, only the ratio heuristic.
const res = makeResponse({
choices: [{ message: { content: null, reasoning_content: "Some reasoning" } }],
usage: { completion_tokens: 1024, reasoning_tokens: 645 },
});
const out = await validateResponseQuality(res, false, silentLog);
assert.equal(out.valid, true, "should stay valid: no finish_reason signal, ratio under 90%");
});
test("client-audit-2026-09-01: finish_reason:stop + empty content + reasoning → ratio heuristic still applies unchanged", async () => {
const res = makeResponse({
choices: [
{
message: { content: null, reasoning_content: "Deep reasoning" },
finish_reason: "stop",
},
],
usage: { completion_tokens: 4096, reasoning_tokens: 3800 },
});
const out = await validateResponseQuality(res, false, silentLog);
assert.equal(out.valid, false, "finish_reason:stop doesn't short-circuit — ratio (>90%) still applies");
assert.match(out.reason ?? "", /reasoning consumed/i);
});
test("#3587 edge: completion_tokens=0 → safe (no division by zero)", async () => {
const res = makeResponse({
choices: [