diff --git a/src/lib/combos/testHealth.ts b/src/lib/combos/testHealth.ts index 15959e6913..f7ac5f66cf 100644 --- a/src/lib/combos/testHealth.ts +++ b/src/lib/combos/testHealth.ts @@ -4,6 +4,10 @@ function asRecord(value: unknown): JsonRecord { return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {}; } +function joinNonEmpty(parts: string[]) { + return parts.filter(Boolean).join("\n").trim(); +} + function extractTextFromContent(content: unknown): string { if (typeof content === "string") return content.trim(); @@ -28,12 +32,85 @@ function extractTextFromContent(content: unknown): string { .trim(); } +function extractReasoningText(record: JsonRecord): string { + const reasoningDetails = Array.isArray(record.reasoning_details) ? record.reasoning_details : []; + const detailText = reasoningDetails + .map((detail) => { + const detailRecord = asRecord(detail); + const detailType = typeof detailRecord.type === "string" ? detailRecord.type : ""; + const text = + typeof detailRecord.text === "string" + ? detailRecord.text.trim() + : typeof detailRecord.content === "string" + ? detailRecord.content.trim() + : ""; + + if ( + text && + (detailType === "" || + detailType === "reasoning" || + detailType === "reasoning.text" || + detailType === "thinking") + ) { + return text; + } + + return ""; + }) + .filter(Boolean); + + return joinNonEmpty([ + typeof record.reasoning_content === "string" ? record.reasoning_content.trim() : "", + typeof record.reasoning === "string" ? record.reasoning.trim() : "", + typeof record.reasoning_text === "string" ? record.reasoning_text.trim() : "", + joinNonEmpty(detailText), + ]); +} + +function getUsageReasoningTokens(body: JsonRecord): number { + const usage = asRecord(body.usage); + if (!usage) return 0; + + const completionDetails = asRecord(usage.completion_tokens_details); + const topLevelReasoning = + typeof usage.reasoning_tokens === "number" && Number.isFinite(usage.reasoning_tokens) + ? usage.reasoning_tokens + : 0; + const detailedReasoning = + typeof completionDetails.reasoning_tokens === "number" && + Number.isFinite(completionDetails.reasoning_tokens) + ? completionDetails.reasoning_tokens + : 0; + + return Math.max(topLevelReasoning, detailedReasoning); +} + +function hasReasoningOnlyCompletion(body: JsonRecord): boolean { + if (!Array.isArray(body.choices) || body.choices.length === 0) return false; + if (getUsageReasoningTokens(body) <= 0) return false; + + return body.choices.some((choice) => { + const choiceRecord = asRecord(choice); + const message = asRecord(choiceRecord.message); + const finishReason = + typeof choiceRecord.finish_reason === "string" ? choiceRecord.finish_reason : ""; + + if (!message || message.role !== "assistant") return false; + if (!finishReason) return false; + if (extractTextFromContent(message.content)) return false; + if (extractReasoningText(message)) return false; + return true; + }); +} + export function buildComboTestRequestBody(modelStr: string) { return { model: modelStr, messages: [{ role: "user", content: "Reply with OK only." }], - // Keep this close to a real client request without inflating cost. - max_tokens: 16, + // Give reasoning-heavy models enough headroom to emit a tiny visible answer + // without turning the smoke test into a full-cost real request. + max_tokens: 64, + temperature: 0, stream: false, }; } @@ -52,6 +129,9 @@ export function extractComboTestResponseText(responseBody: unknown): string { const messageText = extractTextFromContent(message.content); if (messageText) return messageText; + const reasoningText = extractReasoningText(message); + if (reasoningText) return reasoningText; + if (typeof choiceRecord.text === "string" && choiceRecord.text.trim()) { return choiceRecord.text.trim(); } @@ -63,8 +143,21 @@ export function extractComboTestResponseText(responseBody: unknown): string { const itemRecord = asRecord(item); const contentText = extractTextFromContent(itemRecord.content); if (contentText) return contentText; + + const reasoningText = extractReasoningText(itemRecord); + if (reasoningText) return reasoningText; } } - return extractTextFromContent(body.content); + const topLevelText = extractTextFromContent(body.content); + if (topLevelText) return topLevelText; + + const topLevelReasoning = extractReasoningText(body); + if (topLevelReasoning) return topLevelReasoning; + + if (hasReasoningOnlyCompletion(body)) { + return "[reasoning-only completion]"; + } + + return ""; } diff --git a/tests/unit/combo-test-health.test.mjs b/tests/unit/combo-test-health.test.mjs index 2471656d46..19de579271 100644 --- a/tests/unit/combo-test-health.test.mjs +++ b/tests/unit/combo-test-health.test.mjs @@ -9,7 +9,8 @@ test("combo test helper builds a realistic smoke payload", () => { assert.equal(body.model, "openrouter/openai/gpt-5.4"); assert.equal(body.messages[0].content, "Reply with OK only."); - assert.equal(body.max_tokens, 16); + assert.equal(body.max_tokens, 64); + assert.equal(body.temperature, 0); assert.equal(body.stream, false); }); @@ -46,6 +47,62 @@ test("combo test helper extracts text from block-based responses", () => { assert.equal(text, "OK\nConfirmed."); }); +test("combo test helper extracts reasoning content when visible text is absent", () => { + const text = extractComboTestResponseText({ + choices: [ + { + message: { + role: "assistant", + content: null, + reasoning_content: "Working through the request.\nOK", + }, + }, + ], + }); + + assert.equal(text, "Working through the request.\nOK"); +}); + +test("combo test helper extracts reasoning_text aliases from GitHub-style responses", () => { + const text = extractComboTestResponseText({ + choices: [ + { + message: { + role: "assistant", + content: "", + reasoning_text: "Reasoning trace", + }, + }, + ], + }); + + assert.equal(text, "Reasoning trace"); +}); + +test("combo test helper treats reasoning-only completions as a healthy signal", () => { + const text = extractComboTestResponseText({ + choices: [ + { + finish_reason: "length", + message: { + role: "assistant", + content: "", + }, + }, + ], + usage: { + prompt_tokens: 6, + completion_tokens: 12, + total_tokens: 18, + completion_tokens_details: { + reasoning_tokens: 12, + }, + }, + }); + + assert.equal(text, "[reasoning-only completion]"); +}); + test("combo test helper returns empty string when no text content exists", () => { const text = extractComboTestResponseText({ choices: [ diff --git a/tests/unit/combo-test-route.test.mjs b/tests/unit/combo-test-route.test.mjs index 2cadf2ec1c..3f09834152 100644 --- a/tests/unit/combo-test-route.test.mjs +++ b/tests/unit/combo-test-route.test.mjs @@ -86,6 +86,8 @@ test("combo test route marks a model healthy only when it returns assistant text assert.match(fetchCalls[0].init.headers["X-Request-Id"], /^combo-test-/); assert.equal(forwardedBody.model, "openrouter/openai/gpt-5.4"); assert.equal(forwardedBody.messages[0].content, "Reply with OK only."); + assert.equal(forwardedBody.max_tokens, 64); + assert.equal(forwardedBody.temperature, 0); assert.equal(body.resolvedBy, "openrouter/openai/gpt-5.4"); assert.equal(body.results[0].status, "ok"); assert.equal(body.results[0].responseText, "OK"); @@ -122,6 +124,45 @@ test("combo test route treats empty successful responses as failures", async () assert.match(body.results[0].error, /no text content/i); }); +test("combo test route accepts reasoning-only completions as healthy smoke-test responses", async () => { + await createTestCombo(); + + globalThis.fetch = async () => + new Response( + JSON.stringify({ + choices: [ + { + finish_reason: "length", + message: { + role: "assistant", + content: "", + }, + }, + ], + usage: { + prompt_tokens: 6, + completion_tokens: 12, + total_tokens: 18, + completion_tokens_details: { + reasoning_tokens: 12, + }, + }, + }), + { + status: 200, + headers: { "content-type": "application/json" }, + } + ); + + const response = await route.POST(makeRequest()); + const body = await response.json(); + + assert.equal(response.status, 200); + assert.equal(body.resolvedBy, "openrouter/openai/gpt-5.4"); + assert.equal(body.results[0].status, "ok"); + assert.equal(body.results[0].responseText, "[reasoning-only completion]"); +}); + test("combo test route surfaces provider errors instead of downgrading them to reachability", async () => { await createTestCombo();