diff --git a/src/app/api/combos/test/route.ts b/src/app/api/combos/test/route.ts index fb9888ea29..5e93979a79 100644 --- a/src/app/api/combos/test/route.ts +++ b/src/app/api/combos/test/route.ts @@ -5,6 +5,87 @@ import { getComboByName } from "@/lib/localDb"; import { testComboSchema } from "@/shared/validation/schemas"; import { isValidationFailure, validateBody } from "@/shared/validation/helpers"; +async function testComboModel(modelStr, internalUrl) { + const startTime = Date.now(); + try { + // Send a minimal but real chat request through the same internal + // endpoint an external OpenAI-compatible client would use. + const testBody = buildComboTestRequestBody(modelStr); + + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 20000); + + let res; + try { + res = await fetch(internalUrl, { + method: "POST", + headers: { + "Content-Type": "application/json", + // Internal dashboard tests still use the normal /v1 pipeline but + // bypass REQUIRE_API_KEY so admins can test with local session auth. + "X-Internal-Test": "combo-health-check", + // Force a fresh execution path so combo tests cannot be satisfied by + // OmniRoute's semantic cache or other request reuse layers. + "X-OmniRoute-No-Cache": "true", + "X-Request-Id": `combo-test-${randomUUID()}`, + }, + body: JSON.stringify(testBody), + signal: controller.signal, + }); + } finally { + clearTimeout(timeout); + } + + const latencyMs = Date.now() - startTime; + + if (res.ok) { + let responseBody = null; + try { + responseBody = await res.json(); + } catch { + responseBody = null; + } + + const responseText = extractComboTestResponseText(responseBody); + if (!responseText) { + return { + model: modelStr, + status: "error", + statusCode: res.status, + error: "Provider returned HTTP 200 but no text content.", + latencyMs, + }; + } + + return { model: modelStr, status: "ok", latencyMs, responseText }; + } + + let errorMsg = ""; + try { + const errBody = await res.json(); + errorMsg = errBody?.error?.message || errBody?.error || res.statusText; + } catch { + errorMsg = res.statusText; + } + + return { + model: modelStr, + status: "error", + statusCode: res.status, + error: errorMsg, + latencyMs, + }; + } catch (error) { + const latencyMs = Date.now() - startTime; + return { + model: modelStr, + status: "error", + error: error.name === "AbortError" ? "Timeout (20s)" : error.message, + latencyMs, + }; + } +} + /** * POST /api/combos/test - Quick test a combo * Sends a real chat completion request through each model in the combo @@ -44,93 +125,11 @@ export async function POST(request) { return NextResponse.json({ error: "Combo has no models" }, { status: 400 }); } - const results = []; - let resolvedBy = null; - - // Test each model sequentially - for (const modelStr of models) { - const startTime = Date.now(); - try { - // Send a minimal but real chat request through the same internal - // endpoint an external OpenAI-compatible client would use. - const testBody = buildComboTestRequestBody(modelStr); - - const internalUrl = `${getBaseUrl(request)}/v1/chat/completions`; - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), 20000); - - let res; - try { - res = await fetch(internalUrl, { - method: "POST", - headers: { - "Content-Type": "application/json", - // Internal dashboard tests still use the normal /v1 pipeline but - // bypass REQUIRE_API_KEY so admins can test with local session auth. - "X-Internal-Test": "combo-health-check", - // Force a fresh execution path so combo tests cannot be satisfied by - // OmniRoute's semantic cache or other request reuse layers. - "X-OmniRoute-No-Cache": "true", - "X-Request-Id": `combo-test-${randomUUID()}`, - }, - body: JSON.stringify(testBody), - signal: controller.signal, - }); - } finally { - clearTimeout(timeout); - } - - const latencyMs = Date.now() - startTime; - - if (res.ok) { - let responseBody = null; - try { - responseBody = await res.json(); - } catch { - responseBody = null; - } - - const responseText = extractComboTestResponseText(responseBody); - if (!responseText) { - results.push({ - model: modelStr, - status: "error", - statusCode: res.status, - error: "Provider returned HTTP 200 but no text content.", - latencyMs, - }); - continue; - } - - results.push({ model: modelStr, status: "ok", latencyMs, responseText }); - if (!resolvedBy) resolvedBy = modelStr; - } else { - let errorMsg = ""; - try { - const errBody = await res.json(); - errorMsg = errBody?.error?.message || errBody?.error || res.statusText; - } catch { - errorMsg = res.statusText; - } - - results.push({ - model: modelStr, - status: "error", - statusCode: res.status, - error: errorMsg, - latencyMs, - }); - } - } catch (error) { - const latencyMs = Date.now() - startTime; - results.push({ - model: modelStr, - status: "error", - error: error.name === "AbortError" ? "Timeout (20s)" : error.message, - latencyMs, - }); - } - } + const internalUrl = `${getBaseUrl(request)}/v1/chat/completions`; + const results = await Promise.all( + models.map((modelStr) => testComboModel(modelStr, internalUrl)) + ); + const resolvedBy = results.find((result) => result.status === "ok")?.model || null; return NextResponse.json({ comboName, diff --git a/src/lib/combos/testHealth.ts b/src/lib/combos/testHealth.ts index 15959e6913..f7ac5f66cf 100644 --- a/src/lib/combos/testHealth.ts +++ b/src/lib/combos/testHealth.ts @@ -4,6 +4,10 @@ function asRecord(value: unknown): JsonRecord { return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {}; } +function joinNonEmpty(parts: string[]) { + return parts.filter(Boolean).join("\n").trim(); +} + function extractTextFromContent(content: unknown): string { if (typeof content === "string") return content.trim(); @@ -28,12 +32,85 @@ function extractTextFromContent(content: unknown): string { .trim(); } +function extractReasoningText(record: JsonRecord): string { + const reasoningDetails = Array.isArray(record.reasoning_details) ? record.reasoning_details : []; + const detailText = reasoningDetails + .map((detail) => { + const detailRecord = asRecord(detail); + const detailType = typeof detailRecord.type === "string" ? detailRecord.type : ""; + const text = + typeof detailRecord.text === "string" + ? detailRecord.text.trim() + : typeof detailRecord.content === "string" + ? detailRecord.content.trim() + : ""; + + if ( + text && + (detailType === "" || + detailType === "reasoning" || + detailType === "reasoning.text" || + detailType === "thinking") + ) { + return text; + } + + return ""; + }) + .filter(Boolean); + + return joinNonEmpty([ + typeof record.reasoning_content === "string" ? record.reasoning_content.trim() : "", + typeof record.reasoning === "string" ? record.reasoning.trim() : "", + typeof record.reasoning_text === "string" ? record.reasoning_text.trim() : "", + joinNonEmpty(detailText), + ]); +} + +function getUsageReasoningTokens(body: JsonRecord): number { + const usage = asRecord(body.usage); + if (!usage) return 0; + + const completionDetails = asRecord(usage.completion_tokens_details); + const topLevelReasoning = + typeof usage.reasoning_tokens === "number" && Number.isFinite(usage.reasoning_tokens) + ? usage.reasoning_tokens + : 0; + const detailedReasoning = + typeof completionDetails.reasoning_tokens === "number" && + Number.isFinite(completionDetails.reasoning_tokens) + ? completionDetails.reasoning_tokens + : 0; + + return Math.max(topLevelReasoning, detailedReasoning); +} + +function hasReasoningOnlyCompletion(body: JsonRecord): boolean { + if (!Array.isArray(body.choices) || body.choices.length === 0) return false; + if (getUsageReasoningTokens(body) <= 0) return false; + + return body.choices.some((choice) => { + const choiceRecord = asRecord(choice); + const message = asRecord(choiceRecord.message); + const finishReason = + typeof choiceRecord.finish_reason === "string" ? choiceRecord.finish_reason : ""; + + if (!message || message.role !== "assistant") return false; + if (!finishReason) return false; + if (extractTextFromContent(message.content)) return false; + if (extractReasoningText(message)) return false; + return true; + }); +} + export function buildComboTestRequestBody(modelStr: string) { return { model: modelStr, messages: [{ role: "user", content: "Reply with OK only." }], - // Keep this close to a real client request without inflating cost. - max_tokens: 16, + // Give reasoning-heavy models enough headroom to emit a tiny visible answer + // without turning the smoke test into a full-cost real request. + max_tokens: 64, + temperature: 0, stream: false, }; } @@ -52,6 +129,9 @@ export function extractComboTestResponseText(responseBody: unknown): string { const messageText = extractTextFromContent(message.content); if (messageText) return messageText; + const reasoningText = extractReasoningText(message); + if (reasoningText) return reasoningText; + if (typeof choiceRecord.text === "string" && choiceRecord.text.trim()) { return choiceRecord.text.trim(); } @@ -63,8 +143,21 @@ export function extractComboTestResponseText(responseBody: unknown): string { const itemRecord = asRecord(item); const contentText = extractTextFromContent(itemRecord.content); if (contentText) return contentText; + + const reasoningText = extractReasoningText(itemRecord); + if (reasoningText) return reasoningText; } } - return extractTextFromContent(body.content); + const topLevelText = extractTextFromContent(body.content); + if (topLevelText) return topLevelText; + + const topLevelReasoning = extractReasoningText(body); + if (topLevelReasoning) return topLevelReasoning; + + if (hasReasoningOnlyCompletion(body)) { + return "[reasoning-only completion]"; + } + + return ""; } diff --git a/tests/unit/combo-test-health.test.mjs b/tests/unit/combo-test-health.test.mjs index 2471656d46..19de579271 100644 --- a/tests/unit/combo-test-health.test.mjs +++ b/tests/unit/combo-test-health.test.mjs @@ -9,7 +9,8 @@ test("combo test helper builds a realistic smoke payload", () => { assert.equal(body.model, "openrouter/openai/gpt-5.4"); assert.equal(body.messages[0].content, "Reply with OK only."); - assert.equal(body.max_tokens, 16); + assert.equal(body.max_tokens, 64); + assert.equal(body.temperature, 0); assert.equal(body.stream, false); }); @@ -46,6 +47,62 @@ test("combo test helper extracts text from block-based responses", () => { assert.equal(text, "OK\nConfirmed."); }); +test("combo test helper extracts reasoning content when visible text is absent", () => { + const text = extractComboTestResponseText({ + choices: [ + { + message: { + role: "assistant", + content: null, + reasoning_content: "Working through the request.\nOK", + }, + }, + ], + }); + + assert.equal(text, "Working through the request.\nOK"); +}); + +test("combo test helper extracts reasoning_text aliases from GitHub-style responses", () => { + const text = extractComboTestResponseText({ + choices: [ + { + message: { + role: "assistant", + content: "", + reasoning_text: "Reasoning trace", + }, + }, + ], + }); + + assert.equal(text, "Reasoning trace"); +}); + +test("combo test helper treats reasoning-only completions as a healthy signal", () => { + const text = extractComboTestResponseText({ + choices: [ + { + finish_reason: "length", + message: { + role: "assistant", + content: "", + }, + }, + ], + usage: { + prompt_tokens: 6, + completion_tokens: 12, + total_tokens: 18, + completion_tokens_details: { + reasoning_tokens: 12, + }, + }, + }); + + assert.equal(text, "[reasoning-only completion]"); +}); + test("combo test helper returns empty string when no text content exists", () => { const text = extractComboTestResponseText({ choices: [ diff --git a/tests/unit/combo-test-route.test.mjs b/tests/unit/combo-test-route.test.mjs index 2cadf2ec1c..3a75a3322f 100644 --- a/tests/unit/combo-test-route.test.mjs +++ b/tests/unit/combo-test-route.test.mjs @@ -86,6 +86,8 @@ test("combo test route marks a model healthy only when it returns assistant text assert.match(fetchCalls[0].init.headers["X-Request-Id"], /^combo-test-/); assert.equal(forwardedBody.model, "openrouter/openai/gpt-5.4"); assert.equal(forwardedBody.messages[0].content, "Reply with OK only."); + assert.equal(forwardedBody.max_tokens, 64); + assert.equal(forwardedBody.temperature, 0); assert.equal(body.resolvedBy, "openrouter/openai/gpt-5.4"); assert.equal(body.results[0].status, "ok"); assert.equal(body.results[0].responseText, "OK"); @@ -122,6 +124,45 @@ test("combo test route treats empty successful responses as failures", async () assert.match(body.results[0].error, /no text content/i); }); +test("combo test route accepts reasoning-only completions as healthy smoke-test responses", async () => { + await createTestCombo(); + + globalThis.fetch = async () => + new Response( + JSON.stringify({ + choices: [ + { + finish_reason: "length", + message: { + role: "assistant", + content: "", + }, + }, + ], + usage: { + prompt_tokens: 6, + completion_tokens: 12, + total_tokens: 18, + completion_tokens_details: { + reasoning_tokens: 12, + }, + }, + }), + { + status: 200, + headers: { "content-type": "application/json" }, + } + ); + + const response = await route.POST(makeRequest()); + const body = await response.json(); + + assert.equal(response.status, 200); + assert.equal(body.resolvedBy, "openrouter/openai/gpt-5.4"); + assert.equal(body.results[0].status, "ok"); + assert.equal(body.results[0].responseText, "[reasoning-only completion]"); +}); + test("combo test route surfaces provider errors instead of downgrading them to reachability", async () => { await createTestCombo(); @@ -148,3 +189,67 @@ test("combo test route surfaces provider errors instead of downgrading them to r assert.equal(body.results[0].error, "Upstream rejected this request shape"); assert.equal("probeMethod" in body.results[0], false); }); + +test("combo test route launches model probes concurrently while preserving combo order", async () => { + await createTestCombo(["provider/first", "provider/second", "provider/third"]); + + const fetchCalls = []; + const resolvers = []; + globalThis.fetch = (url, init = {}) => + new Promise((resolve) => { + fetchCalls.push({ url: String(url), init }); + resolvers.push(resolve); + }); + + const responsePromise = route.POST(makeRequest()); + await new Promise((resolve) => setTimeout(resolve, 0)); + + assert.equal(fetchCalls.length, 3); + assert.deepEqual( + fetchCalls.map(({ init }) => JSON.parse(init.body).model), + ["provider/first", "provider/second", "provider/third"] + ); + + resolvers[2]( + new Response( + JSON.stringify({ + choices: [{ message: { role: "assistant", content: "THIRD" } }], + }), + { status: 200, headers: { "content-type": "application/json" } } + ) + ); + resolvers[1]( + new Response( + JSON.stringify({ + choices: [{ message: { role: "assistant", content: "SECOND" } }], + }), + { status: 200, headers: { "content-type": "application/json" } } + ) + ); + resolvers[0]( + new Response( + JSON.stringify({ + choices: [{ message: { role: "assistant", content: "FIRST" } }], + }), + { status: 200, headers: { "content-type": "application/json" } } + ) + ); + + const response = await responsePromise; + const body = await response.json(); + + assert.equal(response.status, 200); + assert.equal(body.resolvedBy, "provider/first"); + assert.deepEqual( + body.results.map((result) => ({ + model: result.model, + status: result.status, + responseText: result.responseText, + })), + [ + { model: "provider/first", status: "ok", responseText: "FIRST" }, + { model: "provider/second", status: "ok", responseText: "SECOND" }, + { model: "provider/third", status: "ok", responseText: "THIRD" }, + ] + ); +});