Merge pull request #828 from rdself/coder/fix-combo-test-false-negative

Fix combo test false negatives and parallelize model probes

src/app/api/combos/test/route.ts

@@ -5,6 +5,87 @@ import { getComboByName } from "@/lib/localDb";
 import { testComboSchema } from "@/shared/validation/schemas";
 import { isValidationFailure, validateBody } from "@/shared/validation/helpers";
 
+async function testComboModel(modelStr, internalUrl) {
+  const startTime = Date.now();
+  try {
+    // Send a minimal but real chat request through the same internal
+    // endpoint an external OpenAI-compatible client would use.
+    const testBody = buildComboTestRequestBody(modelStr);
+
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), 20000);
+
+    let res;
+    try {
+      res = await fetch(internalUrl, {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          // Internal dashboard tests still use the normal /v1 pipeline but
+          // bypass REQUIRE_API_KEY so admins can test with local session auth.
+          "X-Internal-Test": "combo-health-check",
+          // Force a fresh execution path so combo tests cannot be satisfied by
+          // OmniRoute's semantic cache or other request reuse layers.
+          "X-OmniRoute-No-Cache": "true",
+          "X-Request-Id": `combo-test-${randomUUID()}`,
+        },
+        body: JSON.stringify(testBody),
+        signal: controller.signal,
+      });
+    } finally {
+      clearTimeout(timeout);
+    }
+
+    const latencyMs = Date.now() - startTime;
+
+    if (res.ok) {
+      let responseBody = null;
+      try {
+        responseBody = await res.json();
+      } catch {
+        responseBody = null;
+      }
+
+      const responseText = extractComboTestResponseText(responseBody);
+      if (!responseText) {
+        return {
+          model: modelStr,
+          status: "error",
+          statusCode: res.status,
+          error: "Provider returned HTTP 200 but no text content.",
+          latencyMs,
+        };
+      }
+
+      return { model: modelStr, status: "ok", latencyMs, responseText };
+    }
+
+    let errorMsg = "";
+    try {
+      const errBody = await res.json();
+      errorMsg = errBody?.error?.message || errBody?.error || res.statusText;
+    } catch {
+      errorMsg = res.statusText;
+    }
+
+    return {
+      model: modelStr,
+      status: "error",
+      statusCode: res.status,
+      error: errorMsg,
+      latencyMs,
+    };
+  } catch (error) {
+    const latencyMs = Date.now() - startTime;
+    return {
+      model: modelStr,
+      status: "error",
+      error: error.name === "AbortError" ? "Timeout (20s)" : error.message,
+      latencyMs,
+    };
+  }
+}
+
 /**
  * POST /api/combos/test - Quick test a combo
  * Sends a real chat completion request through each model in the combo
@@ -44,93 +125,11 @@ export async function POST(request) {
       return NextResponse.json({ error: "Combo has no models" }, { status: 400 });
     }
 
-    const results = [];
-    let resolvedBy = null;
-
-    // Test each model sequentially
-    for (const modelStr of models) {
-      const startTime = Date.now();
-      try {
-        // Send a minimal but real chat request through the same internal
-        // endpoint an external OpenAI-compatible client would use.
-        const testBody = buildComboTestRequestBody(modelStr);
-
-        const internalUrl = `${getBaseUrl(request)}/v1/chat/completions`;
-        const controller = new AbortController();
-        const timeout = setTimeout(() => controller.abort(), 20000);
-
-        let res;
-        try {
-          res = await fetch(internalUrl, {
-            method: "POST",
-            headers: {
-              "Content-Type": "application/json",
-              // Internal dashboard tests still use the normal /v1 pipeline but
-              // bypass REQUIRE_API_KEY so admins can test with local session auth.
-              "X-Internal-Test": "combo-health-check",
-              // Force a fresh execution path so combo tests cannot be satisfied by
-              // OmniRoute's semantic cache or other request reuse layers.
-              "X-OmniRoute-No-Cache": "true",
-              "X-Request-Id": `combo-test-${randomUUID()}`,
-            },
-            body: JSON.stringify(testBody),
-            signal: controller.signal,
-          });
-        } finally {
-          clearTimeout(timeout);
-        }
-
-        const latencyMs = Date.now() - startTime;
-
-        if (res.ok) {
-          let responseBody = null;
-          try {
-            responseBody = await res.json();
-          } catch {
-            responseBody = null;
-          }
-
-          const responseText = extractComboTestResponseText(responseBody);
-          if (!responseText) {
-            results.push({
-              model: modelStr,
-              status: "error",
-              statusCode: res.status,
-              error: "Provider returned HTTP 200 but no text content.",
-              latencyMs,
-            });
-            continue;
-          }
-
-          results.push({ model: modelStr, status: "ok", latencyMs, responseText });
-          if (!resolvedBy) resolvedBy = modelStr;
-        } else {
-          let errorMsg = "";
-          try {
-            const errBody = await res.json();
-            errorMsg = errBody?.error?.message || errBody?.error || res.statusText;
-          } catch {
-            errorMsg = res.statusText;
-          }
-
-          results.push({
-            model: modelStr,
-            status: "error",
-            statusCode: res.status,
-            error: errorMsg,
-            latencyMs,
-          });
-        }
-      } catch (error) {
-        const latencyMs = Date.now() - startTime;
-        results.push({
-          model: modelStr,
-          status: "error",
-          error: error.name === "AbortError" ? "Timeout (20s)" : error.message,
-          latencyMs,
-        });
-      }
-    }
+    const internalUrl = `${getBaseUrl(request)}/v1/chat/completions`;
+    const results = await Promise.all(
+      models.map((modelStr) => testComboModel(modelStr, internalUrl))
+    );
+    const resolvedBy = results.find((result) => result.status === "ok")?.model || null;
 
     return NextResponse.json({
       comboName,

src/lib/combos/testHealth.ts

@@ -4,6 +4,10 @@ function asRecord(value: unknown): JsonRecord {
   return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {};
 }
 
+function joinNonEmpty(parts: string[]) {
+  return parts.filter(Boolean).join("\n").trim();
+}
+
 function extractTextFromContent(content: unknown): string {
   if (typeof content === "string") return content.trim();
 
@@ -28,12 +32,85 @@ function extractTextFromContent(content: unknown): string {
     .trim();
 }
 
+function extractReasoningText(record: JsonRecord): string {
+  const reasoningDetails = Array.isArray(record.reasoning_details) ? record.reasoning_details : [];
+  const detailText = reasoningDetails
+    .map((detail) => {
+      const detailRecord = asRecord(detail);
+      const detailType = typeof detailRecord.type === "string" ? detailRecord.type : "";
+      const text =
+        typeof detailRecord.text === "string"
+          ? detailRecord.text.trim()
+          : typeof detailRecord.content === "string"
+            ? detailRecord.content.trim()
+            : "";
+
+      if (
+        text &&
+        (detailType === "" ||
+          detailType === "reasoning" ||
+          detailType === "reasoning.text" ||
+          detailType === "thinking")
+      ) {
+        return text;
+      }
+
+      return "";
+    })
+    .filter(Boolean);
+
+  return joinNonEmpty([
+    typeof record.reasoning_content === "string" ? record.reasoning_content.trim() : "",
+    typeof record.reasoning === "string" ? record.reasoning.trim() : "",
+    typeof record.reasoning_text === "string" ? record.reasoning_text.trim() : "",
+    joinNonEmpty(detailText),
+  ]);
+}
+
+function getUsageReasoningTokens(body: JsonRecord): number {
+  const usage = asRecord(body.usage);
+  if (!usage) return 0;
+
+  const completionDetails = asRecord(usage.completion_tokens_details);
+  const topLevelReasoning =
+    typeof usage.reasoning_tokens === "number" && Number.isFinite(usage.reasoning_tokens)
+      ? usage.reasoning_tokens
+      : 0;
+  const detailedReasoning =
+    typeof completionDetails.reasoning_tokens === "number" &&
+    Number.isFinite(completionDetails.reasoning_tokens)
+      ? completionDetails.reasoning_tokens
+      : 0;
+
+  return Math.max(topLevelReasoning, detailedReasoning);
+}
+
+function hasReasoningOnlyCompletion(body: JsonRecord): boolean {
+  if (!Array.isArray(body.choices) || body.choices.length === 0) return false;
+  if (getUsageReasoningTokens(body) <= 0) return false;
+
+  return body.choices.some((choice) => {
+    const choiceRecord = asRecord(choice);
+    const message = asRecord(choiceRecord.message);
+    const finishReason =
+      typeof choiceRecord.finish_reason === "string" ? choiceRecord.finish_reason : "";
+
+    if (!message || message.role !== "assistant") return false;
+    if (!finishReason) return false;
+    if (extractTextFromContent(message.content)) return false;
+    if (extractReasoningText(message)) return false;
+    return true;
+  });
+}
+
 export function buildComboTestRequestBody(modelStr: string) {
   return {
     model: modelStr,
     messages: [{ role: "user", content: "Reply with OK only." }],
-    // Keep this close to a real client request without inflating cost.
-    max_tokens: 16,
+    // Give reasoning-heavy models enough headroom to emit a tiny visible answer
+    // without turning the smoke test into a full-cost real request.
+    max_tokens: 64,
+    temperature: 0,
     stream: false,
   };
 }
@@ -52,6 +129,9 @@ export function extractComboTestResponseText(responseBody: unknown): string {
       const messageText = extractTextFromContent(message.content);
       if (messageText) return messageText;
 
+      const reasoningText = extractReasoningText(message);
+      if (reasoningText) return reasoningText;
+
       if (typeof choiceRecord.text === "string" && choiceRecord.text.trim()) {
         return choiceRecord.text.trim();
       }
@@ -63,8 +143,21 @@ export function extractComboTestResponseText(responseBody: unknown): string {
       const itemRecord = asRecord(item);
       const contentText = extractTextFromContent(itemRecord.content);
       if (contentText) return contentText;
+
+      const reasoningText = extractReasoningText(itemRecord);
+      if (reasoningText) return reasoningText;
     }
   }
 
-  return extractTextFromContent(body.content);
+  const topLevelText = extractTextFromContent(body.content);
+  if (topLevelText) return topLevelText;
+
+  const topLevelReasoning = extractReasoningText(body);
+  if (topLevelReasoning) return topLevelReasoning;
+
+  if (hasReasoningOnlyCompletion(body)) {
+    return "[reasoning-only completion]";
+  }
+
+  return "";
 }

tests/unit/combo-test-health.test.mjs

@@ -9,7 +9,8 @@ test("combo test helper builds a realistic smoke payload", () => {
 
   assert.equal(body.model, "openrouter/openai/gpt-5.4");
   assert.equal(body.messages[0].content, "Reply with OK only.");
-  assert.equal(body.max_tokens, 16);
+  assert.equal(body.max_tokens, 64);
+  assert.equal(body.temperature, 0);
   assert.equal(body.stream, false);
 });
 
@@ -46,6 +47,62 @@ test("combo test helper extracts text from block-based responses", () => {
   assert.equal(text, "OK\nConfirmed.");
 });
 
+test("combo test helper extracts reasoning content when visible text is absent", () => {
+  const text = extractComboTestResponseText({
+    choices: [
+      {
+        message: {
+          role: "assistant",
+          content: null,
+          reasoning_content: "Working through the request.\nOK",
+        },
+      },
+    ],
+  });
+
+  assert.equal(text, "Working through the request.\nOK");
+});
+
+test("combo test helper extracts reasoning_text aliases from GitHub-style responses", () => {
+  const text = extractComboTestResponseText({
+    choices: [
+      {
+        message: {
+          role: "assistant",
+          content: "",
+          reasoning_text: "Reasoning trace",
+        },
+      },
+    ],
+  });
+
+  assert.equal(text, "Reasoning trace");
+});
+
+test("combo test helper treats reasoning-only completions as a healthy signal", () => {
+  const text = extractComboTestResponseText({
+    choices: [
+      {
+        finish_reason: "length",
+        message: {
+          role: "assistant",
+          content: "",
+        },
+      },
+    ],
+    usage: {
+      prompt_tokens: 6,
+      completion_tokens: 12,
+      total_tokens: 18,
+      completion_tokens_details: {
+        reasoning_tokens: 12,
+      },
+    },
+  });
+
+  assert.equal(text, "[reasoning-only completion]");
+});
+
 test("combo test helper returns empty string when no text content exists", () => {
   const text = extractComboTestResponseText({
     choices: [

tests/unit/combo-test-route.test.mjs

@@ -86,6 +86,8 @@ test("combo test route marks a model healthy only when it returns assistant text
   assert.match(fetchCalls[0].init.headers["X-Request-Id"], /^combo-test-/);
   assert.equal(forwardedBody.model, "openrouter/openai/gpt-5.4");
   assert.equal(forwardedBody.messages[0].content, "Reply with OK only.");
+  assert.equal(forwardedBody.max_tokens, 64);
+  assert.equal(forwardedBody.temperature, 0);
   assert.equal(body.resolvedBy, "openrouter/openai/gpt-5.4");
   assert.equal(body.results[0].status, "ok");
   assert.equal(body.results[0].responseText, "OK");
@@ -122,6 +124,45 @@ test("combo test route treats empty successful responses as failures", async ()
   assert.match(body.results[0].error, /no text content/i);
 });
 
+test("combo test route accepts reasoning-only completions as healthy smoke-test responses", async () => {
+  await createTestCombo();
+
+  globalThis.fetch = async () =>
+    new Response(
+      JSON.stringify({
+        choices: [
+          {
+            finish_reason: "length",
+            message: {
+              role: "assistant",
+              content: "",
+            },
+          },
+        ],
+        usage: {
+          prompt_tokens: 6,
+          completion_tokens: 12,
+          total_tokens: 18,
+          completion_tokens_details: {
+            reasoning_tokens: 12,
+          },
+        },
+      }),
+      {
+        status: 200,
+        headers: { "content-type": "application/json" },
+      }
+    );
+
+  const response = await route.POST(makeRequest());
+  const body = await response.json();
+
+  assert.equal(response.status, 200);
+  assert.equal(body.resolvedBy, "openrouter/openai/gpt-5.4");
+  assert.equal(body.results[0].status, "ok");
+  assert.equal(body.results[0].responseText, "[reasoning-only completion]");
+});
+
 test("combo test route surfaces provider errors instead of downgrading them to reachability", async () => {
   await createTestCombo();
 
@@ -148,3 +189,67 @@ test("combo test route surfaces provider errors instead of downgrading them to r
   assert.equal(body.results[0].error, "Upstream rejected this request shape");
   assert.equal("probeMethod" in body.results[0], false);
 });
+
+test("combo test route launches model probes concurrently while preserving combo order", async () => {
+  await createTestCombo(["provider/first", "provider/second", "provider/third"]);
+
+  const fetchCalls = [];
+  const resolvers = [];
+  globalThis.fetch = (url, init = {}) =>
+    new Promise((resolve) => {
+      fetchCalls.push({ url: String(url), init });
+      resolvers.push(resolve);
+    });
+
+  const responsePromise = route.POST(makeRequest());
+  await new Promise((resolve) => setTimeout(resolve, 0));
+
+  assert.equal(fetchCalls.length, 3);
+  assert.deepEqual(
+    fetchCalls.map(({ init }) => JSON.parse(init.body).model),
+    ["provider/first", "provider/second", "provider/third"]
+  );
+
+  resolvers[2](
+    new Response(
+      JSON.stringify({
+        choices: [{ message: { role: "assistant", content: "THIRD" } }],
+      }),
+      { status: 200, headers: { "content-type": "application/json" } }
+    )
+  );
+  resolvers[1](
+    new Response(
+      JSON.stringify({
+        choices: [{ message: { role: "assistant", content: "SECOND" } }],
+      }),
+      { status: 200, headers: { "content-type": "application/json" } }
+    )
+  );
+  resolvers[0](
+    new Response(
+      JSON.stringify({
+        choices: [{ message: { role: "assistant", content: "FIRST" } }],
+      }),
+      { status: 200, headers: { "content-type": "application/json" } }
+    )
+  );
+
+  const response = await responsePromise;
+  const body = await response.json();
+
+  assert.equal(response.status, 200);
+  assert.equal(body.resolvedBy, "provider/first");
+  assert.deepEqual(
+    body.results.map((result) => ({
+      model: result.model,
+      status: result.status,
+      responseText: result.responseText,
+    })),
+    [
+      { model: "provider/first", status: "ok", responseText: "FIRST" },
+      { model: "provider/second", status: "ok", responseText: "SECOND" },
+      { model: "provider/third", status: "ok", responseText: "THIRD" },
+    ]
+  );
+});