fix: skip non-chat generation models in Test all models (#13409)

"Test all models" no longer sends image/music/video generation-only models through a chat completion, which triggered real billable generations (#13376). `detectTestKind` flags them via `isNonChatGeneration` and `runSingleModelTest` skips them; chat+image models, embeddings and models without metadata are unaffected (8 new cases plus updated `model-test-runner` shapes).

Validated in one consolidated batch of this series (37 PRs boarded together on `release/v3.8.51`): `typecheck:core`, `check:open-sse-typecheck` and `check:dashboard-typecheck` clean; ESLint clean on every changed file; file-size, complexity, cognitive-complexity, changelog-integrity, docs-counts, docs-sync and migration-numbering gates green (only the pre-existing `open-sse/utils/stream.ts` file-size red remains, inherited from the base); 3,743 focused `node:test` cases plus 34 vitest cases green.

Thanks @KooshaPari!

src/lib/api/modelTestRunner.ts

@@ -306,7 +306,20 @@ export function detectTestKind(modelStr: string, customModel: any, nodeApiType?:
     (apiFormat === "responses" ||
       nodeType === "responses" ||
       supportedEndpoints.includes("responses"));
-  return { isRerank, isEmbedding, isAudioTranscription, isResponses };
+  // Non-chat generation endpoints (image, music, video) should NOT be dispatched
+  // as chat completions — they incur billable generation costs (#13376).
+  const isNonChatGeneration =
+    !isAudioTranscription &&
+    !isRerank &&
+    !isEmbedding &&
+    !isResponses &&
+    supportedEndpoints.length > 0 &&
+    !supportedEndpoints.includes("chat") &&
+    (supportedEndpoints.includes("images") ||
+      supportedEndpoints.includes("music") ||
+      supportedEndpoints.includes("videos"));
+
+  return { isRerank, isEmbedding, isAudioTranscription, isResponses, isNonChatGeneration };
 }
 
 /**
@@ -465,11 +478,20 @@ export async function runSingleModelTest(
     findCustomModelMetadata(providerId, fullModelStr),
     findProviderNodeApiType(providerId),
   ]);
-  const { isRerank, isEmbedding, isAudioTranscription, isResponses } = detectTestKind(
+  const { isRerank, isEmbedding, isAudioTranscription, isResponses, isNonChatGeneration } =
-    fullModelStr,
+    detectTestKind(fullModelStr, customModel, nodeApiType);
-    customModel,
+
-    nodeApiType
+  // #13376: Skip image/music/video generation models — dispatching them as
-  );
+  // chat completions incurs real billable generations the operator never asked for.
+  if (isNonChatGeneration) {
+    return {
+      modelId: fullModelStr,
+      status: "error",
+      latencyMs: 0,
+      error:
+        "Skipped: non-chat generation model (images/music/video) — use the corresponding generation endpoint instead",
+    };
+  }
 
   const testBody = isRerank
     ? {

tests/unit/model-test-modality-guard-13376.test.ts

@@ -0,0 +1,90 @@
+// #13376 — "Test all models" dispatched image/music/video generation models
+// as chat completions, incurring real billable generations the operator never
+// asked for. This test verifies detectTestKind flags non-chat generation models
+// and runSingleModelTest skips them.
+
+import test from "node:test";
+import assert from "node:assert/strict";
+
+import { detectTestKind } from "../../src/lib/api/modelTestRunner.ts";
+
+const serial = { concurrency: false };
+
+test("#13376 — detectTestKind flags image-generation-only models", serial, () => {
+  const result = detectTestKind(
+    "openai/dall-e-3",
+    { supportedEndpoints: ["images"] },
+    undefined
+  );
+  assert.equal(result.isNonChatGeneration, true, "image-only model should be flagged");
+  assert.equal(result.isEmbedding, false);
+  assert.equal(result.isRerank, false);
+});
+
+test("#13376 — detectTestKind flags music-generation-only models", serial, () => {
+  const result = detectTestKind(
+    "suno/suno-v3",
+    { supportedEndpoints: ["music"] },
+    undefined
+  );
+  assert.equal(result.isNonChatGeneration, true, "music-only model should be flagged");
+});
+
+test("#13376 — detectTestKind flags video-generation-only models", serial, () => {
+  const result = detectTestKind(
+    "runway/runway-gen3",
+    { supportedEndpoints: ["videos"] },
+    undefined
+  );
+  assert.equal(result.isNonChatGeneration, true, "video-only model should be flagged");
+});
+
+test("#13376 — detectTestKind does NOT flag chat+image models", serial, () => {
+  const result = detectTestKind(
+    "openai/gpt-4o",
+    { supportedEndpoints: ["chat", "images"] },
+    undefined
+  );
+  assert.equal(result.isNonChatGeneration, false, "chat-capable model should NOT be flagged");
+});
+
+test("#13376 — detectTestKind does NOT flag models with no supportedEndpoints", serial, () => {
+  const result = detectTestKind(
+    "openai/gpt-4o",
+    { supportedEndpoints: [] },
+    undefined
+  );
+  assert.equal(
+    result.isNonChatGeneration,
+    false,
+    "model with empty supportedEndpoints should NOT be flagged"
+  );
+});
+
+test("#13376 — detectTestKind does NOT flag plain chat models", serial, () => {
+  const result = detectTestKind(
+    "anthropic/claude-3.5-sonnet",
+    { supportedEndpoints: ["chat"] },
+    undefined
+  );
+  assert.equal(result.isNonChatGeneration, false, "chat-only model should NOT be flagged");
+});
+
+test("#13376 — detectTestKind does NOT flag embedding models", serial, () => {
+  const result = detectTestKind(
+    "openai/text-embedding-3-small",
+    { supportedEndpoints: ["embeddings"] },
+    undefined
+  );
+  assert.equal(result.isNonChatGeneration, false, "embedding model should NOT be flagged");
+  assert.equal(result.isEmbedding, true);
+});
+
+test("#13376 — detectTestKind does NOT flag models with no customModel metadata", serial, () => {
+  const result = detectTestKind("openai/gpt-4o", undefined, undefined);
+  assert.equal(
+    result.isNonChatGeneration,
+    false,
+    "model without metadata should NOT be flagged"
+  );
+});

tests/unit/model-test-runner.test.ts

@@ -75,6 +75,7 @@ test("detectTestKind defaults to a plain chat test for ordinary models", () => {
     isEmbedding: false,
     isAudioTranscription: false,
     isResponses: false,
+    isNonChatGeneration: false,
   });
 });
 
@@ -97,6 +98,7 @@ test("detectTestKind detects rerank by id and by metadata, and rerank wins over
     isEmbedding: false,
     isAudioTranscription: false,
     isResponses: false,
+    isNonChatGeneration: false,
   });
   // apiFormat metadata drives detection even when the id is opaque
   assert.equal(detectTestKind("vendor/opaque-model", { apiFormat: "rerank" }).isRerank, true);
@@ -119,6 +121,7 @@ test("detectTestKind detects audio transcription from metadata, and it wins over
     isEmbedding: false,
     isAudioTranscription: true,
     isResponses: false,
+    isNonChatGeneration: false,
   });
   assert.equal(
     detectTestKind("vendor/opaque-model", { supportedEndpoints: ["audio-transcriptions"] })
@@ -156,6 +159,7 @@ test("detectTestKind falls back to the provider node's configured apiType", () =
     isEmbedding: false,
     isAudioTranscription: false,
     isResponses: false,
+    isNonChatGeneration: false,
   });
 
   // Per-model metadata still wins when present.