diff --git a/src/lib/api/modelTestRunner.ts b/src/lib/api/modelTestRunner.ts index 59790cc111..760ed8224f 100644 --- a/src/lib/api/modelTestRunner.ts +++ b/src/lib/api/modelTestRunner.ts @@ -306,7 +306,20 @@ export function detectTestKind(modelStr: string, customModel: any, nodeApiType?: (apiFormat === "responses" || nodeType === "responses" || supportedEndpoints.includes("responses")); - return { isRerank, isEmbedding, isAudioTranscription, isResponses }; + // Non-chat generation endpoints (image, music, video) should NOT be dispatched + // as chat completions — they incur billable generation costs (#13376). + const isNonChatGeneration = + !isAudioTranscription && + !isRerank && + !isEmbedding && + !isResponses && + supportedEndpoints.length > 0 && + !supportedEndpoints.includes("chat") && + (supportedEndpoints.includes("images") || + supportedEndpoints.includes("music") || + supportedEndpoints.includes("videos")); + + return { isRerank, isEmbedding, isAudioTranscription, isResponses, isNonChatGeneration }; } /** @@ -465,11 +478,20 @@ export async function runSingleModelTest( findCustomModelMetadata(providerId, fullModelStr), findProviderNodeApiType(providerId), ]); - const { isRerank, isEmbedding, isAudioTranscription, isResponses } = detectTestKind( - fullModelStr, - customModel, - nodeApiType - ); + const { isRerank, isEmbedding, isAudioTranscription, isResponses, isNonChatGeneration } = + detectTestKind(fullModelStr, customModel, nodeApiType); + + // #13376: Skip image/music/video generation models — dispatching them as + // chat completions incurs real billable generations the operator never asked for. + if (isNonChatGeneration) { + return { + modelId: fullModelStr, + status: "error", + latencyMs: 0, + error: + "Skipped: non-chat generation model (images/music/video) — use the corresponding generation endpoint instead", + }; + } const testBody = isRerank ? { diff --git a/tests/unit/model-test-modality-guard-13376.test.ts b/tests/unit/model-test-modality-guard-13376.test.ts new file mode 100644 index 0000000000..f506883e25 --- /dev/null +++ b/tests/unit/model-test-modality-guard-13376.test.ts @@ -0,0 +1,90 @@ +// #13376 — "Test all models" dispatched image/music/video generation models +// as chat completions, incurring real billable generations the operator never +// asked for. This test verifies detectTestKind flags non-chat generation models +// and runSingleModelTest skips them. + +import test from "node:test"; +import assert from "node:assert/strict"; + +import { detectTestKind } from "../../src/lib/api/modelTestRunner.ts"; + +const serial = { concurrency: false }; + +test("#13376 — detectTestKind flags image-generation-only models", serial, () => { + const result = detectTestKind( + "openai/dall-e-3", + { supportedEndpoints: ["images"] }, + undefined + ); + assert.equal(result.isNonChatGeneration, true, "image-only model should be flagged"); + assert.equal(result.isEmbedding, false); + assert.equal(result.isRerank, false); +}); + +test("#13376 — detectTestKind flags music-generation-only models", serial, () => { + const result = detectTestKind( + "suno/suno-v3", + { supportedEndpoints: ["music"] }, + undefined + ); + assert.equal(result.isNonChatGeneration, true, "music-only model should be flagged"); +}); + +test("#13376 — detectTestKind flags video-generation-only models", serial, () => { + const result = detectTestKind( + "runway/runway-gen3", + { supportedEndpoints: ["videos"] }, + undefined + ); + assert.equal(result.isNonChatGeneration, true, "video-only model should be flagged"); +}); + +test("#13376 — detectTestKind does NOT flag chat+image models", serial, () => { + const result = detectTestKind( + "openai/gpt-4o", + { supportedEndpoints: ["chat", "images"] }, + undefined + ); + assert.equal(result.isNonChatGeneration, false, "chat-capable model should NOT be flagged"); +}); + +test("#13376 — detectTestKind does NOT flag models with no supportedEndpoints", serial, () => { + const result = detectTestKind( + "openai/gpt-4o", + { supportedEndpoints: [] }, + undefined + ); + assert.equal( + result.isNonChatGeneration, + false, + "model with empty supportedEndpoints should NOT be flagged" + ); +}); + +test("#13376 — detectTestKind does NOT flag plain chat models", serial, () => { + const result = detectTestKind( + "anthropic/claude-3.5-sonnet", + { supportedEndpoints: ["chat"] }, + undefined + ); + assert.equal(result.isNonChatGeneration, false, "chat-only model should NOT be flagged"); +}); + +test("#13376 — detectTestKind does NOT flag embedding models", serial, () => { + const result = detectTestKind( + "openai/text-embedding-3-small", + { supportedEndpoints: ["embeddings"] }, + undefined + ); + assert.equal(result.isNonChatGeneration, false, "embedding model should NOT be flagged"); + assert.equal(result.isEmbedding, true); +}); + +test("#13376 — detectTestKind does NOT flag models with no customModel metadata", serial, () => { + const result = detectTestKind("openai/gpt-4o", undefined, undefined); + assert.equal( + result.isNonChatGeneration, + false, + "model without metadata should NOT be flagged" + ); +}); diff --git a/tests/unit/model-test-runner.test.ts b/tests/unit/model-test-runner.test.ts index c8853b3c3a..3f3fd1ef6b 100644 --- a/tests/unit/model-test-runner.test.ts +++ b/tests/unit/model-test-runner.test.ts @@ -75,6 +75,7 @@ test("detectTestKind defaults to a plain chat test for ordinary models", () => { isEmbedding: false, isAudioTranscription: false, isResponses: false, + isNonChatGeneration: false, }); }); @@ -97,6 +98,7 @@ test("detectTestKind detects rerank by id and by metadata, and rerank wins over isEmbedding: false, isAudioTranscription: false, isResponses: false, + isNonChatGeneration: false, }); // apiFormat metadata drives detection even when the id is opaque assert.equal(detectTestKind("vendor/opaque-model", { apiFormat: "rerank" }).isRerank, true); @@ -119,6 +121,7 @@ test("detectTestKind detects audio transcription from metadata, and it wins over isEmbedding: false, isAudioTranscription: true, isResponses: false, + isNonChatGeneration: false, }); assert.equal( detectTestKind("vendor/opaque-model", { supportedEndpoints: ["audio-transcriptions"] }) @@ -156,6 +159,7 @@ test("detectTestKind falls back to the provider node's configured apiType", () = isEmbedding: false, isAudioTranscription: false, isResponses: false, + isNonChatGeneration: false, }); // Per-model metadata still wins when present.