diff --git a/changelog.d/fixes/2032-openai-compatible-check-404-warning.md b/changelog.d/fixes/2032-openai-compatible-check-404-warning.md new file mode 100644 index 0000000000..5933bf065f --- /dev/null +++ b/changelog.d/fixes/2032-openai-compatible-check-404-warning.md @@ -0,0 +1 @@ +- **fix(providers):** the OpenAI-compatible "Check" validation flow now surfaces a warning when the chat-completions probe returns `404` (e.g. `model_not_found`) instead of silently passing as `Valid` — a bogus/non-standard model id (Featherless/OpenRouter-style `vendor/model` typos) previously went undetected at Check time and only surfaced once a real request tripped the per-model lockout. (thanks @advane204f) diff --git a/src/lib/providers/validation/openaiFormat.ts b/src/lib/providers/validation/openaiFormat.ts index c9cc575ef8..cce3931f1d 100644 --- a/src/lib/providers/validation/openaiFormat.ts +++ b/src/lib/providers/validation/openaiFormat.ts @@ -459,6 +459,31 @@ export async function validateOpenAICompatibleProvider({ apiKey, providerSpecifi }; } + // #2032: a 404 on the chat probe commonly means the requested model id + // does not exist at this provider (OpenAI-compatible `model_not_found`, + // e.g. Featherless/OpenRouter-style `vendor/model` typos). Credentials + // are still valid (the endpoint responded), but silently passing this + // hides the bad model id from the user until a real request later trips + // the per-model lockout — surface it as a warning at Check time instead. + if (chatRes.status === 404) { + let modelNotFoundDetail = ""; + try { + const body: any = await chatRes.json(); + const err = body?.error; + if (typeof err?.message === "string" && err.message.trim()) { + modelNotFoundDetail = `: ${err.message.trim()}`; + } + } catch { + // Non-JSON or unreadable body — fall through with the generic warning. + } + return { + valid: true, + error: null, + method: "inference_available", + warning: `Model ID may not exist at this provider (404)${modelNotFoundDetail}`, + }; + } + // 4xx other than auth (e.g. 400 bad model, 422) usually means auth passed if (chatRes.status >= 400 && chatRes.status < 500) { return { diff --git a/tests/unit/t25-provider-validation-modelid-fallback.test.ts b/tests/unit/t25-provider-validation-modelid-fallback.test.ts index 7912d30d96..03b1f68c0b 100644 --- a/tests/unit/t25-provider-validation-modelid-fallback.test.ts +++ b/tests/unit/t25-provider-validation-modelid-fallback.test.ts @@ -114,3 +114,50 @@ test("T25: fallback chat probe treats 429 as valid credentials with warning", as globalThis.fetch = originalFetch; } }); + +// decolua/9router#2032: OpenAI-compatible "Check" silently passed for ANY +// non-empty Model ID because a chat-probe 404 (model_not_found) fell through +// the generic "4xx other than auth" branch with no warning. The user only +// discovered the bad model id after a real request tripped the per-model +// lockout. A 404 must surface a warning at Check time instead of a bare pass. +test("T25 / #2032: fallback chat probe surfaces a warning on 404 model_not_found", async () => { + const originalFetch = globalThis.fetch; + + globalThis.fetch = async (url) => { + if (String(url).endsWith("/models")) { + return new Response(JSON.stringify({ error: "Not Found" }), { status: 404 }); + } + return new Response( + JSON.stringify({ + error: { + message: "The model glm-5.2 does not exist.", + type: "invalid_request_error", + param: null, + code: "model_not_found", + }, + }), + { status: 404 } + ); + }; + + try { + const result = await validateProviderApiKey({ + provider: "openai-compatible-chat-t25-model-not-found", + apiKey: "sk-test", + providerSpecificData: { + baseUrl: "https://api.example.com/v1", + validationModelId: "glm-5.2", + }, + }); + + // Credentials themselves are fine (404 is not an auth failure), so this + // still resolves as valid — but MUST carry an actionable warning instead + // of a silent pass, so the user learns about the bad model id at Check + // time rather than after the first real request gets locked out. + assert.equal(result.valid, true); + assert.equal(result.method, "inference_available"); + assert.match(result.warning, /model.*(?:not found|does not exist|glm-5\.2)/i); + } finally { + globalThis.fetch = originalFetch; + } +});