diff --git a/config/quality/file-size-baseline.json b/config/quality/file-size-baseline.json index 1b94003389..f0e4848b59 100644 --- a/config/quality/file-size-baseline.json +++ b/config/quality/file-size-baseline.json @@ -201,7 +201,7 @@ "src/lib/evals/evalRunner.ts": 961, "src/lib/memory/retrieval.ts": 1171, "src/lib/modelsDevSync.ts": 934, - "src/lib/providers/validation.ts": 4522, + "src/lib/providers/validation.ts": 4523, "src/lib/tailscaleTunnel.ts": 1202, "src/lib/usage/callLogs.ts": 975, "src/lib/usage/providerLimits.ts": 950, diff --git a/src/lib/providers/validation.ts b/src/lib/providers/validation.ts index 2578bc911e..13040f3b8b 100644 --- a/src/lib/providers/validation.ts +++ b/src/lib/providers/validation.ts @@ -2376,7 +2376,12 @@ async function validateAnthropicCompatibleProvider({ providerSpecificData ); - // Step 1: Try GET /models + // Step 1: Best-effort GET /models probe. /models is NOT part of the Anthropic API spec + // and many compatible proxies either 404, 401, or 403 on /models even with a valid key — + // so a 401/403 here must NOT mark the credentials invalid. Only a 2xx is a positive + // signal that the proxy DOES implement /models AND the key was accepted; everything else + // (including auth-shaped statuses) falls through to the authoritative POST /v1/messages + // probe below. Ported from decolua/9router 584cf66a. try { const modelsRes = await validationRead( joinBaseUrlAndPath(baseUrl, providerSpecificData?.modelsPath || "/models"), @@ -2390,15 +2395,11 @@ async function validateAnthropicCompatibleProvider({ if (modelsRes.ok) { return { valid: true, error: null }; } - - if (modelsRes.status === 401 || modelsRes.status === 403) { - return { valid: false, error: "Invalid API key" }; - } } catch { // /models fetch failed — fall through to messages test } - // Step 2: Fallback — try a minimal messages request + // Step 2: Authoritative probe — POST /v1/messages with max_tokens=1. const testModelId = providerSpecificData?.validationModelId || "claude-3-5-sonnet-20241022"; try { const messagesRes = await validationWrite( diff --git a/tests/unit/anthropic-compat-validation-v1-messages.test.ts b/tests/unit/anthropic-compat-validation-v1-messages.test.ts new file mode 100644 index 0000000000..2ef51cf726 --- /dev/null +++ b/tests/unit/anthropic-compat-validation-v1-messages.test.ts @@ -0,0 +1,80 @@ +// Regression test for the anthropic-compatible connection validator. +// +// Upstream fix: GET /models is not part of the Anthropic API spec; many +// compatible proxies either 404, 401, or 403 on /models even with a valid +// API key. The connection test must therefore not reject the credentials +// solely on a 401/403 from /models — it must fall back to POST /v1/messages +// (the canonical Anthropic auth probe) and treat any non-401/403 messages +// response as proof that the key was accepted. +// +// Ported from decolua/9router 584cf66a (Co-author: Rehan Choirul). + +import test from "node:test"; +import assert from "node:assert/strict"; + +const { validateProviderApiKey } = await import("../../src/lib/providers/validation.ts"); + +const originalFetch = globalThis.fetch; + +test.afterEach(() => { + globalThis.fetch = originalFetch; +}); + +test( + "anthropic-compatible validation falls back to /messages when /models returns 403", + async () => { + const calls: { url: string; method: string }[] = []; + globalThis.fetch = async (url: any, init: any = {}) => { + const u = String(url); + const method = String(init?.method || "GET").toUpperCase(); + calls.push({ url: u, method }); + if (u.endsWith("/models")) { + return new Response(JSON.stringify({ error: "forbidden on models" }), { status: 403 }); + } + // /messages: upstream accepts the key but rejects the toy payload with 400. + return new Response(JSON.stringify({ error: "bad request" }), { status: 400 }); + }; + + const result = await validateProviderApiKey({ + provider: "anthropic-compatible-403-on-models", + apiKey: "sk-test", + providerSpecificData: { baseUrl: "https://proxy.example.com/v1/messages" }, + }); + + // BEFORE the fix this returned { valid: false, error: "Invalid API key" } + // because validateAnthropicCompatibleProvider short-circuited on the 403 + // from GET /models without ever probing POST /v1/messages. + assert.equal(result.valid, true, "403 on /models alone must NOT mark the key invalid"); + assert.equal(result.error, null); + + // The validator must actually exercise the messages endpoint. + const messagesCall = calls.find( + (call) => call.url.endsWith("/messages") && call.method === "POST" + ); + assert.ok(messagesCall, "expected a POST /messages probe after /models 403"); + } +); + +test( + "anthropic-compatible validation still rejects when /messages itself returns 401", + async () => { + // Symmetry guard: the fix must NOT make every 403/401 pass. Only the + // messages probe is authoritative — if it also rejects auth, the key is bad. + globalThis.fetch = async (url: any) => { + const u = String(url); + if (u.endsWith("/models")) { + return new Response(JSON.stringify({ error: "no models endpoint" }), { status: 403 }); + } + return new Response(JSON.stringify({ error: "invalid_api_key" }), { status: 401 }); + }; + + const result = await validateProviderApiKey({ + provider: "anthropic-compatible-truly-bad-key", + apiKey: "sk-bad", + providerSpecificData: { baseUrl: "https://proxy.example.com/v1/messages" }, + }); + + assert.equal(result.valid, false); + assert.equal(result.error, "Invalid API key"); + } +); diff --git a/tests/unit/provider-validation-branches.test.ts b/tests/unit/provider-validation-branches.test.ts index a54b8aaa5c..9cd98eb72e 100644 --- a/tests/unit/provider-validation-branches.test.ts +++ b/tests/unit/provider-validation-branches.test.ts @@ -168,7 +168,10 @@ test("anthropic-compatible validation requires a base URL", async () => { assert.match(result.error, /No base URL configured/i); }); -test("anthropic-compatible validation rejects invalid keys from /models", async () => { +test("anthropic-compatible validation rejects invalid keys (auth-fail on both /models and /messages)", async () => { + // After the 584cf66a port, /models alone is not authoritative — many compatible + // proxies 401/403 on /models even with a valid key. To prove the key is bad we + // require an auth-shaped failure on POST /v1/messages too. const calls = []; globalThis.fetch = async (url) => { calls.push(String(url)); @@ -183,7 +186,10 @@ test("anthropic-compatible validation rejects invalid keys from /models", async assert.equal(result.valid, false); assert.equal(result.error, "Invalid API key"); - assert.deepEqual(calls, ["https://api.example.com/v1/models"]); + assert.deepEqual(calls, [ + "https://api.example.com/v1/models", + "https://api.example.com/v1/messages", + ]); }); test("anthropic-compatible validation falls back to /messages and treats 400 as auth success", async () => {