fix(test): validate anthropic-compatible connections via POST /v1/messages (#4657)

Integrated into release/v3.8.36 — anthropic-compat validation via POST /v1/messages (port 584cf66a), rebuilt clean + baseline; release-green

config/quality/file-size-baseline.json

@@ -201,7 +201,7 @@
     "src/lib/evals/evalRunner.ts": 961,
     "src/lib/memory/retrieval.ts": 1171,
     "src/lib/modelsDevSync.ts": 934,
-    "src/lib/providers/validation.ts": 4522,
+    "src/lib/providers/validation.ts": 4523,
     "src/lib/tailscaleTunnel.ts": 1202,
     "src/lib/usage/callLogs.ts": 975,
     "src/lib/usage/providerLimits.ts": 950,

src/lib/providers/validation.ts

@@ -2376,7 +2376,12 @@ async function validateAnthropicCompatibleProvider({
     providerSpecificData
   );
 
-  // Step 1: Try GET /models
+  // Step 1: Best-effort GET /models probe. /models is NOT part of the Anthropic API spec
+  // and many compatible proxies either 404, 401, or 403 on /models even with a valid key —
+  // so a 401/403 here must NOT mark the credentials invalid. Only a 2xx is a positive
+  // signal that the proxy DOES implement /models AND the key was accepted; everything else
+  // (including auth-shaped statuses) falls through to the authoritative POST /v1/messages
+  // probe below. Ported from decolua/9router 584cf66a.
   try {
     const modelsRes = await validationRead(
       joinBaseUrlAndPath(baseUrl, providerSpecificData?.modelsPath || "/models"),
@@ -2390,15 +2395,11 @@ async function validateAnthropicCompatibleProvider({
     if (modelsRes.ok) {
       return { valid: true, error: null };
     }
-
-    if (modelsRes.status === 401 || modelsRes.status === 403) {
-      return { valid: false, error: "Invalid API key" };
-    }
   } catch {
     // /models fetch failed — fall through to messages test
   }
 
-  // Step 2: Fallback — try a minimal messages request
+  // Step 2: Authoritative probe — POST /v1/messages with max_tokens=1.
   const testModelId = providerSpecificData?.validationModelId || "claude-3-5-sonnet-20241022";
   try {
     const messagesRes = await validationWrite(

tests/unit/anthropic-compat-validation-v1-messages.test.ts

@@ -0,0 +1,80 @@
+// Regression test for the anthropic-compatible connection validator.
+//
+// Upstream fix: GET /models is not part of the Anthropic API spec; many
+// compatible proxies either 404, 401, or 403 on /models even with a valid
+// API key. The connection test must therefore not reject the credentials
+// solely on a 401/403 from /models — it must fall back to POST /v1/messages
+// (the canonical Anthropic auth probe) and treat any non-401/403 messages
+// response as proof that the key was accepted.
+//
+// Ported from decolua/9router 584cf66a (Co-author: Rehan Choirul).
+
+import test from "node:test";
+import assert from "node:assert/strict";
+
+const { validateProviderApiKey } = await import("../../src/lib/providers/validation.ts");
+
+const originalFetch = globalThis.fetch;
+
+test.afterEach(() => {
+  globalThis.fetch = originalFetch;
+});
+
+test(
+  "anthropic-compatible validation falls back to /messages when /models returns 403",
+  async () => {
+    const calls: { url: string; method: string }[] = [];
+    globalThis.fetch = async (url: any, init: any = {}) => {
+      const u = String(url);
+      const method = String(init?.method || "GET").toUpperCase();
+      calls.push({ url: u, method });
+      if (u.endsWith("/models")) {
+        return new Response(JSON.stringify({ error: "forbidden on models" }), { status: 403 });
+      }
+      // /messages: upstream accepts the key but rejects the toy payload with 400.
+      return new Response(JSON.stringify({ error: "bad request" }), { status: 400 });
+    };
+
+    const result = await validateProviderApiKey({
+      provider: "anthropic-compatible-403-on-models",
+      apiKey: "sk-test",
+      providerSpecificData: { baseUrl: "https://proxy.example.com/v1/messages" },
+    });
+
+    // BEFORE the fix this returned { valid: false, error: "Invalid API key" }
+    // because validateAnthropicCompatibleProvider short-circuited on the 403
+    // from GET /models without ever probing POST /v1/messages.
+    assert.equal(result.valid, true, "403 on /models alone must NOT mark the key invalid");
+    assert.equal(result.error, null);
+
+    // The validator must actually exercise the messages endpoint.
+    const messagesCall = calls.find(
+      (call) => call.url.endsWith("/messages") && call.method === "POST"
+    );
+    assert.ok(messagesCall, "expected a POST /messages probe after /models 403");
+  }
+);
+
+test(
+  "anthropic-compatible validation still rejects when /messages itself returns 401",
+  async () => {
+    // Symmetry guard: the fix must NOT make every 403/401 pass. Only the
+    // messages probe is authoritative — if it also rejects auth, the key is bad.
+    globalThis.fetch = async (url: any) => {
+      const u = String(url);
+      if (u.endsWith("/models")) {
+        return new Response(JSON.stringify({ error: "no models endpoint" }), { status: 403 });
+      }
+      return new Response(JSON.stringify({ error: "invalid_api_key" }), { status: 401 });
+    };
+
+    const result = await validateProviderApiKey({
+      provider: "anthropic-compatible-truly-bad-key",
+      apiKey: "sk-bad",
+      providerSpecificData: { baseUrl: "https://proxy.example.com/v1/messages" },
+    });
+
+    assert.equal(result.valid, false);
+    assert.equal(result.error, "Invalid API key");
+  }
+);

tests/unit/provider-validation-branches.test.ts

@@ -168,7 +168,10 @@ test("anthropic-compatible validation requires a base URL", async () => {
   assert.match(result.error, /No base URL configured/i);
 });
 
-test("anthropic-compatible validation rejects invalid keys from /models", async () => {
+test("anthropic-compatible validation rejects invalid keys (auth-fail on both /models and /messages)", async () => {
+  // After the 584cf66a port, /models alone is not authoritative — many compatible
+  // proxies 401/403 on /models even with a valid key. To prove the key is bad we
+  // require an auth-shaped failure on POST /v1/messages too.
   const calls = [];
   globalThis.fetch = async (url) => {
     calls.push(String(url));
@@ -183,7 +186,10 @@ test("anthropic-compatible validation rejects invalid keys from /models", async
 
   assert.equal(result.valid, false);
   assert.equal(result.error, "Invalid API key");
-  assert.deepEqual(calls, ["https://api.example.com/v1/models"]);
+  assert.deepEqual(calls, [
+    "https://api.example.com/v1/models",
+    "https://api.example.com/v1/messages",
+  ]);
 });
 
 test("anthropic-compatible validation falls back to /messages and treats 400 as auth success", async () => {