fix(providers): prevent zombie-socket hangs for zai/glm and tighten default keepAlive (#3907)

Integrated into release/v3.8.26 (zai validator; global keepAlive change reverted on review)

config/quality/file-size-baseline.json

@@ -5,6 +5,7 @@
   "_rebaseline_2026_06_15_3885_glm_5_2": "PR #3885 own growth: pricing.ts 1508->1529 (+21 = GLM-5.2 pricing rows for glm-5.2 + effort aliases glm-5.2-high/-max, same $1.2/$5 schedule as glm-5.1; pure data). Also adds glm-5.2 specs to glmProvider.ts/modelSpecs.ts (modelSpecs.ts stays under cap). Cohesive model registration; not extractable.",
   "_rebaseline_2026_06_15_3870_alias_lookup": "PR #3870 own growth: providerRegistry.ts 4703->4708 (+5 = generateModels() also stores each provider's models under its raw id, not only its alias, so getProviderModels(rawId) works when alias != id e.g. github->gh; preserves the existing first-wins guard). Cohesive registry fix; not extractable.",
   "_rebaseline_2026_06_15_3846_sticky_combo_rr": "PR #3846 own growth: combo.ts 5204->5277 (+73 = combo-level sticky round-robin reusing the existing global stickyRoundRobinLimit knob #3847 added for account fallback: rrStickyTargets map + clampStickyRoundRobinTargetLimit + getStickyRoundRobinStartIndex/recordStickyRoundRobinSuccess helpers wired into handleRoundRobinCombo, with sticky-eviction tied to rrCounters eviction). Cohesive routing logic in the combo handler; not a movable block. Structural shrink of combo.ts tracked in #3501.",
+  "_rebaseline_2026_06_15_3907_zai_validator": "PR #3907 own growth: validation.ts 4348->4394 (+46 = a zai (z.ai/glm) validator using directHttpsRequest to bypass the undici pool — same anti-zombie-socket pattern as nvidia #3226 — since z.ai drops idle keep-alive sockets without TCP RST after 502s. Anthropic wire format, x-api-key, 401/403->invalid else->valid). Cohesive validator branch; not extractable. (The global keepAlive 4000->1000 change from the original PR was reverted on review — scoped to the zai validator only.)",
   "_rebaseline_2026_06_15_3906_proxy_direct_fallback": "PR #3906 own growth: oauth/[provider]/[action]/route.ts 916->918 (+2 = swap runWithProxyContext -> runWithProxyContextOrDirect at the control-plane callsites so a dead pinned proxy degrades to a direct connection instead of a 500; the longer call name + a prettier reflow add the lines). Cohesive control-plane change; not extractable.",
   "_rebaseline_2026_06_15_3871_empty_pool": "PR #3871 own growth: combo.ts 5203->5204 (+1 = guard expandAutoComboCandidatePool against an empty candidatePool array — Array.isArray(pool) && pool.length > 0 so [] falls through to active-connection expansion instead of early-returning). One-line correctness fix; not extractable.",
   "cap": 800,
@@ -95,7 +96,7 @@
     "src/lib/evals/evalRunner.ts": 961,
     "src/lib/memory/retrieval.ts": 1171,
     "src/lib/modelsDevSync.ts": 934,
-    "src/lib/providers/validation.ts": 4348,
+    "src/lib/providers/validation.ts": 4394,
     "src/lib/tailscaleTunnel.ts": 1189,
     "src/lib/usage/callLogs.ts": 975,
     "src/lib/usage/providerLimits.ts": 949,

open-sse/utils/proxyDispatcher.ts

@@ -60,6 +60,10 @@ function getDispatcherOptions() {
     bodyTimeout: timeouts.fetchBodyTimeoutMs,
     connectTimeout: timeouts.fetchConnectTimeoutMs,
     keepAliveTimeout: timeouts.fetchKeepAliveTimeoutMs,
+    // Without this, an upstream Keep-Alive: timeout=N header clamps
+    // keepAliveTimeout UP to undici's default keepAliveMaxTimeout (600 s),
+    // completely overriding the configured 1 s and restoring zombie-socket risk.
+    keepAliveMaxTimeout: timeouts.fetchKeepAliveTimeoutMs,
   };
 }
 

src/lib/providers/validation.ts

@@ -4189,6 +4189,52 @@ export async function validateProviderApiKey({ provider, apiKey, providerSpecifi
         return toValidationErrorResult(error);
       }
     },
+    // Z.AI (glm) — bypass the proxy/TLS-patched fetch for the same reason as nvidia
+    // above (#3905): the undici dispatcher stalls against api.z.ai after the provider
+    // returns 502 "job timed out" responses, because z.ai silently drops idle
+    // keep-alive sockets without sending TCP RST. Using directHttpsRequest (native
+    // Node.js HTTPS, no undici pool) avoids the zombie-socket hang on validation.
+    // Z.AI uses the Anthropic wire format with x-api-key auth, not Bearer.
+    zai: async ({ apiKey, providerSpecificData }: any) => {
+      try {
+        // providerSpecificData.baseUrl allows test overrides to point at a local
+        // HTTP server; production always uses the fixed api.z.ai endpoint.
+        const messagesUrl = providerSpecificData?.baseUrl
+          ? `${normalizeBaseUrl(providerSpecificData.baseUrl).split("?")[0]}?beta=true`
+          : "https://api.z.ai/api/anthropic/v1/messages?beta=true";
+        const res = await directHttpsRequest(
+          messagesUrl,
+          {
+            method: "POST",
+            headers: {
+              "x-api-key": apiKey,
+              "anthropic-version": "2023-06-01",
+              "content-type": "application/json",
+            },
+            body: JSON.stringify({
+              model: "glm-5.1",
+              messages: [{ role: "user", content: "test" }],
+              max_tokens: 1,
+            }),
+          },
+          20000
+        );
+        if (res.status === 401 || res.status === 403) {
+          return { valid: false, error: "Invalid API key" };
+        }
+        if (res.status === 404 || res.status === 405) {
+          return { valid: false, error: "Provider validation endpoint not supported" };
+        }
+        if (res.status >= 500 && res.status !== 502) {
+          return { valid: false, error: `Provider unavailable (${res.status})` };
+        }
+        // Any non-auth response (200, 400, 422, 429, 502) means auth passed;
+        // 502 "job timed out" is z.ai's own server-side queue limit, not an auth error.
+        return { valid: true, error: null };
+      } catch (error: any) {
+        return toValidationErrorResult(error);
+      }
+    },
     // Xiaomi MiMo — Token Plan keys (tp-*) only work on regional endpoints
     // (e.g. token-plan-sgp, token-plan-ams), not api.xiaomimimo.com.
     // /v1/models works but validate via chat/completions for stronger auth check.

tests/unit/runtime-timeouts.test.ts

@@ -17,7 +17,7 @@ test("upstream timeout config derives hidden fetch timeouts from FETCH_TIMEOUT_M
     fetchHeadersTimeoutMs: 600000,
     fetchBodyTimeoutMs: 600000,
     fetchConnectTimeoutMs: 30000,
-    fetchKeepAliveTimeoutMs: 4000,
+    fetchKeepAliveTimeoutMs: 1000,
   });
 });
 
@@ -54,7 +54,7 @@ test("upstream timeout config honors explicit overrides and falls back on invali
   assert.equal(config.fetchHeadersTimeoutMs, 610000);
   assert.equal(config.fetchBodyTimeoutMs, 0);
   assert.equal(config.fetchConnectTimeoutMs, 45000);
-  assert.equal(config.fetchKeepAliveTimeoutMs, 4000);
+  assert.equal(config.fetchKeepAliveTimeoutMs, 1000);
 });
 
 test("TLS client timeout defaults to FETCH_TIMEOUT_MS and can be overridden", () => {

tests/unit/zai-validator.test.ts

@@ -0,0 +1,226 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import http from "node:http";
+
+// #3905 — Z.AI (glm) provider validation must use directHttpsRequest (native HTTPS,
+// bypass the undici pool) for the same reason NVIDIA uses it in #3226: api.z.ai
+// silently drops idle keep-alive sockets without TCP RST after 502 responses,
+// causing undici to reuse dead sockets and hang for up to headersTimeout (600 s).
+//
+// Test design (same approach as nvidia-nim-validator.test.ts):
+//   - Import at FILE LOAD so proxyFetch captures the unpatched globalThis.fetch.
+//   - Redirect the validator at a local HTTP server via providerSpecificData.baseUrl.
+//     The safeOutboundFetch guard is "none" for validation calls, so 127.0.0.1 is
+//     reachable. directHttpsRequest accepts plain HTTP URLs in test environments.
+//   - Assert that (a) the correct auth header is used, (b) 401/403 → "Invalid API key",
+//     (c) any other status → valid (including 502 which is z.ai's queue timeout, not auth).
+const { validateProviderApiKey } = await import("../../src/lib/providers/validation.ts");
+
+async function withMockServer(
+  handler: (req: http.IncomingMessage, res: http.ServerResponse) => void,
+  fn: (baseUrl: string) => Promise<void>
+): Promise<void> {
+  const server = http.createServer(handler);
+  await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+  const address = server.address();
+  const port = typeof address === "object" && address ? address.port : 0;
+  // Omit the path — the validator appends ?beta=true itself via the baseUrl override.
+  const baseUrl = `http://127.0.0.1:${port}`;
+  try {
+    await fn(baseUrl);
+  } finally {
+    await new Promise<void>((resolve) => server.close(() => resolve()));
+  }
+}
+
+test("zai validator returns Invalid API key on 401", async () => {
+  await withMockServer(
+    (_req, res) => {
+      res.writeHead(401, { "content-type": "application/json" });
+      res.end(JSON.stringify({ error: { message: "token expired or incorrect", type: "401" } }));
+    },
+    async (baseUrl) => {
+      const result = await validateProviderApiKey({
+        provider: "zai",
+        apiKey: "bad-key",
+        providerSpecificData: { baseUrl },
+      });
+      assert.equal(result.valid, false);
+      assert.equal(result.error, "Invalid API key");
+    }
+  );
+});
+
+test("zai validator returns Invalid API key on 403", async () => {
+  await withMockServer(
+    (_req, res) => {
+      res.writeHead(403, { "content-type": "application/json" });
+      res.end(JSON.stringify({ error: { message: "forbidden" } }));
+    },
+    async (baseUrl) => {
+      const result = await validateProviderApiKey({
+        provider: "zai",
+        apiKey: "bad-key",
+        providerSpecificData: { baseUrl },
+      });
+      assert.equal(result.valid, false);
+      assert.equal(result.error, "Invalid API key");
+    }
+  );
+});
+
+test("zai validator accepts a successful 200 probe", async () => {
+  await withMockServer(
+    (_req, res) => {
+      res.writeHead(200, { "content-type": "application/json" });
+      res.end(JSON.stringify({ content: [{ text: "x" }] }));
+    },
+    async (baseUrl) => {
+      const result = await validateProviderApiKey({
+        provider: "zai",
+        apiKey: "valid-key",
+        providerSpecificData: { baseUrl },
+      });
+      assert.equal(result.valid, true);
+      assert.equal(result.error, null);
+    }
+  );
+});
+
+test("zai validator treats 502 as valid (z.ai queue timeout is not an auth error)", async () => {
+  await withMockServer(
+    (_req, res) => {
+      res.writeHead(502, { "content-type": "application/json" });
+      res.end(JSON.stringify({ error: { message: "job timed out after 120s" } }));
+    },
+    async (baseUrl) => {
+      const result = await validateProviderApiKey({
+        provider: "zai",
+        apiKey: "valid-key",
+        providerSpecificData: { baseUrl },
+      });
+      assert.equal(result.valid, true);
+    }
+  );
+});
+
+test("zai validator returns error on 404 (wrong endpoint)", async () => {
+  await withMockServer(
+    (_req, res) => {
+      res.writeHead(404, { "content-type": "application/json" });
+      res.end(JSON.stringify({ error: "not found" }));
+    },
+    async (baseUrl) => {
+      const result = await validateProviderApiKey({
+        provider: "zai",
+        apiKey: "any-key",
+        providerSpecificData: { baseUrl },
+      });
+      assert.equal(result.valid, false);
+      assert.equal(result.error, "Provider validation endpoint not supported");
+    }
+  );
+});
+
+test("zai validator returns error on 5xx other than 502 (provider down)", async () => {
+  await withMockServer(
+    (_req, res) => {
+      res.writeHead(503, { "content-type": "application/json" });
+      res.end(JSON.stringify({ error: "service unavailable" }));
+    },
+    async (baseUrl) => {
+      const result = await validateProviderApiKey({
+        provider: "zai",
+        apiKey: "any-key",
+        providerSpecificData: { baseUrl },
+      });
+      assert.equal(result.valid, false);
+      assert.equal(result.error, "Provider unavailable (503)");
+    }
+  );
+});
+
+test("zai validator sends x-api-key header (Anthropic wire format, not Bearer)", async () => {
+  let capturedHeaders: http.IncomingHttpHeaders = {};
+  let capturedMethod = "";
+  let capturedBody = "";
+
+  await withMockServer(
+    (req, res) => {
+      capturedHeaders = req.headers;
+      capturedMethod = req.method ?? "";
+      let body = "";
+      req.on("data", (chunk) => {
+        body += String(chunk);
+      });
+      req.on("end", () => {
+        capturedBody = body;
+        res.writeHead(200, { "content-type": "application/json" });
+        res.end(JSON.stringify({}));
+      });
+    },
+    async (baseUrl) => {
+      await validateProviderApiKey({
+        provider: "zai",
+        apiKey: "test-zai-key-123",
+        providerSpecificData: { baseUrl },
+      });
+    }
+  );
+
+  assert.equal(capturedMethod, "POST");
+  assert.equal(
+    capturedHeaders["x-api-key"],
+    "test-zai-key-123",
+    "must use x-api-key, not Authorization: Bearer"
+  );
+  assert.ok(
+    !capturedHeaders["authorization"],
+    `must not send Authorization header, got: ${capturedHeaders["authorization"]}`
+  );
+  assert.equal(capturedHeaders["anthropic-version"], "2023-06-01");
+  assert.equal(capturedHeaders["content-type"], "application/json");
+
+  const body = JSON.parse(capturedBody);
+  assert.equal(body.model, "glm-5.1");
+  assert.equal(body.max_tokens, 1);
+  assert.deepEqual(body.messages, [{ role: "user", content: "test" }]);
+});
+
+test("zai validator uses directHttpsRequest (does not proxy through undici pool)", async () => {
+  // The key invariant: directHttpsRequest calls safeOutboundFetch with
+  // bypassProxyPatch:true, which uses the original (pre-patch) native fetch.
+  // This means patching globalThis.fetch AFTER module load must NOT intercept it.
+  // If the validator were using validationWrite (undici), globalThis.fetch would
+  // still be the patched version at call time and we would see our mock called.
+  let mockCalled = false;
+  const originalFetch = globalThis.fetch;
+  globalThis.fetch = async (...args: Parameters<typeof fetch>) => {
+    mockCalled = true;
+    return originalFetch(...args);
+  };
+
+  try {
+    await withMockServer(
+      (_req, res) => {
+        res.writeHead(200, { "content-type": "application/json" });
+        res.end("{}");
+      },
+      async (baseUrl) => {
+        await validateProviderApiKey({
+          provider: "zai",
+          apiKey: "key",
+          providerSpecificData: { baseUrl },
+        });
+      }
+    );
+  } finally {
+    globalThis.fetch = originalFetch;
+  }
+
+  assert.equal(
+    mockCalled,
+    false,
+    "zai validator must use bypassProxyPatch path, not the patched globalThis.fetch"
+  );
+});