feat(chat): dynamic tool limit detection with proactive truncation (#2061)

Integrated into release/v3.8.0

open-sse/config/constants.ts

@@ -184,3 +184,6 @@ export const DEFAULT_API_LIMITS = {
 
 // Skip patterns - requests containing these texts will bypass provider
 export const SKIP_PATTERNS = ["Please write a 5-10 word title for the following conversation:"];
+
+// Default maximum number of tools allowed in a request (OpenAI default)
+export const MAX_TOOLS_LIMIT = 128;

open-sse/handlers/chatCore.ts

@@ -30,6 +30,7 @@ import {
 import {
   COOLDOWN_MS,
   HTTP_STATUS,
+  MAX_TOOLS_LIMIT,
   PROVIDER_MAX_TOKENS,
   STREAM_IDLE_TIMEOUT_MS,
 } from "../config/constants.ts";
@@ -83,6 +84,12 @@ import { getCacheMetrics } from "@/lib/db/settings.ts";
 import { getCachedSettings } from "@/lib/db/readCache";
 import { cacheReasoningFromAssistantMessage } from "../services/reasoningCache.ts";
 import { sanitizeOpenAITool } from "../services/toolSchemaSanitizer.ts";
+import {
+  getEffectiveToolLimit,
+  setDetectedToolLimit,
+  parseToolLimitFromError,
+  shouldDetectLimit,
+} from "../services/toolLimitDetector.ts";
 
 import {
   parseCodexQuotaHeaders,
@@ -2681,6 +2688,20 @@ export async function handleChatCore({
         );
       }
 
+      const effectiveToolLimit = getEffectiveToolLimit(provider);
+      if (
+        effectiveToolLimit < MAX_TOOLS_LIMIT &&
+        Array.isArray(bodyToSend.tools) &&
+        bodyToSend.tools.length > effectiveToolLimit
+      ) {
+        const truncatedTools = bodyToSend.tools.slice(0, effectiveToolLimit);
+        bodyToSend = { ...bodyToSend, tools: truncatedTools };
+        log?.debug?.(
+          "TOOL_LIMIT",
+          `Truncated ${bodyToSend.tools.length} tools to ${effectiveToolLimit} for ${provider}`
+        );
+      }
+
       // Qwen OAuth rejects requests without a non-empty `user` field.
       // Some minimal OpenAI-compatible clients omit it, so we backfill a
       // stable default only for OAuth mode (API key mode is unaffected).
@@ -3052,6 +3073,18 @@ export async function handleChatCore({
     upstreamErrorParsed = true;
   }
 
+  const errorMessageForToolDetection =
+    typeof upstreamErrorBody === "string"
+      ? upstreamErrorBody
+      : JSON.stringify(upstreamErrorBody ?? {});
+  if (shouldDetectLimit(errorMessageForToolDetection, parsedStatusCode)) {
+    const detectedLimit = parseToolLimitFromError(errorMessageForToolDetection);
+    if (detectedLimit) {
+      setDetectedToolLimit(provider, detectedLimit);
+      log?.info?.("TOOL_LIMIT", `Detected tool limit ${detectedLimit} for ${provider}`);
+    }
+  }
+
   const isQwenExpiredError =
     provider === "qwen" &&
     parsedStatusCode === HTTP_STATUS.BAD_REQUEST &&

open-sse/services/toolLimitDetector.ts

@@ -0,0 +1,63 @@
+import { MAX_TOOLS_LIMIT } from "../config/constants.ts";
+
+const DETECTED_LIMITS = new Map<string, { limit: number; timestamp: number }>();
+const TTL_MS = 24 * 60 * 60 * 1000;
+const DEFAULT_LIMIT = MAX_TOOLS_LIMIT;
+
+export function getEffectiveToolLimit(provider: string): number {
+  const cached = DETECTED_LIMITS.get(provider);
+  if (cached && Date.now() - cached.timestamp < TTL_MS) {
+    return cached.limit;
+  }
+  return DEFAULT_LIMIT;
+}
+
+export function setDetectedToolLimit(provider: string, limit: number): void {
+  const current = getEffectiveToolLimit(provider);
+  if (limit < current) {
+    DETECTED_LIMITS.set(provider, { limit, timestamp: Date.now() });
+  }
+}
+
+const TOOL_LIMIT_PATTERNS = [
+  /'tools':\s*maximum\s+number\s+of\s+items\s+is\s+(\d+)/i,
+  /Maximum\s+number\s+of\s+tools\s+(?:allowed\s+)?(?:is\s+)?(\d+)/i,
+  /Too\s+many\s+tools\.?\s*(?:Maximum\s+)?(\d+)/i,
+  /tool.*limit.*(\d+)/i,
+  /tools.*exceeded.*(\d+)/i,
+];
+
+export function parseToolLimitFromError(errorMessage: string): number | null {
+  for (const pattern of TOOL_LIMIT_PATTERNS) {
+    const match = errorMessage.match(pattern);
+    if (match && match[1]) {
+      const limit = parseInt(match[1], 10);
+      if (limit > 0 && limit <= 10000) {
+        return limit;
+      }
+    }
+  }
+  return null;
+}
+
+const TOOL_LIMIT_ERROR_INDICATORS = [
+  "maximum number of tools",
+  "too many tools",
+  "tools limit",
+  "'tools'",
+  "maximum number of items",
+];
+
+export function shouldDetectLimit(errorMessage: string, statusCode: number): boolean {
+  if (statusCode !== 400) return false;
+  const lower = errorMessage.toLowerCase();
+  return TOOL_LIMIT_ERROR_INDICATORS.some((indicator) => lower.includes(indicator));
+}
+
+export function getDetectedToolLimit(provider: string): number {
+  return getEffectiveToolLimit(provider);
+}
+
+export function clearDetectedLimits(): void {
+  DETECTED_LIMITS.clear();
+}

tests/unit/tool-limit-detector.test.ts

@@ -0,0 +1,61 @@
+/**
+ * Unit tests for the tool limit detector.
+ */
+
+import { describe, it, beforeEach } from "node:test";
+import assert from "node:assert/strict";
+
+import {
+  getEffectiveToolLimit,
+  setDetectedToolLimit,
+  parseToolLimitFromError,
+  shouldDetectLimit,
+  clearDetectedLimits,
+} from "../../open-sse/services/toolLimitDetector.ts";
+
+describe("toolLimitDetector", () => {
+  beforeEach(() => {
+    clearDetectedLimits();
+  });
+
+  it("should return default limit when no cached value", () => {
+    assert.strictEqual(getEffectiveToolLimit("openai"), 128);
+  });
+
+  it("should return cached limit when available", () => {
+    setDetectedToolLimit("openai", 100);
+    assert.strictEqual(getEffectiveToolLimit("openai"), 100);
+  });
+
+  it("should only update cache when limit is lower", () => {
+    setDetectedToolLimit("openai", 100);
+    setDetectedToolLimit("openai", 120);
+    assert.strictEqual(getEffectiveToolLimit("openai"), 100);
+  });
+
+  it("should parse tool limit from OpenAI error message", () => {
+    const result = parseToolLimitFromError("'tools': maximum number of items is 128");
+    assert.strictEqual(result, 128);
+  });
+
+  it("should parse tool limit from alternative format", () => {
+    const result = parseToolLimitFromError("Maximum number of tools allowed is 64");
+    assert.strictEqual(result, 64);
+  });
+
+  it("should return null for non-tool errors", () => {
+    const result = parseToolLimitFromError("Invalid API key");
+    assert.strictEqual(result, null);
+  });
+
+  it("should detect tool limit errors for 400 status", () => {
+    assert.strictEqual(shouldDetectLimit("Maximum number of tools is 128", 400), true);
+    assert.strictEqual(shouldDetectLimit("Too many tools provided", 400), true);
+    assert.strictEqual(shouldDetectLimit("Invalid API key", 400), false);
+  });
+
+  it("should not detect for non-400 errors", () => {
+    assert.strictEqual(shouldDetectLimit("Maximum number of tools is 128", 500), false);
+    assert.strictEqual(shouldDetectLimit("Maximum number of tools is 128", 429), false);
+  });
+});