feat(combo): filter auto-combo candidates by context window (#1808)

open-sse/services/combo.ts

@@ -35,6 +35,7 @@ import {
   type ScoringWeights,
 } from "./autoCombo/scoring.ts";
 import { supportsToolCalling } from "./modelCapabilities.ts";
+import { estimateTokens } from "./contextManager.ts";
 import { getSessionConnection } from "./sessionManager.ts";
 import { generateRoutingHints } from "./manifestAdapter";
 import type { RoutingHint } from "./manifestAdapter";
@@ -1721,6 +1722,32 @@ export async function handleComboChat({
       }
     }
 
+    // Context-window pre-filter (#1808)
+    // Estimate input tokens once; exclude candidates whose known context limit is too small.
+    // Uses the same 4-chars-per-token heuristic as contextManager.ts::compressContext().
+    // Null/unknown limits are treated as "include" to avoid incorrectly dropping valid targets.
+    const estimatedInputTokens = estimateTokens(body?.messages ?? []);
+    if (estimatedInputTokens > 0) {
+      const filteredByContext = eligibleTargets.filter((target) => {
+        const limit = getModelContextLimitForModelString(target.modelStr);
+        if (limit === null || limit === undefined) return true; // unknown — include to be safe
+        return limit >= estimatedInputTokens;
+      });
+      if (filteredByContext.length > 0) {
+        log.debug(
+          "COMBO",
+          `Auto strategy: context-window filter kept ${filteredByContext.length}/${eligibleTargets.length} candidates (est. ${estimatedInputTokens} tokens)`
+        );
+        eligibleTargets = filteredByContext;
+      } else {
+        log.warn(
+          "COMBO",
+          `Auto strategy: all candidates filtered by context-window policy (est. ${estimatedInputTokens} tokens), falling back to full pool`
+        );
+        // eligibleTargets intentionally unchanged — same fallback contract as tool-calling filter
+      }
+    }
+
     const prompt = extractPromptForIntent(body);
     const systemPrompt =
       typeof combo?.system_message === "string" ? combo.system_message : undefined;
@@ -1775,7 +1802,7 @@ export async function handleComboChat({
         try {
           const decision = selectWithStrategy(
             candidates,
-            { taskType, requestHasTools, lastKnownGoodProvider },
+            { taskType, requestHasTools, lastKnownGoodProvider, estimatedInputTokens },
             routingStrategy
           );
           selectedProvider = decision.provider;

tests/unit/combo-context-window-filter.test.ts

@@ -0,0 +1,207 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+
+// Test cases for the auto-combo context window pre-filter (#1808)
+// Filters out models whose context window is too small for the estimated input tokens
+
+interface Target {
+  modelStr: string;
+}
+
+interface FilterResult {
+  result: Target[];
+  didFallback: boolean;
+}
+
+/**
+ * Simulates the context-window filter logic from combo.ts
+ * Filters out candidates whose known context limit is smaller than estimated input tokens.
+ * Null/unknown limits are treated as "include" to avoid incorrectly dropping valid targets.
+ */
+function contextWindowFilter(
+  eligibleTargets: Target[],
+  estimatedInputTokens: number,
+  getLimitFn: (modelStr: string) => number | null
+): FilterResult {
+  if (estimatedInputTokens <= 0) {
+    return { result: eligibleTargets, didFallback: false };
+  }
+
+  const filtered = eligibleTargets.filter((target) => {
+    const limit = getLimitFn(target.modelStr);
+    if (limit === null || limit === undefined) return true;
+    return limit >= estimatedInputTokens;
+  });
+
+  if (filtered.length > 0) {
+    return { result: filtered, didFallback: false };
+  }
+
+  return { result: eligibleTargets, didFallback: true };
+}
+
+test("TC-1: large input exceeds small models — only large-context candidates survive", () => {
+  const targets: Target[] = [
+    { modelStr: "openai/gpt-4o-mini" },
+    { modelStr: "openai/gpt-4o" },
+    { modelStr: "anthropic/claude-3-5" },
+  ];
+  const limits: Record<string, number> = {
+    "openai/gpt-4o-mini": 8192,
+    "openai/gpt-4o": 32768,
+    "anthropic/claude-3-5": 131072,
+  };
+
+  const { result, didFallback } = contextWindowFilter(
+    targets,
+    20000,
+    (m) => limits[m] ?? null
+  );
+
+  assert.equal(result.length, 2, "Should keep 2 models with context >= 20k");
+  assert.ok(
+    result.every((t) => t.modelStr !== "openai/gpt-4o-mini"),
+    "Should exclude gpt-4o-mini (8k)"
+  );
+  assert.equal(didFallback, false, "Should not fallback when matches found");
+});
+
+test("TC-2: all candidates too small — fallback to full pool", () => {
+  const targets: Target[] = [
+    { modelStr: "a/small1" },
+    { modelStr: "a/small2" },
+    { modelStr: "a/small3" },
+  ];
+
+  const { result, didFallback } = contextWindowFilter(
+    targets,
+    20000,
+    () => 4096
+  );
+
+  assert.equal(result.length, 3, "Should preserve all targets when all filtered");
+  assert.equal(didFallback, true, "Should indicate fallback occurred");
+});
+
+test("TC-3: null-limit candidates always included", () => {
+  const targets: Target[] = [
+    { modelStr: "a/unknown1" },
+    { modelStr: "a/small" },
+    { modelStr: "a/unknown2" },
+  ];
+  const limits: Record<string, number | null> = {
+    "a/unknown1": null,
+    "a/small": 4096,
+    "a/unknown2": null,
+  };
+
+  const { result, didFallback } = contextWindowFilter(
+    targets,
+    20000,
+    (m) => limits[m] ?? null
+  );
+
+  assert.equal(result.length, 2, "Should include 2 null-limit models, exclude small");
+  assert.ok(
+    result.every((t) => t.modelStr !== "a/small"),
+    "Should exclude model with insufficient context"
+  );
+  assert.equal(didFallback, false, "Should not fallback");
+});
+
+test("TC-4: zero estimated tokens — filter is skipped, pool unchanged", () => {
+  const targets: Target[] = [
+    { modelStr: "a/m1" },
+    { modelStr: "a/m2" },
+  ];
+
+  const { result, didFallback } = contextWindowFilter(
+    targets,
+    0,
+    () => 4096
+  );
+
+  assert.equal(result.length, 2, "Should not filter when tokens = 0");
+  assert.equal(didFallback, false);
+});
+
+test("TC-5: exact context limit match passes", () => {
+  const targets: Target[] = [
+    { modelStr: "a/exact" },
+    { modelStr: "a/small" },
+  ];
+  const limits: Record<string, number> = {
+    "a/exact": 10000,
+    "a/small": 4096,
+  };
+
+  const { result } = contextWindowFilter(
+    targets,
+    10000,
+    (m) => limits[m] ?? null
+  );
+
+  assert.equal(result.length, 1, "Should include model with exact limit match");
+  assert.deepEqual(result[0], { modelStr: "a/exact" });
+});
+
+test("TC-6: undefined limit (not null) treated as unknown — included", () => {
+  const targets: Target[] = [
+    { modelStr: "a/unknown" },
+  ];
+
+  const { result } = contextWindowFilter(
+    targets,
+    5000,
+    (): (number | null) => undefined
+  );
+
+  assert.equal(result.length, 1, "Should include model with undefined limit");
+});
+
+test("TC-7: negative estimated tokens treated as 0 — no filtering", () => {
+  const targets: Target[] = [
+    { modelStr: "a/m1" },
+    { modelStr: "a/m2" },
+  ];
+
+  const { result } = contextWindowFilter(
+    targets,
+    -100,
+    () => 4096
+  );
+
+  assert.equal(result.length, 2, "Should not filter on negative tokens");
+});
+
+test("TC-8: mixed limits scenario", () => {
+  const targets: Target[] = [
+    { modelStr: "openai/gpt-3.5" },
+    { modelStr: "openai/gpt-4" },
+    { modelStr: "anthropic/claude" },
+    { modelStr: "google/gemini" },
+  ];
+  const limits: Record<string, number | null> = {
+    "openai/gpt-3.5": 4096,
+    "openai/gpt-4": 8192,
+    "anthropic/claude": null, // unknown
+    "google/gemini": 32768,
+  };
+
+  const { result, didFallback } = contextWindowFilter(
+    targets,
+    5000,
+    (m) => limits[m] ?? null
+  );
+
+  assert.equal(result.length, 3, "Should keep gpt-4 (8k), claude (unknown), gemini (32k)");
+  assert.ok(
+    result.every((t) => t.modelStr !== "openai/gpt-3.5"),
+    "Should exclude gpt-3.5 (4k)"
+  );
+  assert.ok(
+    result.some((t) => t.modelStr === "anthropic/claude"),
+    "Should keep unknown-limit model"
+  );
+  assert.equal(didFallback, false);
+});