feat: adaptive keepalive threshold for web-session providers (#3397)

Integrated into release/v3.8.16

open-sse/utils/keepaliveThreshold.ts

@@ -0,0 +1,62 @@
+/**
+ * Adaptive keepalive threshold resolver for streaming routes.
+ *
+ * Web-session and anonymous-fallback providers are slower to produce the first
+ * byte because they route through browser sessions or public rate-limited
+ * endpoints. The default 2 s keepalive threshold is too aggressive for these
+ * providers — the keepalive stream commits before the upstream has a chance to
+ * respond, adding unnecessary SSE framing overhead.
+ *
+ * `resolveKeepaliveThreshold(model)` inspects the model prefix and returns a
+ * longer threshold (15 s) for known-slow providers, or the default (2 s) for
+ * everything else.
+ */
+
+import { NOAUTH_PROVIDERS } from "@/shared/constants/providers";
+import { APIKEY_PROVIDERS } from "@/shared/constants/providers";
+import { WEB_COOKIE_PROVIDERS } from "@/shared/constants/providers";
+import { WEB_SESSION_CREDENTIAL_REQUIREMENTS } from "@/shared/providers/webSessionCredentials";
+
+const DEFAULT_THRESHOLD_MS = 2_000;
+const SLOW_THRESHOLD_MS = 15_000;
+
+const SLOW_PROVIDER_IDS: Set<string> = new Set();
+
+function addSlowProvider(id: string, alias?: string) {
+  SLOW_PROVIDER_IDS.add(id);
+  if (typeof alias === "string" && alias) SLOW_PROVIDER_IDS.add(alias);
+}
+
+for (const [id, def] of Object.entries(NOAUTH_PROVIDERS)) {
+  if ((def as Record<string, unknown>).noAuth === true) {
+    addSlowProvider(id, (def as Record<string, unknown>).alias as string | undefined);
+  }
+}
+
+for (const [id, def] of Object.entries(APIKEY_PROVIDERS)) {
+  if ((def as Record<string, unknown>).anonymousFallback === true) {
+    addSlowProvider(id, (def as Record<string, unknown>).alias as string | undefined);
+  }
+}
+
+for (const [id, def] of Object.entries(WEB_COOKIE_PROVIDERS)) {
+  addSlowProvider(id, (def as Record<string, unknown>).alias as string | undefined);
+}
+
+for (const id of Object.keys(WEB_SESSION_CREDENTIAL_REQUIREMENTS)) {
+  SLOW_PROVIDER_IDS.add(id);
+}
+
+export const SLOW_KEEPALIVE_PROVIDERS: ReadonlySet<string> = SLOW_PROVIDER_IDS;
+
+export function resolveKeepaliveThreshold(model: string | undefined | null): number {
+  if (!model || typeof model !== "string") return DEFAULT_THRESHOLD_MS;
+
+  const slashIndex = model.indexOf("/");
+  if (slashIndex <= 0) return DEFAULT_THRESHOLD_MS;
+
+  const prefix = model.slice(0, slashIndex);
+  if (SLOW_PROVIDER_IDS.has(prefix)) return SLOW_THRESHOLD_MS;
+
+  return DEFAULT_THRESHOLD_MS;
+}

src/app/api/v1/responses/route.ts

@@ -3,6 +3,7 @@ import { withEarlyStreamKeepalive } from "@omniroute/open-sse/utils/earlyStreamK
 import { resolveResponsesApiModel } from "@/app/api/internal/codex-responses-ws/modelResolution";
 import { getModelInfo } from "@/sse/services/model";
 import { getComboByName } from "@/lib/db/combos";
+import { resolveKeepaliveThreshold } from "@omniroute/open-sse/utils/keepaliveThreshold";
 
 // NOTE: We do NOT call initTranslators() here — the translator registry is
 // bootstrapped at module level inside open-sse/translator/index.ts when it
@@ -68,7 +69,19 @@ export async function POST(request) {
   const resolved = await withCodexPreferredModel(request);
   const accept = String(request.headers?.get?.("accept") || "").toLowerCase();
   if (accept.includes("text/event-stream")) {
-    return await withEarlyStreamKeepalive(handleChat(resolved), { signal: request.signal });
+    // Adaptive threshold: web-session and anonymous-fallback providers are slower
+    // to produce the first byte, so use a longer keepalive threshold (15s vs 2s).
+    let model;
+    try {
+      const body = await resolved.clone().json().catch(() => null);
+      model = body?.model;
+    } catch {
+    }
+    const thresholdMs = resolveKeepaliveThreshold(model);
+    return await withEarlyStreamKeepalive(handleChat(resolved), {
+      signal: request.signal,
+      thresholdMs,
+    });
   }
   return await handleChat(resolved);
 }

tests/unit/keepalive-threshold.test.ts

@@ -0,0 +1,75 @@
+/**
+ * Adaptive keepalive threshold — Unit Tests (PR5 of issue #3368)
+ *
+ * Run: node --import tsx/esm --test tests/unit/keepalive-threshold.test.ts
+ */
+
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import {
+  resolveKeepaliveThreshold,
+  SLOW_KEEPALIVE_PROVIDERS,
+} from "../../open-sse/utils/keepaliveThreshold.ts";
+
+describe("resolveKeepaliveThreshold", () => {
+  it("returns 2000ms default for undefined model", () => {
+    assert.equal(resolveKeepaliveThreshold(undefined), 2000);
+  });
+
+  it("returns 2000ms default for null model", () => {
+    assert.equal(resolveKeepaliveThreshold(null), 2000);
+  });
+
+  it("returns 2000ms default for empty string", () => {
+    assert.equal(resolveKeepaliveThreshold(""), 2000);
+  });
+
+  it("returns 2000ms default for model without prefix", () => {
+    assert.equal(resolveKeepaliveThreshold("gpt-4"), 2000);
+  });
+
+  it("returns 2000ms default for normal API-key provider", () => {
+    assert.equal(resolveKeepaliveThreshold("openai/gpt-4"), 2000);
+    assert.equal(resolveKeepaliveThreshold("anthropic/claude-sonnet-4"), 2000);
+    assert.equal(resolveKeepaliveThreshold("deepseek/deepseek-chat"), 2000);
+  });
+
+  it("returns 15000ms for anonymous fallback provider (pollinations)", () => {
+    assert.equal(resolveKeepaliveThreshold("pollinations/gpt-5"), 15000);
+  });
+
+  it("returns 15000ms for anonymous fallback provider alias (pol)", () => {
+    assert.equal(resolveKeepaliveThreshold("pol/gpt-5"), 15000);
+  });
+
+  it("returns 15000ms for anonymous fallback provider (opencode-zen)", () => {
+    assert.equal(resolveKeepaliveThreshold("opencode-zen/gpt-4"), 15000);
+  });
+
+  it("returns 15000ms for web-session provider (chatgpt-web)", () => {
+    assert.equal(resolveKeepaliveThreshold("chatgpt-web/gpt-5"), 15000);
+  });
+
+  it("returns 15000ms for web-session provider (grok-web)", () => {
+    assert.equal(resolveKeepaliveThreshold("grok-web/grok-4"), 15000);
+  });
+
+  it("returns 15000ms for web-session provider (claude-web)", () => {
+    assert.equal(resolveKeepaliveThreshold("claude-web/claude-sonnet-4"), 15000);
+  });
+
+  it("SLOW_KEEPALIVE_PROVIDERS set contains expected providers", () => {
+    assert.ok(SLOW_KEEPALIVE_PROVIDERS.has("pollinations"));
+    assert.ok(SLOW_KEEPALIVE_PROVIDERS.has("pol"));
+    assert.ok(SLOW_KEEPALIVE_PROVIDERS.has("opencode-zen"));
+    assert.ok(SLOW_KEEPALIVE_PROVIDERS.has("chatgpt-web"));
+    assert.ok(SLOW_KEEPALIVE_PROVIDERS.has("grok-web"));
+    assert.ok(SLOW_KEEPALIVE_PROVIDERS.has("claude-web"));
+  });
+
+  it("SLOW_KEEPALIVE_PROVIDERS does not contain normal providers", () => {
+    assert.ok(!SLOW_KEEPALIVE_PROVIDERS.has("openai"));
+    assert.ok(!SLOW_KEEPALIVE_PROVIDERS.has("anthropic"));
+    assert.ok(!SLOW_KEEPALIVE_PROVIDERS.has("deepseek"));
+  });
+});