diff --git a/open-sse/utils/keepaliveThreshold.ts b/open-sse/utils/keepaliveThreshold.ts new file mode 100644 index 0000000000..9dd6d32938 --- /dev/null +++ b/open-sse/utils/keepaliveThreshold.ts @@ -0,0 +1,62 @@ +/** + * Adaptive keepalive threshold resolver for streaming routes. + * + * Web-session and anonymous-fallback providers are slower to produce the first + * byte because they route through browser sessions or public rate-limited + * endpoints. The default 2 s keepalive threshold is too aggressive for these + * providers — the keepalive stream commits before the upstream has a chance to + * respond, adding unnecessary SSE framing overhead. + * + * `resolveKeepaliveThreshold(model)` inspects the model prefix and returns a + * longer threshold (15 s) for known-slow providers, or the default (2 s) for + * everything else. + */ + +import { NOAUTH_PROVIDERS } from "@/shared/constants/providers"; +import { APIKEY_PROVIDERS } from "@/shared/constants/providers"; +import { WEB_COOKIE_PROVIDERS } from "@/shared/constants/providers"; +import { WEB_SESSION_CREDENTIAL_REQUIREMENTS } from "@/shared/providers/webSessionCredentials"; + +const DEFAULT_THRESHOLD_MS = 2_000; +const SLOW_THRESHOLD_MS = 15_000; + +const SLOW_PROVIDER_IDS: Set = new Set(); + +function addSlowProvider(id: string, alias?: string) { + SLOW_PROVIDER_IDS.add(id); + if (typeof alias === "string" && alias) SLOW_PROVIDER_IDS.add(alias); +} + +for (const [id, def] of Object.entries(NOAUTH_PROVIDERS)) { + if ((def as Record).noAuth === true) { + addSlowProvider(id, (def as Record).alias as string | undefined); + } +} + +for (const [id, def] of Object.entries(APIKEY_PROVIDERS)) { + if ((def as Record).anonymousFallback === true) { + addSlowProvider(id, (def as Record).alias as string | undefined); + } +} + +for (const [id, def] of Object.entries(WEB_COOKIE_PROVIDERS)) { + addSlowProvider(id, (def as Record).alias as string | undefined); +} + +for (const id of Object.keys(WEB_SESSION_CREDENTIAL_REQUIREMENTS)) { + SLOW_PROVIDER_IDS.add(id); +} + +export const SLOW_KEEPALIVE_PROVIDERS: ReadonlySet = SLOW_PROVIDER_IDS; + +export function resolveKeepaliveThreshold(model: string | undefined | null): number { + if (!model || typeof model !== "string") return DEFAULT_THRESHOLD_MS; + + const slashIndex = model.indexOf("/"); + if (slashIndex <= 0) return DEFAULT_THRESHOLD_MS; + + const prefix = model.slice(0, slashIndex); + if (SLOW_PROVIDER_IDS.has(prefix)) return SLOW_THRESHOLD_MS; + + return DEFAULT_THRESHOLD_MS; +} diff --git a/src/app/api/v1/responses/route.ts b/src/app/api/v1/responses/route.ts index b737ae01c1..a6c5d8218c 100644 --- a/src/app/api/v1/responses/route.ts +++ b/src/app/api/v1/responses/route.ts @@ -3,6 +3,7 @@ import { withEarlyStreamKeepalive } from "@omniroute/open-sse/utils/earlyStreamK import { resolveResponsesApiModel } from "@/app/api/internal/codex-responses-ws/modelResolution"; import { getModelInfo } from "@/sse/services/model"; import { getComboByName } from "@/lib/db/combos"; +import { resolveKeepaliveThreshold } from "@omniroute/open-sse/utils/keepaliveThreshold"; // NOTE: We do NOT call initTranslators() here — the translator registry is // bootstrapped at module level inside open-sse/translator/index.ts when it @@ -68,7 +69,19 @@ export async function POST(request) { const resolved = await withCodexPreferredModel(request); const accept = String(request.headers?.get?.("accept") || "").toLowerCase(); if (accept.includes("text/event-stream")) { - return await withEarlyStreamKeepalive(handleChat(resolved), { signal: request.signal }); + // Adaptive threshold: web-session and anonymous-fallback providers are slower + // to produce the first byte, so use a longer keepalive threshold (15s vs 2s). + let model; + try { + const body = await resolved.clone().json().catch(() => null); + model = body?.model; + } catch { + } + const thresholdMs = resolveKeepaliveThreshold(model); + return await withEarlyStreamKeepalive(handleChat(resolved), { + signal: request.signal, + thresholdMs, + }); } return await handleChat(resolved); } diff --git a/tests/unit/keepalive-threshold.test.ts b/tests/unit/keepalive-threshold.test.ts new file mode 100644 index 0000000000..a8973d2de9 --- /dev/null +++ b/tests/unit/keepalive-threshold.test.ts @@ -0,0 +1,75 @@ +/** + * Adaptive keepalive threshold — Unit Tests (PR5 of issue #3368) + * + * Run: node --import tsx/esm --test tests/unit/keepalive-threshold.test.ts + */ + +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { + resolveKeepaliveThreshold, + SLOW_KEEPALIVE_PROVIDERS, +} from "../../open-sse/utils/keepaliveThreshold.ts"; + +describe("resolveKeepaliveThreshold", () => { + it("returns 2000ms default for undefined model", () => { + assert.equal(resolveKeepaliveThreshold(undefined), 2000); + }); + + it("returns 2000ms default for null model", () => { + assert.equal(resolveKeepaliveThreshold(null), 2000); + }); + + it("returns 2000ms default for empty string", () => { + assert.equal(resolveKeepaliveThreshold(""), 2000); + }); + + it("returns 2000ms default for model without prefix", () => { + assert.equal(resolveKeepaliveThreshold("gpt-4"), 2000); + }); + + it("returns 2000ms default for normal API-key provider", () => { + assert.equal(resolveKeepaliveThreshold("openai/gpt-4"), 2000); + assert.equal(resolveKeepaliveThreshold("anthropic/claude-sonnet-4"), 2000); + assert.equal(resolveKeepaliveThreshold("deepseek/deepseek-chat"), 2000); + }); + + it("returns 15000ms for anonymous fallback provider (pollinations)", () => { + assert.equal(resolveKeepaliveThreshold("pollinations/gpt-5"), 15000); + }); + + it("returns 15000ms for anonymous fallback provider alias (pol)", () => { + assert.equal(resolveKeepaliveThreshold("pol/gpt-5"), 15000); + }); + + it("returns 15000ms for anonymous fallback provider (opencode-zen)", () => { + assert.equal(resolveKeepaliveThreshold("opencode-zen/gpt-4"), 15000); + }); + + it("returns 15000ms for web-session provider (chatgpt-web)", () => { + assert.equal(resolveKeepaliveThreshold("chatgpt-web/gpt-5"), 15000); + }); + + it("returns 15000ms for web-session provider (grok-web)", () => { + assert.equal(resolveKeepaliveThreshold("grok-web/grok-4"), 15000); + }); + + it("returns 15000ms for web-session provider (claude-web)", () => { + assert.equal(resolveKeepaliveThreshold("claude-web/claude-sonnet-4"), 15000); + }); + + it("SLOW_KEEPALIVE_PROVIDERS set contains expected providers", () => { + assert.ok(SLOW_KEEPALIVE_PROVIDERS.has("pollinations")); + assert.ok(SLOW_KEEPALIVE_PROVIDERS.has("pol")); + assert.ok(SLOW_KEEPALIVE_PROVIDERS.has("opencode-zen")); + assert.ok(SLOW_KEEPALIVE_PROVIDERS.has("chatgpt-web")); + assert.ok(SLOW_KEEPALIVE_PROVIDERS.has("grok-web")); + assert.ok(SLOW_KEEPALIVE_PROVIDERS.has("claude-web")); + }); + + it("SLOW_KEEPALIVE_PROVIDERS does not contain normal providers", () => { + assert.ok(!SLOW_KEEPALIVE_PROVIDERS.has("openai")); + assert.ok(!SLOW_KEEPALIVE_PROVIDERS.has("anthropic")); + assert.ok(!SLOW_KEEPALIVE_PROVIDERS.has("deepseek")); + }); +});