fix(sse): use brand-neutral keepalive placeholders (#8888)

Validated in local merge-train T4 (HouMinXi+Zartharas+Andrian+artickc)
This commit is contained in:
Andrew B.
2026-08-05 21:52:27 -05:00
committed by GitHub
parent a1edde420e
commit 48b5c7fb81
7 changed files with 51 additions and 26 deletions

View File

@@ -1,13 +1,18 @@
/**
* Early SSE keepalive wrapper for streaming route handlers.
* @file earlyStreamKeepalive.ts
* @description Early SSE keepalive wrapper so short idle-read clients stay connected
* while the handler waits on upstream first-byte (reasoning models, combo failover).
*
* @changes
* - [2026-07-28] [Cursor Grok 4.5] - Scrub omniroute from client-facing keepalive id/model/comment frames
* - [2026-07-28] [Cursor Grok 4.5] - Neutralize Responses startup thinking text (no OmniRoute brand leak)
*
* Strict HTTP clients (notably Codex CLI's `reqwest`, which has a ~5s idle-read
* timeout) drop the connection if no bytes arrive shortly after the request.
* OmniRoute, however, holds the streaming response until `ensureStreamReadiness`
* observes the upstream's first useful byte — which can exceed 5s for reasoning
* models that "think" before emitting any token (#2544). `curl` has no such
* idle timeout, so it was never affected, which is why the bug looked
* client-specific.
* The proxy holds the streaming response until `ensureStreamReadiness` observes
* the upstream's first useful byte — which can exceed 5s for reasoning models
* that "think" before emitting any token (#2544). `curl` has no such idle
* timeout, so it was never affected, which is why the bug looked client-specific.
*
* This wrapper keeps the connection warm without disturbing the handler's
* internal logic (combo failover, stream readiness, account cooldown all still
@@ -27,12 +32,13 @@
*/
const ENCODER = new TextEncoder();
const KEEPALIVE_FRAME = ENCODER.encode(": omniroute-keepalive\n\n");
const KEEPALIVE_FRAME = ENCODER.encode(": keepalive\n\n");
// OpenAI-compatible keepalive: a syntactically valid empty streaming chunk.
// Some OpenAI-compatible clients parse every non-empty SSE line as JSON and
// reject legal SSE comments before their first provider chunk arrives.
// id/model stay brand-neutral — these frames go to the client, not upstream.
export const OPENAI_KEEPALIVE_FRAME = ENCODER.encode(
'data: {"id":"omniroute-keepalive","object":"chat.completion.chunk","created":0,"model":"omniroute","choices":[{"index":0,"delta":{},"finish_reason":null}]}\n\n'
'data: {"id":"chatcmpl-keepalive","object":"chat.completion.chunk","created":0,"model":"keepalive","choices":[{"index":0,"delta":{},"finish_reason":null}]}\n\n'
);
// The first slow-path frame must be a valid OpenAI chunk without creating
// visible reasoning that clients persist into the conversation.
@@ -51,8 +57,9 @@ export const ANTHROPIC_PING_FRAME = ENCODER.encode('event: ping\ndata: {"type":"
// real upstream response — once it arrives — starts its own independent
// response.created lifecycle from scratch; this placeholder item never
// carries a response_id and isn't meant to be continued.
const RESPONSES_STARTUP_ITEM_ID = "rs_omniroute_keepalive";
const STARTUP_THINKING_TEXT = "OmniRoute: got request, sending to provider";
const RESPONSES_STARTUP_ITEM_ID = "rs_keepalive";
// Brand-neutral placeholder — clients persist this as visible reasoning.
const STARTUP_THINKING_TEXT = "✨";
export const RESPONSES_STARTUP_THINKING_FRAME = ENCODER.encode(
[
{
@@ -144,7 +151,7 @@ export type EarlyStreamKeepaliveOptions = {
signal?: AbortSignal | null;
/**
* Frame emitted on each keepalive tick. Defaults to an SSE comment
* (`: omniroute-keepalive`). Anthropic-format routes (/v1/messages) must pass
* (`: keepalive`). Anthropic-format routes (/v1/messages) must pass
* `ANTHROPIC_PING_FRAME` instead, because Anthropic clients ignore SSE comments
* for their stream watchdog and only a real `event: ping` keeps them from aborting.
*/

View File

@@ -1,3 +1,10 @@
/**
* @file sseHeartbeat.ts
* @description Mid-stream SSE heartbeat transform (comment / Anthropic ping / OpenAI chunk).
*
* @changes
* - [2026-07-28] [Cursor Grok 4.5] - Brand-neutral default OpenAI keepalive id/model
*/
export const DEFAULT_SSE_HEARTBEAT_INTERVAL_MS = 15_000;
export const HEARTBEAT_SHAPES = {
@@ -37,10 +44,10 @@ function buildHeartbeatPayload(
return 'data: {"type":"response.in_progress"}\n\n';
case HEARTBEAT_SHAPES.OPENAI_CHUNK: {
const payload = {
id: opts.chunkId ?? "omniroute-keepalive",
id: opts.chunkId ?? "chatcmpl-keepalive",
object: "chat.completion.chunk",
created: Math.floor(Date.now() / 1000),
model: opts.chunkModel ?? "omniroute",
model: opts.chunkModel ?? "keepalive",
choices: [{ index: 0, delta: {}, finish_reason: null }],
};
return `data: ${JSON.stringify(payload)}\n\n`;

View File

@@ -1,5 +1,9 @@
/**
* tests/integration/live-gemini-agentic-loop.test.ts
* @file live-gemini-agentic-loop.test.ts
* @description Live 3-turn Gemini combo agentic loop with cooldown-wait + keepalive.
*
* @changes
* - [2026-07-28] [Cursor Grok 4.5] - Match brand-neutral STARTUP_THINKING_TEXT (✨)
*
* Live test: a REAL, streaming, 3-turn agentic tool-calling flow against the
* "default" gemini combo (strategy=auto, 2 gemma-4 targets), scripted to
@@ -58,9 +62,9 @@ const TURN_TIMEOUT_MS = 700_000;
const FILLER_TOKENS_PER_TURN = 10_000;
const MODEL_A = "gemma-4-31b-it";
const MODEL_B = "gemma-4-26b-a4b-it";
const SYNTHETIC_MODEL_MARKER = "omniroute";
const SYNTHETIC_MODEL_MARKER = "keepalive";
// Must match STARTUP_THINKING_TEXT in open-sse/utils/earlyStreamKeepalive.ts.
const STARTUP_THINKING_SUBSTRING = "OmniRoute:";
const STARTUP_THINKING_SUBSTRING = "";
// Node's global fetch (undici) has its own client-side headersTimeout that
// defaults to 300_000ms — the SAME order of magnitude as comboCooldownWait's

View File

@@ -276,7 +276,7 @@ test("chat completions route emits early keepalive while waiting for stream read
const body = await readAll(response);
assert.match(
body,
/data: \{"id":"omniroute-keepalive","object":"chat\.completion\.chunk"/
/data: \{"id":"chatcmpl-keepalive","object":"chat\.completion\.chunk"/
);
assert.match(body, /OK/);
assert.match(body, /\[DONE\]/);

View File

@@ -38,7 +38,7 @@ test("combo test helper builds a small streaming model probe", () => {
test("combo test helper ignores keepalives and extracts streamed model content", () => {
const text = extractComboTestStreamText(
': omniroute-keepalive\n\ndata: {"choices":[{"delta":{"content":"O"}}]}\n\n' +
': keepalive\n\ndata: {"choices":[{"delta":{"content":"O"}}]}\n\n' +
'data: {"choices":[{"delta":{"content":"K"}}]}\n\ndata: [DONE]\n\n'
);
assert.equal(text, "OK");

View File

@@ -1,3 +1,10 @@
/**
* @file early-stream-keepalive.test.ts
* @description Unit tests for withEarlyStreamKeepalive (fast/slow path, frames, abort).
*
* @changes
* - [2026-07-28] [Cursor Grok 4.5] - Assert brand-neutral startup thinking text (✨)
*/
import test from "node:test";
import assert from "node:assert/strict";
@@ -59,7 +66,7 @@ test("slow handler emits early keepalive then forwards the real body (#2544)", a
assert.match(result.headers.get("content-type") || "", /text\/event-stream/);
const body = await readAll(result);
assert.match(body, /: omniroute-keepalive/, "should emit a keepalive comment before the body");
assert.match(body, /: keepalive/, "should emit a keepalive comment before the body");
assert.match(body, /event: response\.created/, "should forward the real upstream body");
assert.match(body, /data: \[DONE\]/);
});
@@ -95,7 +102,7 @@ test("slow handler emits the custom OpenAI keepalive chunk before the body", asy
});
const body = await readAll(result);
assert.doesNotMatch(body, /: omniroute-keepalive/);
assert.doesNotMatch(body, /: keepalive\n/);
const firstFrame = body.split("\n\n")[0];
assert.doesNotThrow(() => JSON.parse(firstFrame.slice("data: ".length)));
assert.match(body, /data: \[DONE\]/);
@@ -204,9 +211,9 @@ test("RESPONSES_STARTUP_THINKING_FRAME is a self-closed synthetic reasoning item
assert.equal(partAdded.data.item_id, itemId);
assert.equal(delta.data.item_id, itemId);
assert.equal(delta.data.delta, "OmniRoute: got request, sending to provider");
assert.equal(delta.data.delta, "");
assert.equal(partDone.data.item_id, itemId);
assert.equal(partDone.data.part.text, "OmniRoute: got request, sending to provider");
assert.equal(partDone.data.part.text, "");
});
test("slow handler emits the Responses API startup frame before the real body", async () => {
@@ -225,7 +232,7 @@ test("slow handler emits the Responses API startup frame before the real body",
const body = await readAll(result);
assert.match(body, /event: response\.output_item\.added/);
assert.match(body, /OmniRoute: got request, sending to provider/);
assert.match(body, //);
assert.match(body, /event: response\.reasoning_summary_part\.done/);
assert.match(body, /event: response\.created/, "should forward the real upstream body");
assert.match(body, /data: \[DONE\]/);
@@ -247,7 +254,7 @@ test("slow handler emits the custom keepaliveFrame (Anthropic ping) before the b
const body = await readAll(result);
assert.match(body, /event: ping\ndata: {"type":"ping"}/, "should emit a real ping event");
assert.doesNotMatch(body, /: omniroute-keepalive/, "must not fall back to the comment frame");
assert.doesNotMatch(body, /: keepalive\n/, "must not fall back to the comment frame");
assert.match(body, /event: message_start/, "should forward the real upstream body");
});
@@ -272,7 +279,7 @@ test("slow handler that errors emits an in-band error frame (#2544)", async () =
assert.equal(result.status, 200, "already committed to 200 SSE before the error surfaced");
const body = await readAll(result);
assert.match(body, /: omniroute-keepalive/);
assert.match(body, /: keepalive/);
assert.match(body, /event: error/);
assert.match(body, /rate limited/);
});

View File

@@ -103,7 +103,7 @@ test("integration: openai-chunk heartbeat is valid JSON parseable by SDKs", asyn
const { value, done } = await readWithTimeout(reader);
if (done) break;
const chunk = decodeChunk(value);
if (chunk.startsWith("data: ") && chunk.includes("omniroute-keepalive")) {
if (chunk.startsWith("data: ") && chunk.includes("chatcmpl-keepalive")) {
const jsonStr = chunk.slice(6, chunk.indexOf("\n\n"));
const parsed = JSON.parse(jsonStr); // must not throw
assert.equal(parsed.object, "chat.completion.chunk");