mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-08 00:02:20 +03:00
fix(sse): use brand-neutral keepalive placeholders (#8888)
Validated in local merge-train T4 (HouMinXi+Zartharas+Andrian+artickc)
This commit is contained in:
@@ -1,13 +1,18 @@
|
||||
/**
|
||||
* Early SSE keepalive wrapper for streaming route handlers.
|
||||
* @file earlyStreamKeepalive.ts
|
||||
* @description Early SSE keepalive wrapper so short idle-read clients stay connected
|
||||
* while the handler waits on upstream first-byte (reasoning models, combo failover).
|
||||
*
|
||||
* @changes
|
||||
* - [2026-07-28] [Cursor Grok 4.5] - Scrub omniroute from client-facing keepalive id/model/comment frames
|
||||
* - [2026-07-28] [Cursor Grok 4.5] - Neutralize Responses startup thinking text (no OmniRoute brand leak)
|
||||
*
|
||||
* Strict HTTP clients (notably Codex CLI's `reqwest`, which has a ~5s idle-read
|
||||
* timeout) drop the connection if no bytes arrive shortly after the request.
|
||||
* OmniRoute, however, holds the streaming response until `ensureStreamReadiness`
|
||||
* observes the upstream's first useful byte — which can exceed 5s for reasoning
|
||||
* models that "think" before emitting any token (#2544). `curl` has no such
|
||||
* idle timeout, so it was never affected, which is why the bug looked
|
||||
* client-specific.
|
||||
* The proxy holds the streaming response until `ensureStreamReadiness` observes
|
||||
* the upstream's first useful byte — which can exceed 5s for reasoning models
|
||||
* that "think" before emitting any token (#2544). `curl` has no such idle
|
||||
* timeout, so it was never affected, which is why the bug looked client-specific.
|
||||
*
|
||||
* This wrapper keeps the connection warm without disturbing the handler's
|
||||
* internal logic (combo failover, stream readiness, account cooldown all still
|
||||
@@ -27,12 +32,13 @@
|
||||
*/
|
||||
|
||||
const ENCODER = new TextEncoder();
|
||||
const KEEPALIVE_FRAME = ENCODER.encode(": omniroute-keepalive\n\n");
|
||||
const KEEPALIVE_FRAME = ENCODER.encode(": keepalive\n\n");
|
||||
// OpenAI-compatible keepalive: a syntactically valid empty streaming chunk.
|
||||
// Some OpenAI-compatible clients parse every non-empty SSE line as JSON and
|
||||
// reject legal SSE comments before their first provider chunk arrives.
|
||||
// id/model stay brand-neutral — these frames go to the client, not upstream.
|
||||
export const OPENAI_KEEPALIVE_FRAME = ENCODER.encode(
|
||||
'data: {"id":"omniroute-keepalive","object":"chat.completion.chunk","created":0,"model":"omniroute","choices":[{"index":0,"delta":{},"finish_reason":null}]}\n\n'
|
||||
'data: {"id":"chatcmpl-keepalive","object":"chat.completion.chunk","created":0,"model":"keepalive","choices":[{"index":0,"delta":{},"finish_reason":null}]}\n\n'
|
||||
);
|
||||
// The first slow-path frame must be a valid OpenAI chunk without creating
|
||||
// visible reasoning that clients persist into the conversation.
|
||||
@@ -51,8 +57,9 @@ export const ANTHROPIC_PING_FRAME = ENCODER.encode('event: ping\ndata: {"type":"
|
||||
// real upstream response — once it arrives — starts its own independent
|
||||
// response.created lifecycle from scratch; this placeholder item never
|
||||
// carries a response_id and isn't meant to be continued.
|
||||
const RESPONSES_STARTUP_ITEM_ID = "rs_omniroute_keepalive";
|
||||
const STARTUP_THINKING_TEXT = "OmniRoute: got request, sending to provider";
|
||||
const RESPONSES_STARTUP_ITEM_ID = "rs_keepalive";
|
||||
// Brand-neutral placeholder — clients persist this as visible reasoning.
|
||||
const STARTUP_THINKING_TEXT = "✨";
|
||||
export const RESPONSES_STARTUP_THINKING_FRAME = ENCODER.encode(
|
||||
[
|
||||
{
|
||||
@@ -144,7 +151,7 @@ export type EarlyStreamKeepaliveOptions = {
|
||||
signal?: AbortSignal | null;
|
||||
/**
|
||||
* Frame emitted on each keepalive tick. Defaults to an SSE comment
|
||||
* (`: omniroute-keepalive`). Anthropic-format routes (/v1/messages) must pass
|
||||
* (`: keepalive`). Anthropic-format routes (/v1/messages) must pass
|
||||
* `ANTHROPIC_PING_FRAME` instead, because Anthropic clients ignore SSE comments
|
||||
* for their stream watchdog and only a real `event: ping` keeps them from aborting.
|
||||
*/
|
||||
|
||||
@@ -1,3 +1,10 @@
|
||||
/**
|
||||
* @file sseHeartbeat.ts
|
||||
* @description Mid-stream SSE heartbeat transform (comment / Anthropic ping / OpenAI chunk).
|
||||
*
|
||||
* @changes
|
||||
* - [2026-07-28] [Cursor Grok 4.5] - Brand-neutral default OpenAI keepalive id/model
|
||||
*/
|
||||
export const DEFAULT_SSE_HEARTBEAT_INTERVAL_MS = 15_000;
|
||||
|
||||
export const HEARTBEAT_SHAPES = {
|
||||
@@ -37,10 +44,10 @@ function buildHeartbeatPayload(
|
||||
return 'data: {"type":"response.in_progress"}\n\n';
|
||||
case HEARTBEAT_SHAPES.OPENAI_CHUNK: {
|
||||
const payload = {
|
||||
id: opts.chunkId ?? "omniroute-keepalive",
|
||||
id: opts.chunkId ?? "chatcmpl-keepalive",
|
||||
object: "chat.completion.chunk",
|
||||
created: Math.floor(Date.now() / 1000),
|
||||
model: opts.chunkModel ?? "omniroute",
|
||||
model: opts.chunkModel ?? "keepalive",
|
||||
choices: [{ index: 0, delta: {}, finish_reason: null }],
|
||||
};
|
||||
return `data: ${JSON.stringify(payload)}\n\n`;
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
/**
|
||||
* tests/integration/live-gemini-agentic-loop.test.ts
|
||||
* @file live-gemini-agentic-loop.test.ts
|
||||
* @description Live 3-turn Gemini combo agentic loop with cooldown-wait + keepalive.
|
||||
*
|
||||
* @changes
|
||||
* - [2026-07-28] [Cursor Grok 4.5] - Match brand-neutral STARTUP_THINKING_TEXT (✨)
|
||||
*
|
||||
* Live test: a REAL, streaming, 3-turn agentic tool-calling flow against the
|
||||
* "default" gemini combo (strategy=auto, 2 gemma-4 targets), scripted to
|
||||
@@ -58,9 +62,9 @@ const TURN_TIMEOUT_MS = 700_000;
|
||||
const FILLER_TOKENS_PER_TURN = 10_000;
|
||||
const MODEL_A = "gemma-4-31b-it";
|
||||
const MODEL_B = "gemma-4-26b-a4b-it";
|
||||
const SYNTHETIC_MODEL_MARKER = "omniroute";
|
||||
const SYNTHETIC_MODEL_MARKER = "keepalive";
|
||||
// Must match STARTUP_THINKING_TEXT in open-sse/utils/earlyStreamKeepalive.ts.
|
||||
const STARTUP_THINKING_SUBSTRING = "OmniRoute:";
|
||||
const STARTUP_THINKING_SUBSTRING = "✨";
|
||||
|
||||
// Node's global fetch (undici) has its own client-side headersTimeout that
|
||||
// defaults to 300_000ms — the SAME order of magnitude as comboCooldownWait's
|
||||
|
||||
@@ -276,7 +276,7 @@ test("chat completions route emits early keepalive while waiting for stream read
|
||||
const body = await readAll(response);
|
||||
assert.match(
|
||||
body,
|
||||
/data: \{"id":"omniroute-keepalive","object":"chat\.completion\.chunk"/
|
||||
/data: \{"id":"chatcmpl-keepalive","object":"chat\.completion\.chunk"/
|
||||
);
|
||||
assert.match(body, /OK/);
|
||||
assert.match(body, /\[DONE\]/);
|
||||
|
||||
@@ -38,7 +38,7 @@ test("combo test helper builds a small streaming model probe", () => {
|
||||
|
||||
test("combo test helper ignores keepalives and extracts streamed model content", () => {
|
||||
const text = extractComboTestStreamText(
|
||||
': omniroute-keepalive\n\ndata: {"choices":[{"delta":{"content":"O"}}]}\n\n' +
|
||||
': keepalive\n\ndata: {"choices":[{"delta":{"content":"O"}}]}\n\n' +
|
||||
'data: {"choices":[{"delta":{"content":"K"}}]}\n\ndata: [DONE]\n\n'
|
||||
);
|
||||
assert.equal(text, "OK");
|
||||
|
||||
@@ -1,3 +1,10 @@
|
||||
/**
|
||||
* @file early-stream-keepalive.test.ts
|
||||
* @description Unit tests for withEarlyStreamKeepalive (fast/slow path, frames, abort).
|
||||
*
|
||||
* @changes
|
||||
* - [2026-07-28] [Cursor Grok 4.5] - Assert brand-neutral startup thinking text (✨)
|
||||
*/
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
@@ -59,7 +66,7 @@ test("slow handler emits early keepalive then forwards the real body (#2544)", a
|
||||
assert.match(result.headers.get("content-type") || "", /text\/event-stream/);
|
||||
|
||||
const body = await readAll(result);
|
||||
assert.match(body, /: omniroute-keepalive/, "should emit a keepalive comment before the body");
|
||||
assert.match(body, /: keepalive/, "should emit a keepalive comment before the body");
|
||||
assert.match(body, /event: response\.created/, "should forward the real upstream body");
|
||||
assert.match(body, /data: \[DONE\]/);
|
||||
});
|
||||
@@ -95,7 +102,7 @@ test("slow handler emits the custom OpenAI keepalive chunk before the body", asy
|
||||
});
|
||||
|
||||
const body = await readAll(result);
|
||||
assert.doesNotMatch(body, /: omniroute-keepalive/);
|
||||
assert.doesNotMatch(body, /: keepalive\n/);
|
||||
const firstFrame = body.split("\n\n")[0];
|
||||
assert.doesNotThrow(() => JSON.parse(firstFrame.slice("data: ".length)));
|
||||
assert.match(body, /data: \[DONE\]/);
|
||||
@@ -204,9 +211,9 @@ test("RESPONSES_STARTUP_THINKING_FRAME is a self-closed synthetic reasoning item
|
||||
|
||||
assert.equal(partAdded.data.item_id, itemId);
|
||||
assert.equal(delta.data.item_id, itemId);
|
||||
assert.equal(delta.data.delta, "OmniRoute: got request, sending to provider");
|
||||
assert.equal(delta.data.delta, "✨");
|
||||
assert.equal(partDone.data.item_id, itemId);
|
||||
assert.equal(partDone.data.part.text, "OmniRoute: got request, sending to provider");
|
||||
assert.equal(partDone.data.part.text, "✨");
|
||||
});
|
||||
|
||||
test("slow handler emits the Responses API startup frame before the real body", async () => {
|
||||
@@ -225,7 +232,7 @@ test("slow handler emits the Responses API startup frame before the real body",
|
||||
|
||||
const body = await readAll(result);
|
||||
assert.match(body, /event: response\.output_item\.added/);
|
||||
assert.match(body, /OmniRoute: got request, sending to provider/);
|
||||
assert.match(body, /✨/);
|
||||
assert.match(body, /event: response\.reasoning_summary_part\.done/);
|
||||
assert.match(body, /event: response\.created/, "should forward the real upstream body");
|
||||
assert.match(body, /data: \[DONE\]/);
|
||||
@@ -247,7 +254,7 @@ test("slow handler emits the custom keepaliveFrame (Anthropic ping) before the b
|
||||
|
||||
const body = await readAll(result);
|
||||
assert.match(body, /event: ping\ndata: {"type":"ping"}/, "should emit a real ping event");
|
||||
assert.doesNotMatch(body, /: omniroute-keepalive/, "must not fall back to the comment frame");
|
||||
assert.doesNotMatch(body, /: keepalive\n/, "must not fall back to the comment frame");
|
||||
assert.match(body, /event: message_start/, "should forward the real upstream body");
|
||||
});
|
||||
|
||||
@@ -272,7 +279,7 @@ test("slow handler that errors emits an in-band error frame (#2544)", async () =
|
||||
assert.equal(result.status, 200, "already committed to 200 SSE before the error surfaced");
|
||||
|
||||
const body = await readAll(result);
|
||||
assert.match(body, /: omniroute-keepalive/);
|
||||
assert.match(body, /: keepalive/);
|
||||
assert.match(body, /event: error/);
|
||||
assert.match(body, /rate limited/);
|
||||
});
|
||||
|
||||
@@ -103,7 +103,7 @@ test("integration: openai-chunk heartbeat is valid JSON parseable by SDKs", asyn
|
||||
const { value, done } = await readWithTimeout(reader);
|
||||
if (done) break;
|
||||
const chunk = decodeChunk(value);
|
||||
if (chunk.startsWith("data: ") && chunk.includes("omniroute-keepalive")) {
|
||||
if (chunk.startsWith("data: ") && chunk.includes("chatcmpl-keepalive")) {
|
||||
const jsonStr = chunk.slice(6, chunk.indexOf("\n\n"));
|
||||
const parsed = JSON.parse(jsonStr); // must not throw
|
||||
assert.equal(parsed.object, "chat.completion.chunk");
|
||||
|
||||
Reference in New Issue
Block a user