Compare commits

..

1 Commits

Author SHA1 Message Date
diegosouzapw
1430ded910 fix(sse): require Responses-shaped body before native OpenAI-compatible passthrough (#12129)
The internal context-handoff/universal-handoff summary request is built in Chat
Completions shape and dispatched through the same handleSingleModel closure
that still carries the original client request's /responses endpoint. This
made resolveChatCoreRequestFormat resolve sourceFormat to "openai-responses"
purely from the inherited endpoint path, and
shouldUseNativeOpenAICompatibleResponsesPassthrough then skipped chat->responses
translation entirely for any openai-compatible-* connection with
apiType: "responses", regardless of the actual body shape. The upstream ended
up receiving the raw Chat Completions body (messages) on /v1/responses with no
input, so the request was rejected outright.

shouldUseNativeOpenAICompatibleResponsesPassthrough now takes the request body
and requires it to actually look Responses-shaped (input present, messages
absent) before allowing the native-passthrough fast path. A genuine client
Responses-API request still gets the zero-translation shortcut; an
internally-synthesized chat-shaped body is now routed through the normal
chat->responses translation layer instead.
2026-09-10 15:51:31 -03:00
13 changed files with 107 additions and 300 deletions

View File

@@ -1646,7 +1646,6 @@ CURSOR_USER_AGENT="Cursor/3.4"
# ── TLS client (wreq-js fingerprint proxy) ──
# TLS_CLIENT_TIMEOUT_MS=600000 # Inherits from FETCH_TIMEOUT_MS by default
# TLS_FIRST_BYTE_WATCHDOG_MS=10000 # #12656: bounds time-to-first-byte on the wreq body (0 disables)
# ── API Bridge (/v1 proxy server) ──
# API_BRIDGE_PROXY_TIMEOUT_MS=600000 # Proxy hop timeout (default: 10min)

View File

@@ -0,0 +1 @@
- fix(sse): require Responses-shaped body before native OpenAI-compatible passthrough (#12129)

View File

@@ -1 +0,0 @@
- fix(sse): add first-byte watchdog to the TLS-fingerprint transport so a stalled wreq body falls back instead of hanging for minutes (#12656)

View File

@@ -729,7 +729,6 @@ REQUEST_TIMEOUT_MS (global override)
│ ├─→ FETCH_HEADERS_TIMEOUT_MS (inherits from FETCH_TIMEOUT_MS)
│ ├─→ FETCH_BODY_TIMEOUT_MS (inherits from FETCH_TIMEOUT_MS)
│ ├─→ TLS_CLIENT_TIMEOUT_MS (inherits from FETCH_TIMEOUT_MS)
│ │ └── TLS_FIRST_BYTE_WATCHDOG_MS (independent, default: 10000)
│ ├── FETCH_CONNECT_TIMEOUT_MS (independent, default: 30000)
│ └── FETCH_KEEPALIVE_TIMEOUT_MS (independent, default: 4000)
├─→ STREAM_IDLE_TIMEOUT_MS (inherits from REQUEST_TIMEOUT_MS, default: 600000)
@@ -767,7 +766,6 @@ REQUEST_TIMEOUT_MS (global override)
| `FETCH_CONNECT_TIMEOUT_MS` | `30000` | TCP connection establishment timeout. |
| `FETCH_KEEPALIVE_TIMEOUT_MS` | `4000` | Keep-alive socket idle timeout. |
| `TLS_CLIENT_TIMEOUT_MS` | = `FETCH_TIMEOUT_MS` | TLS fingerprint proxy (wreq-js) timeout. |
| `TLS_FIRST_BYTE_WATCHDOG_MS` | `10000` | Bounds time-to-first-byte on the wreq-js TLS-fingerprint transport's body specifically; `TLS_CLIENT_TIMEOUT_MS` alone cannot catch a stalled body since it resolves as soon as headers arrive (#12656). A timeout cancels the wreq reader and falls back to the direct/proxy dispatcher; `0` disables the watchdog. |
| `API_BRIDGE_PROXY_TIMEOUT_MS` | `30000` | Proxy hop timeout for `/v1` bridge requests. |
| `FIRECRAWL_BASE_URL` | `https://api.firecrawl.dev` | Point the Firecrawl web-fetch executor at a self-hosted instance (API key optional off-cloud). |
| `FIRECRAWL_TIMEOUT_MS` | `30000` | Per-request timeout for the Firecrawl web-fetch executor. |

View File

@@ -31,13 +31,6 @@ unavailable; a caller may explicitly select a fallback outside this wrapper.
- Proxy resolution (priority): `HTTPS_PROXY``HTTP_PROXY``ALL_PROXY` (also lower-case)
- Timeout: `TLS_CLIENT_TIMEOUT_MS` (inherits from `FETCH_TIMEOUT_MS`, default 600000)
- `wreq-js` Response is fetch-compatible (`headers`, `text()`, `json()`, `clone()`, `body`).
- First-byte watchdog (`open-sse/utils/tlsFirstByteWatchdog.ts`, #12656): `TlsClient.fetch()`
resolves as soon as upstream headers arrive, so `TLS_CLIENT_TIMEOUT_MS` alone cannot bound a
body that never yields a first byte. `guardTlsFirstByte()` races the body's first `read()`
against `TLS_FIRST_BYTE_WATCHDOG_MS` (default `10000`, `0` disables it); a healthy body is
unaffected, while a stalled body cancels the wreq reader and lets `proxyFetch`'s existing
TLS-fallback logic fall through to the direct/proxy dispatcher (a non-replay-safe request, e.g.
a POST with a body, still throws instead of being silently retried).
### Web-cookie provider transport — wreq-js 3.2.0

View File

@@ -731,6 +731,7 @@ export async function handleChatCore({
sourceFormat,
endpointPath,
providerSpecificData: credentials?.providerSpecificData,
body,
});
const responsesInputItems = Array.isArray(body?.input) ? body.input : [];
const customToolNames = collectCustomToolNamesForSourceFormat(

View File

@@ -53,19 +53,35 @@ export function stampNativeResponsesPassthroughBody(
return { ...body, _nativeOpenAICompatibleResponsesPassthrough: true };
}
// A body only qualifies for the native-Responses passthrough fast path when it is
// actually shaped like a Responses API request (`input`, no `messages`). Endpoint
// path alone is not sufficient: an internally-synthesized Chat Completions-shaped
// body (e.g. the context-handoff summary request) can be dispatched through a
// closure that still carries the original client request's `/responses` endpoint,
// which otherwise makes `sourceFormat` resolve to "openai-responses" even though
// the body itself was never translated. See issue #12129.
function isResponsesShapedBody(body: unknown): boolean {
if (!body || typeof body !== "object") return false;
const candidate = body as Record<string, unknown>;
return candidate.input !== undefined && candidate.messages === undefined;
}
export function shouldUseNativeOpenAICompatibleResponsesPassthrough({
provider,
sourceFormat,
endpointPath,
providerSpecificData,
body,
}: {
provider?: string | null;
sourceFormat?: string | null;
endpointPath?: string | null;
providerSpecificData?: unknown;
body?: unknown;
}): boolean {
if (!provider?.startsWith("openai-compatible-")) return false;
if (sourceFormat !== FORMATS.OPENAI_RESPONSES) return false;
if (body !== undefined && !isResponsesShapedBody(body)) return false;
if (providerSpecificData && typeof providerSpecificData === "object") {
const psd = providerSpecificData as Record<string, unknown>;
if (psd.apiType === "responses" || psd._omnirouteForceResponsesUpstream === true) {

View File

@@ -13,7 +13,7 @@ import {
proxyConfigToUrl,
proxyUrlForLogs,
} from "./proxyDispatcher.ts";
import tlsClient, { type TlsFetchOptions, guardTlsFirstByte } from "./tlsClient.ts";
import tlsClient, { type TlsFetchOptions } from "./tlsClient.ts";
import { isProxyReachable } from "@/lib/proxyHealth";
import {
isControlPlaneProxyDirectFallbackEnabled,
@@ -807,7 +807,7 @@ async function patchedFetch(
...tlsProfileForProvider(tlsStore?.provider),
});
if (tlsStore) tlsStore.used = true;
return await guardTlsFirstByte(response);
return response;
} catch (error) {
if (isCallerAbort(error, getEffectiveSignal(input, options))) throw error;
const sessionHadCookies =
@@ -1100,7 +1100,7 @@ async function patchedFetch(
...tlsProfileForProvider(tlsStore?.provider),
});
if (tlsStore) tlsStore.used = true;
return await guardTlsFirstByte(response);
return response;
} catch (error) {
if (isCallerAbort(error, getEffectiveSignal(input, options))) throw error;
const sessionHadCookies =

View File

@@ -1,9 +1,6 @@
import { createHash } from "node:crypto";
import * as nodeModule from "node:module";
import { getTlsClientTimeoutConfig } from "@/shared/utils/runtimeTimeouts";
// #12656 — re-exported so proxyFetch.ts (frozen at its file-size cap) can
// import the first-byte watchdog alongside TlsClient without adding a line.
export { guardTlsFirstByte } from "./tlsFirstByteWatchdog.ts";
const runtimeRequire = nodeModule.createRequire(import.meta.url);

View File

@@ -1,115 +0,0 @@
import { getTlsFirstByteWatchdogMs } from "@/shared/utils/runtimeTimeouts";
// #12656 — the wreq-js TLS-fingerprint transport resolves the Response as
// soon as upstream headers arrive, with zero protection around how long the
// caller then waits for the body's first byte. The only timing guard on that
// path, TlsClient's flat `timeout`, defaults to 600_000ms — matching the
// reported 90-600s stall window exactly. This module races the body's first
// `read()` against a short, env-overridable watchdog: a healthy body is
// completely unaffected (bytes already buffered are replayed through a
// passthrough stream, nothing is dropped), while a body that never yields
// within the deadline cancels the wreq reader and throws so the caller
// (proxyFetch's existing TLS-fallback catch blocks) can fall back to the
// direct/proxy dispatcher instead of hanging for minutes.
export const TLS_FIRST_BYTE_WATCHDOG_TIMEOUT_CODE = "TLS_FIRST_BYTE_WATCHDOG_TIMEOUT";
type BodyReader = ReadableStreamDefaultReader<Uint8Array>;
type FirstReadResult = ReadableStreamReadResult<Uint8Array>;
function createWatchdogTimeoutError(timeoutMs: number): Error & { code: string } {
const err = new Error(
`TLS fingerprint transport produced no first byte within ${timeoutMs}ms`
) as Error & { code: string };
err.name = "TimeoutError";
err.code = TLS_FIRST_BYTE_WATCHDOG_TIMEOUT_CODE;
return err;
}
export function isTlsFirstByteWatchdogTimeout(err: unknown): boolean {
return (
!!err &&
typeof err === "object" &&
"code" in err &&
(err as { code?: unknown }).code === TLS_FIRST_BYTE_WATCHDOG_TIMEOUT_CODE
);
}
async function raceFirstChunk(reader: BodyReader, timeoutMs: number): Promise<FirstReadResult> {
let timer: ReturnType<typeof setTimeout> | undefined;
const timeoutPromise = new Promise<never>((_, reject) => {
timer = setTimeout(() => reject(createWatchdogTimeoutError(timeoutMs)), timeoutMs);
timer.unref?.();
});
try {
return await Promise.race([reader.read(), timeoutPromise]);
} finally {
clearTimeout(timer);
}
}
async function pumpRemainingChunks(
reader: BodyReader,
controller: ReadableStreamDefaultController<Uint8Array>
): Promise<void> {
try {
for (;;) {
const { done, value } = await reader.read();
if (done) {
controller.close();
return;
}
if (value) controller.enqueue(value);
}
} catch (error) {
controller.error(error);
}
}
function buildPassthroughStream(
reader: BodyReader,
firstChunk: FirstReadResult
): ReadableStream<Uint8Array> {
return new ReadableStream<Uint8Array>({
start(controller) {
if (firstChunk.value) controller.enqueue(firstChunk.value);
if (firstChunk.done) {
controller.close();
return;
}
void pumpRemainingChunks(reader, controller);
},
cancel(reason) {
void reader.cancel(reason).catch(() => {});
},
});
}
/**
* Guard a TLS-fingerprint Response's first body byte with a short watchdog.
* Resolves with an equivalent Response (status/headers preserved) whose body
* has already produced at least one byte, or throws
* TLS_FIRST_BYTE_WATCHDOG_TIMEOUT after cancelling the reader so the caller
* can fall back to another transport.
*/
export async function guardTlsFirstByte(
response: Response,
timeoutMs: number = getTlsFirstByteWatchdogMs()
): Promise<Response> {
if (!timeoutMs || timeoutMs <= 0 || !response.body) return response;
const reader = response.body.getReader();
let firstChunk: FirstReadResult;
try {
firstChunk = await raceFirstChunk(reader, timeoutMs);
} catch (error) {
await reader.cancel(error).catch(() => {});
throw error;
}
return new Response(buildPassthroughStream(reader, firstChunk), {
status: response.status,
statusText: response.statusText,
headers: response.headers,
});
}

View File

@@ -35,14 +35,6 @@ export const DEFAULT_MAIN_SERVER_HEADERS_TIMEOUT_MS = 66_000;
// failure, wait this long for the real completion to land. Set to 0 to
// disable and restore the old immediate-fail behavior.
export const DEFAULT_STREAM_DISCONNECT_GRACE_PERIOD_MS = 10_000;
// #12656 — the wreq-js TLS-fingerprint transport resolves the Response as
// soon as upstream headers arrive; the only timing guard on the body itself
// was TlsClient's flat `timeout` (defaults to DEFAULT_FETCH_TIMEOUT_MS =
// 600_000ms), matching the reporter's observed 90-600s stall range exactly.
// This bounds time-to-first-byte specifically for that transport so a wedged
// wreq body falls back fast instead of riding the 10-minute ceiling. Set to
// 0 to disable the watchdog entirely.
export const DEFAULT_TLS_FIRST_BYTE_WATCHDOG_MS = 10_000;
function hasEnvValue(env: EnvSource, name: string): boolean {
const raw = env[name];
@@ -220,16 +212,6 @@ export function getTlsClientTimeoutConfig(
};
}
export function getTlsFirstByteWatchdogMs(
env: EnvSource = process.env,
logger?: TimeoutLogger
): number {
return readTimeoutMs(env, "TLS_FIRST_BYTE_WATCHDOG_MS", DEFAULT_TLS_FIRST_BYTE_WATCHDOG_MS, {
allowZero: true,
logger,
});
}
export function getApiBridgeTimeoutConfig(
env: EnvSource = process.env,
logger?: TimeoutLogger

View File

@@ -0,0 +1,86 @@
// Regression test for issue #12129: an internal context-handoff summary request (built in
// Chat Completions shape -- `messages`, no `input`) is dispatched through the SAME
// handleSingleModel closure that carries the ORIGINAL client request's endpoint.
// When that original endpoint matched `/responses` and the resolved handoff-model
// provider is an openai-compatible-* connection configured with apiType "responses",
// the pipeline used to decide the body was already native-Responses-shaped and skip
// chat->responses translation entirely (`_nativeOpenAICompatibleResponsesPassthrough`),
// so the upstream received `messages` on `/v1/responses` and rejected it with zero input.
//
// Fix: `shouldUseNativeOpenAICompatibleResponsesPassthrough` now requires the body to
// actually look Responses-shaped (`input` present, `messages` absent) before allowing
// the passthrough fast path, so an internally-synthesized chat-shaped body is routed
// through the normal chat->responses translation layer instead.
import assert from "node:assert/strict";
import { test } from "node:test";
import { resolveChatCoreRequestFormat } from "../../open-sse/handlers/chatCore/requestFormat.ts";
import { shouldUseNativeOpenAICompatibleResponsesPassthrough } from "../../open-sse/handlers/chatCore/passthroughHelpers.ts";
test("internal chat-shaped handoff body is no longer treated as native Responses passthrough", () => {
const clientRawRequest = {
endpoint: "/v1/responses",
headers: new Headers(),
};
const summaryBody = {
model: "some-handoff-model",
messages: [{ role: "user", content: "Summarize this conversation." }],
stream: false,
max_tokens: 800,
temperature: 0.1,
_omnirouteSkipContextRelay: true,
_omnirouteInternalRequest: "context-handoff",
};
const { sourceFormat, endpointPath } = resolveChatCoreRequestFormat({
clientRawRequest,
body: summaryBody,
provider: "openai-compatible-responses-cliproxy",
userAgent: null,
});
assert.equal(sourceFormat, "openai-responses");
assert.equal(endpointPath, "/v1/responses");
const providerSpecificData = { apiType: "responses" };
const nativePassthrough = shouldUseNativeOpenAICompatibleResponsesPassthrough({
provider: "openai-compatible-responses-cliproxy",
sourceFormat,
endpointPath,
providerSpecificData,
body: summaryBody,
});
assert.equal(
nativePassthrough,
false,
"fixed: chat-shaped internal body must not take the native-Responses passthrough shortcut"
);
assert.equal((summaryBody as Record<string, unknown>).input, undefined);
assert.ok(Array.isArray(summaryBody.messages) && summaryBody.messages.length > 0);
});
test("genuine Responses-shaped body still takes the native passthrough fast path", () => {
const genuineResponsesBody = {
model: "gpt-5.6-sol",
input: [{ role: "user", content: [{ type: "input_text", text: "Hello" }] }],
stream: false,
};
const nativePassthrough = shouldUseNativeOpenAICompatibleResponsesPassthrough({
provider: "openai-compatible-responses-cliproxy",
sourceFormat: "openai-responses",
endpointPath: "/v1/responses",
providerSpecificData: { apiType: "responses" },
body: genuineResponsesBody,
});
assert.equal(
nativePassthrough,
true,
"a genuine Responses-shaped client body must keep the zero-translation fast path"
);
});

View File

@@ -1,150 +0,0 @@
import assert from "node:assert/strict";
import test from "node:test";
import {
proxyFetch,
runWithTlsTracking,
setTlsClientForTest,
} from "../../open-sse/utils/proxyFetch.ts";
import type { TlsFetchOptions } from "../../open-sse/utils/tlsClient.ts";
// #12656 — when ENABLE_TLS_FINGERPRINT=true, the wreq-js TLS-fingerprint
// transport used to return the Response as soon as headers resolved, with no
// guard on how long the caller then waited for the body's first byte (the
// only timing control, TlsClient's flat `timeout`, defaults to 600_000ms).
// These tests promote the RED probe from the #12656 plan-file into a
// permanent regression suite for the first-byte watchdog added in
// open-sse/utils/tlsFirstByteWatchdog.ts.
type EnvState = Record<string, string | undefined>;
const ENV_KEYS = [
"ENABLE_TLS_FINGERPRINT",
"TLS_FINGERPRINT_PROVIDERS",
"TLS_FIRST_BYTE_WATCHDOG_MS",
] as const;
async function withEnv(env: EnvState, fn: () => Promise<void> | void): Promise<void> {
const prior = Object.fromEntries(ENV_KEYS.map((key) => [key, process.env[key]]));
for (const key of ENV_KEYS) {
if (env[key] === undefined) delete process.env[key];
else process.env[key] = env[key];
}
try {
await fn();
} finally {
for (const key of ENV_KEYS) {
if (prior[key] === undefined) delete process.env[key];
else process.env[key] = prior[key];
}
setTlsClientForTest(null);
}
}
function fakeTlsClient(fetch: (url: string, options?: TlsFetchOptions) => Promise<Response>) {
return { available: true, fetch };
}
function neverYieldingBody(): ReadableStream<Uint8Array> {
return new ReadableStream<Uint8Array>({
pull() {
// Never enqueue, never close — simulates the reported wreq stall.
},
});
}
test("#12656 (a) a stalled wreq body falls back to the direct dispatcher within the watchdog window", async () => {
await withEnv({ ENABLE_TLS_FINGERPRINT: "true", TLS_FIRST_BYTE_WATCHDOG_MS: "80" }, async () => {
setTlsClientForTest(
fakeTlsClient(
async () =>
new Response(neverYieldingBody(), {
status: 200,
headers: { "content-type": "text/event-stream" },
})
)
);
let dispatcherCalls = 0;
const startedAt = Date.now();
const tracked = await runWithTlsTracking("openai", () =>
proxyFetch(
"https://example-provider.test/v1/chat/completions",
{ method: "GET" },
{
undiciFetch: async () => {
dispatcherCalls++;
return new Response("fallback-body", { status: 200 });
},
}
)
);
const elapsedMs = Date.now() - startedAt;
assert.equal(dispatcherCalls, 1);
assert.equal(await tracked.result.text(), "fallback-body");
// Well under the OLD 600_000ms flat TlsClient timeout — proves the
// watchdog fired instead of riding the default request timeout.
assert.ok(elapsedMs < 5_000, `expected fast fallback, took ${elapsedMs}ms`);
// tlsStore.used is flipped back to false on the fallback path in
// proxyFetch's existing catch block, same as any other TLS failure.
assert.equal(tracked.tlsFingerprintUsed, false);
});
});
test("#12656 (b) a healthy/fast wreq body is unaffected by the watchdog", async () => {
await withEnv({ ENABLE_TLS_FINGERPRINT: "true", TLS_FIRST_BYTE_WATCHDOG_MS: "80" }, async () => {
setTlsClientForTest(fakeTlsClient(async () => new Response("healthy-body", { status: 200 })));
let dispatcherCalls = 0;
const tracked = await runWithTlsTracking("openai", () =>
proxyFetch(
"https://example-provider.test/v1/chat/completions",
{ method: "GET" },
{
undiciFetch: async () => {
dispatcherCalls++;
return new Response("fallback-body", { status: 200 });
},
}
)
);
assert.equal(dispatcherCalls, 0);
assert.equal(await tracked.result.text(), "healthy-body");
assert.equal(tracked.tlsFingerprintUsed, true);
});
});
test("#12656 (c) a non-replay-safe POST throws on watchdog timeout instead of silently retrying", async () => {
await withEnv({ ENABLE_TLS_FINGERPRINT: "true", TLS_FIRST_BYTE_WATCHDOG_MS: "80" }, async () => {
setTlsClientForTest(
fakeTlsClient(
async () =>
new Response(neverYieldingBody(), {
status: 200,
headers: { "content-type": "text/event-stream" },
})
)
);
let dispatcherCalls = 0;
await assert.rejects(
runWithTlsTracking("openai", () =>
proxyFetch(
"https://example-provider.test/v1/chat/completions",
{ method: "POST", body: "{}" },
{
undiciFetch: async () => {
dispatcherCalls++;
return new Response("unexpected", { status: 200 });
},
}
)
),
(error: Error) =>
error.message === "TLS fingerprint request failed; request is not safe to replay"
);
assert.equal(dispatcherCalls, 0);
});
});