diff --git a/CHANGELOG.md b/CHANGELOG.md index 1918a8a5b6..7031b13fa1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ _In development — bullets added per PR; finalized at release._ - **fix(executors): strip `client_metadata` from forwarded body for Cerebras and Mistral** — Cerebras returns 400 (`wrong_api_format`) and Mistral returns 422 (`extra_forbidden`) when the passthrough body carries `client_metadata` (an OpenAI Codex / Claude CLI field with no equivalent on these upstreams). The default executor now drops it for these two providers before sending downstream; other providers (notably `openai`/`codex`) keep it. (thanks @saurabh321gupta) - **fix(codebuddy):** only send reasoning params when the client requests reasoning. (thanks @anki1kr) +- **fix(sse):** keep streaming for forceStream providers when a JSON client requests it. Providers marked `forceStream:true` reject `stream:false` upstream (HTTP 400); `resolveStreamFlag` now guards against this so stream-only providers keep streaming even when the client sends `Accept: application/json` or `stream:false`. (thanks @anki1kr) --- diff --git a/open-sse/config/providers/shared.ts b/open-sse/config/providers/shared.ts index 0447b7bb23..5c0d7e5fb1 100644 --- a/open-sse/config/providers/shared.ts +++ b/open-sse/config/providers/shared.ts @@ -126,6 +126,12 @@ export interface RegistryEntry { defaultContextLength?: number; /** Optional session pool config for rate limit management */ poolConfig?: Record; + /** + * When true, the provider rejects non-streaming requests (HTTP 400). + * resolveStreamFlag will keep streaming even when the client requests JSON; + * OmniRoute accumulates the stream and converts it to a JSON body for the client. (#2081) + */ + forceStream?: boolean; } export interface LegacyProvider { diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 86f143b27c..f97931cfa3 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -99,7 +99,7 @@ import { normalizeClaudeAdaptiveThinking } from "../services/claudeAdaptiveThink import { normalizeClaudeHaikuConstraints } from "../services/claudeHaikuConstraints.ts"; import { echoModelInObject } from "../services/responseModelEcho.ts"; import { stripGpt5SamplingWhenReasoning } from "../services/gpt5SamplingGuard.ts"; -import { getUnsupportedParams } from "../config/providerRegistry.ts"; +import { getUnsupportedParams, REGISTRY } from "../config/providerRegistry.ts"; import { supportsMaxTokens } from "@/lib/modelCapabilities.ts"; import { normalizeThinkingForModel } from "@/shared/constants/modelSpecs.ts"; import { @@ -783,12 +783,17 @@ export async function handleChatCore({ // sourceFormat="claude" applies the Anthropic Messages spec default (stream=false // when body omits stream), preventing STREAM_EARLY_EOF on /v1/messages when // clients send Accept: */* without an explicit stream flag. + // providerRequiresStreaming: providers with forceStream:true reject stream:false + // upstream (HTTP 400); keep streaming so OmniRoute can convert the stream to JSON + // for the client via handleForcedSSEToJson. (#2081) + const providerRequiresStreaming = REGISTRY[provider]?.forceStream === true; const stream = nativeCodexPassthrough && isCompactResponsesEndpoint(endpointPath) ? false : resolveStreamFlag(body?.stream, acceptHeader, sourceFormat, { userAgent: streamUserAgent, streamDefaultMode: apiKeyInfo?.streamDefaultMode, + providerRequiresStreaming, }); // `settings` is already consolidated once near the top of handleChatCore diff --git a/open-sse/utils/aiSdkCompat.ts b/open-sse/utils/aiSdkCompat.ts index 5d84d1b52e..088e898b82 100644 --- a/open-sse/utils/aiSdkCompat.ts +++ b/open-sse/utils/aiSdkCompat.ts @@ -7,6 +7,13 @@ export type StreamDefaultMode = "legacy" | "json"; export interface ResolveStreamFlagOptions { userAgent?: unknown; streamDefaultMode?: unknown; + /** + * When true, the provider rejects non-streaming requests (e.g. forceStream providers + * such as CodeBuddy). resolveStreamFlag will keep streaming even when the client sends + * Accept: application/json or stream:false; the caller is responsible for accumulating + * the stream and converting it to a JSON response for the client. (#2081) + */ + providerRequiresStreaming?: boolean; } function normalizeResolveStreamFlagOptions(optionsOrUserAgent?: unknown): ResolveStreamFlagOptions { @@ -35,7 +42,9 @@ export function clientWantsJsonResponse(acceptHeader: unknown): boolean { /** * Resolves stream behavior from request body + Accept header. - * Priority: explicit `stream: true/false` in body wins. + * Priority: explicit `stream: true/false` in body wins, UNLESS the provider + * requires streaming (`providerRequiresStreaming: true`) — in that case the + * result is always `true` regardless of client preference (#2081). * Accept header only acts as fallback when stream is not explicitly set. * Fixes #656: clients sending both `stream: true` and `Accept: application/json` * should still get streaming responses — body intent takes precedence. @@ -53,11 +62,18 @@ export function resolveStreamFlag( sourceFormat?: string, optionsOrUserAgent?: unknown ): boolean { - // Explicit body value always wins + const options = normalizeResolveStreamFlagOptions(optionsOrUserAgent); + + // Stream-only providers must keep streaming even when the client asked for JSON; + // OmniRoute accumulates the provider stream and converts it to JSON for the client + // downstream (handleForcedSSEToJson). Sending stream:false to such a provider + // returns HTTP 400. (#2081) + if (options.providerRequiresStreaming) return true; + + // Explicit body value always wins (for non-stream-only providers) if (bodyStream === true) return true; if (bodyStream === false) return false; - const options = normalizeResolveStreamFlagOptions(optionsOrUserAgent); const streamDefaultMode = normalizeStreamDefaultMode(options.streamDefaultMode); const acceptsEventStream = diff --git a/tests/unit/resolve-stream-flag.test.ts b/tests/unit/resolve-stream-flag.test.ts new file mode 100644 index 0000000000..caf544b960 --- /dev/null +++ b/tests/unit/resolve-stream-flag.test.ts @@ -0,0 +1,73 @@ +// Port of upstream #2081 — forceStream (stream-only) providers must keep streaming even +// when the client asks for a non-streaming/JSON response. OmniRoute then accumulates the +// provider stream and returns a normal JSON body to the client (handleForcedSSEToJson). +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { resolveStreamFlag } from "../../open-sse/utils/aiSdkCompat.ts"; + +describe("resolveStreamFlag — forceStream / providerRequiresStreaming guard (#2081)", () => { + it("keeps streaming for a forceStream provider even when client prefers JSON and sets stream:false", () => { + // The bug: Accept: application/json + stream:false used to override providerRequiresStreaming, + // sending stream:false to a stream-only provider (e.g. CodeBuddy) → HTTP 400. + const result = resolveStreamFlag( + false, // body.stream = false + "application/json", // Accept header + undefined, // sourceFormat + { providerRequiresStreaming: true } + ); + assert.equal(result, true, "stream-only provider must stay streaming even when client prefers JSON"); + }); + + it("non-forceStream provider: client prefers JSON + stream:false → non-streaming (unchanged behavior)", () => { + const result = resolveStreamFlag( + false, + "application/json", + undefined, + { providerRequiresStreaming: false } + ); + assert.equal(result, false, "normal provider should respect client JSON preference"); + }); + + it("forceStream provider: no explicit stream flag → streams by default", () => { + const result = resolveStreamFlag( + undefined, + undefined, + undefined, + { providerRequiresStreaming: true } + ); + assert.equal(result, true); + }); + + it("ordinary provider with no special flags streams by default (backward compat)", () => { + const result = resolveStreamFlag(undefined, undefined); + assert.equal(result, true); + }); + + it("forceStream provider: client explicitly sends stream:true → stays true", () => { + const result = resolveStreamFlag( + true, + "application/json", + undefined, + { providerRequiresStreaming: true } + ); + assert.equal(result, true); + }); + + it("forceStream provider: client sends Accept: text/event-stream + stream:false → stays true", () => { + // SSE Accept header alone shouldn't be needed for stream-only providers, + // but providerRequiresStreaming should still force true. + const result = resolveStreamFlag( + false, + "text/event-stream", + undefined, + { providerRequiresStreaming: true } + ); + assert.equal(result, true); + }); + + it("without providerRequiresStreaming option, JSON client + stream:false still gets non-streaming", () => { + // Verify backward compatibility — no regression for callers that don't pass the option + const result = resolveStreamFlag(false, "application/json"); + assert.equal(result, false); + }); +});