diff --git a/changelog.d/fixes/13002-fetch-start-cap-per-provider.md b/changelog.d/fixes/13002-fetch-start-cap-per-provider.md new file mode 100644 index 0000000000..4eb91e6688 --- /dev/null +++ b/changelog.d/fixes/13002-fetch-start-cap-per-provider.md @@ -0,0 +1 @@ +- **fix(streaming):** allow a per-provider override of the fetch-start (headers-wait) timeout cap so providers that buffer the full generation before the first byte (e.g. `command-code`, `opencode-go`) are not cut off at the global 110s cap; the same two entries also gain a reasoning-safe `requestDefaults.maxTokens` of 16384 so thinking models such as `z-ai/glm-5.3-flash` are not cut off mid-reasoning ([#13002](https://github.com/diegosouzapw/OmniRoute/pull/13002)) — thanks @alvinveroy diff --git a/open-sse/config/providerRegistry.ts b/open-sse/config/providerRegistry.ts index 08a3f4b099..20b46cf35b 100644 --- a/open-sse/config/providerRegistry.ts +++ b/open-sse/config/providerRegistry.ts @@ -54,6 +54,14 @@ export function generateLegacyProviders(): Record { if (typeof entry.timeoutMs === "number") { p.timeoutMs = entry.timeoutMs; } + // #11526 follow-up: the headers-wait cap override must reach the executor's + // LegacyProvider config — dropping it here silently fell back to the 110s + // global cap even for providers whose registry entry overrides it (observed: + // opencode-go deepseek thinking generations 504ing at exactly 110s despite + // fetchStartTimeoutCapMs on the registry entry). + if (typeof entry.fetchStartTimeoutCapMs === "number") { + p.fetchStartTimeoutCapMs = entry.fetchStartTimeoutCapMs; + } // Headers const mergedHeaders = { diff --git a/open-sse/config/providers/registry/command-code/index.ts b/open-sse/config/providers/registry/command-code/index.ts index f298bcc21f..2a8bd998c8 100644 --- a/open-sse/config/providers/registry/command-code/index.ts +++ b/open-sse/config/providers/registry/command-code/index.ts @@ -17,6 +17,13 @@ export const command_codeProvider: RegistryEntry = { // The discovery response is a partial routing catalog; static registry // entries omitted from it can still be accepted by the gateway. liveCatalogAuthoritative: false, + // Reasoning models (e.g. z-ai/glm-5.3-flash) exhaust small client budgets on + // thinking before emitting content; default to a reasoning-safe budget. + requestDefaults: { maxTokens: 16_384 }, + // Console Go / Command Code gateways buffer entire generations — no upstream + // bytes flow until the model finishes thinking. Streaming needs a headers-wait + // ceiling well above the 110s global cap for long reasoning generations. + fetchStartTimeoutCapMs: 600_000, authType: "apikey", authHeader: "Authorization", authPrefix: "Bearer ", diff --git a/open-sse/config/providers/registry/opencode/go/index.ts b/open-sse/config/providers/registry/opencode/go/index.ts index dcc428037f..8bf7c65e99 100644 --- a/open-sse/config/providers/registry/opencode/go/index.ts +++ b/open-sse/config/providers/registry/opencode/go/index.ts @@ -13,6 +13,13 @@ export const opencode_goProvider: RegistryEntry = { authHeader: "Authorization", authPrefix: "Bearer", defaultContextLength: 200000, + // glm-5.3-flash and other always-thinking models need a generous output + // budget or reasoning consumes every token before content is emitted. + requestDefaults: { maxTokens: 16_384 }, + // Console Go / Command Code gateways buffer entire generations — no upstream + // bytes flow until the model finishes thinking. Streaming needs a headers-wait + // ceiling well above the 110s global cap for long reasoning generations. + fetchStartTimeoutCapMs: 600_000, models: [ // Port from decolua/9router 8efacc11: align with official Go endpoints — // glm-5.2 is now advertised and Kimi chat traffic must route through diff --git a/open-sse/config/providers/shared.ts b/open-sse/config/providers/shared.ts index 8007b12627..88725a0d51 100644 --- a/open-sse/config/providers/shared.ts +++ b/open-sse/config/providers/shared.ts @@ -183,6 +183,11 @@ export interface RegistryEntry { chatPath?: string; clientVersion?: string; timeoutMs?: number; + /** Headers-wait ceiling override for streaming requests (#11526). Gateways + * that buffer entire generations (Console Go / Command Code) need this well + * above the 110s global cap — generateLegacyProviders() copies it into the + * executor's LegacyProvider config. */ + fetchStartTimeoutCapMs?: number; passthroughModels?: boolean; /** * Whether a non-empty synchronized live model list is exhaustive enough @@ -281,6 +286,7 @@ export interface LegacyProvider { chatPath?: string; clientVersion?: string; timeoutMs?: number; + fetchStartTimeoutCapMs?: number; } export const buildModels = (ids: readonly string[]): RegistryModel[] => diff --git a/open-sse/executors/base.ts b/open-sse/executors/base.ts index 50612683d5..cdbd1046a7 100644 --- a/open-sse/executors/base.ts +++ b/open-sse/executors/base.ts @@ -162,6 +162,7 @@ export type ProviderConfig = { headers?: Record; requestDefaults?: ProviderRequestDefaults; timeoutMs?: number; + fetchStartTimeoutCapMs?: number; format?: string; }; @@ -915,6 +916,10 @@ export class BaseExecutor { const fetchStartTimeoutPolicy = resolveFetchStartTimeout({ baseTimeoutMs: this.getTimeoutMs(), stream, + // Providers with non-incremental upstreams (whole generation buffered + // behind the gateway, e.g. opencode-go's Console Go GLM tier) can take + // minutes before first bytes; the registry overrides the 110s cap. + capMs: this.config?.fetchStartTimeoutCapMs, }); const fetchStartTimeoutMs = fetchStartTimeoutPolicy.timeoutMs; if (fetchStartTimeoutPolicy.capped) { diff --git a/tests/unit/fetch-start-cap-legacy-projection.test.ts b/tests/unit/fetch-start-cap-legacy-projection.test.ts new file mode 100644 index 0000000000..5dc748c78b --- /dev/null +++ b/tests/unit/fetch-start-cap-legacy-projection.test.ts @@ -0,0 +1,37 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +// #11526 follow-up: generateLegacyProviders() must copy fetchStartTimeoutCapMs +// into the executor's LegacyProvider config. Dropping the field silently fell +// back to the 110s global headers-wait cap even for providers whose registry +// entry overrides it — opencode-go deepseek thinking generations 504ed at +// exactly 110s despite fetchStartTimeoutCapMs on the registry entry (the +// buffered Console Go gateway sends no bytes until the model finishes +// thinking, so the headers phase legitimately spans minutes). + +const { generateLegacyProviders } = await import("../../open-sse/config/providerRegistry.ts"); + +test("generateLegacyProviders copies fetchStartTimeoutCapMs into the executor config", () => { + const providers = generateLegacyProviders(); + + for (const id of ["opencode-go", "command-code"]) { + const legacy = providers[id]; + assert.ok(legacy, `${id} must exist in the legacy provider map`); + assert.equal( + legacy.fetchStartTimeoutCapMs, + 600_000, + `${id} buffered-gateway cap must be 600s (registry override must reach the executor)` + ); + } +}); + +test("generateLegacyProviders omits the cap when the registry entry does not set it", () => { + const providers = generateLegacyProviders(); + const plain = providers["openai"]; + if (!plain) return; // registry shape change guard — only assert when present + assert.equal( + plain.fetchStartTimeoutCapMs, + undefined, + "providers without an override keep the global 110s default" + ); +});