From 0c3d33899de82adee89a0cad64b6274fe2edf0d8 Mon Sep 17 00:00:00 2001 From: Vitalii <90065738+one-vs@users.noreply.github.com> Date: Thu, 14 May 2026 04:53:26 -0400 Subject: [PATCH] Fix Azure AI Foundry provider connection handling (#2236) Integrated into release/v3.8.0 with unit tests for Azure-AI /responses routing --- .dockerignore | 4 +- open-sse/executors/default.ts | 32 +++++-- open-sse/handlers/chatCore.ts | 55 ++++++++++- .../dashboard/providers/[id]/page.tsx | 2 + src/sse/handlers/chat.ts | 3 +- src/sse/handlers/chatHelpers.ts | 14 ++- src/sse/services/model.ts | 16 +++- .../unit/executor-azure-ai-responses.test.ts | 93 +++++++++++++++++++ 8 files changed, 201 insertions(+), 18 deletions(-) create mode 100644 tests/unit/executor-azure-ai-responses.test.ts diff --git a/.dockerignore b/.dockerignore index 0d921136da..fb2d2a7d3a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -38,7 +38,9 @@ playwright-report blob-report # Documentation (not needed in container) -!docs/ +# Keep the docs directory itself in context so openapi.yaml can be re-included. +docs/* +!docs/openapi.yaml *.md !README.md diff --git a/open-sse/executors/default.ts b/open-sse/executors/default.ts index 728fa9de02..0b033f468d 100644 --- a/open-sse/executors/default.ts +++ b/open-sse/executors/default.ts @@ -11,6 +11,7 @@ import { getGigachatAccessToken } from "../services/gigachatAuth.ts"; import { getRegistryEntry } from "../config/providerRegistry.ts"; import { applyProviderRequestDefaults } from "../services/providerRequestDefaults.ts"; import { + detectFormat, getOpenAICompatibleType, getTargetFormat, isClaudeCodeCompatible, @@ -147,8 +148,12 @@ export class DefaultExecutor extends BaseExecutor { return normalizeDataRobotChatUrl(baseUrl); } case "azure-ai": { + const forceResponses = + credentials?.providerSpecificData?._omnirouteForceResponsesUpstream === true; const apiType = - credentials?.providerSpecificData?.apiType === "responses" ? "responses" : "chat"; + forceResponses || credentials?.providerSpecificData?.apiType === "responses" + ? "responses" + : "chat"; const baseUrl = credentials?.providerSpecificData?.baseUrl || this.config.baseUrl; return normalizeAzureAiChatUrl(baseUrl, apiType); } @@ -161,8 +166,12 @@ export class DefaultExecutor extends BaseExecutor { return normalizeWatsonxChatUrl(baseUrl); } case "oci": { + const forceResponses = + credentials?.providerSpecificData?._omnirouteForceResponsesUpstream === true; const apiType = - credentials?.providerSpecificData?.apiType === "responses" ? "responses" : "chat"; + forceResponses || credentials?.providerSpecificData?.apiType === "responses" + ? "responses" + : "chat"; const baseUrl = credentials?.providerSpecificData?.baseUrl || this.config.baseUrl; return normalizeOciChatUrl(baseUrl, apiType); } @@ -393,6 +402,11 @@ export class DefaultExecutor extends BaseExecutor { transformRequest(model, body, stream, credentials) { const cleanedBody = super.transformRequest(model, body, stream, credentials); let withDefaults = applyProviderRequestDefaults(cleanedBody, this.config.requestDefaults); + const targetFormat = getTargetFormat(this.provider, credentials?.providerSpecificData); + const requestFormat = + withDefaults && typeof withDefaults === "object" && !Array.isArray(withDefaults) + ? detectFormat(withDefaults as Record) + : "openai"; if (typeof withDefaults === "object" && withDefaults !== null && !Array.isArray(withDefaults)) { if (this.provider?.startsWith?.("anthropic-compatible-")) { @@ -401,10 +415,7 @@ export class DefaultExecutor extends BaseExecutor { delete withoutStreamOptions.stream_options; withDefaults = withoutStreamOptions; } - } else if ( - stream && - getTargetFormat(this.provider, credentials?.providerSpecificData) === "openai" - ) { + } else if (stream && targetFormat === "openai" && requestFormat !== "openai-responses") { if (!credentials?.providerSpecificData?.disableStreamOptions) { withDefaults = { ...withDefaults, @@ -418,10 +429,17 @@ export class DefaultExecutor extends BaseExecutor { delete withoutStreamOptions.stream_options; withDefaults = withoutStreamOptions; } + } else if ( + (targetFormat === "openai-responses" || requestFormat === "openai-responses") && + Object.prototype.hasOwnProperty.call(withDefaults, "stream_options") + ) { + const withoutStreamOptions = { ...withDefaults } as Record; + delete withoutStreamOptions.stream_options; + withDefaults = withoutStreamOptions; } // #1961: Map max_tokens -> max_completion_tokens for recent OpenAI models - if (getTargetFormat(this.provider, credentials?.providerSpecificData) === "openai") { + if (targetFormat === "openai") { const isRecentOpenAI = /^(o1|o3|o4|gpt-5)/i.test(model); if (isRecentOpenAI && withDefaults && typeof withDefaults === "object") { const defaultsRecord = withDefaults as Record; diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index b926be19c3..90b4b1a784 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -1164,6 +1164,16 @@ export async function handleChatCore({ cachedSettings = null, }) { let { provider, model, extendedContext } = modelInfo; + // apiFormat is an optional custom-model marker injected by getModelInfo for + // providers whose models can route to /chat/completions or /responses + // (Azure AI Foundry, OCI generic OpenAI). It's not on the base ModelInfo + // shape, so we read it via a structural narrowing without `as any`. + const apiFormat: string | undefined = + modelInfo && typeof modelInfo === "object" && "apiFormat" in modelInfo + ? typeof (modelInfo as { apiFormat?: unknown }).apiFormat === "string" + ? ((modelInfo as { apiFormat?: string }).apiFormat as string) + : undefined + : undefined; const requestedModel = typeof body?.model === "string" && body.model.trim().length > 0 ? body.model : model; const isModelScope = () => isModelScopeProvider(provider, credentials?.providerSpecificData); @@ -1401,7 +1411,9 @@ export async function handleChatCore({ const alias = PROVIDER_ID_TO_ALIAS[provider] || provider; const modelTargetFormat = getModelTargetFormat(alias, resolvedModel); const targetFormat = - modelTargetFormat || getTargetFormat(provider, credentials?.providerSpecificData); + apiFormat === "responses" + ? FORMATS.OPENAI_RESPONSES + : modelTargetFormat || getTargetFormat(provider, credentials?.providerSpecificData); const { body: bodyWithWebSearchFallback, fallback: webSearchFallbackPlan } = prepareWebSearchFallbackBody(body as Record, { provider, @@ -2754,6 +2766,12 @@ export async function handleChatCore({ delete translatedBody.store; } + // Chat clients may send stream_options.include_usage, but OpenAI Responses + // upstreams (including Azure AI Foundry /responses) reject stream_options. + if (targetFormat === FORMATS.OPENAI_RESPONSES && "stream_options" in translatedBody) { + delete translatedBody.stream_options; + } + // Provider-specific max_tokens caps (#711) // Some providers reject requests when max_tokens exceeds their API limit. // Cap before sending to avoid upstream HTTP 400 errors. @@ -2839,12 +2857,41 @@ export async function handleChatCore({ ? { ...credentials, requestEndpointPath: endpointPath } : credentials; - if (!ccSessionId) return nextCredentials; + const providerSpecificData = + nextCredentials?.providerSpecificData && + typeof nextCredentials.providerSpecificData === "object" + ? { ...nextCredentials.providerSpecificData } + : {}; + + // Some providers (Azure AI Foundry, OCI OpenAI-compatible) choose upstream + // endpoint path from providerSpecificData.apiType. When a model routes to + // OpenAI Responses format, force apiType=responses unless explicitly set. + if ( + targetFormat === FORMATS.OPENAI_RESPONSES && + (provider === "azure-ai" || provider === "oci") && + providerSpecificData.apiType !== "responses" + ) { + providerSpecificData.apiType = "responses"; + } + + if ( + targetFormat === FORMATS.OPENAI_RESPONSES && + (provider === "azure-ai" || provider === "oci") + ) { + providerSpecificData._omnirouteForceResponsesUpstream = true; + } + + const withApiType = { + ...nextCredentials, + providerSpecificData, + }; + + if (!ccSessionId) return withApiType; return { - ...nextCredentials, + ...withApiType, providerSpecificData: { - ...(nextCredentials?.providerSpecificData || {}), + ...(withApiType?.providerSpecificData || {}), ccSessionId, }, }; diff --git a/src/app/(dashboard)/dashboard/providers/[id]/page.tsx b/src/app/(dashboard)/dashboard/providers/[id]/page.tsx index e80c848b38..063183b372 100644 --- a/src/app/(dashboard)/dashboard/providers/[id]/page.tsx +++ b/src/app/(dashboard)/dashboard/providers/[id]/page.tsx @@ -5923,6 +5923,7 @@ function ConnectionRow({ const CONFIGURABLE_BASE_URL_PROVIDERS = new Set([ "azure-openai", + "azure-ai", "bailian-coding-plan", "xiaomi-mimo", "heroku", @@ -5934,6 +5935,7 @@ const CONFIGURABLE_BASE_URL_PROVIDERS = new Set([ const DEFAULT_PROVIDER_BASE_URLS: Record = { "azure-openai": "https://example-resource.openai.azure.com", + "azure-ai": "https://example-resource.services.ai.azure.com/openai/v1", "bailian-coding-plan": "https://coding-intl.dashscope.aliyuncs.com/apps/anthropic/v1", "xiaomi-mimo": "https://token-plan-sgp.xiaomimimo.com/v1", "searxng-search": "http://localhost:8888/search", diff --git a/src/sse/handlers/chat.ts b/src/sse/handlers/chat.ts index 9a9bb63a90..03d257337c 100644 --- a/src/sse/handlers/chat.ts +++ b/src/sse/handlers/chat.ts @@ -661,7 +661,7 @@ async function handleSingleModelChat( }); } - const { provider, model, sourceFormat, targetFormat, extendedContext } = resolved; + const { provider, model, sourceFormat, targetFormat, extendedContext, apiFormat } = resolved; const forceLiveComboTest = runtimeOptions.forceLiveComboTest === true; const hasForcedConnection = typeof runtimeOptions.forcedConnectionId === "string" && @@ -901,6 +901,7 @@ async function handleSingleModelChat( comboStepId: runtimeOptions.comboStepId ?? null, comboExecutionKey: runtimeOptions.comboExecutionKey ?? runtimeOptions.comboStepId ?? null, extendedContext, + modelApiFormat: apiFormat, providerProfile, cachedSettings: runtimeOptions.cachedSettings, }); diff --git a/src/sse/handlers/chatHelpers.ts b/src/sse/handlers/chatHelpers.ts index fdac283a01..ec4dc2774a 100644 --- a/src/sse/handlers/chatHelpers.ts +++ b/src/sse/handlers/chatHelpers.ts @@ -207,9 +207,16 @@ export async function resolveModelOrError( } const { provider, model, extendedContext } = modelInfo; + // apiFormat: optional custom-model marker — see chatCore.ts for shape narrowing rationale. + const apiFormat: string | undefined = + modelInfo && typeof modelInfo === "object" && "apiFormat" in modelInfo + ? typeof (modelInfo as { apiFormat?: unknown }).apiFormat === "string" + ? ((modelInfo as { apiFormat?: string }).apiFormat as string) + : undefined + : undefined; const providerAlias = PROVIDER_ID_TO_ALIAS[provider] || provider; let targetFormat = getModelTargetFormat(providerAlias, model) || getTargetFormat(provider); - if ((modelInfo as any).apiFormat === "responses") { + if (apiFormat === "responses") { targetFormat = "openai-responses"; log.info("ROUTING", `Custom model apiFormat=responses → targetFormat=openai-responses`); } @@ -221,7 +228,7 @@ export async function resolveModelOrError( log.info("ROUTING", `Provider: ${provider}, Model: ${model}${ctxTag}`); } - return { provider, model, sourceFormat, targetFormat, extendedContext }; + return { provider, model, sourceFormat, targetFormat, extendedContext, apiFormat }; } export async function checkPipelineGates( @@ -292,6 +299,7 @@ export async function executeChatWithBreaker({ comboStepId, comboExecutionKey, extendedContext, + modelApiFormat, providerProfile, cachedSettings, }: any): Promise<{ result: any; tlsFingerprintUsed: boolean }> { @@ -302,7 +310,7 @@ export async function executeChatWithBreaker({ runWithProxyContext(proxyInfo?.proxy || null, () => (handleChatCore as any)({ body: { ...body, model: `${provider}/${model}` }, - modelInfo: { provider, model, extendedContext }, + modelInfo: { provider, model, extendedContext, apiFormat: modelApiFormat }, credentials: refreshedCredentials, log: handlerLog, clientRawRequest, diff --git a/src/sse/services/model.ts b/src/sse/services/model.ts index e1f2b64d23..68cc9f867c 100644 --- a/src/sse/services/model.ts +++ b/src/sse/services/model.ts @@ -43,6 +43,18 @@ export async function getModelInfo(modelStr) { const parsed = parseModel(modelStr); const { extendedContext } = parsed; + const attachCustomApiFormat = async (info: any) => { + if (!info?.provider || !info?.model) return info; + const apiFormat = await lookupCustomModelApiFormat(String(info.provider), String(info.model)); + if (apiFormat) { + return { + ...info, + apiFormat, + }; + } + return info; + }; + // Check custom provider nodes first (for both alias and non-alias formats) if (parsed.providerAlias || parsed.provider) { // Ensure prefixToCheck is always a concise identifier, not a full model string @@ -94,10 +106,10 @@ export async function getModelInfo(modelStr) { } if (!parsed.isAlias) { - return getModelInfoCore(modelStr, null); + return await attachCustomApiFormat(await getModelInfoCore(modelStr, null)); } - return getModelInfoCore(modelStr, getModelAliases); + return await attachCustomApiFormat(await getModelInfoCore(modelStr, getModelAliases)); } /** diff --git a/tests/unit/executor-azure-ai-responses.test.ts b/tests/unit/executor-azure-ai-responses.test.ts new file mode 100644 index 0000000000..72f9a6bbf0 --- /dev/null +++ b/tests/unit/executor-azure-ai-responses.test.ts @@ -0,0 +1,93 @@ +/** + * Tests for Azure AI Foundry + OCI generic-OpenAI providers with + * apiFormat=responses routing (PR #2236). + * + * Covers: + * 1. transformRequest strips stream_options when routing to /responses. + * 2. buildUrl picks the /responses path when + * providerSpecificData._omnirouteForceResponsesUpstream === true. + * 3. The default chat path is preserved for non-responses requests. + */ +import test from "node:test"; +import assert from "node:assert/strict"; + +import { DefaultExecutor } from "@omniroute/open-sse/executors/default.ts"; + +test("DefaultExecutor.transformRequest strips stream_options for openai-responses target (azure-ai)", () => { + const executor = new DefaultExecutor("azure-ai"); + const body = { + model: "gpt-4.1", + input: "hi", + stream_options: { include_usage: true }, + }; + + // providerSpecificData with apiType=responses signals openai-responses target. + const result = executor.transformRequest("gpt-4.1", body, true, { + providerSpecificData: { + baseUrl: "https://example-resource.services.ai.azure.com/openai/v1", + apiType: "responses", + }, + }); + + assert.equal( + (result as { stream_options?: unknown }).stream_options, + undefined, + "stream_options must be stripped on /responses path" + ); +}); + +test("DefaultExecutor.buildUrl forces /responses path when _omnirouteForceResponsesUpstream=true", () => { + const executor = new DefaultExecutor("azure-ai"); + + const forced = executor.buildUrl("gpt-4.1", true, 0, { + providerSpecificData: { + baseUrl: "https://example-resource.services.ai.azure.com/openai/v1", + _omnirouteForceResponsesUpstream: true, + }, + }); + + // Even without apiType: "responses" on credentials, the force flag must win. + assert.match( + forced, + /\/responses(\?|$)/, + "Forced upstream flag must route to the /responses endpoint" + ); + + const defaultPath = executor.buildUrl("gpt-4.1", true, 0, { + providerSpecificData: { + baseUrl: "https://example-resource.services.ai.azure.com/openai/v1", + }, + }); + + assert.match( + defaultPath, + /\/chat\/completions(\?|$)/, + "Default azure-ai path must be /chat/completions when force flag is absent" + ); +}); + +test("DefaultExecutor: apiType=responses on credentials still routes to /responses (azure-ai)", () => { + const executor = new DefaultExecutor("azure-ai"); + + const result = executor.buildUrl("gpt-4.1", true, 0, { + providerSpecificData: { + baseUrl: "https://example-resource.services.ai.azure.com/openai/v1", + apiType: "responses", + }, + }); + + assert.match(result, /\/responses(\?|$)/); +}); + +test("DefaultExecutor: OCI generic-OpenAI honors force-responses flag", () => { + const executor = new DefaultExecutor("oci"); + + const forced = executor.buildUrl("openai.gpt-oss-20b", true, 0, { + providerSpecificData: { + baseUrl: "https://inference.generativeai.us-ashburn-1.oci.oraclecloud.com", + _omnirouteForceResponsesUpstream: true, + }, + }); + + assert.match(forced, /\/responses(\?|$)/, "OCI must also honor the responses force flag"); +});