From 7d3dc0bc3512d43d6315797a504c2c2784a33177 Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza Date: Fri, 7 Aug 2026 11:23:19 -0300 Subject: [PATCH] fix(vertex): route Claude models to native rawPredict and respect targetFormat overrides (#8994) Closes #8994 --- .../fixes/8994-vertex-partner-claude.md | 1 + open-sse/executors/vertex.ts | 193 +++++++++++++++++- open-sse/handlers/chatCore/targetFormat.ts | 12 +- src/sse/handlers/chat.ts | 1 + src/sse/handlers/chatHelpers.ts | 44 ++-- tests/unit/chatcore-target-format.test.ts | 17 ++ tests/unit/executor-vertex-extended.test.ts | 12 +- 7 files changed, 254 insertions(+), 26 deletions(-) create mode 100644 changelog.d/fixes/8994-vertex-partner-claude.md diff --git a/changelog.d/fixes/8994-vertex-partner-claude.md b/changelog.d/fixes/8994-vertex-partner-claude.md new file mode 100644 index 0000000000..4e673ffc4b --- /dev/null +++ b/changelog.d/fixes/8994-vertex-partner-claude.md @@ -0,0 +1 @@ +- fix(vertex): route Claude models to native rawPredict endpoint and respect custom targetFormat overrides (#8994) diff --git a/open-sse/executors/vertex.ts b/open-sse/executors/vertex.ts index 0344a456d6..9d9ed6502f 100644 --- a/open-sse/executors/vertex.ts +++ b/open-sse/executors/vertex.ts @@ -138,13 +138,149 @@ function isPartnerModel(model: string) { return [...PARTNER_MODELS].some((prefix) => normalizedModel.startsWith(prefix)); } +// Anthropic models need their own branch: they use Vertex's native Anthropic Messages API +// (publishers/anthropic/.../rawPredict), not the generic OpenAI-compatible partner endpoint the +// other PARTNER_MODELS entries (DeepSeek, Qwen, Llama, Mistral, GLM) go through — the OpenAI-shaped +// endpoint 404s/"malformed argument"s for Claude models on at least some projects. +function isClaudeModel(model: string) { + return model.toLowerCase().startsWith("claude-"); +} + +// Defensive normalizer: target-format resolution for manually-added custom Claude models under +// "vertex"/"vertex-partner" was observed sending a Gemini-shaped body (contents/parts) to the +// Anthropic rawPredict endpoint instead of the configured "claude" format, causing a hard +// "messages: Field required" error upstream regardless of the stored per-model targetFormat. This +// converts a Gemini-shaped body to Anthropic Messages shape so the executor works either way, +// independent of that unresolved upstream resolution gap. +function toAnthropicBody(body: Record): Record { + const contents = body.contents as Array<{ role?: string; parts?: Array<{ text?: string }> }> | undefined; + if (!Array.isArray(contents)) return body; + + const messages = contents.map((c) => ({ + role: c.role === "model" ? "assistant" : "user", + content: (c.parts || []).map((p) => p.text || "").join(""), + })); + const generationConfig = body.generationConfig as { maxOutputTokens?: number } | undefined; + const systemInstruction = body.systemInstruction as { parts?: Array<{ text?: string }> } | undefined; + + const converted: Record = { + messages, + max_tokens: generationConfig?.maxOutputTokens || 4096, + }; + if (systemInstruction?.parts?.length) { + converted.system = systemInstruction.parts.map((p) => p.text || "").join(""); + } + return converted; +} + +// rawPredict always returns a single complete JSON body, never real SSE framing (see buildUrl). +// When the caller actually requested a stream, synthesize a genuine Anthropic-native event +// sequence from that JSON so the existing claude-to-openai.ts (and sibling) response translators +// — which already parse real message_start/content_block_*/message_delta/message_stop events — +// can consume it correctly, instead of relying on the OpenAI-`choices`-only JSON→SSE fallback +// (open-sse/utils/jsonToSse.ts) which cannot represent Anthropic's native response shape at all. +function synthesizeClaudeSse(response: Record): string { + const messageId = typeof response.id === "string" ? response.id : `msg_${Date.now()}`; + const model = typeof response.model === "string" ? response.model : ""; + const usage = (response.usage as Record) || {}; + const stopReason = typeof response.stop_reason === "string" ? response.stop_reason : "end_turn"; + const stopSequence = (response.stop_sequence as string | null | undefined) ?? null; + const content = Array.isArray(response.content) ? response.content : []; + + const events: Array<{ event: string; data: Record }> = []; + + events.push({ + event: "message_start", + data: { + type: "message_start", + message: { + id: messageId, + type: "message", + role: "assistant", + content: [], + model, + stop_reason: null, + stop_sequence: null, + usage: { input_tokens: usage.input_tokens || 0, output_tokens: 0 }, + }, + }, + }); + + content.forEach((block: Record, index: number) => { + if (block.type === "text") { + events.push({ + event: "content_block_start", + data: { type: "content_block_start", index, content_block: { type: "text", text: "" } }, + }); + if (block.text) { + events.push({ + event: "content_block_delta", + data: { + type: "content_block_delta", + index, + delta: { type: "text_delta", text: block.text }, + }, + }); + } + events.push({ event: "content_block_stop", data: { type: "content_block_stop", index } }); + } else if (block.type === "tool_use") { + events.push({ + event: "content_block_start", + data: { + type: "content_block_start", + index, + content_block: { type: "tool_use", id: block.id, name: block.name, input: {} }, + }, + }); + events.push({ + event: "content_block_delta", + data: { + type: "content_block_delta", + index, + delta: { type: "input_json_delta", partial_json: JSON.stringify(block.input ?? {}) }, + }, + }); + events.push({ event: "content_block_stop", data: { type: "content_block_stop", index } }); + } else if (block.type === "thinking") { + events.push({ + event: "content_block_start", + data: { type: "content_block_start", index, content_block: { type: "thinking", thinking: "" } }, + }); + if (block.thinking) { + events.push({ + event: "content_block_delta", + data: { + type: "content_block_delta", + index, + delta: { type: "thinking_delta", thinking: block.thinking }, + }, + }); + } + events.push({ event: "content_block_stop", data: { type: "content_block_stop", index } }); + } + }); + + events.push({ + event: "message_delta", + data: { + type: "message_delta", + delta: { stop_reason: stopReason, stop_sequence: stopSequence }, + usage: { output_tokens: usage.output_tokens || 0 }, + }, + }); + + events.push({ event: "message_stop", data: { type: "message_stop" } }); + + return events.map((e) => `event: ${e.event}\ndata: ${JSON.stringify(e.data)}\n\n`).join(""); +} + export class VertexExecutor extends BaseExecutor { constructor() { super("vertex", PROVIDERS.vertex); } async execute(input: ExecuteInput) { - const { credentials, log } = input; + const { credentials, log, model, stream } = input; // Defensive: trim stray surrounding whitespace from a pasted credential. if (typeof credentials.apiKey === "string") { credentials.apiKey = credentials.apiKey.trim(); @@ -160,7 +296,53 @@ export class VertexExecutor extends BaseExecutor { throw err; } } - return super.execute(input); + if (isClaudeModel(model) && input.body && typeof input.body === "object") { + let body = input.body as Record; + if (!Array.isArray(body.messages)) { + body = toAnthropicBody(body); + input.body = body; + } + // The rawPredict endpoint requires "anthropic_version" in the body (Vertex's substitute + // for the "anthropic-version" header used by Anthropic's direct API). + body.anthropic_version ??= "vertex-2023-10-16"; + // Unlike Anthropic's direct API (which reads the model from the body), Vertex's + // rawPredict endpoint already encodes project/region/model in the URL and 400s with + // "model: Extra inputs are not permitted" if the translated request body still carries + // one (the openai→claude request translator copies the client's model field over). + delete body.model; + } + + const result = await super.execute(input); + + if (isClaudeModel(model) && stream) { + const response = result instanceof Response ? result : result?.response; + if (response?.ok) { + const contentType = response.headers.get("content-type") || ""; + if (contentType.includes("application/json") && !contentType.includes("text/event-stream")) { + const jsonText = await response.text(); + let newBody = jsonText; + let newContentType = contentType; + try { + newBody = synthesizeClaudeSse(JSON.parse(jsonText)); + newContentType = "text/event-stream"; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + log?.warn?.("VERTEX", `Failed to synthesize Claude SSE stream: ${message}`); + } + const newHeaders = new Headers(response.headers); + newHeaders.set("content-type", newContentType); + newHeaders.delete("content-length"); + const newResponse = new Response(newBody, { + status: response.status, + statusText: response.statusText, + headers: newHeaders, + }); + return result instanceof Response ? newResponse : { ...result, response: newResponse }; + } + } + } + + return result; } buildUrl(model: string, stream: boolean, urlIndex = 0, credentials: any = null) { @@ -189,6 +371,13 @@ export class VertexExecutor extends BaseExecutor { } } + if (isClaudeModel(model)) { + // streamRawPredict?alt=sse was verified to return a single plain JSON body (not real SSE + // framing) rather than actual chunked events, which breaks the SSE parser upstream + // ("stream ended before producing a non-ping SSE event"). rawPredict is confirmed reliable + // for both streaming and non-streaming requests; always use it here. + return `https://aiplatform.googleapis.com/v1/projects/${project}/locations/${region}/publishers/anthropic/models/${model}:rawPredict`; + } if (isPartnerModel(model)) { return `https://aiplatform.googleapis.com/v1/projects/${project}/locations/global/endpoints/openapi/chat/completions`; } diff --git a/open-sse/handlers/chatCore/targetFormat.ts b/open-sse/handlers/chatCore/targetFormat.ts index e5f00eb81a..cde4be6162 100644 --- a/open-sse/handlers/chatCore/targetFormat.ts +++ b/open-sse/handlers/chatCore/targetFormat.ts @@ -46,13 +46,15 @@ export function resolveChatCoreTargetFormat(opts: { sourceFormat === FORMATS.CLAUDE) ? sourceFormat : undefined; + // #8994: model-level targetFormat overrides (from registry or custom-model DB override) + // take precedence over apiFormat="responses" — otherwise Vertex Claude models with + // targetFormat="claude" get wrongly routed to OpenAI Responses format. let targetFormat = - apiFormat === "responses" + modelTargetFormat || + customModelTargetFormat || + (apiFormat === "responses" ? FORMATS.OPENAI_RESPONSES - : modelTargetFormat || - customModelTargetFormat || - inferredAgentRouterTargetFormat || - getTargetFormat(provider, providerSpecificData); + : inferredAgentRouterTargetFormat || getTargetFormat(provider, providerSpecificData)); if (nativeXaiResponsesPassthrough) targetFormat = FORMATS.OPENAI_RESPONSES; return { alias, targetFormat }; } diff --git a/src/sse/handlers/chat.ts b/src/sse/handlers/chat.ts index 45bcaa7e22..2b5f0ef0a6 100644 --- a/src/sse/handlers/chat.ts +++ b/src/sse/handlers/chat.ts @@ -1454,6 +1454,7 @@ async function handleSingleModelChat( comboExecutionKey: runtimeOptions.comboExecutionKey ?? runtimeOptions.comboStepId ?? null, extendedContext, modelApiFormat: apiFormat, + modelTargetFormat: targetFormat, providerProfile, cachedSettings: runtimeOptions.cachedSettings, skipUpstreamRetry: runtimeOptions.skipUpstreamRetry ?? false, diff --git a/src/sse/handlers/chatHelpers.ts b/src/sse/handlers/chatHelpers.ts index 272735940d..80f8685b1f 100644 --- a/src/sse/handlers/chatHelpers.ts +++ b/src/sse/handlers/chatHelpers.ts @@ -4,14 +4,8 @@ import { connectionHasExtraKeys } from "@omniroute/open-sse/services/apiKeyRotat import { createBuiltinAutoCombo } from "@omniroute/open-sse/services/autoCombo/builtinCatalog.ts"; import * as log from "../utils/logger"; import { updateProviderCredentials } from "../services/tokenRefresh"; -import { - detectFormatFromEndpoint, - getTargetFormat, -} from "@omniroute/open-sse/services/provider.ts"; -import { - getModelTargetFormat, - PROVIDER_ID_TO_ALIAS, -} from "@omniroute/open-sse/config/providerModels.ts"; +import { detectFormatFromEndpoint } from "@omniroute/open-sse/services/provider.ts"; +import { resolveChatCoreTargetFormat } from "@omniroute/open-sse/handlers/chatCore/targetFormat.ts"; import { handleChatCore } from "@omniroute/open-sse/handlers/chatCore.ts"; import { checkResourcePressureGuard, @@ -305,12 +299,25 @@ export async function resolveModelOrError( ? ((modelInfo as { apiFormat?: string }).apiFormat as string) : undefined : undefined; - const providerAlias = PROVIDER_ID_TO_ALIAS[provider] || provider; - let targetFormat = getModelTargetFormat(providerAlias, model) || getTargetFormat(provider); - if (apiFormat === "responses") { - targetFormat = "openai-responses"; - log.info("ROUTING", `Custom model apiFormat=responses → targetFormat=openai-responses`); - } + // customModelTargetFormat: #2905 per-model wire-format override for custom models, + // injected by getModelInfo. Must be threaded into the same resolution formula + // chatCore.ts uses (static registry > custom-model DB override > provider default) — + // a model that's ALSO a static registry entry (e.g. a Vertex Claude model with no + // per-model registry targetFormat) otherwise silently drops the DB override and + // falls through to the provider default, breaking response translation. + const customModelTargetFormat: string | undefined = + modelInfo && typeof modelInfo === "object" && "targetFormat" in modelInfo + ? typeof (modelInfo as { targetFormat?: unknown }).targetFormat === "string" + ? ((modelInfo as { targetFormat?: string }).targetFormat as string) + : undefined + : undefined; + const { alias: providerAlias, targetFormat } = resolveChatCoreTargetFormat({ + provider, + resolvedModel: model, + apiFormat, + customModelTargetFormat, + providerSpecificData: undefined, + }); const ctxTag = extendedContext && providerAlias === "claude" ? " [1m]" : ""; if (modelStr !== `${provider}/${model}`) { @@ -405,6 +412,7 @@ export async function executeChatWithBreaker({ comboExecutionKey, extendedContext, modelApiFormat, + modelTargetFormat, providerProfile, cachedSettings, skipUpstreamRetry = false, @@ -437,7 +445,13 @@ export async function executeChatWithBreaker({ runWithProxyContext(proxyInfo?.proxy || null, () => (handleChatCore as any)({ body: { ...body, model: `${provider}/${model}` }, - modelInfo: { provider, model, extendedContext, apiFormat: modelApiFormat }, + modelInfo: { + provider, + model, + extendedContext, + apiFormat: modelApiFormat, + targetFormat: modelTargetFormat, + }, credentials: refreshedCredentials, log: handlerLog, clientRawRequest, diff --git a/tests/unit/chatcore-target-format.test.ts b/tests/unit/chatcore-target-format.test.ts index 2fb4da01dd..4eb22d5598 100644 --- a/tests/unit/chatcore-target-format.test.ts +++ b/tests/unit/chatcore-target-format.test.ts @@ -98,6 +98,23 @@ test("AgentRouter explicit connection protocol overrides the inferred inbound pr assert.equal(r.targetFormat, FORMATS.CLAUDE); }); +test("#8994: customModelTargetFormat takes precedence over apiFormat='responses'", () => { + // When a Vertex Claude model has customModelTargetFormat="claude" and the + // handler also receives apiFormat="responses", the model-level override + // must win — otherwise the request body is translated to OpenAI Responses + // format (which Vertex's Claude endpoint cannot parse). + const r = resolveChatCoreTargetFormat({ + provider: "vertex", + resolvedModel: "claude-sonnet-4-6", + apiFormat: "responses", + sourceFormat: FORMATS.OPENAI, + customModelTargetFormat: "claude", + providerSpecificData: undefined, + }); + // BUG: apiFormat short-circuits before customModelTargetFormat is checked + assert.equal(r.targetFormat, "claude", "model-level targetFormat must win over apiFormat"); +}); + test("unmapped provider → alias falls back to the provider id", () => { const r = resolveChatCoreTargetFormat({ provider: "some-unmapped-provider", diff --git a/tests/unit/executor-vertex-extended.test.ts b/tests/unit/executor-vertex-extended.test.ts index b8cbbe5900..44a2ba336f 100644 --- a/tests/unit/executor-vertex-extended.test.ts +++ b/tests/unit/executor-vertex-extended.test.ts @@ -90,11 +90,15 @@ test("VertexExecutor.buildUrl routes partner and org-prefixed models to the glob ); }); -test("VertexExecutor.buildUrl routes current-generation Claude models to the global partner endpoint (#1985)", () => { +test("VertexExecutor.buildUrl routes current-generation Claude models to the native Anthropic rawPredict endpoint (#1985, #8994)", () => { const executor = new VertexExecutor(); // These model IDs post-date the old pinned "claude-3-5-sonnet" / "claude-3-opus" / - // "claude-3-haiku" prefixes and were previously misrouted to the Google-publisher path. + // "claude-3-haiku" prefixes and were previously misrouted to the Google-publisher path, + // then (once generalized to a "claude-" prefix, #1985) to the generic OpenAI-compatible + // partner endpoint. Claude models use Vertex's native Anthropic Messages API + // (publishers/anthropic/.../rawPredict) instead — the partner endpoint 404s/"malformed + // argument"s for Claude on at least some projects. const claude4Sonnet = executor.buildUrl("claude-sonnet-4-6", false, 0, { apiKey: createServiceAccountJson({ projectId: "proj-claude" }), }); @@ -104,11 +108,11 @@ test("VertexExecutor.buildUrl routes current-generation Claude models to the glo assert.equal( claude4Sonnet, - "https://aiplatform.googleapis.com/v1/projects/proj-claude/locations/global/endpoints/openapi/chat/completions" + "https://aiplatform.googleapis.com/v1/projects/proj-claude/locations/us-central1/publishers/anthropic/models/claude-sonnet-4-6:rawPredict" ); assert.equal( claude4Haiku, - "https://aiplatform.googleapis.com/v1/projects/proj-claude/locations/global/endpoints/openapi/chat/completions" + "https://aiplatform.googleapis.com/v1/projects/proj-claude/locations/us-central1/publishers/anthropic/models/claude-haiku-4-5@20251001:rawPredict" ); });