fix(vertex): route Claude models to native rawPredict and respect targetFormat overrides (#8994)

Closes #8994

changelog.d/fixes/8994-vertex-partner-claude.md

@@ -0,0 +1 @@
+- fix(vertex): route Claude models to native rawPredict endpoint and respect custom targetFormat overrides (#8994)

open-sse/executors/vertex.ts

@@ -138,13 +138,149 @@ function isPartnerModel(model: string) {
   return [...PARTNER_MODELS].some((prefix) => normalizedModel.startsWith(prefix));
 }
 
+// Anthropic models need their own branch: they use Vertex's native Anthropic Messages API
+// (publishers/anthropic/.../rawPredict), not the generic OpenAI-compatible partner endpoint the
+// other PARTNER_MODELS entries (DeepSeek, Qwen, Llama, Mistral, GLM) go through — the OpenAI-shaped
+// endpoint 404s/"malformed argument"s for Claude models on at least some projects.
+function isClaudeModel(model: string) {
+  return model.toLowerCase().startsWith("claude-");
+}
+
+// Defensive normalizer: target-format resolution for manually-added custom Claude models under
+// "vertex"/"vertex-partner" was observed sending a Gemini-shaped body (contents/parts) to the
+// Anthropic rawPredict endpoint instead of the configured "claude" format, causing a hard
+// "messages: Field required" error upstream regardless of the stored per-model targetFormat. This
+// converts a Gemini-shaped body to Anthropic Messages shape so the executor works either way,
+// independent of that unresolved upstream resolution gap.
+function toAnthropicBody(body: Record<string, unknown>): Record<string, unknown> {
+  const contents = body.contents as Array<{ role?: string; parts?: Array<{ text?: string }> }> | undefined;
+  if (!Array.isArray(contents)) return body;
+
+  const messages = contents.map((c) => ({
+    role: c.role === "model" ? "assistant" : "user",
+    content: (c.parts || []).map((p) => p.text || "").join(""),
+  }));
+  const generationConfig = body.generationConfig as { maxOutputTokens?: number } | undefined;
+  const systemInstruction = body.systemInstruction as { parts?: Array<{ text?: string }> } | undefined;
+
+  const converted: Record<string, unknown> = {
+    messages,
+    max_tokens: generationConfig?.maxOutputTokens || 4096,
+  };
+  if (systemInstruction?.parts?.length) {
+    converted.system = systemInstruction.parts.map((p) => p.text || "").join("");
+  }
+  return converted;
+}
+
+// rawPredict always returns a single complete JSON body, never real SSE framing (see buildUrl).
+// When the caller actually requested a stream, synthesize a genuine Anthropic-native event
+// sequence from that JSON so the existing claude-to-openai.ts (and sibling) response translators
+// — which already parse real message_start/content_block_*/message_delta/message_stop events —
+// can consume it correctly, instead of relying on the OpenAI-`choices`-only JSON→SSE fallback
+// (open-sse/utils/jsonToSse.ts) which cannot represent Anthropic's native response shape at all.
+function synthesizeClaudeSse(response: Record<string, unknown>): string {
+  const messageId = typeof response.id === "string" ? response.id : `msg_${Date.now()}`;
+  const model = typeof response.model === "string" ? response.model : "";
+  const usage = (response.usage as Record<string, unknown>) || {};
+  const stopReason = typeof response.stop_reason === "string" ? response.stop_reason : "end_turn";
+  const stopSequence = (response.stop_sequence as string | null | undefined) ?? null;
+  const content = Array.isArray(response.content) ? response.content : [];
+
+  const events: Array<{ event: string; data: Record<string, unknown> }> = [];
+
+  events.push({
+    event: "message_start",
+    data: {
+      type: "message_start",
+      message: {
+        id: messageId,
+        type: "message",
+        role: "assistant",
+        content: [],
+        model,
+        stop_reason: null,
+        stop_sequence: null,
+        usage: { input_tokens: usage.input_tokens || 0, output_tokens: 0 },
+      },
+    },
+  });
+
+  content.forEach((block: Record<string, unknown>, index: number) => {
+    if (block.type === "text") {
+      events.push({
+        event: "content_block_start",
+        data: { type: "content_block_start", index, content_block: { type: "text", text: "" } },
+      });
+      if (block.text) {
+        events.push({
+          event: "content_block_delta",
+          data: {
+            type: "content_block_delta",
+            index,
+            delta: { type: "text_delta", text: block.text },
+          },
+        });
+      }
+      events.push({ event: "content_block_stop", data: { type: "content_block_stop", index } });
+    } else if (block.type === "tool_use") {
+      events.push({
+        event: "content_block_start",
+        data: {
+          type: "content_block_start",
+          index,
+          content_block: { type: "tool_use", id: block.id, name: block.name, input: {} },
+        },
+      });
+      events.push({
+        event: "content_block_delta",
+        data: {
+          type: "content_block_delta",
+          index,
+          delta: { type: "input_json_delta", partial_json: JSON.stringify(block.input ?? {}) },
+        },
+      });
+      events.push({ event: "content_block_stop", data: { type: "content_block_stop", index } });
+    } else if (block.type === "thinking") {
+      events.push({
+        event: "content_block_start",
+        data: { type: "content_block_start", index, content_block: { type: "thinking", thinking: "" } },
+      });
+      if (block.thinking) {
+        events.push({
+          event: "content_block_delta",
+          data: {
+            type: "content_block_delta",
+            index,
+            delta: { type: "thinking_delta", thinking: block.thinking },
+          },
+        });
+      }
+      events.push({ event: "content_block_stop", data: { type: "content_block_stop", index } });
+    }
+  });
+
+  events.push({
+    event: "message_delta",
+    data: {
+      type: "message_delta",
+      delta: { stop_reason: stopReason, stop_sequence: stopSequence },
+      usage: { output_tokens: usage.output_tokens || 0 },
+    },
+  });
+
+  events.push({ event: "message_stop", data: { type: "message_stop" } });
+
+  return events.map((e) => `event: ${e.event}\ndata: ${JSON.stringify(e.data)}\n\n`).join("");
+}
+
 export class VertexExecutor extends BaseExecutor {
   constructor() {
     super("vertex", PROVIDERS.vertex);
   }
 
   async execute(input: ExecuteInput) {
-    const { credentials, log } = input;
+    const { credentials, log, model, stream } = input;
     // Defensive: trim stray surrounding whitespace from a pasted credential.
     if (typeof credentials.apiKey === "string") {
       credentials.apiKey = credentials.apiKey.trim();
@@ -160,7 +296,53 @@ export class VertexExecutor extends BaseExecutor {
         throw err;
       }
     }
-    return super.execute(input);
+    if (isClaudeModel(model) && input.body && typeof input.body === "object") {
+      let body = input.body as Record<string, unknown>;
+      if (!Array.isArray(body.messages)) {
+        body = toAnthropicBody(body);
+        input.body = body;
+      }
+      // The rawPredict endpoint requires "anthropic_version" in the body (Vertex's substitute
+      // for the "anthropic-version" header used by Anthropic's direct API).
+      body.anthropic_version ??= "vertex-2023-10-16";
+      // Unlike Anthropic's direct API (which reads the model from the body), Vertex's
+      // rawPredict endpoint already encodes project/region/model in the URL and 400s with
+      // "model: Extra inputs are not permitted" if the translated request body still carries
+      // one (the openai→claude request translator copies the client's model field over).
+      delete body.model;
+    }
+
+    const result = await super.execute(input);
+
+    if (isClaudeModel(model) && stream) {
+      const response = result instanceof Response ? result : result?.response;
+      if (response?.ok) {
+        const contentType = response.headers.get("content-type") || "";
+        if (contentType.includes("application/json") && !contentType.includes("text/event-stream")) {
+          const jsonText = await response.text();
+          let newBody = jsonText;
+          let newContentType = contentType;
+          try {
+            newBody = synthesizeClaudeSse(JSON.parse(jsonText));
+            newContentType = "text/event-stream";
+          } catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            log?.warn?.("VERTEX", `Failed to synthesize Claude SSE stream: ${message}`);
+          }
+          const newHeaders = new Headers(response.headers);
+          newHeaders.set("content-type", newContentType);
+          newHeaders.delete("content-length");
+          const newResponse = new Response(newBody, {
+            status: response.status,
+            statusText: response.statusText,
+            headers: newHeaders,
+          });
+          return result instanceof Response ? newResponse : { ...result, response: newResponse };
+        }
+      }
+    }
+
+    return result;
   }
 
   buildUrl(model: string, stream: boolean, urlIndex = 0, credentials: any = null) {
@@ -189,6 +371,13 @@ export class VertexExecutor extends BaseExecutor {
       }
     }
 
+    if (isClaudeModel(model)) {
+      // streamRawPredict?alt=sse was verified to return a single plain JSON body (not real SSE
+      // framing) rather than actual chunked events, which breaks the SSE parser upstream
+      // ("stream ended before producing a non-ping SSE event"). rawPredict is confirmed reliable
+      // for both streaming and non-streaming requests; always use it here.
+      return `https://aiplatform.googleapis.com/v1/projects/${project}/locations/${region}/publishers/anthropic/models/${model}:rawPredict`;
+    }
     if (isPartnerModel(model)) {
       return `https://aiplatform.googleapis.com/v1/projects/${project}/locations/global/endpoints/openapi/chat/completions`;
     }

open-sse/handlers/chatCore/targetFormat.ts

@@ -46,13 +46,15 @@ export function resolveChatCoreTargetFormat(opts: {
       sourceFormat === FORMATS.CLAUDE)
       ? sourceFormat
       : undefined;
+  // #8994: model-level targetFormat overrides (from registry or custom-model DB override)
+  // take precedence over apiFormat="responses" — otherwise Vertex Claude models with
+  // targetFormat="claude" get wrongly routed to OpenAI Responses format.
   let targetFormat =
-    apiFormat === "responses"
+    modelTargetFormat ||
+    customModelTargetFormat ||
+    (apiFormat === "responses"
       ? FORMATS.OPENAI_RESPONSES
-      : modelTargetFormat ||
-        customModelTargetFormat ||
-        inferredAgentRouterTargetFormat ||
-        getTargetFormat(provider, providerSpecificData);
+      : inferredAgentRouterTargetFormat || getTargetFormat(provider, providerSpecificData));
   if (nativeXaiResponsesPassthrough) targetFormat = FORMATS.OPENAI_RESPONSES;
   return { alias, targetFormat };
 }

src/sse/handlers/chat.ts

@@ -1454,6 +1454,7 @@ async function handleSingleModelChat(
         comboExecutionKey: runtimeOptions.comboExecutionKey ?? runtimeOptions.comboStepId ?? null,
         extendedContext,
         modelApiFormat: apiFormat,
+        modelTargetFormat: targetFormat,
         providerProfile,
         cachedSettings: runtimeOptions.cachedSettings,
         skipUpstreamRetry: runtimeOptions.skipUpstreamRetry ?? false,

src/sse/handlers/chatHelpers.ts

@@ -4,14 +4,8 @@ import { connectionHasExtraKeys } from "@omniroute/open-sse/services/apiKeyRotat
 import { createBuiltinAutoCombo } from "@omniroute/open-sse/services/autoCombo/builtinCatalog.ts";
 import * as log from "../utils/logger";
 import { updateProviderCredentials } from "../services/tokenRefresh";
-import {
-  detectFormatFromEndpoint,
-  getTargetFormat,
-} from "@omniroute/open-sse/services/provider.ts";
-import {
-  getModelTargetFormat,
-  PROVIDER_ID_TO_ALIAS,
-} from "@omniroute/open-sse/config/providerModels.ts";
+import { detectFormatFromEndpoint } from "@omniroute/open-sse/services/provider.ts";
+import { resolveChatCoreTargetFormat } from "@omniroute/open-sse/handlers/chatCore/targetFormat.ts";
 import { handleChatCore } from "@omniroute/open-sse/handlers/chatCore.ts";
 import {
   checkResourcePressureGuard,
@@ -305,12 +299,25 @@ export async function resolveModelOrError(
         ? ((modelInfo as { apiFormat?: string }).apiFormat as string)
         : undefined
       : undefined;
-  const providerAlias = PROVIDER_ID_TO_ALIAS[provider] || provider;
-  let targetFormat = getModelTargetFormat(providerAlias, model) || getTargetFormat(provider);
-  if (apiFormat === "responses") {
-    targetFormat = "openai-responses";
-    log.info("ROUTING", `Custom model apiFormat=responses → targetFormat=openai-responses`);
-  }
+  // customModelTargetFormat: #2905 per-model wire-format override for custom models,
+  // injected by getModelInfo. Must be threaded into the same resolution formula
+  // chatCore.ts uses (static registry > custom-model DB override > provider default) —
+  // a model that's ALSO a static registry entry (e.g. a Vertex Claude model with no
+  // per-model registry targetFormat) otherwise silently drops the DB override and
+  // falls through to the provider default, breaking response translation.
+  const customModelTargetFormat: string | undefined =
+    modelInfo && typeof modelInfo === "object" && "targetFormat" in modelInfo
+      ? typeof (modelInfo as { targetFormat?: unknown }).targetFormat === "string"
+        ? ((modelInfo as { targetFormat?: string }).targetFormat as string)
+        : undefined
+      : undefined;
+  const { alias: providerAlias, targetFormat } = resolveChatCoreTargetFormat({
+    provider,
+    resolvedModel: model,
+    apiFormat,
+    customModelTargetFormat,
+    providerSpecificData: undefined,
+  });
 
   const ctxTag = extendedContext && providerAlias === "claude" ? " [1m]" : "";
   if (modelStr !== `${provider}/${model}`) {
@@ -405,6 +412,7 @@ export async function executeChatWithBreaker({
   comboExecutionKey,
   extendedContext,
   modelApiFormat,
+  modelTargetFormat,
   providerProfile,
   cachedSettings,
   skipUpstreamRetry = false,
@@ -437,7 +445,13 @@ export async function executeChatWithBreaker({
         runWithProxyContext(proxyInfo?.proxy || null, () =>
           (handleChatCore as any)({
             body: { ...body, model: `${provider}/${model}` },
-            modelInfo: { provider, model, extendedContext, apiFormat: modelApiFormat },
+            modelInfo: {
+              provider,
+              model,
+              extendedContext,
+              apiFormat: modelApiFormat,
+              targetFormat: modelTargetFormat,
+            },
             credentials: refreshedCredentials,
             log: handlerLog,
             clientRawRequest,

tests/unit/chatcore-target-format.test.ts

@@ -98,6 +98,23 @@ test("AgentRouter explicit connection protocol overrides the inferred inbound pr
   assert.equal(r.targetFormat, FORMATS.CLAUDE);
 });
 
+test("#8994: customModelTargetFormat takes precedence over apiFormat='responses'", () => {
+  // When a Vertex Claude model has customModelTargetFormat="claude" and the
+  // handler also receives apiFormat="responses", the model-level override
+  // must win — otherwise the request body is translated to OpenAI Responses
+  // format (which Vertex's Claude endpoint cannot parse).
+  const r = resolveChatCoreTargetFormat({
+    provider: "vertex",
+    resolvedModel: "claude-sonnet-4-6",
+    apiFormat: "responses",
+    sourceFormat: FORMATS.OPENAI,
+    customModelTargetFormat: "claude",
+    providerSpecificData: undefined,
+  });
+  // BUG: apiFormat short-circuits before customModelTargetFormat is checked
+  assert.equal(r.targetFormat, "claude", "model-level targetFormat must win over apiFormat");
+});
+
 test("unmapped provider → alias falls back to the provider id", () => {
   const r = resolveChatCoreTargetFormat({
     provider: "some-unmapped-provider",

tests/unit/executor-vertex-extended.test.ts

@@ -90,11 +90,15 @@ test("VertexExecutor.buildUrl routes partner and org-prefixed models to the glob
   );
 });
 
-test("VertexExecutor.buildUrl routes current-generation Claude models to the global partner endpoint (#1985)", () => {
+test("VertexExecutor.buildUrl routes current-generation Claude models to the native Anthropic rawPredict endpoint (#1985, #8994)", () => {
   const executor = new VertexExecutor();
 
   // These model IDs post-date the old pinned "claude-3-5-sonnet" / "claude-3-opus" /
-  // "claude-3-haiku" prefixes and were previously misrouted to the Google-publisher path.
+  // "claude-3-haiku" prefixes and were previously misrouted to the Google-publisher path,
+  // then (once generalized to a "claude-" prefix, #1985) to the generic OpenAI-compatible
+  // partner endpoint. Claude models use Vertex's native Anthropic Messages API
+  // (publishers/anthropic/.../rawPredict) instead — the partner endpoint 404s/"malformed
+  // argument"s for Claude on at least some projects.
   const claude4Sonnet = executor.buildUrl("claude-sonnet-4-6", false, 0, {
     apiKey: createServiceAccountJson({ projectId: "proj-claude" }),
   });
@@ -104,11 +108,11 @@ test("VertexExecutor.buildUrl routes current-generation Claude models to the glo
 
   assert.equal(
     claude4Sonnet,
-    "https://aiplatform.googleapis.com/v1/projects/proj-claude/locations/global/endpoints/openapi/chat/completions"
+    "https://aiplatform.googleapis.com/v1/projects/proj-claude/locations/us-central1/publishers/anthropic/models/claude-sonnet-4-6:rawPredict"
   );
   assert.equal(
     claude4Haiku,
-    "https://aiplatform.googleapis.com/v1/projects/proj-claude/locations/global/endpoints/openapi/chat/completions"
+    "https://aiplatform.googleapis.com/v1/projects/proj-claude/locations/us-central1/publishers/anthropic/models/claude-haiku-4-5@20251001:rawPredict"
   );
 });