diff --git a/CHANGELOG.md b/CHANGELOG.md index 216a6681dc..82399f1a32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,55 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 --- +## [1.1.0] β€” 2026-02-18 + +> ### πŸ”§ API Compatibility & SDK Hardening +> +> Response sanitization, role normalization, and structured output improvements for strict OpenAI SDK compatibility and cross-provider robustness. + +### πŸ›‘οΈ Response Sanitization (NEW) + +- **Response sanitizer module** β€” New `responseSanitizer.ts` strips non-standard fields (`x_groq`, `usage_breakdown`, `service_tier`, etc.) from all OpenAI-format provider responses, fixing OpenAI Python SDK v1.83+ Pydantic validation failures that returned raw strings instead of parsed `ChatCompletion` objects +- **Streaming chunk sanitization** β€” Passthrough streaming mode now sanitizes each SSE chunk in real-time via `sanitizeStreamingChunk()`, ensuring strict `chat.completion.chunk` schema compliance +- **ID/Object/Usage normalization** β€” Ensures `id`, `object`, `created`, `model`, `choices`, and `usage` fields always exist with correct types +- **Usage field cleanup** β€” Strips non-standard usage sub-fields, keeps only `prompt_tokens`, `completion_tokens`, `total_tokens`, and OpenAI detail fields + +### 🧠 Think Tag Extraction (NEW) + +- **`` tag extraction** β€” Automatically extracts `...` blocks from thinking model responses (DeepSeek R1, Kimi K2 Thinking, etc.) into OpenAI's standard `reasoning_content` field +- **Streaming think-tag stripping** β€” Real-time `` extraction in passthrough SSE stream, preventing JSON parsing errors in downstream tools +- **Preserves native reasoning** β€” Providers that already send `reasoning_content` natively (e.g., OpenAI o1) are not overwritten + +### πŸ”„ Role Normalization (NEW) + +- **`developer` β†’ `system` conversion** β€” OpenAI's new `developer` role is automatically converted to `system` for all non-OpenAI providers (Claude, Gemini, Kiro, etc.) +- **`system` β†’ `user` merging** β€” For models that reject the `system` role (GLM, ERNIE), system messages are intelligently merged into the first user message with clear delimiters +- **Model-aware normalization** β€” Uses model name prefix matching (`glm-*`, `ernie-*`) for compatibility decisions, avoiding hardcoded provider-level flags + +### πŸ“ Structured Output for Gemini (NEW) + +- **`response_format` β†’ Gemini conversion** β€” OpenAI's `json_schema` structured output is now translated to Gemini's `responseMimeType` + `responseSchema` in the translator pipeline +- **`json_object` support** β€” `response_format: { type: "json_object" }` maps to Gemini's `application/json` MIME type +- **Schema cleanup** β€” Automatically removes unsupported JSON Schema keywords (`$schema`, `additionalProperties`) for Gemini compatibility + +### πŸ“ Files Added + +| File | Purpose | +| ---------------------------------------- | ---------------------------------------------------------------------- | +| `open-sse/handlers/responseSanitizer.ts` | Response field stripping, think-tag extraction, ID/usage normalization | +| `open-sse/services/roleNormalizer.ts` | Developerβ†’system, systemβ†’user role conversion pipeline | + +### πŸ“ Files Modified + +| File | Change | +| ------------------------------------------------- | ------------------------------------------------------------------------------- | +| `open-sse/handlers/chatCore.ts` | Integrated response sanitizer for non-streaming OpenAI responses | +| `open-sse/utils/stream.ts` | Integrated streaming chunk sanitizer + think-tag extraction in passthrough mode | +| `open-sse/translator/index.ts` | Integrated role normalizer into the request translation pipeline | +| `open-sse/translator/request/openai-to-gemini.ts` | Added `response_format` β†’ `responseMimeType`/`responseSchema` conversion | + +--- + ## [1.0.0] β€” 2026-02-18 > ### πŸŽ‰ First Major Release β€” OmniRoute 1.0 @@ -138,4 +187,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 --- +[1.1.0]: https://github.com/diegosouzapw/OmniRoute/releases/tag/v1.1.0 [1.0.0]: https://github.com/diegosouzapw/OmniRoute/releases/tag/v1.0.0 diff --git a/README.md b/README.md index 45de17a898..c5149725ac 100644 --- a/README.md +++ b/README.md @@ -340,7 +340,7 @@ Access via: WhatsApp, Telegram, Slack, Discord, iMessage, Signal... | ------------------------------- | ------------------------------------------------------------------------------ | | 🎯 **Smart 4-Tier Fallback** | Auto-route: Subscription β†’ API Key β†’ Cheap β†’ Free | | πŸ“Š **Real-Time Quota Tracking** | Live token count + reset countdown per provider | -| πŸ”„ **Format Translation** | OpenAI ↔ Claude ↔ Gemini ↔ Cursor ↔ Kiro seamless | +| πŸ”„ **Format Translation** | OpenAI ↔ Claude ↔ Gemini ↔ Cursor ↔ Kiro seamless + response sanitization | | πŸ‘₯ **Multi-Account Support** | Multiple accounts per provider with intelligent selection | | πŸ”„ **Auto Token Refresh** | OAuth tokens refresh automatically with retry | | 🎨 **Custom Combos** | 6 strategies: fill-first, round-robin, p2c, random, least-used, cost-optimized | @@ -429,6 +429,10 @@ Seamless translation between formats: - **OpenAI** ↔ **Claude** ↔ **Gemini** ↔ **OpenAI Responses** - Your CLI tool sends OpenAI format β†’ OmniRoute translates β†’ Provider receives native format - Works with any tool that supports custom OpenAI endpoints +- **Response sanitization** β€” Strips non-standard fields for strict OpenAI SDK compatibility +- **Role normalization** β€” `developer` β†’ `system` for non-OpenAI; `system` β†’ `user` for GLM/ERNIE models +- **Think tag extraction** β€” `` blocks β†’ `reasoning_content` for thinking models +- **Structured output** β€” `json_schema` β†’ Gemini's `responseMimeType`/`responseSchema` ### πŸ‘₯ Multi-Account Support diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index ca157e9956..5ceb57ebe5 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -1,6 +1,6 @@ # OmniRoute Architecture -_Last updated: 2026-02-17_ +_Last updated: 2026-02-18_ ## Executive Summary @@ -17,6 +17,9 @@ Core capabilities: - Embedding generation via `/v1/embeddings` (6 providers, 9 models) - Image generation via `/v1/images/generations` (4 providers, 9 models) - Think tag parsing (`...`) for reasoning models +- Response sanitization for strict OpenAI SDK compatibility +- Role normalization (developerβ†’system, systemβ†’user) for cross-provider compatibility +- Structured output conversion (json_schema β†’ Gemini responseSchema) - Local persistence for providers, keys, aliases, combos, settings, pricing - Usage/cost tracking and request logging - Optional cloud sync for multi-device/state sync @@ -180,6 +183,8 @@ Main flow modules: - Embedding provider registry: `open-sse/config/embeddingRegistry.ts` - Image generation handler: `open-sse/handlers/imageGeneration.ts` - Image provider registry: `open-sse/config/imageRegistry.ts` +- Response sanitization: `open-sse/handlers/responseSanitizer.ts` +- Role normalization: `open-sse/services/roleNormalizer.ts` Services (business logic): @@ -647,6 +652,13 @@ Source Format β†’ OpenAI (hub) β†’ Target Format Translations are selected dynamically based on source payload shape and provider target format. +Additional processing layers in the translation pipeline: + +- **Response sanitization** β€” Strips non-standard fields from OpenAI-format responses (both streaming and non-streaming) to ensure strict SDK compliance +- **Role normalization** β€” Converts `developer` β†’ `system` for non-OpenAI targets; merges `system` β†’ `user` for models that reject the system role (GLM, ERNIE) +- **Think tag extraction** β€” Parses `...` blocks from content into `reasoning_content` field +- **Structured output** β€” Converts OpenAI `response_format.json_schema` to Gemini's `responseMimeType` + `responseSchema` + ## Supported API Endpoints | Endpoint | Format | Handler | diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index 365d991df4..612942e5d8 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -177,6 +177,10 @@ Use **Dashboard β†’ Translator** to debug format translation issues: - **Thinking tags not appearing** β€” Check if the target provider supports thinking and the thinking budget setting - **Tool calls dropping** β€” Some format translations may strip unsupported fields; verify in Playground mode - **System prompt missing** β€” Claude and Gemini handle system prompts differently; check translation output +- **SDK returns raw string instead of object** β€” Fixed in v1.1.0: response sanitizer now strips non-standard fields (`x_groq`, `usage_breakdown`, etc.) that cause OpenAI SDK Pydantic validation failures +- **GLM/ERNIE rejects `system` role** β€” Fixed in v1.1.0: role normalizer automatically merges system messages into user messages for incompatible models +- **`developer` role not recognized** β€” Fixed in v1.1.0: automatically converted to `system` for non-OpenAI providers +- **`json_schema` not working with Gemini** β€” Fixed in v1.1.0: `response_format` is now converted to Gemini's `responseMimeType` + `responseSchema` --- diff --git a/next.config.mjs b/next.config.mjs index f8ae4922b0..e122958835 100644 --- a/next.config.mjs +++ b/next.config.mjs @@ -2,6 +2,7 @@ const nextConfig = { turbopack: {}, output: "standalone", + serverExternalPackages: ["better-sqlite3"], transpilePackages: ["@omniroute/open-sse"], allowedDevOrigins: ["192.168.*"], typescript: { diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index ac1f736918..86fc9c99b4 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -24,6 +24,7 @@ import { getExecutor } from "../executors/index.ts"; import { translateNonStreamingResponse } from "./responseTranslator.ts"; import { extractUsageFromResponse } from "./usageExtractor.ts"; import { parseSSEToOpenAIResponse, parseSSEToResponsesOutput } from "./sseParser.ts"; +import { sanitizeOpenAIResponse } from "./responseSanitizer.ts"; import { withRateLimit, updateFromHeaders, @@ -471,10 +472,17 @@ export async function handleChatCore({ } // Translate response to client's expected format (usually OpenAI) - const translatedResponse = needsTranslation(targetFormat, sourceFormat) + let translatedResponse = needsTranslation(targetFormat, sourceFormat) ? translateNonStreamingResponse(responseBody, targetFormat, sourceFormat) : responseBody; + // Sanitize response for OpenAI SDK compatibility + // Strips non-standard fields (x_groq, usage_breakdown, service_tier, etc.) + // Extracts tags into reasoning_content + if (sourceFormat === FORMATS.OPENAI) { + translatedResponse = sanitizeOpenAIResponse(translatedResponse); + } + // Add buffer and filter usage for client (to prevent CLI context errors) if (translatedResponse?.usage) { const buffered = addBufferToUsage(translatedResponse.usage); diff --git a/open-sse/handlers/responseSanitizer.ts b/open-sse/handlers/responseSanitizer.ts new file mode 100644 index 0000000000..950ac35fb8 --- /dev/null +++ b/open-sse/handlers/responseSanitizer.ts @@ -0,0 +1,269 @@ +/** + * Response Sanitizer β€” Normalizes LLM responses to strict OpenAI SDK format. + * + * Fixes Issues: + * 1. Strips non-standard fields (x_groq, usage_breakdown, service_tier) that + * break OpenAI Python SDK v1.83+ Pydantic validation (returns str instead of object) + * 2. Extracts tags from thinking models into reasoning_content + * 3. Normalizes response id, object, and usage fields + * 4. Converts developer role β†’ system for non-OpenAI providers + */ + +// ── Standard OpenAI ChatCompletion fields ────────────────────────────────── +const ALLOWED_TOP_LEVEL_FIELDS = new Set([ + "id", + "object", + "created", + "model", + "choices", + "usage", + "system_fingerprint", +]); + +const ALLOWED_USAGE_FIELDS = new Set([ + "prompt_tokens", + "completion_tokens", + "total_tokens", + "prompt_tokens_details", + "completion_tokens_details", +]); + +const ALLOWED_MESSAGE_FIELDS = new Set([ + "role", + "content", + "tool_calls", + "function_call", + "refusal", + "reasoning_content", +]); + +const ALLOWED_CHOICE_FIELDS = new Set(["index", "message", "delta", "finish_reason", "logprobs"]); + +// ── Think tag regex ──────────────────────────────────────────────────────── +// Matches ... blocks (greedy, dotAll) +const THINK_TAG_REGEX = /([\s\S]*?)<\/think>/gi; + +/** + * Extract blocks from text content and return separated parts. + * @returns {{ content: string, thinking: string | null }} + */ +export function extractThinkingFromContent(text: string): { + content: string; + thinking: string | null; +} { + if (!text || typeof text !== "string") { + return { content: text || "", thinking: null }; + } + + const thinkingParts: string[] = []; + let hasThinkTags = false; + + const cleaned = text.replace(THINK_TAG_REGEX, (_, thinkContent) => { + hasThinkTags = true; + const trimmed = thinkContent.trim(); + if (trimmed) { + thinkingParts.push(trimmed); + } + return ""; + }); + + if (!hasThinkTags) { + return { content: text, thinking: null }; + } + + return { + content: cleaned.trim(), + thinking: thinkingParts.length > 0 ? thinkingParts.join("\n\n") : null, + }; +} + +/** + * Sanitize a non-streaming OpenAI ChatCompletion response. + * Strips non-standard fields and normalizes required fields. + */ +export function sanitizeOpenAIResponse(body: any): any { + if (!body || typeof body !== "object") return body; + + // Build sanitized response with only allowed top-level fields + const sanitized: Record = {}; + + // Ensure required fields exist + sanitized.id = normalizeResponseId(body.id); + sanitized.object = body.object || "chat.completion"; + sanitized.created = body.created || Math.floor(Date.now() / 1000); + sanitized.model = body.model || "unknown"; + + // Sanitize choices + if (Array.isArray(body.choices)) { + sanitized.choices = body.choices.map((choice: any, idx: number) => sanitizeChoice(choice, idx)); + } else { + sanitized.choices = []; + } + + // Sanitize usage + if (body.usage && typeof body.usage === "object") { + sanitized.usage = sanitizeUsage(body.usage); + } + + // Keep system_fingerprint if present (it's a valid OpenAI field) + if (body.system_fingerprint) { + sanitized.system_fingerprint = body.system_fingerprint; + } + + return sanitized; +} + +/** + * Sanitize a single choice object. + */ +function sanitizeChoice(choice: any, defaultIndex: number): any { + const sanitized: Record = { + index: choice.index ?? defaultIndex, + finish_reason: choice.finish_reason || null, + }; + + // Sanitize message (non-streaming) or delta (streaming) + if (choice.message) { + sanitized.message = sanitizeMessage(choice.message); + } + if (choice.delta) { + sanitized.delta = sanitizeMessage(choice.delta); + } + + // Keep logprobs if present + if (choice.logprobs !== undefined) { + sanitized.logprobs = choice.logprobs; + } + + return sanitized; +} + +/** + * Sanitize a message object, extracting tags if present. + */ +function sanitizeMessage(msg: any): any { + if (!msg || typeof msg !== "object") return msg; + + const sanitized: Record = {}; + + // Copy only allowed fields + if (msg.role) sanitized.role = msg.role; + if (msg.refusal !== undefined) sanitized.refusal = msg.refusal; + + // Handle content β€” extract tags + if (typeof msg.content === "string") { + const { content, thinking } = extractThinkingFromContent(msg.content); + sanitized.content = content; + + // Set reasoning_content from tags (if not already set) + if (thinking && !msg.reasoning_content) { + sanitized.reasoning_content = thinking; + } + } else if (msg.content !== undefined) { + sanitized.content = msg.content; + } + + // Preserve existing reasoning_content (from providers that natively support it) + if (msg.reasoning_content && !sanitized.reasoning_content) { + sanitized.reasoning_content = msg.reasoning_content; + } + + // Preserve tool_calls + if (msg.tool_calls) { + sanitized.tool_calls = msg.tool_calls; + } + + // Preserve function_call (legacy) + if (msg.function_call) { + sanitized.function_call = msg.function_call; + } + + return sanitized; +} + +/** + * Sanitize usage object β€” keep only standard fields. + */ +function sanitizeUsage(usage: any): any { + if (!usage || typeof usage !== "object") return usage; + + const sanitized: Record = {}; + for (const key of ALLOWED_USAGE_FIELDS) { + if (usage[key] !== undefined) { + sanitized[key] = usage[key]; + } + } + + // Ensure required fields + if (sanitized.prompt_tokens === undefined) sanitized.prompt_tokens = 0; + if (sanitized.completion_tokens === undefined) sanitized.completion_tokens = 0; + if (sanitized.total_tokens === undefined) { + sanitized.total_tokens = sanitized.prompt_tokens + sanitized.completion_tokens; + } + + return sanitized; +} + +/** + * Normalize response ID to use chatcmpl- prefix. + */ +function normalizeResponseId(id: any): string { + if (!id || typeof id !== "string") { + return `chatcmpl-${crypto.randomUUID().replace(/-/g, "").slice(0, 29)}`; + } + // Already correct format + if (id.startsWith("chatcmpl-")) return id; + // Keep custom IDs but don't break them + return id; +} + +/** + * Sanitize a streaming SSE chunk for passthrough mode. + * Lighter than full sanitization β€” only strips problematic extra fields. + */ +export function sanitizeStreamingChunk(parsed: any): any { + if (!parsed || typeof parsed !== "object") return parsed; + + // Build sanitized chunk + const sanitized: Record = {}; + + // Keep only standard fields + if (parsed.id !== undefined) sanitized.id = parsed.id; + sanitized.object = parsed.object || "chat.completion.chunk"; + if (parsed.created !== undefined) sanitized.created = parsed.created; + if (parsed.model !== undefined) sanitized.model = parsed.model; + + // Sanitize choices with delta + if (Array.isArray(parsed.choices)) { + sanitized.choices = parsed.choices.map((choice: any) => { + const c: Record = { + index: choice.index ?? 0, + }; + if (choice.delta !== undefined) { + c.delta = {}; + const delta = choice.delta; + if (delta.role !== undefined) c.delta.role = delta.role; + if (delta.content !== undefined) c.delta.content = delta.content; + if (delta.reasoning_content !== undefined) + c.delta.reasoning_content = delta.reasoning_content; + if (delta.tool_calls !== undefined) c.delta.tool_calls = delta.tool_calls; + if (delta.function_call !== undefined) c.delta.function_call = delta.function_call; + } + if (choice.finish_reason !== undefined) c.finish_reason = choice.finish_reason; + if (choice.logprobs !== undefined) c.logprobs = choice.logprobs; + return c; + }); + } + + // Sanitize usage if present + if (parsed.usage && typeof parsed.usage === "object") { + sanitized.usage = sanitizeUsage(parsed.usage); + } + + // Keep system_fingerprint if present + if (parsed.system_fingerprint) { + sanitized.system_fingerprint = parsed.system_fingerprint; + } + + return sanitized; +} diff --git a/open-sse/services/roleNormalizer.ts b/open-sse/services/roleNormalizer.ts new file mode 100644 index 0000000000..57b4f8bce9 --- /dev/null +++ b/open-sse/services/roleNormalizer.ts @@ -0,0 +1,169 @@ +/** + * Role Normalizer β€” Converts message roles for provider compatibility. + * + * Fixes Issues: + * 1. GLM/ZhipuAI rejects `system` role β†’ merged into first `user` message + * 2. OpenAI `developer` role not understood by non-OpenAI providers β†’ normalized to `system` + * 3. Some providers don't support `system` role at all β†’ prepended to user message + * + * Provider capability matrix is defined here rather than in the registry to + * avoid breaking changes to the existing RegistryEntry interface. + */ + +// ── Provider capabilities ────────────────────────────────────────────────── + +/** + * Providers that do NOT support the `system` role in messages. + * For these, system messages are merged into the first user message. + * + * Note: This applies only to OpenAI-format passthrough providers. + * Claude and Gemini have their own system message handling in dedicated translators. + */ +const PROVIDERS_WITHOUT_SYSTEM_ROLE = new Set([ + // Known to reject system role (from troubleshooting report) + // GLM uses Claude format, so this is handled through claude translator + // But if accessed through OpenAI-format providers like nvidia, it needs this: +]); + +/** + * Models that are known to reject the `system` role regardless of provider. + * Uses prefix matching (e.g., "glm-" matches "glm-4.7", "glm-4.5", etc.) + */ +const MODELS_WITHOUT_SYSTEM_ROLE = [ + "glm-", // ZhipuAI GLM models + "ernie-", // Baidu ERNIE models +]; + +/** + * Check if a provider+model combo supports the system role. + */ +function supportsSystemRole(provider: string, model: string): boolean { + if (PROVIDERS_WITHOUT_SYSTEM_ROLE.has(provider)) return false; + + const modelLower = (model || "").toLowerCase(); + for (const prefix of MODELS_WITHOUT_SYSTEM_ROLE) { + if (modelLower.startsWith(prefix)) return false; + } + + return true; +} + +/** + * Normalize the `developer` role to `system` for non-OpenAI providers. + * OpenAI introduced `developer` as a replacement for `system` in newer models, + * but most other providers still expect `system`. + * + * @param messages - Array of messages + * @param targetFormat - The target format (e.g., "openai", "claude", "gemini") + * @returns Modified messages array + */ +export function normalizeDeveloperRole(messages: any[], targetFormat: string): any[] { + if (!Array.isArray(messages)) return messages; + + // For OpenAI format, keep developer role as-is (it's valid) + // For all other formats, convert developer β†’ system + if (targetFormat === "openai") return messages; + + return messages.map((msg) => { + if (msg.role === "developer") { + return { ...msg, role: "system" }; + } + return msg; + }); +} + +/** + * Convert `system` messages to user messages for providers that don't support + * the system role. The system content is prepended to the first user message + * with a clear delimiter. + * + * @param messages - Array of messages + * @param provider - Provider name + * @param model - Model name + * @returns Modified messages array + */ +export function normalizeSystemRole(messages: any[], provider: string, model: string): any[] { + if (!Array.isArray(messages) || messages.length === 0) return messages; + if (supportsSystemRole(provider, model)) return messages; + + // Extract system messages + const systemMessages = messages.filter((m) => m.role === "system" || m.role === "developer"); + if (systemMessages.length === 0) return messages; + + // Build system content string + const systemContent = systemMessages + .map((m) => { + if (typeof m.content === "string") return m.content; + if (Array.isArray(m.content)) { + return m.content + .filter((c: any) => c.type === "text") + .map((c: any) => c.text) + .join("\n"); + } + return ""; + }) + .filter(Boolean) + .join("\n\n"); + + if (!systemContent) { + return messages.filter((m) => m.role !== "system" && m.role !== "developer"); + } + + // Remove system messages and merge into first user message + const nonSystemMessages = messages.filter((m) => m.role !== "system" && m.role !== "developer"); + + // Find first user message and prepend system content + const firstUserIdx = nonSystemMessages.findIndex((m) => m.role === "user"); + if (firstUserIdx >= 0) { + const userMsg = nonSystemMessages[firstUserIdx]; + const userContent = + typeof userMsg.content === "string" + ? userMsg.content + : Array.isArray(userMsg.content) + ? userMsg.content + .filter((c: any) => c.type === "text") + .map((c: any) => c.text) + .join("\n") + : ""; + + nonSystemMessages[firstUserIdx] = { + ...userMsg, + content: `[System Instructions]\n${systemContent}\n\n[User Message]\n${userContent}`, + }; + } else { + // No user message found β€” insert as a user message at the beginning + nonSystemMessages.unshift({ + role: "user", + content: `[System Instructions]\n${systemContent}`, + }); + } + + return nonSystemMessages; +} + +/** + * Full role normalization pipeline. + * Call this before sending the request to the provider. + * + * @param messages - Array of messages + * @param provider - Provider name/id + * @param model - Model name + * @param targetFormat - Target API format + * @returns Normalized messages array + */ +export function normalizeRoles( + messages: any[], + provider: string, + model: string, + targetFormat: string +): any[] { + if (!Array.isArray(messages)) return messages; + + // Step 1: Normalize developer β†’ system (for non-OpenAI formats) + let result = normalizeDeveloperRole(messages, targetFormat); + + // Step 2: Normalize system β†’ user (for providers that don't support system role) + result = normalizeSystemRole(result, provider, model); + + return result; +} diff --git a/open-sse/translator/index.ts b/open-sse/translator/index.ts index 6d2cd82f69..20ba2d906d 100644 --- a/open-sse/translator/index.ts +++ b/open-sse/translator/index.ts @@ -4,6 +4,7 @@ import { prepareClaudeRequest } from "./helpers/claudeHelper.ts"; import { filterToOpenAIFormat } from "./helpers/openaiHelper.ts"; import { normalizeThinkingConfig } from "../services/provider.ts"; import { applyThinkingBudget } from "../services/thinkingBudget.ts"; +import { normalizeRoles } from "../services/roleNormalizer.ts"; // Registry for translators. // NOTE: translator modules import this file and call register() at module-load time. @@ -132,6 +133,11 @@ export function translateRequest( // Fix missing tool responses (insert empty tool_result if needed) fixMissingToolResponses(result); + // Normalize roles: developerβ†’system for non-OpenAI, systemβ†’user for incompatible models + if (result.messages && Array.isArray(result.messages)) { + result.messages = normalizeRoles(result.messages, provider || "", model || "", targetFormat); + } + // If same format, skip translation steps if (sourceFormat !== targetFormat) { // Check for direct translation path first (e.g., Claude β†’ Gemini) diff --git a/open-sse/translator/request/openai-to-gemini.ts b/open-sse/translator/request/openai-to-gemini.ts index 124155d6aa..7714481409 100644 --- a/open-sse/translator/request/openai-to-gemini.ts +++ b/open-sse/translator/request/openai-to-gemini.ts @@ -199,6 +199,24 @@ function openaiToGeminiBase(model, body, stream) { } } + // Convert response_format to Gemini's responseMimeType/responseSchema + if (body.response_format) { + if (body.response_format.type === "json_schema" && body.response_format.json_schema) { + result.generationConfig.responseMimeType = "application/json"; + // Extract the schema (may be nested under .schema key) + const schema = body.response_format.json_schema.schema || body.response_format.json_schema; + if (schema && typeof schema === "object") { + // Remove unsupported keywords for Gemini (it uses a subset of JSON Schema) + const { $schema, additionalProperties, ...cleanSchema } = schema; + result.generationConfig.responseSchema = cleanSchema; + } + } else if (body.response_format.type === "json_object") { + result.generationConfig.responseMimeType = "application/json"; + } else if (body.response_format.type === "text") { + result.generationConfig.responseMimeType = "text/plain"; + } + } + return result; } diff --git a/open-sse/utils/stream.ts b/open-sse/utils/stream.ts index 21dee8579e..927d1604c3 100644 --- a/open-sse/utils/stream.ts +++ b/open-sse/utils/stream.ts @@ -12,6 +12,10 @@ import { } from "./usageTracking.ts"; import { parseSSELine, hasValuableContent, fixInvalidId, formatSSE } from "./streamHelpers.ts"; import { STREAM_IDLE_TIMEOUT_MS, HTTP_STATUS } from "../config/constants.ts"; +import { + sanitizeStreamingChunk, + extractThinkingFromContent, +} from "../handlers/responseSanitizer.ts"; export { COLORS, formatSSE }; @@ -130,7 +134,10 @@ export function createSSEStream(options: any = {}) { if (trimmed.startsWith("data:") && trimmed.slice(5).trim() !== "[DONE]") { try { - const parsed = JSON.parse(trimmed.slice(5).trim()); + let parsed = JSON.parse(trimmed.slice(5).trim()); + + // Sanitize: strip non-standard fields for OpenAI SDK compatibility + parsed = sanitizeStreamingChunk(parsed); const idFixed = fixInvalidId(parsed); @@ -139,6 +146,16 @@ export function createSSEStream(options: any = {}) { } const delta = parsed.choices?.[0]?.delta; + + // Extract tags from streaming content + if (delta?.content && typeof delta.content === "string") { + const { content, thinking } = extractThinkingFromContent(delta.content); + delta.content = content; + if (thinking && !delta.reasoning_content) { + delta.reasoning_content = thinking; + } + } + const content = delta?.content || delta?.reasoning_content; if (content && typeof content === "string") { totalContentLength += content.length;