import { translateResponse, initState } from "../translator/index.ts"; import { FORMATS } from "../translator/formats.ts"; import { trackPendingRequest, appendRequestLog } from "@/lib/usageDb"; import { extractUsage, hasValidUsage, estimateUsage, logUsage, addBufferToUsage, filterUsageForFormat, COLORS, } from "./usageTracking.ts"; import { parseSSELine, hasValuableContent, fixInvalidId, formatSSE, unwrapGeminiChunk, } from "./streamHelpers.ts"; import { calculateCost } from "@/lib/usage/costCalculator"; import { buildOmniRouteSseMetadataComment } from "@/domain/omnirouteResponseMeta"; import { createStructuredSSECollector, buildStreamSummaryFromEvents, } from "./streamPayloadCollector.ts"; import { STREAM_IDLE_TIMEOUT_MS, FETCH_BODY_TIMEOUT_MS, HTTP_STATUS } from "../config/constants.ts"; import { sanitizeStreamingChunk, extractThinkingFromContent, } from "../handlers/responseSanitizer.ts"; import { buildErrorBody } from "./error.ts"; /** * Race a response body read against a timeout. * Prevents indefinite hangs when the upstream sends headers but stalls on the body. */ export function withBodyTimeout( promise: Promise, timeoutMs: number = FETCH_BODY_TIMEOUT_MS ): Promise { if (timeoutMs <= 0) return promise; let timer: ReturnType; const timeout = new Promise((_, reject) => { timer = setTimeout(() => { const err = new Error(`Response body read timeout after ${timeoutMs}ms`); err.name = "BodyTimeoutError"; reject(err); }, timeoutMs); }); return Promise.race([promise, timeout]).finally(() => clearTimeout(timer)) as Promise; } export { COLORS, formatSSE }; type JsonRecord = Record; export const PENDING_REQUEST_CLEARED_MARKER = "__omniroutePendingRequestCleared"; function markPendingRequestCleared(error: Error): Error { (error as Error & Record)[PENDING_REQUEST_CLEARED_MARKER] = true; return error; } function buildResponsesOutputItemKey(item: unknown): string | null { if (!item || typeof item !== "object" || Array.isArray(item)) { return null; } const record = item as JsonRecord; const type = typeof record.type === "string" ? record.type : ""; const id = typeof record.id === "string" ? record.id : ""; const callId = typeof record.call_id === "string" ? record.call_id : ""; const outputIndex = typeof record.output_index === "number" ? record.output_index : ""; const name = typeof record.name === "string" ? record.name : ""; if (!type && !id && !callId) { return null; } return `${type}:${id}:${callId}:${outputIndex}:${name}`; } function pushUniqueResponsesOutputItems(target: unknown[], items: readonly unknown[]) { const seen = new Set(); for (const existingItem of target) { const key = buildResponsesOutputItemKey(existingItem); if (key) { seen.add(key); } } for (const item of items) { const key = buildResponsesOutputItemKey(item); if (key && seen.has(key)) { continue; } target.push(item); if (key) { seen.add(key); } } } type StreamLogger = { appendProviderChunk?: (value: string) => void; appendConvertedChunk?: (value: string) => void; appendOpenAIChunk?: (value: string) => void; }; type StreamCompletePayload = { status: number; usage: unknown; /** Minimal response body for call log (streaming: usage + note; non-streaming not used) */ responseBody?: unknown; providerPayload?: unknown; clientPayload?: unknown; }; type StreamFailurePayload = { status: number; message: string; code?: string; type?: string; }; type StreamOptions = { mode?: string; targetFormat?: string; sourceFormat?: string; clientResponseFormat?: string | null; copilotCompatibleReasoning?: boolean; provider?: string | null; reqLogger?: StreamLogger | null; toolNameMap?: unknown; model?: string | null; connectionId?: string | null; apiKeyInfo?: unknown; body?: unknown; onComplete?: ((payload: StreamCompletePayload) => void) | null; onFailure?: ((payload: StreamFailurePayload) => void | Promise) | null; }; type TranslateState = ReturnType & { provider?: string | null; toolNameMap?: unknown; signatureNamespace?: string | null; usage?: unknown; finishReason?: unknown; copilotCompatibleReasoning?: boolean; /** Accumulated message content for call log response body */ accumulatedContent?: string; upstreamError?: { status: number; type: string; code: string; message: string; } | null; }; type ToolCall = { id: string | null; index: number; type: string; function: { name: string; arguments: string }; }; type UsageTokenRecord = Record; function asRecord(value: unknown): JsonRecord { return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {}; } const STREAM_SUMMARY_TEXT_LIMIT = 64 * 1024; function appendBoundedText(current: string, next: string): string { if (!next) return current; const combined = current + next; if (combined.length <= STREAM_SUMMARY_TEXT_LIMIT) return combined; return combined.slice(-STREAM_SUMMARY_TEXT_LIMIT); } function stripZeroWidth(value: unknown): unknown { if (typeof value === "string") { return value.replace(/[\u200B-\u200D\uFEFF]/g, ""); } if (Array.isArray(value)) { return value.map((item) => stripZeroWidth(item)); } if (value && typeof value === "object") { return Object.fromEntries( Object.entries(value as Record).map(([key, item]) => [ key, stripZeroWidth(item), ]) ); } return value; } function parseTextualToolCallCandidate( text: unknown ): { kind: "complete"; name: string; args: unknown } | { kind: "partial" } | null { if (typeof text !== "string") return null; const normalized = text.replace(/[\u200B-\u200D\uFEFF]/g, ""); const toolCallIndex = normalized.lastIndexOf("[Tool call:"); if (toolCallIndex < 0) return null; const candidate = normalized.slice(toolCallIndex); const headerMatch = candidate.match(/^\[Tool call:\s*([^\]\n]+)\]\s*\nArguments:\s*/); if (!headerMatch) return { kind: "partial" }; const name = headerMatch[1]?.trim(); const rawArgs = candidate.slice(headerMatch[0].length).trim(); if (!name || !rawArgs) return { kind: "partial" }; const decoders = [ (value: string) => value, (value: string) => { if (value.startsWith('"') && value.endsWith('"')) { const decoded = JSON.parse(value); return typeof decoded === "string" ? decoded : value; } return value; }, ]; for (const decode of decoders) { try { const decoded = decode(rawArgs); const parsed = JSON.parse(decoded); return { kind: "complete", name, args: stripZeroWidth(parsed) }; } catch {} } return { kind: "partial" }; } function parseTextualToolCallFromContent(text: unknown): { name: string; args: unknown } | null { const candidate = parseTextualToolCallCandidate(text); return candidate?.kind === "complete" ? { name: candidate.name, args: candidate.args } : null; } function containsTextualToolCallCandidate(text: unknown): boolean { return parseTextualToolCallCandidate(text) !== null; } function containsMalformedTextualToolCall(text: unknown): boolean { if (typeof text !== "string") return false; return text.replace(/[\u200B-\u200D\uFEFF]/g, "").includes("[Tool call:"); } function extractAllowedToolNames(body: unknown): Set | null { const record = asRecord(body); const tools = record.tools; if (!Array.isArray(tools)) return null; const names = new Set(); for (const tool of tools) { if (!tool || typeof tool !== "object" || Array.isArray(tool)) continue; const item = tool as JsonRecord; const directName = typeof item.name === "string" ? item.name.trim() : ""; const fn = item.function && typeof item.function === "object" && !Array.isArray(item.function) ? (item.function as JsonRecord) : null; const functionName = typeof fn?.name === "string" ? fn.name.trim() : ""; const name = functionName || directName; if (name) names.add(name); } return names.size > 0 ? names : null; } function collectPassthroughTextualToolCall( text: string, toolCalls: Map, allowedToolNames?: Set | null ): ToolCall | null { const parsed = parseTextualToolCallFromContent(text); if (!parsed) return null; if (allowedToolNames?.size && !allowedToolNames.has(parsed.name)) return null; const key = `textual:${toolCalls.size}`; const toolCall: ToolCall = { id: `call_${Date.now()}_${toolCalls.size}`, index: toolCalls.size, type: "function", function: { name: parsed.name, arguments: JSON.stringify(parsed.args || {}), }, }; toolCalls.set(key, toolCall); return toolCall; } function toStreamingToolCallDelta(toolCall: ToolCall) { return { index: toolCall.index, id: toolCall.id, type: toolCall.type, function: { name: toolCall.function.name, arguments: toolCall.function.arguments, }, }; } function toResponsesFunctionCallItem(toolCall: ToolCall) { return { type: "function_call", id: toolCall.id || `fc_${toolCall.index}`, call_id: toolCall.id || `call_${toolCall.index}`, name: toolCall.function.name, arguments: toolCall.function.arguments, status: "completed", }; } function buildResponsesFunctionCallEvents(toolCall: ToolCall) { const item = toResponsesFunctionCallItem(toolCall); return [ { type: "response.output_item.added", output_index: toolCall.index, item, }, { type: "response.function_call_arguments.done", item_id: item.id, output_index: toolCall.index, arguments: toolCall.function.arguments, }, { type: "response.output_item.done", output_index: toolCall.index, item, }, ]; } function formatSSEDataEvents(events: unknown[]) { return events.map((event) => `data: ${JSON.stringify(event)}\n`).join("\n"); } function toChatCompletionChunkWithToolCall(base: JsonRecord, toolCall: ToolCall) { const choice = asRecord(Array.isArray(base.choices) ? base.choices[0] : null); const delta = { ...asRecord(choice.delta) }; delete delta.content; delete delta.reasoning_content; return { ...base, choices: [ { ...choice, index: typeof choice.index === "number" ? choice.index : 0, delta: { ...delta, tool_calls: [toStreamingToolCallDelta(toolCall)], }, finish_reason: null, }, ], }; } function toResponsesCompletedWithToolCalls(parsed: JsonRecord, toolCalls: ToolCall[]) { const response = asRecord(parsed.response); const existingOutput = Array.isArray(response.output) ? response.output : []; return { ...parsed, response: { ...response, output: [ ...existingOutput, ...toolCalls.map((toolCall) => toResponsesFunctionCallItem(toolCall)), ], }, }; } function toStreamFailureStatus(value: unknown): number | null { if (typeof value === "number" && Number.isInteger(value) && value >= 400 && value <= 599) { return value; } if (typeof value === "string" && /^\d{3}$/.test(value.trim())) { const parsed = Number(value.trim()); return parsed >= 400 && parsed <= 599 ? parsed : null; } return null; } function looksLikeStreamRateLimit(code: string, type: string, message: string): boolean { const haystack = `${code} ${type} ${message}`.toLowerCase(); return ( haystack.includes("usage_limit_reached") || haystack.includes("rate_limit") || haystack.includes("rate limit") || haystack.includes("quota") || haystack.includes("too many requests") || haystack.includes("limit reached") || haystack.includes("limit has been reached") ); } function normalizeStreamFailurePayload(payload: unknown): StreamFailurePayload | null { const record = payload && typeof payload === "object" ? (payload as JsonRecord) : {}; const response = asRecord(record.response); const error = Object.keys(asRecord(response.error)).length ? asRecord(response.error) : Object.keys(asRecord(record.error)).length ? asRecord(record.error) : record; const code = typeof error.code === "string" ? error.code : "upstream_error"; const type = typeof error.type === "string" ? error.type : undefined; const message = typeof error.message === "string" && error.message.trim() ? error.message : typeof record.message === "string" && record.message.trim() ? record.message : "Upstream failure"; const status = toStreamFailureStatus(error.status_code) ?? toStreamFailureStatus(error.status) ?? toStreamFailureStatus(response.status_code) ?? toStreamFailureStatus(response.status) ?? toStreamFailureStatus(record.status_code) ?? toStreamFailureStatus(record.status) ?? (looksLikeStreamRateLimit(code, type || "", message) ? 429 : 502); return { status, message, code, ...(type ? { type } : {}), }; } type ClaudeEmptyResponseLifecycle = { hasMessageStart: boolean; hasContentBlock: boolean; hasMessageDelta: boolean; hasMessageStop: boolean; hasError: boolean; syntheticContentInjected: boolean; warningLogged: boolean; }; const SYNTHETIC_CLAUDE_EMPTY_RESPONSE_TEXT = "[Proxy Error] The upstream API returned an empty response. Please retry the request."; function createClaudeEmptyResponseLifecycle(): ClaudeEmptyResponseLifecycle { return { hasMessageStart: false, hasContentBlock: false, hasMessageDelta: false, hasMessageStop: false, hasError: false, syntheticContentInjected: false, warningLogged: false, }; } function getClaudeEventType(payload: unknown): string | null { if (!payload || typeof payload !== "object") return null; const type = (payload as JsonRecord).type; return typeof type === "string" ? type : null; } function isClaudeEventPayload(payload: unknown): payload is JsonRecord { return getClaudeEventType(payload) !== null; } function updateClaudeEmptyResponseLifecycle( lifecycle: ClaudeEmptyResponseLifecycle, payload: unknown ) { const type = getClaudeEventType(payload); if (!type) return; switch (type) { case "message_start": lifecycle.hasMessageStart = true; break; case "content_block_start": case "content_block_delta": case "content_block_stop": lifecycle.hasContentBlock = true; break; case "message_delta": lifecycle.hasMessageDelta = true; break; case "message_stop": lifecycle.hasMessageStop = true; break; case "error": lifecycle.hasError = true; break; default: break; } } function hasClaudeAssistantLifecycle(lifecycle: ClaudeEmptyResponseLifecycle): boolean { return lifecycle.hasMessageStart || lifecycle.hasMessageDelta || lifecycle.hasMessageStop; } function shouldInjectClaudeEmptyResponseBeforeCurrentEvent( lifecycle: ClaudeEmptyResponseLifecycle, payload: unknown ): boolean { const type = getClaudeEventType(payload); if (!type || lifecycle.hasError || lifecycle.hasContentBlock) return false; if (!hasClaudeAssistantLifecycle(lifecycle)) return false; return type === "message_delta" || type === "message_stop"; } function shouldInjectClaudeEmptyResponseOnFlush(lifecycle: ClaudeEmptyResponseLifecycle): boolean { if (lifecycle.hasError || lifecycle.hasContentBlock) return false; return hasClaudeAssistantLifecycle(lifecycle); } function shouldInjectClaudeMissingFinalizersOnFlush( lifecycle: ClaudeEmptyResponseLifecycle ): boolean { if (lifecycle.hasError || !lifecycle.syntheticContentInjected) return false; return !lifecycle.hasMessageDelta || !lifecycle.hasMessageStop; } function buildSyntheticClaudeEmptyResponseEvents( lifecycle: ClaudeEmptyResponseLifecycle, model: string | null, options: { includeContentBlock?: boolean; includeMessageDelta?: boolean; includeMessageStop?: boolean; } = {} ): JsonRecord[] { const { includeContentBlock = true, includeMessageDelta = false, includeMessageStop = false, } = options; const events: JsonRecord[] = []; const resolvedModel = typeof model === "string" && model ? model : "unknown"; if (includeContentBlock) { if (!lifecycle.hasMessageStart) { events.push({ type: "message_start", message: { id: `msg_synthetic_${Date.now()}`, type: "message", role: "assistant", model: resolvedModel, content: [], stop_reason: null, stop_sequence: null, usage: { input_tokens: 0, output_tokens: 0 }, }, }); } events.push( { type: "content_block_start", index: 0, content_block: { type: "text", text: "" }, }, { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: SYNTHETIC_CLAUDE_EMPTY_RESPONSE_TEXT, }, }, { type: "content_block_stop", index: 0, } ); } if (includeMessageDelta) { events.push({ type: "message_delta", delta: { stop_reason: "end_turn", stop_sequence: null }, usage: { input_tokens: 0, output_tokens: 0 }, }); } if (includeMessageStop) { events.push({ type: "message_stop" }); } return events; } function getOpenAIIntermediateChunks(value: unknown): unknown[] { if (!value || typeof value !== "object") return []; const candidate = (value as JsonRecord)._openaiIntermediate; return Array.isArray(candidate) ? candidate : []; } function restoreClaudePassthroughToolUseName(parsed: JsonRecord, toolNameMap: unknown): boolean { if (!(toolNameMap instanceof Map)) return false; if (!parsed || typeof parsed !== "object") return false; const block = parsed.content_block && typeof parsed.content_block === "object" ? (parsed.content_block as JsonRecord) : null; if (!block || block.type !== "tool_use" || typeof block.name !== "string") return false; const restoredName = toolNameMap.get(block.name) ?? block.name; if (restoredName === block.name) return false; block.name = restoredName; return true; } // Note: TextDecoder/TextEncoder are created per-stream inside createSSEStream() // to avoid shared state issues with concurrent streams (TextDecoder with {stream:true} // maintains internal buffering state between decode() calls). /** * Stream modes */ const STREAM_MODE = { TRANSLATE: "translate", // Full translation between formats PASSTHROUGH: "passthrough", // No translation, normalize output, extract usage }; /** * Lifecycle event types in OpenAI Responses API streams whose `response` * payload is a snapshot of the request (echoes back `instructions` + `tools`). */ const RESPONSES_LIFECYCLE_EVENT_TYPES = new Set([ "response.created", "response.in_progress", "response.completed", ]); /** * Backfill `parsed.response.output` on a `response.completed` event from the * snapshots accumulated as the stream progressed (`response.output_item.done`). * * Why: when the upstream request runs with `store: false`, OpenAI's Responses * API leaves `response.output` empty in the final `response.completed` * snapshot — clients that rebuild assistant messages from that snapshot * (notably the GitHub Copilot CLI 1.0.36) end up with `choices: []` and never * trigger tool execution. Codex CLI and others that consume per-item events * are unaffected; backfilling the array makes both styles work. * * Returns true when `parsed.response.output` was empty and got replaced, so * the caller can re-serialize. */ export function backfillResponsesCompletedOutput( parsed: unknown, collectedItems: readonly unknown[] ): boolean { if (!collectedItems.length) return false; if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return false; const obj = parsed as Record; if (obj.type !== "response.completed") return false; const resp = obj.response; if (!resp || typeof resp !== "object" || Array.isArray(resp)) return false; const r = resp as Record; const existing = r.output; if (Array.isArray(existing) && existing.length > 0) return false; r.output = collectedItems.slice(); return true; } /** * Strip the request echo (`instructions`, `tools`) from `parsed.response` * on Responses API lifecycle events. * * Why: those fields can balloon the SSE message past 100 KB when the request * carries large tool definitions / instructions. Some clients (notably the * GitHub Copilot CLI) cannot process oversized SSE events and stop rendering * mid-stream. The fields are pure echo of the original request — clients * already hold the original locally — so removing them is observably safe. * * Returns true when the payload was modified and must be re-serialized. */ export function stripResponsesLifecycleEcho(parsed: unknown): boolean { if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return false; const obj = parsed as Record; if (typeof obj.type !== "string" || !RESPONSES_LIFECYCLE_EVENT_TYPES.has(obj.type)) { return false; } const resp = obj.response; if (!resp || typeof resp !== "object" || Array.isArray(resp)) return false; const r = resp as Record; let changed = false; if ("instructions" in r) { delete r.instructions; changed = true; } if ("tools" in r) { delete r.tools; changed = true; } return changed; } /** * Create unified SSE transform stream with idle timeout protection. * If the upstream provider stops sending data for STREAM_IDLE_TIMEOUT_MS, * the stream emits an error event and closes to prevent indefinite hanging. * * @param {object} options * @param {string} options.mode - Stream mode: translate, passthrough * @param {string} options.targetFormat - Provider format (for translate mode) * @param {string} options.sourceFormat - Client format (for translate mode) * @param {string} options.provider - Provider name * @param {object} options.reqLogger - Request logger instance * @param {string} options.model - Model name * @param {string} options.connectionId - Connection ID for usage tracking * @param {object|null} options.apiKeyInfo - API key metadata for usage attribution * @param {object} options.body - Request body (for input token estimation) * @param {function} options.onComplete - Callback when stream finishes: ({ status, usage }) => void */ export function createSSEStream(options: StreamOptions = {}) { const { mode = STREAM_MODE.TRANSLATE, targetFormat, sourceFormat, clientResponseFormat = null, copilotCompatibleReasoning = false, provider = null, reqLogger = null, toolNameMap = null, model = null, connectionId = null, apiKeyInfo = null, body = null, onComplete = null, onFailure = null, } = options; const signatureNamespace = connectionId; const clientExpectsResponsesStream = (mode === STREAM_MODE.PASSTHROUGH ? clientResponseFormat === FORMATS.OPENAI_RESPONSES : sourceFormat === FORMATS.OPENAI_RESPONSES) === true; // Clients whose SSE protocol terminates naturally on the last // provider-shape event (not on a `data: [DONE]` line). Emitting // `[DONE]` to these clients produces a parser error in the SDK and // breaks follow-up turns (Capy/Anthropic SDK: text gets stuck in the // "Thought" area; subsequent /v1/messages calls retry into a corrupt // state). Skip the `[DONE]` for these formats. const clientExpectsClaudeStream = (mode === STREAM_MODE.PASSTHROUGH ? clientResponseFormat === FORMATS.CLAUDE : sourceFormat === FORMATS.CLAUDE) === true; // Single source of truth for the [DONE] decision, used at both emission // sites below. Only OpenAI Chat Completions clients expect [DONE]; // Responses API and Anthropic SSE terminate on their own protocol events // (response.completed / message_stop respectively). const shouldEmitDoneTerminator = !clientExpectsResponsesStream && !clientExpectsClaudeStream; let buffer = ""; let usage: UsageTokenRecord | null = null; /** Passthrough (OpenAI CC shape): saw tool_calls in stream before finish_reason */ let passthroughHasToolCalls = false; /** Passthrough: accumulate tool_calls deltas for call log responseBody */ const passthroughToolCalls = new Map(); let passthroughToolCallSeq = 0; const allowedToolNames = extractAllowedToolNames(body); let skipPassthroughEvent = false; // State for translate mode (accumulatedContent for call log response body) const state: TranslateState | null = mode === STREAM_MODE.TRANSLATE ? { ...(initState(sourceFormat) as TranslateState), provider, toolNameMap, signatureNamespace, copilotCompatibleReasoning, accumulatedContent: "", } : null; // Track content length for usage estimation (both modes) let totalContentLength = 0; // Passthrough: accumulate content and reasoning separately for call log response body let passthroughAccumulatedContent = ""; let passthroughAccumulatedReasoning = ""; let passthroughBufferedTextualToolCallContent = ""; // Passthrough Responses SSE: snapshots of items seen via `response.output_item.done`, // used to backfill `response.completed.response.output` when upstream returns it // empty (which happens when `store: false` — see backfillResponsesCompletedOutput). const passthroughResponsesOutputItems: unknown[] = []; const passthroughResponsesPendingFunctionCalls = new Map(); let passthroughResponsesId: string | null = null; let passthroughResponsesCurrentFunctionCallKey: string | null = null; const passthroughResponsesReasoningSummarySeen = new Set(); const streamStartedAt = Date.now(); // Guard against duplicate [DONE] events — ensures exactly one per stream let doneSent = false; const providerPayloadCollector = createStructuredSSECollector({ stage: "provider_response", }); const clientPayloadCollector = createStructuredSSECollector({ stage: "client_response", }); // Per-stream instances to avoid shared state with concurrent streams const decoder = new TextDecoder(); const encoder = new TextEncoder(); // Idle timeout state — closes stream if provider stops sending data let lastChunkTime = Date.now(); let idleTimer: ReturnType | null = null; let streamTimedOut = false; const claudeEmptyResponseLifecycle = createClaudeEmptyResponseLifecycle(); let pendingPassthroughEventLine: string | null = null; let pendingPassthroughEventEmitted = false; const clearIdleTimer = () => { if (idleTimer) { clearInterval(idleTimer); idleTimer = null; } }; const clearPendingPassthroughEvent = () => { pendingPassthroughEventLine = null; pendingPassthroughEventEmitted = false; }; const maybePrefixPendingPassthroughEvent = (output: string, line: string) => { if (!pendingPassthroughEventLine || !line.startsWith("data:")) { return output; } if (!pendingPassthroughEventEmitted) { pendingPassthroughEventEmitted = true; return `${pendingPassthroughEventLine}\n${output}`; } return output; }; const applyTextualToolCallStreamingGuard = (parsed: Record) => { const choice = Array.isArray((parsed as JsonRecord).choices) ? (((parsed as JsonRecord).choices as unknown[])[0] as JsonRecord | undefined) : undefined; const delta = asRecord(choice?.delta); let textualToolCallConverted = false; if (typeof delta?.content === "string") { const incomingContent = delta.content; const bufferedCandidate = passthroughBufferedTextualToolCallContent + incomingContent; if ( passthroughBufferedTextualToolCallContent || containsTextualToolCallCandidate(incomingContent) ) { const parsedCandidate = parseTextualToolCallCandidate(bufferedCandidate); if (parsedCandidate?.kind === "complete") { const collectedToolCall = collectPassthroughTextualToolCall( bufferedCandidate, passthroughToolCalls, allowedToolNames ); if (collectedToolCall) { parsed = toChatCompletionChunkWithToolCall( parsed as JsonRecord, collectedToolCall ) as typeof parsed; passthroughHasToolCalls = true; } else { delete delta.content; delete delta.reasoning_content; } textualToolCallConverted = true; passthroughBufferedTextualToolCallContent = ""; } else if (parsedCandidate?.kind === "partial") { passthroughBufferedTextualToolCallContent = appendBoundedText( passthroughBufferedTextualToolCallContent, incomingContent ); textualToolCallConverted = true; delta.content = ""; } else { passthroughAccumulatedContent = appendBoundedText( passthroughAccumulatedContent, passthroughBufferedTextualToolCallContent + incomingContent ); passthroughBufferedTextualToolCallContent = ""; } } else { passthroughAccumulatedContent = appendBoundedText( passthroughAccumulatedContent, incomingContent ); } } return { parsed, textualToolCallConverted }; }; const emitSyntheticClaudeEmptyResponse = ( controller: TransformStreamDefaultController, options: { includeContentBlock?: boolean; includeMessageDelta?: boolean; includeMessageStop?: boolean; } = {} ) => { const events = buildSyntheticClaudeEmptyResponseEvents( claudeEmptyResponseLifecycle, model, options ); if (events.length === 0) return; if (!claudeEmptyResponseLifecycle.warningLogged) { claudeEmptyResponseLifecycle.warningLogged = true; console.warn( `[STREAM] Injecting synthetic Claude SSE response for empty upstream output (${provider || "provider"}:${model || "unknown"})` ); } if (options.includeContentBlock !== false) { claudeEmptyResponseLifecycle.syntheticContentInjected = true; if (!passthroughAccumulatedContent.trim()) { passthroughAccumulatedContent = SYNTHETIC_CLAUDE_EMPTY_RESPONSE_TEXT; } if (state?.accumulatedContent !== undefined && !state.accumulatedContent.trim()) { state.accumulatedContent = SYNTHETIC_CLAUDE_EMPTY_RESPONSE_TEXT; } } for (const event of events) { updateClaudeEmptyResponseLifecycle(claudeEmptyResponseLifecycle, event); clientPayloadCollector.push(event); const output = formatSSE(event, FORMATS.CLAUDE); reqLogger?.appendConvertedChunk?.(output); controller.enqueue(encoder.encode(output)); } }; const emitTranslatedClientItem = ( controller: TransformStreamDefaultController, item: Record ) => { let itemSanitized: Record = item; const isResponsesEvent = typeof item?.event === "string" && item.event.startsWith("response."); if (sourceFormat === FORMATS.OPENAI && !isResponsesEvent) { itemSanitized = sanitizeStreamingChunk(itemSanitized) as Record; const delta = itemSanitized?.choices?.[0]?.delta; if (delta?.content && typeof delta.content === "string") { const { content, thinking } = extractThinkingFromContent(delta.content); delta.content = content; if (thinking && !delta.reasoning_content) { delta.reasoning_content = thinking; } } } if (!hasValuableContent(itemSanitized, sourceFormat)) { return; } const isFinishChunk = itemSanitized.type === "message_delta" || itemSanitized.choices?.[0]?.finish_reason; if ( state?.finishReason && isFinishChunk && !hasValidUsage(itemSanitized.usage) && totalContentLength > 0 ) { const estimated = estimateUsage(body, totalContentLength, sourceFormat); itemSanitized.usage = filterUsageForFormat(estimated, sourceFormat); state.usage = estimated; } else if (state?.finishReason && isFinishChunk && state.usage) { const buffered = addBufferToUsage(state.usage); itemSanitized.usage = filterUsageForFormat(buffered, sourceFormat); } if ( sourceFormat === FORMATS.CLAUDE && shouldInjectClaudeEmptyResponseBeforeCurrentEvent(claudeEmptyResponseLifecycle, itemSanitized) ) { const eventType = getClaudeEventType(itemSanitized); emitSyntheticClaudeEmptyResponse(controller, { includeContentBlock: true, includeMessageDelta: eventType === "message_stop" && !claudeEmptyResponseLifecycle.hasMessageDelta, includeMessageStop: false, }); } if (sourceFormat === FORMATS.CLAUDE && isClaudeEventPayload(itemSanitized)) { updateClaudeEmptyResponseLifecycle(claudeEmptyResponseLifecycle, itemSanitized); } const output = formatSSE(itemSanitized, sourceFormat); clientPayloadCollector.push(itemSanitized); reqLogger?.appendConvertedChunk?.(output); controller.enqueue(encoder.encode(output)); }; const emitFinalSseMetadata = async ( controller: TransformStreamDefaultController, finalUsage: UsageTokenRecord | Record | null | undefined ) => { const costUsd = finalUsage ? await calculateCost(provider, model, finalUsage) : 0; const comment = buildOmniRouteSseMetadataComment({ provider, model, cacheHit: false, latencyMs: Date.now() - streamStartedAt, usage: finalUsage, costUsd, }); if (!comment) return; reqLogger?.appendConvertedChunk?.(comment); controller.enqueue(encoder.encode(comment)); }; const getResponsesReasoningKey = (payload: Record): string | null => { if (typeof payload.item_id === "string" && payload.item_id) { return payload.item_id; } const item = payload.item && typeof payload.item === "object" && !Array.isArray(payload.item) ? (payload.item as Record) : null; if (item && typeof item.id === "string" && item.id) { return item.id; } const responseId = typeof payload.response_id === "string" && payload.response_id ? payload.response_id : passthroughResponsesId; const outputIndex = typeof payload.output_index === "number" && Number.isInteger(payload.output_index) ? payload.output_index : null; return responseId !== null && outputIndex !== null ? `${responseId}:${outputIndex}` : null; }; const getResponsesReasoningSummaryText = (item: Record): string => { return Array.isArray(item.summary) ? item.summary .map((part) => { if (!part || typeof part !== "object" || Array.isArray(part)) { return ""; } return typeof (part as Record).text === "string" ? ((part as Record).text as string) : ""; }) .join("") : ""; }; const ensureVisibleResponsesReasoningSummary = (payload: Record): boolean => { const item = payload.item && typeof payload.item === "object" && !Array.isArray(payload.item) ? (payload.item as Record) : null; if (!item || item.type !== "reasoning") { return false; } if (getResponsesReasoningSummaryText(item)) { return false; } const hasEncryptedReasoning = typeof item.encrypted_content === "string" && item.encrypted_content.length > 0; if (!hasEncryptedReasoning) { return false; } item.summary = [ { type: "summary_text", text: "Codex is reasoning, but the upstream Responses API exposed this reasoning block only as encrypted state. OmniRoute cannot recover the private reasoning text.", }, ]; return true; }; const emitSyntheticResponsesReasoningSummary = ( controller: TransformStreamDefaultController, payload: Record ) => { const item = payload.item && typeof payload.item === "object" && !Array.isArray(payload.item) ? (payload.item as Record) : null; if (!item || item.type !== "reasoning") { return; } ensureVisibleResponsesReasoningSummary(payload); const visibleSummary = getResponsesReasoningSummaryText(item); if (!visibleSummary) { return; } const reasoningKey = getResponsesReasoningKey(payload); if (!reasoningKey || passthroughResponsesReasoningSummarySeen.has(reasoningKey)) { return; } passthroughResponsesReasoningSummarySeen.add(reasoningKey); const itemId = typeof item.id === "string" && item.id ? item.id : reasoningKey; const outputIndex = typeof payload.output_index === "number" && Number.isInteger(payload.output_index) ? payload.output_index : 0; const syntheticEvents = [ { event: "response.reasoning_summary_text.delta", body: { type: "response.reasoning_summary_text.delta", item_id: itemId, output_index: outputIndex, summary_index: 0, delta: visibleSummary, }, }, { event: "response.reasoning_summary_part.done", body: { type: "response.reasoning_summary_part.done", item_id: itemId, output_index: outputIndex, summary_index: 0, part: { type: "summary_text", text: visibleSummary }, }, }, ]; for (const syntheticEvent of syntheticEvents) { clientPayloadCollector.push(syntheticEvent.body); const output = `event: ${syntheticEvent.event}\ndata: ${JSON.stringify(syntheticEvent.body)}\n\n`; reqLogger?.appendConvertedChunk?.(output); controller.enqueue(encoder.encode(output)); } }; return new TransformStream( { start(controller) { // Start idle watchdog — checks every 10s if provider has stopped sending if (STREAM_IDLE_TIMEOUT_MS > 0) { idleTimer = setInterval(() => { if (!streamTimedOut && Date.now() - lastChunkTime > STREAM_IDLE_TIMEOUT_MS) { streamTimedOut = true; clearIdleTimer(); const timeoutMsg = `[STREAM] Idle timeout: no data from ${provider || "provider"} for ${STREAM_IDLE_TIMEOUT_MS}ms (model: ${model || "unknown"})`; console.warn(timeoutMsg); trackPendingRequest(model, provider, connectionId, false); appendRequestLog({ model, provider, connectionId, status: `FAILED ${HTTP_STATUS.GATEWAY_TIMEOUT}`, }).catch(() => {}); const timeoutError = new Error(timeoutMsg); timeoutError.name = "StreamIdleTimeoutError"; controller.error(markPendingRequestCleared(timeoutError)); } }, 10_000); } }, transform(chunk, controller) { if (streamTimedOut) return; lastChunkTime = Date.now(); const text = decoder.decode(chunk, { stream: true }); buffer += text; reqLogger?.appendProviderChunk?.(text); const lines = buffer.split("\n"); buffer = lines.pop() || ""; for (const line of lines) { const trimmed = line.trim(); // Passthrough mode: normalize and forward if (mode === STREAM_MODE.PASSTHROUGH) { let output; let injectedUsage = false; let clientPayload: unknown = null; let failurePayload: StreamFailurePayload | null = null; if (skipPassthroughEvent) { if (!trimmed) { skipPassthroughEvent = false; clearPendingPassthroughEvent(); } continue; } // Drop whole keepalive event blocks — strict OpenAI-compatible SDKs // try to JSON.parse empty keepalive payloads and crash. if (/^event:\s*keepalive\b/i.test(trimmed)) { skipPassthroughEvent = true; clearPendingPassthroughEvent(); continue; } if (/^event:/i.test(trimmed)) { if (pendingPassthroughEventLine && !pendingPassthroughEventEmitted) { const pendingOutput = `${pendingPassthroughEventLine}\n`; reqLogger?.appendConvertedChunk?.(pendingOutput); controller.enqueue(encoder.encode(pendingOutput)); } const eventType = trimmed.replace(/^event:\s*/i, ""); if ( shouldInjectClaudeEmptyResponseBeforeCurrentEvent(claudeEmptyResponseLifecycle, { type: eventType, }) ) { emitSyntheticClaudeEmptyResponse(controller, { includeContentBlock: true, includeMessageDelta: eventType === "message_stop" && !claudeEmptyResponseLifecycle.hasMessageDelta, includeMessageStop: false, }); } pendingPassthroughEventLine = line; pendingPassthroughEventEmitted = false; continue; } if (trimmed.startsWith("data:")) { const providerPayload = parseSSELine(trimmed); if (providerPayload) { providerPayloadCollector.push(providerPayload); if ((providerPayload as { done?: unknown }).done === true) { continue; } } } if (trimmed.startsWith("data:") && trimmed.slice(5).trim() === "[DONE]") { continue; } if (trimmed.startsWith("data:") && trimmed.slice(5).trim() !== "[DONE]") { try { let parsed = JSON.parse(trimmed.slice(5).trim()); // Detect Responses SSE payloads (have a `type` field like "response.created", // "response.output_item.added", etc.) and skip Chat Completions-specific // sanitization to avoid corrupting the stream for Responses-native clients. const isResponsesSSE = parsed.type && typeof parsed.type === "string" && parsed.type.startsWith("response."); // Detect Claude SSE payloads. Includes "ping" and "error" to ensure // they bypass the Chat Completions sanitization path which would // incorrectly process or drop them. const isClaudeSSE = parsed.type && typeof parsed.type === "string" && (parsed.type.startsWith("message") || parsed.type.startsWith("content_block") || parsed.type === "ping" || parsed.type === "error"); if (isResponsesSSE) { const responseId = typeof parsed.response?.id === "string" ? parsed.response.id : typeof parsed.response_id === "string" ? parsed.response_id : null; if (responseId) { passthroughResponsesId = responseId; } // Responses SSE: only extract usage, forward payload as-is const extracted = extractUsage(parsed); if (extracted) { usage = extracted; } // Keep generic Responses deltas for fallback usage estimates, // but only visible text deltas may become assistant content in // logs/replay payloads. if (typeof parsed.delta === "string") { totalContentLength += parsed.delta.length; } if ( parsed.type === "response.output_text.delta" && typeof parsed.delta === "string" ) { const incomingDelta = parsed.delta; const bufferedCandidate = passthroughBufferedTextualToolCallContent + incomingDelta; if ( passthroughBufferedTextualToolCallContent || containsTextualToolCallCandidate(incomingDelta) ) { const parsedCandidate = parseTextualToolCallCandidate(bufferedCandidate); if (parsedCandidate?.kind === "complete") { const collectedToolCall = collectPassthroughTextualToolCall( bufferedCandidate, passthroughToolCalls, allowedToolNames ); if (collectedToolCall) { passthroughHasToolCalls = true; const responseToolCallEvents = buildResponsesFunctionCallEvents(collectedToolCall); output = formatSSEDataEvents(responseToolCallEvents); clientPayloadCollector.push(...responseToolCallEvents); reqLogger?.appendConvertedChunk?.(output); controller.enqueue(encoder.encode(output)); injectedUsage = true; } else { output = `data: ${JSON.stringify(parsed)} `; injectedUsage = true; } passthroughBufferedTextualToolCallContent = ""; parsed.delta = ""; } else if (parsedCandidate?.kind === "partial") { passthroughBufferedTextualToolCallContent = appendBoundedText( passthroughBufferedTextualToolCallContent, incomingDelta ); parsed.delta = ""; output = `data: ${JSON.stringify(parsed)}\n`; injectedUsage = true; } else { passthroughAccumulatedContent = appendBoundedText( passthroughAccumulatedContent, passthroughBufferedTextualToolCallContent + incomingDelta ); passthroughBufferedTextualToolCallContent = ""; } } else { passthroughAccumulatedContent = appendBoundedText( passthroughAccumulatedContent, incomingDelta ); } } if (parsed.type === "response.failed") { failurePayload = normalizeStreamFailurePayload(parsed); } if ( parsed.type === "response.reasoning_summary_text.delta" || parsed.type === "response.reasoning_summary_text.done" || parsed.type === "response.reasoning_summary_part.done" ) { const reasoningKey = getResponsesReasoningKey(parsed); if (reasoningKey) { passthroughResponsesReasoningSummarySeen.add(reasoningKey); } } if ( parsed.type === "response.output_item.added" && parsed.item?.type === "function_call" ) { const item = parsed.item && typeof parsed.item === "object" && !Array.isArray(parsed.item) ? { ...(parsed.item as JsonRecord) } : null; const pendingKey = item && typeof item.id === "string" ? item.id : item && typeof item.call_id === "string" ? item.call_id : null; if (item && pendingKey) { if (typeof item.arguments !== "string") { item.arguments = ""; } passthroughResponsesPendingFunctionCalls.set(pendingKey, item); passthroughResponsesCurrentFunctionCallKey = pendingKey; } } if (parsed.type === "response.function_call_arguments.delta") { const pendingKey = typeof parsed.item_id === "string" ? parsed.item_id : passthroughResponsesCurrentFunctionCallKey; const pending = pendingKey ? passthroughResponsesPendingFunctionCalls.get(pendingKey) : undefined; if (pending && typeof parsed.delta === "string") { const previousArgs = typeof pending.arguments === "string" ? pending.arguments : ""; pending.arguments = previousArgs + parsed.delta; } } if (parsed.type === "response.function_call_arguments.done") { const pendingKey = typeof parsed.item_id === "string" ? parsed.item_id : passthroughResponsesCurrentFunctionCallKey; const pending = pendingKey ? passthroughResponsesPendingFunctionCalls.get(pendingKey) : undefined; if (pending) { if (typeof parsed.arguments === "string") { pending.arguments = parsed.arguments; } pushUniqueResponsesOutputItems(passthroughResponsesOutputItems, [pending]); } } // Capture each completed output item so the final // response.completed snapshot can be backfilled when upstream // returns an empty `output` (happens with store: false). if (parsed.type === "response.output_item.done" && parsed.item) { const reasoningSummaryInjected = ensureVisibleResponsesReasoningSummary(parsed); emitSyntheticResponsesReasoningSummary(controller, parsed); pushUniqueResponsesOutputItems(passthroughResponsesOutputItems, [parsed.item]); if (reasoningSummaryInjected) { output = `data: ${JSON.stringify(parsed)}\n`; injectedUsage = true; } if (parsed.item?.type === "function_call") { const pendingKey = typeof parsed.item.id === "string" ? parsed.item.id : typeof parsed.item.call_id === "string" ? parsed.item.call_id : null; if (pendingKey) { passthroughResponsesPendingFunctionCalls.delete(pendingKey); if (passthroughResponsesCurrentFunctionCallKey === pendingKey) { passthroughResponsesCurrentFunctionCallKey = null; } } } } if ( parsed.type === "response.completed" && Array.isArray(parsed.response?.output) && parsed.response.output.length > 0 ) { pushUniqueResponsesOutputItems( passthroughResponsesOutputItems, parsed.response.output ); } if ( parsed.type === "response.completed" && passthroughResponsesPendingFunctionCalls.size > 0 ) { pushUniqueResponsesOutputItems(passthroughResponsesOutputItems, [ ...passthroughResponsesPendingFunctionCalls.values(), ]); passthroughResponsesPendingFunctionCalls.clear(); passthroughResponsesCurrentFunctionCallKey = null; } // Two transport-level fixes for Responses passthrough: // 1) Strip echoed `instructions` + `tools` from lifecycle // events — they can balloon a single SSE event past // 100 KB and break parsers (e.g. GitHub Copilot CLI). // 2) Backfill `response.completed.response.output` when // upstream sent it empty (store: false) — some clients // build their tool-call list from that snapshot rather // than from per-item events. const textualToolCallBackfilled = parsed.type === "response.completed" && passthroughToolCalls.size > 0; if (textualToolCallBackfilled) { parsed = toResponsesCompletedWithToolCalls(parsed as JsonRecord, [ ...passthroughToolCalls.values(), ]) as typeof parsed; } const stripped = stripResponsesLifecycleEcho(parsed); const backfilled = backfillResponsesCompletedOutput( parsed, passthroughResponsesOutputItems ); if (stripped || backfilled || textualToolCallBackfilled) { output = `data: ${JSON.stringify(parsed)}\n`; injectedUsage = true; } } else if (isClaudeSSE) { // Claude SSE: extract usage, track content, forward as-is const extracted = extractUsage(parsed); if (extracted) { // Non-destructive merge: never overwrite a positive value with 0 // message_start carries input_tokens, message_delta carries output_tokens; if (!usage) usage = {}; const u = usage; const eu = extracted as UsageTokenRecord; if (eu.prompt_tokens > 0) u.prompt_tokens = eu.prompt_tokens; if (eu.completion_tokens > 0) u.completion_tokens = eu.completion_tokens; if (eu.total_tokens > 0) u.total_tokens = eu.total_tokens; if (eu.cache_read_input_tokens) u.cache_read_input_tokens = eu.cache_read_input_tokens; if (eu.cache_creation_input_tokens) u.cache_creation_input_tokens = eu.cache_creation_input_tokens; } if ( shouldInjectClaudeEmptyResponseBeforeCurrentEvent( claudeEmptyResponseLifecycle, parsed ) ) { emitSyntheticClaudeEmptyResponse(controller, { includeContentBlock: true, includeMessageDelta: parsed.type === "message_stop" && !claudeEmptyResponseLifecycle.hasMessageDelta, includeMessageStop: false, }); } updateClaudeEmptyResponseLifecycle(claudeEmptyResponseLifecycle, parsed); const restoredToolName = restoreClaudePassthroughToolUseName(parsed, toolNameMap); // Track content length and accumulate from Claude format if (parsed.delta?.text) { totalContentLength += parsed.delta.text.length; passthroughAccumulatedContent = appendBoundedText( passthroughAccumulatedContent, parsed.delta.text ); } if (parsed.delta?.thinking) { totalContentLength += parsed.delta.thinking.length; passthroughAccumulatedContent = appendBoundedText( passthroughAccumulatedContent, parsed.delta.thinking ); } if (restoredToolName) { output = `data: ${JSON.stringify(parsed)}\n`; injectedUsage = true; } } else { // Chat Completions: full sanitization pipeline // Detect reasoning alias before sanitization strips it const hadReasoningAlias = !!( parsed.choices?.[0]?.delta?.reasoning && typeof parsed.choices[0].delta.reasoning === "string" && !parsed.choices[0].delta.reasoning_content ); parsed = sanitizeStreamingChunk(parsed); const idFixed = fixInvalidId(parsed); if (!hasValuableContent(parsed, FORMATS.OPENAI)) { continue; } const delta = parsed.choices?.[0]?.delta; let textualToolCallConverted = false; // Extract tags from streaming content if (delta?.content && typeof delta.content === "string") { const { content, thinking } = extractThinkingFromContent(delta.content); delta.content = content; if (thinking && !delta.reasoning_content) { delta.reasoning_content = thinking; } } // Split combined reasoning+content deltas into separate SSE events. // Standard OpenAI streaming never mixes both fields in one delta; // clients (e.g. LobeChat) may skip content when reasoning_content // is present, causing the first content token to be lost. if (delta?.reasoning_content && delta?.content) { const reasoningChunk = JSON.parse(JSON.stringify(parsed)); const rDelta = reasoningChunk.choices[0].delta; delete rDelta.content; reasoningChunk.choices[0].finish_reason = null; delete reasoningChunk.usage; const rOutput = `data: ${JSON.stringify(reasoningChunk)}\n`; passthroughAccumulatedReasoning = appendBoundedText( passthroughAccumulatedReasoning, delta.reasoning_content ); totalContentLength += delta.reasoning_content.length; clientPayloadCollector.push(reasoningChunk); reqLogger?.appendConvertedChunk?.(rOutput); controller.enqueue(encoder.encode(rOutput)); controller.enqueue(encoder.encode("\n")); delete delta.reasoning_content; } // Track whether we need to re-serialize (separate from injectedUsage // to avoid blocking subsequent finish_reason / usage mutations) const needsReserialization = hadReasoningAlias || (delta?.content === "" && delta?.reasoning_content); // T18: Track if we saw tool calls & accumulate for call log if (delta?.tool_calls && delta.tool_calls.length > 0) { passthroughHasToolCalls = true; for (const tc of delta.tool_calls) { // Key by index first — id only appears on the first delta in OpenAI streaming let key: string; if (Number.isInteger(tc?.index)) { key = `idx:${tc.index}`; } else if (tc?.id) { key = `id:${tc.id}`; } else { key = `seq:${++passthroughToolCallSeq}`; } const existing = passthroughToolCalls.get(key); const deltaArgs = typeof tc?.function?.arguments === "string" ? tc.function.arguments : ""; if (!existing) { passthroughToolCalls.set(key, { id: tc?.id ?? null, index: Number.isInteger(tc?.index) ? tc.index : passthroughToolCalls.size, type: tc?.type || "function", function: { name: tc?.function?.name || "", arguments: deltaArgs, }, }); } else { if (tc?.id) existing.id = existing.id || tc.id; if (tc?.function?.name && !existing.function.name) existing.function.name = tc.function.name; existing.function.arguments += deltaArgs; } } } const content = delta?.content || delta?.reasoning_content; if (content && typeof content === "string") { totalContentLength += content.length; } { const guarded = applyTextualToolCallStreamingGuard( parsed as Record ); parsed = guarded.parsed as typeof parsed; textualToolCallConverted = guarded.textualToolCallConverted; } if (typeof delta?.reasoning_content === "string") passthroughAccumulatedReasoning = appendBoundedText( passthroughAccumulatedReasoning, delta.reasoning_content ); const extracted = extractUsage(parsed); if (extracted) { usage = extracted; } const isFinishChunk = parsed.choices?.[0]?.finish_reason; // T18: Normalize finish_reason to 'tool_calls' if tool calls were used if ( isFinishChunk && passthroughHasToolCalls && parsed.choices[0].finish_reason !== "tool_calls" ) { parsed.choices[0].finish_reason = "tool_calls"; // If we modify it, we must output the modified object if (!injectedUsage && hasValidUsage(parsed.usage)) { output = `data: ${JSON.stringify(parsed)}\n`; injectedUsage = true; } } if (isFinishChunk && !hasValidUsage(parsed.usage)) { const estimated = estimateUsage(body, totalContentLength, FORMATS.OPENAI); parsed.usage = filterUsageForFormat(estimated, FORMATS.OPENAI); output = `data: ${JSON.stringify(parsed)}\n`; usage = estimated; injectedUsage = true; } else if (isFinishChunk && usage) { const buffered = addBufferToUsage(usage); parsed.usage = filterUsageForFormat(buffered, FORMATS.OPENAI); output = `data: ${JSON.stringify(parsed)}\n`; injectedUsage = true; } else if (textualToolCallConverted) { output = `data: ${JSON.stringify(parsed)}\n`; injectedUsage = true; } else if (idFixed || needsReserialization) { output = `data: ${JSON.stringify(parsed)}\n`; injectedUsage = true; } } clientPayload = parsed; } catch {} } if (!injectedUsage) { if (line.startsWith("data:") && !line.startsWith("data: ")) { output = "data: " + line.slice(5) + "\n"; } else { output = line + "\n"; } } if (!trimmed && pendingPassthroughEventLine && !pendingPassthroughEventEmitted) { output = `${pendingPassthroughEventLine}\n${output}`; pendingPassthroughEventEmitted = true; } output = maybePrefixPendingPassthroughEvent(output, line); if (clientPayload) { clientPayloadCollector.push(clientPayload); } reqLogger?.appendConvertedChunk?.(output); controller.enqueue(encoder.encode(output)); if (failurePayload) { if (onFailure) { try { void onFailure(failurePayload); } catch {} } clearIdleTimer(); trackPendingRequest(model, provider, connectionId, false); controller.error( markPendingRequestCleared(new Error(failurePayload.message || "Upstream failure")) ); return; } if (!trimmed) { clearPendingPassthroughEvent(); } continue; } // Translate mode if (!trimmed) continue; if (state?.upstreamError) { continue; } const parsed = parseSSELine(trimmed); if (!parsed) continue; providerPayloadCollector.push(parsed); if (parsed && parsed.done) { continue; } // Track content length and accumulate for call log (from raw provider chunk, so content is never missed) // Do this before translation so we capture content regardless of translator output shape // Claude format if (parsed.delta?.text) { const t = parsed.delta.text; totalContentLength += t.length; if (state?.accumulatedContent !== undefined && typeof t === "string") state.accumulatedContent = appendBoundedText(state.accumulatedContent, t); } if (parsed.delta?.thinking) { const t = parsed.delta.thinking; totalContentLength += t.length; if (state?.accumulatedContent !== undefined && typeof t === "string") state.accumulatedContent = appendBoundedText(state.accumulatedContent, t); } // OpenAI format if (parsed.choices?.[0]?.delta?.content) { const c = parsed.choices[0].delta.content; if (typeof c === "string") { totalContentLength += c.length; if (state?.accumulatedContent !== undefined) state.accumulatedContent = appendBoundedText(state.accumulatedContent, c); } else if (Array.isArray(c)) { for (const part of c) { if (part?.text && typeof part.text === "string") { totalContentLength += part.text.length; if (state?.accumulatedContent !== undefined) state.accumulatedContent = appendBoundedText( state.accumulatedContent, part.text ); } } } } if (parsed.choices?.[0]?.delta?.reasoning_content) { const r = parsed.choices[0].delta.reasoning_content; if (typeof r === "string") { totalContentLength += r.length; if (state?.accumulatedContent !== undefined) state.accumulatedContent = appendBoundedText(state.accumulatedContent, r); } } // Normalize `reasoning` alias → `reasoning_content` (NVIDIA kimi-k2.5 etc.) if ( parsed.choices?.[0]?.delta?.reasoning && !parsed.choices?.[0]?.delta?.reasoning_content ) { const r = parsed.choices[0].delta.reasoning; if (typeof r === "string") { parsed.choices[0].delta.reasoning_content = r; delete parsed.choices[0].delta.reasoning; totalContentLength += r.length; if (state?.accumulatedContent !== undefined) state.accumulatedContent = appendBoundedText(state.accumulatedContent, r); } } // Gemini / Cloud Code format - may have multiple parts // Cloud Code API wraps in { response: { candidates: [...] } }, so unwrap. // Only applies to Gemini-family formats — skip for OpenAI, Claude, etc. const isGeminiFormat = targetFormat === FORMATS.GEMINI || targetFormat === FORMATS.GEMINI_CLI || targetFormat === FORMATS.ANTIGRAVITY; const geminiChunk = isGeminiFormat ? unwrapGeminiChunk(parsed) : parsed; if (geminiChunk.candidates?.[0]?.content?.parts) { for (const part of geminiChunk.candidates[0].content.parts) { if (part.text && typeof part.text === "string") { totalContentLength += part.text.length; if (state?.accumulatedContent !== undefined) state.accumulatedContent = appendBoundedText(state.accumulatedContent, part.text); } } } // Generic fallback: delta string, top-level content/text (e.g. some SSE payloads) if (state?.accumulatedContent !== undefined) { if (typeof (parsed as JsonRecord).delta === "string") { const d = (parsed as JsonRecord).delta as string; state.accumulatedContent = appendBoundedText(state.accumulatedContent, d); totalContentLength += d.length; } if (typeof (parsed as JsonRecord).content === "string") { const c = (parsed as JsonRecord).content as string; state.accumulatedContent = appendBoundedText(state.accumulatedContent, c); totalContentLength += c.length; } if (typeof (parsed as JsonRecord).text === "string") { const t = (parsed as JsonRecord).text as string; state.accumulatedContent = appendBoundedText(state.accumulatedContent, t); totalContentLength += t.length; } } // Extract usage const extracted = extractUsage(parsed); if (extracted) state.usage = extracted; // Keep original usage for logging // Translate: targetFormat -> openai -> sourceFormat const translated = translateResponse(targetFormat, sourceFormat, parsed, state); // Log OpenAI intermediate chunks (if available) for (const item of getOpenAIIntermediateChunks(translated)) { const openaiOutput = formatSSE(item, FORMATS.OPENAI); reqLogger?.appendOpenAIChunk?.(openaiOutput); } if (translated?.length > 0) { for (const item of translated) { emitTranslatedClientItem(controller, item); } } } }, async flush(controller) { // Clean up idle watchdog timer if (idleTimer) { clearIdleTimer(); } if (streamTimedOut) { return; } trackPendingRequest(model, provider, connectionId, false); try { const remaining = decoder.decode(); if (remaining) buffer += remaining; if (mode === STREAM_MODE.PASSTHROUGH) { const bufferedLine = buffer.trim(); if (skipPassthroughEvent || /^event:\s*keepalive\b/i.test(bufferedLine)) { skipPassthroughEvent = false; clearPendingPassthroughEvent(); } else if (buffer) { let output = buffer; if (buffer.startsWith("data:") && !buffer.startsWith("data: ")) { output = "data: " + buffer.slice(5); } const bufferedPayload = parseSSELine(bufferedLine); if (bufferedPayload) { providerPayloadCollector.push(bufferedPayload); if ( shouldInjectClaudeEmptyResponseBeforeCurrentEvent( claudeEmptyResponseLifecycle, bufferedPayload ) ) { const eventType = getClaudeEventType(bufferedPayload); emitSyntheticClaudeEmptyResponse(controller, { includeContentBlock: true, includeMessageDelta: eventType === "message_stop" && !claudeEmptyResponseLifecycle.hasMessageDelta, includeMessageStop: false, }); } if (isClaudeEventPayload(bufferedPayload)) { updateClaudeEmptyResponseLifecycle(claudeEmptyResponseLifecycle, bufferedPayload); } clientPayloadCollector.push(bufferedPayload); } if (!bufferedLine && pendingPassthroughEventLine && !pendingPassthroughEventEmitted) { output = `${pendingPassthroughEventLine}\n${output}`; pendingPassthroughEventEmitted = true; } output = maybePrefixPendingPassthroughEvent(output, buffer); reqLogger?.appendConvertedChunk?.(output); controller.enqueue(encoder.encode(output)); } if (shouldInjectClaudeEmptyResponseOnFlush(claudeEmptyResponseLifecycle)) { emitSyntheticClaudeEmptyResponse(controller, { includeContentBlock: true, includeMessageDelta: !claudeEmptyResponseLifecycle.hasMessageDelta, includeMessageStop: !claudeEmptyResponseLifecycle.hasMessageStop, }); } else if (shouldInjectClaudeMissingFinalizersOnFlush(claudeEmptyResponseLifecycle)) { emitSyntheticClaudeEmptyResponse(controller, { includeContentBlock: false, includeMessageDelta: !claudeEmptyResponseLifecycle.hasMessageDelta, includeMessageStop: !claudeEmptyResponseLifecycle.hasMessageStop, }); } clearPendingPassthroughEvent(); // Estimate usage if provider didn't return valid usage if (!hasValidUsage(usage) && totalContentLength > 0) { usage = estimateUsage(body, totalContentLength, sourceFormat || FORMATS.OPENAI); } if (hasValidUsage(usage)) { logUsage(provider, usage, model, connectionId, apiKeyInfo); } else { appendRequestLog({ model, provider, connectionId, tokens: null, status: "200 OK", }).catch(() => {}); } if (!doneSent) { await emitFinalSseMetadata(controller, usage); doneSent = true; if (shouldEmitDoneTerminator) { clientPayloadCollector.push({ done: true }); const doneOutput = "data: [DONE]\n\n"; reqLogger?.appendConvertedChunk?.(doneOutput); controller.enqueue(encoder.encode(doneOutput)); } } // Notify caller for call log persistence (include full response body with accumulated content) if (onComplete) { try { const u = usage as Record | null; const prompt = Number(u?.prompt_tokens ?? u?.input_tokens ?? 0); const completion = Number(u?.completion_tokens ?? u?.output_tokens ?? 0); let content = passthroughAccumulatedContent.trim() || ""; const finalBufferedTextualToolCall = passthroughBufferedTextualToolCallContent.trim(); if (finalBufferedTextualToolCall) { if ( collectPassthroughTextualToolCall( finalBufferedTextualToolCall, passthroughToolCalls, allowedToolNames ) ) { passthroughHasToolCalls = true; } passthroughBufferedTextualToolCallContent = ""; } if ( content && collectPassthroughTextualToolCall(content, passthroughToolCalls, allowedToolNames) ) { passthroughHasToolCalls = true; content = ""; } else if (containsMalformedTextualToolCall(content)) { content = ""; } const message: Record = { role: "assistant", content: content || null, }; const reasoning = passthroughAccumulatedReasoning.trim(); if (reasoning) { message.reasoning_content = reasoning; } if (passthroughToolCalls.size > 0) { message.tool_calls = [...passthroughToolCalls.values()].sort( (a, b) => a.index - b.index ); } const responseBody = { choices: [ { message, finish_reason: passthroughHasToolCalls ? "tool_calls" : "stop", }, ], usage: { prompt_tokens: prompt, completion_tokens: completion, total_tokens: prompt + completion, }, _streamed: true, }; onComplete({ status: 200, usage, responseBody, providerPayload: providerPayloadCollector.build( buildStreamSummaryFromEvents( providerPayloadCollector.getEvents(), sourceFormat, model ), { includeEvents: false } ), clientPayload: clientPayloadCollector.build(responseBody, { includeEvents: false, }), }); } catch {} } return; } // Translate mode: process remaining buffer if (buffer.trim()) { const parsed = parseSSELine(buffer.trim()); if (parsed && !parsed.done) { providerPayloadCollector.push(parsed); // Extract usage from remaining buffer — if the usage-bearing event // (e.g. response.completed) is the last SSE line, it ends up here // in the flush handler where extractUsage was not called. // Non-destructive merge: some providers send usage across multiple // events (e.g. prompt_tokens in message_start, completion_tokens // in message_delta). Direct assignment would lose earlier data. const extracted = extractUsage(parsed); if (extracted) { if (!state.usage) { state.usage = extracted; } else { const su = state.usage as Record; const eu = extracted as Record; if (eu.prompt_tokens > 0) su.prompt_tokens = eu.prompt_tokens; if (eu.completion_tokens > 0) su.completion_tokens = eu.completion_tokens; if (eu.total_tokens > 0) su.total_tokens = eu.total_tokens; if (eu.cache_read_input_tokens > 0) su.cache_read_input_tokens = eu.cache_read_input_tokens; if (eu.cache_creation_input_tokens > 0) su.cache_creation_input_tokens = eu.cache_creation_input_tokens; if (eu.cached_tokens > 0) su.cached_tokens = eu.cached_tokens; if (eu.reasoning_tokens > 0) su.reasoning_tokens = eu.reasoning_tokens; } } const translated = translateResponse(targetFormat, sourceFormat, parsed, state); // Log OpenAI intermediate chunks for (const item of getOpenAIIntermediateChunks(translated)) { const openaiOutput = formatSSE(item, FORMATS.OPENAI); reqLogger?.appendOpenAIChunk?.(openaiOutput); } if (translated?.length > 0) { for (const item of translated) { emitTranslatedClientItem(controller, item); } } } } if (state?.upstreamError) { const err = state.upstreamError; trackPendingRequest(model, provider, connectionId, false); if (onFailure) { try { void onFailure({ status: err.status, message: err.message, code: err.code, type: err.type, }); } catch {} } const errorBody = buildErrorBody(err.status, err.message); if (onComplete) { try { onComplete({ status: err.status, usage: state?.usage, responseBody: errorBody, providerPayload: providerPayloadCollector.build( buildStreamSummaryFromEvents( providerPayloadCollector.getEvents(), targetFormat, model ), { includeEvents: false } ), clientPayload: clientPayloadCollector.build(errorBody, { includeEvents: false, }), }); } catch {} } clearIdleTimer(); controller.error( markPendingRequestCleared(new Error(err.message || "Upstream failure")) ); return; } // Flush remaining events (only once at stream end) const flushed = translateResponse(targetFormat, sourceFormat, null, state); // Log OpenAI intermediate chunks for flushed events for (const item of getOpenAIIntermediateChunks(flushed)) { const openaiOutput = formatSSE(item, FORMATS.OPENAI); reqLogger?.appendOpenAIChunk?.(openaiOutput); } if (flushed?.length > 0) { for (const item of flushed) { emitTranslatedClientItem(controller, item); } } if (sourceFormat === FORMATS.CLAUDE) { if (shouldInjectClaudeEmptyResponseOnFlush(claudeEmptyResponseLifecycle)) { emitSyntheticClaudeEmptyResponse(controller, { includeContentBlock: true, includeMessageDelta: !claudeEmptyResponseLifecycle.hasMessageDelta, includeMessageStop: !claudeEmptyResponseLifecycle.hasMessageStop, }); } else if (shouldInjectClaudeMissingFinalizersOnFlush(claudeEmptyResponseLifecycle)) { emitSyntheticClaudeEmptyResponse(controller, { includeContentBlock: false, includeMessageDelta: !claudeEmptyResponseLifecycle.hasMessageDelta, includeMessageStop: !claudeEmptyResponseLifecycle.hasMessageStop, }); } } /** * Usage injection strategy: * Usage data (input/output tokens) is injected into the last content chunk * or the finish_reason chunk rather than sent as a separate SSE event. * This ensures all major clients (Claude CLI, Continue, Cursor) receive * usage data even if they stop reading after the finish signal. * The usage buffer (state.usage) accumulates across chunks and is only * emitted once at stream end when merged into the final translated chunk. */ // Send [DONE] (only if not already sent during transform) if (!doneSent) { await emitFinalSseMetadata(controller, state?.usage as Record | null); doneSent = true; if (shouldEmitDoneTerminator) { clientPayloadCollector.push({ done: true }); const doneOutput = "data: [DONE]\n\n"; reqLogger?.appendConvertedChunk?.(doneOutput); controller.enqueue(encoder.encode(doneOutput)); } } // Estimate usage if provider didn't return valid usage (for translate mode) if (!hasValidUsage(state?.usage) && totalContentLength > 0) { state.usage = estimateUsage(body, totalContentLength, sourceFormat); } if (hasValidUsage(state?.usage)) { logUsage(state.provider || targetFormat, state.usage, model, connectionId, apiKeyInfo); } else { appendRequestLog({ model, provider, connectionId, tokens: null, status: "200 OK", }).catch(() => {}); } // Notify caller for call log persistence (include full response body with accumulated content) if (onComplete) { try { const u = state?.usage as Record | null | undefined; const prompt = Number(u?.prompt_tokens ?? u?.input_tokens ?? 0); const completion = Number(u?.completion_tokens ?? u?.output_tokens ?? 0); let content = (state?.accumulatedContent ?? "").trim() || ""; const normalizedToolCalls: ToolCall[] = state?.toolCalls?.size ? [...state.toolCalls.values()] .map( (tc: Record): ToolCall => ({ id: (tc.id as string) ?? null, index: (tc.index as number) ?? (tc.blockIndex as number) ?? 0, type: (tc.type as string) ?? "function", function: (tc.function as ToolCall["function"]) ?? { name: (tc.name as string) ?? "", arguments: "", }, }) ) .sort((a, b) => a.index - b.index) : []; const textualToolCall = parseTextualToolCallFromContent(content); if (textualToolCall) { normalizedToolCalls.push({ id: `call_${Date.now()}_${normalizedToolCalls.length}`, index: normalizedToolCalls.length, type: "function", function: { name: textualToolCall.name, arguments: JSON.stringify(textualToolCall.args || {}), }, }); content = ""; } else if (containsMalformedTextualToolCall(content)) { content = ""; } const message: Record = { role: "assistant", content: content || null, }; const hasToolCalls = normalizedToolCalls.length > 0; if (hasToolCalls) { message.tool_calls = normalizedToolCalls; } const responseBody = { choices: [ { message, finish_reason: hasToolCalls ? "tool_calls" : "stop", }, ], usage: { prompt_tokens: prompt, completion_tokens: completion, total_tokens: prompt + completion, }, _streamed: true, }; onComplete({ status: 200, usage: state?.usage, responseBody, providerPayload: providerPayloadCollector.build( buildStreamSummaryFromEvents( providerPayloadCollector.getEvents(), targetFormat, model ), { includeEvents: false } ), clientPayload: clientPayloadCollector.build(responseBody, { includeEvents: false, }), }); } catch {} } } catch (error) { console.log(`[STREAM] Error in flush (${model || "unknown"}):`, error.message || error); } }, }, { highWaterMark: 16384 }, { highWaterMark: 16384 } ); } // Convenience functions for backward compatibility export function createSSETransformStreamWithLogger( targetFormat: string, sourceFormat: string, provider: string | null = null, reqLogger: StreamLogger | null = null, toolNameMap: unknown = null, model: string | null = null, connectionId: string | null = null, body: unknown = null, onComplete: ((payload: StreamCompletePayload) => void) | null = null, apiKeyInfo: unknown = null, onFailure: ((payload: StreamFailurePayload) => void | Promise) | null = null, copilotCompatibleReasoning = false ) { return createSSEStream({ mode: STREAM_MODE.TRANSLATE, targetFormat, sourceFormat, provider, reqLogger, toolNameMap, model, connectionId, apiKeyInfo, body, onComplete, onFailure, copilotCompatibleReasoning, }); } export function createPassthroughStreamWithLogger( provider: string | null = null, reqLogger: StreamLogger | null = null, toolNameMap: unknown = null, model: string | null = null, connectionId: string | null = null, body: unknown = null, onComplete: ((payload: StreamCompletePayload) => void) | null = null, apiKeyInfo: unknown = null, onFailure: ((payload: StreamFailurePayload) => void | Promise) | null = null, clientResponseFormat: string | null = null ) { return createSSEStream({ mode: STREAM_MODE.PASSTHROUGH, provider, reqLogger, toolNameMap, model, connectionId, apiKeyInfo, body, onComplete, onFailure, clientResponseFormat, }); }