import { CORS_HEADERS } from "../utils/cors.ts"; import { normalizeHeaders } from "../utils/headers.ts"; import { detectFormatFromEndpoint, getTargetFormat } from "../services/provider.ts"; import { injectSystemPrompt } from "../services/systemPrompt.ts"; import { translateRequest, needsTranslation } from "../translator/index.ts"; import { FORMATS } from "../translator/formats.ts"; import { splitMisplacedToolResults } from "../translator/helpers/claudeHelper.ts"; import { createSSETransformStreamWithLogger, createPassthroughStreamWithLogger, COLORS, withBodyTimeout, } from "../utils/stream.ts"; import { ensureStreamReadiness } from "../utils/streamReadiness.ts"; import { resolveStreamReadinessTimeout } from "../utils/streamReadinessPolicy.ts"; import { createStreamController, pipeWithDisconnect } from "../utils/streamHandler.ts"; import { createSseHeartbeatTransform, shapeForClientFormat } from "../utils/sseHeartbeat.ts"; import { addBufferToUsage, filterUsageForFormat, estimateUsage } from "../utils/usageTracking.ts"; import { refreshWithRetry, isUnrecoverableRefreshError, runWithOnPersist, } from "../services/tokenRefresh.ts"; import { createRequestLogger } from "../utils/requestLogger.ts"; import { getModelTargetFormat, PROVIDER_ID_TO_ALIAS } from "../config/providerModels.ts"; import { DEFAULT_THINKING_CLAUDE_SIGNATURE } from "../config/defaultThinkingSignature.ts"; import { getStripTypesForProviderModel, stripIncompatibleMessageContent, } from "../services/modelStrip.ts"; import { resolveModelAlias } from "../services/modelDeprecation.ts"; import { getUnsupportedParams } from "../config/providerRegistry.ts"; import { supportsMaxTokens } from "@/lib/modelCapabilities.ts"; import { buildErrorBody, createErrorResult, parseUpstreamError, formatProviderError, sanitizeErrorMessage, } from "../utils/error.ts"; import { checkTokenLimits, recordTokenUsage, } from "@omniroute/open-sse/services/tokenLimitCounter.ts"; import { COOLDOWN_MS, HTTP_STATUS, FETCH_TIMEOUT_MS, FETCH_BODY_TIMEOUT_MS, MAX_TOOLS_LIMIT, PROVIDER_MAX_TOKENS, SSE_HEARTBEAT_INTERVAL_MS, STREAM_IDLE_TIMEOUT_MS, STREAM_READINESS_TIMEOUT_MS, ANTIGRAVITY_PRE_RESPONSE_TIMEOUT_CODE, } from "../config/constants.ts"; import { classifyProviderError, PROVIDER_ERROR_TYPES, isEmptyContentResponse, } from "../services/errorClassifier.ts"; import { updateProviderConnection } from "@/lib/db/providers"; import { recordKeyFailure, recordKeySuccess, getInvalidKeyCount, trackConnectionExtraKeys, connectionHasExtraKeys, type KeyHealth, } from "../services/apiKeyRotator.ts"; import { isDetailedLoggingEnabled } from "@/lib/db/detailedLogs"; import { getCallLogPipelineCaptureStreamChunks, getChatLogTextLimit, getChatLogArrayTailItems, getChatLogMaxDepth, getChatLogMaxObjectKeys, } from "@/lib/logEnv"; import { logAuditEvent } from "@/lib/compliance"; import { emit } from "@/lib/events/eventBus"; import { extractProviderWarnings } from "@/lib/compliance/providerAudit"; import { adaptBodyForCompression } from "../services/compression/bodyAdapter.ts"; import { handleBypassRequest } from "../utils/bypassHandler.ts"; import { saveRequestUsage, trackPendingRequest, updatePendingRequest, appendRequestLog, saveCallLog, } from "@/lib/usageDb"; import { formatUsageLog, getLoggedInputTokens, getLoggedOutputTokens, getReasoningTokens, } from "@/lib/usage/tokenAccounting"; import { recordCost } from "@/domain/costRules"; import { calculateCost } from "@/lib/usage/costCalculator"; import { buildOmniRouteResponseMetaHeaders } from "@/domain/omnirouteResponseMeta"; import { CLAUDE_OAUTH_TOOL_PREFIX } from "../translator/request/openai-to-claude.ts"; import { getModelNormalizeToolCallId, getModelPreserveOpenAIDeveloperRole, getModelUpstreamExtraHeaders, getUpstreamProxyConfig, } from "@/lib/localDb"; import { getProviderCredentials, extractSessionAffinityKey } from "@/sse/services/auth"; import { deleteSessionAccountAffinity } from "@/lib/db/sessionAccountAffinity"; import { getExecutor } from "../executors/index.ts"; import { getCacheControlSettings } from "@/lib/cacheControlSettings"; import { guardrailRegistry, resolveDisabledGuardrails } from "@/lib/guardrails"; import { applyConfiguredPayloadRules, resolvePayloadRuleProtocols, } from "../services/payloadRules.ts"; import { shouldPreserveCacheControl, providerSupportsCaching, } from "../utils/cacheControlPolicy.ts"; import { getCachedSettings } from "@/lib/db/readCache"; import { applyCodexGlobalFastServiceTier } from "@/lib/providers/codexFastTier"; import { CPA_FORCE_FAST_MODE_HEADER, shouldRequestClaudeFastMode, } from "@/lib/providers/claudeFastMode"; import { getCodexRequestDefaults, normalizeCodexServiceTier, type CodexServiceTier, } from "@/lib/providers/requestDefaults"; import { cacheReasoningFromAssistantMessage } from "../services/reasoningCache.ts"; import { sanitizeOpenAITool } from "../services/toolSchemaSanitizer.ts"; import { getEffectiveToolLimit, setDetectedToolLimit, parseToolLimitFromError, shouldDetectLimit, } from "../services/toolLimitDetector.ts"; import { parseCodexQuotaHeaders, getCodexModelScope, getCodexDualWindowCooldownMs, isCompactResponsesEndpoint, } from "../executors/codex.ts"; import { invalidateCodexQuotaCache } from "../services/codexQuotaFetcher.ts"; import { translateNonStreamingResponse } from "./responseTranslator.ts"; import { extractUsageFromResponse } from "./usageExtractor.ts"; import { parseSSEToClaudeResponse, parseSSEToOpenAIResponse, parseSSEToResponsesOutput, } from "./sseParser.ts"; import { sanitizeOpenAIResponse, sanitizeResponsesApiResponse } from "./responseSanitizer.ts"; import { withRateLimit, updateFromHeaders, updateFromResponseBody, initializeRateLimits, } from "../services/rateLimitManager.ts"; import { acquire as acquireAccountSemaphore, buildAccountSemaphoreKey, markBlocked as markAccountSemaphoreBlocked, } from "../services/accountSemaphore.ts"; import { lockModel, lockModelIfPerModelQuota } from "../services/accountFallback.ts"; import { generateSignature, getCachedResponse, setCachedResponse, isCacheableForRead, isCacheableForWrite, } from "@/lib/semanticCache"; import { getIdempotencyKey, checkIdempotency, saveIdempotency } from "@/lib/idempotencyLayer"; import { createProgressTransform, wantsProgress } from "../utils/progressTracker.ts"; import { isModelUnavailableError, getNextFamilyFallback, isContextOverflowError, findLargerContextModel, getModelFamily, } from "../services/modelFamilyFallback.ts"; import { computeRequestHash, deduplicate, shouldDeduplicate } from "../services/requestDedup.ts"; import { compressContext, estimateTokens, getTokenLimit } from "../services/contextManager.ts"; import { getBackgroundTaskReason, getDegradedModel, getBackgroundDegradationConfig, } from "../services/backgroundTaskDetector.ts"; import { shouldUseFallback, isFallbackDecision, EMERGENCY_FALLBACK_CONFIG, } from "../services/emergencyFallback.ts"; import type { CompressionConfig } from "../services/compression/types.ts"; import { prepareWebSearchFallbackBody } from "../services/webSearchFallback.ts"; import { resolveExplicitStreamAlias, resolveStreamFlag, stripMarkdownCodeFence, } from "../utils/aiSdkCompat.ts"; import { generateRequestId } from "@/shared/utils/requestId"; import { normalizePayloadForLog } from "@/lib/logPayloads"; import { extractFacts } from "@/lib/memory/extraction"; import { injectMemory, shouldInjectMemory } from "@/lib/memory/injection"; import { retrieveMemories } from "@/lib/memory/retrieval"; import { DEFAULT_MEMORY_SETTINGS, getMemorySettings, toMemoryRetrievalConfig, } from "@/lib/memory/settings"; import { injectSkills } from "@/lib/skills/injection"; import { handleToolCallExecution } from "@/lib/skills/interception"; import { OMNIROUTE_RESPONSE_HEADERS } from "@/shared/constants/headers"; import { buildClaudeCodeCompatibleRequest, isClaudeCodeCompatibleProvider, resolveClaudeCodeCompatibleSessionId, } from "../services/claudeCodeCompatible.ts"; import { setGeminiThoughtSignatureMode } from "../services/geminiThoughtSignatureStore.ts"; import { fetchLiveProviderLimits } from "@/lib/usage/providerLimits"; import { isClaudeExtraUsageBlockEnabled } from "@/lib/providers/claudeExtraUsage"; import { classifyModelScope429, getModelScopeRetryDelayMs, isModelScopeProvider, } from "../services/modelscopePolicy.ts"; const MEMORY_EXTRACTION_TEXT_LIMIT = 64 * 1024; function capMemoryExtractionText(value: string): string { if (value.length <= MEMORY_EXTRACTION_TEXT_LIMIT) return value; return value.slice(-MEMORY_EXTRACTION_TEXT_LIMIT); } function truncateChatLogText(value: string): string { const limit = getChatLogTextLimit(); if (value.length <= limit) return value; const head = value.slice(0, Math.floor(limit / 2)); const tail = value.slice(-Math.ceil(limit / 2)); return `${head}\n[...truncated ${value.length - limit} chars...]\n${tail}`; } function cloneBoundedChatLogPayload(value: unknown, depth = 0): unknown { if (value === null || value === undefined) return value; if (typeof value === "string") return truncateChatLogText(value); if (typeof value !== "object") return value; if (depth >= getChatLogMaxDepth()) return "[MaxDepth]"; const maxTailItems = getChatLogArrayTailItems(); if (Array.isArray(value)) { const retained = value.length > maxTailItems ? value.slice(-maxTailItems) : value; const cloned = retained.map((item) => cloneBoundedChatLogPayload(item, depth + 1)); if (value.length > maxTailItems) { return [ { _omniroute_truncated_array: true, originalLength: value.length, retainedTailItems: maxTailItems, }, ...cloned, ]; } return cloned; } const result: Record = {}; const entries = Object.entries(value as Record); const maxKeys = getChatLogMaxObjectKeys(); for (const [key, item] of maxKeys > 0 ? entries.slice(0, maxKeys) : entries) { result[key] = cloneBoundedChatLogPayload(item, depth + 1); } if (maxKeys > 0 && entries.length > maxKeys) { result._omniroute_truncated_keys = entries.length - maxKeys; } return result; } function isSmallEnoughForSemanticCache(value: unknown): boolean { try { return JSON.stringify(value).length <= 256 * 1024; } catch { return false; } } function extractMemoryTextFromResponse( response: Record | null | undefined ): string { if (!response || typeof response !== "object") return ""; const openAIText = response?.choices?.[0]?.message?.content; if (typeof openAIText === "string") { return capMemoryExtractionText(openAIText.trim()); } if (Array.isArray(response?.content)) { const contentText = response.content .filter( (part: Record) => part?.type === "text" && typeof part?.text === "string" ) .map((part: Record) => String(part.text).trim()) .filter(Boolean) .join("\n"); if (contentText) return capMemoryExtractionText(contentText); } if (typeof response?.output_text === "string") { return capMemoryExtractionText(response.output_text.trim()); } return ""; } function extractMemoryTextFromRequestBody( body: Record | null | undefined ): string { if (!body || typeof body !== "object") return ""; const messages = Array.isArray(body.messages) ? body.messages : null; if (messages && messages.length > 0) { for (let i = messages.length - 1; i >= 0; i -= 1) { const msg = messages[i] as Record; if (msg?.role !== "user") continue; if (typeof msg.content === "string" && msg.content.trim().length > 0) { return capMemoryExtractionText(msg.content.trim()); } if (Array.isArray(msg.content)) { const text = msg.content .map((part: Record) => { if (typeof part?.text === "string") return part.text.trim(); if (part?.type === "input_text" && typeof part?.text === "string") return part.text.trim(); return ""; }) .filter(Boolean) .join("\n") .trim(); if (text) return capMemoryExtractionText(text); } } } const input = Array.isArray(body.input) ? body.input : null; if (input && input.length > 0) { for (let i = input.length - 1; i >= 0; i -= 1) { const item = input[i] as Record; const role = typeof item?.role === "string" ? item.role.trim().toLowerCase() : ""; const itemType = typeof item?.type === "string" ? item.type.trim().toLowerCase() : ""; if (role && role !== "user") continue; if (itemType && itemType !== "message") continue; if (typeof item?.content === "string" && item.content.trim()) { return capMemoryExtractionText(item.content.trim()); } if (Array.isArray(item?.content)) { const text = item.content .map((part: Record) => { if (typeof part?.text === "string") return part.text.trim(); if (part?.type === "input_text" && typeof part?.text === "string") return part.text.trim(); return ""; }) .filter(Boolean) .join("\n") .trim(); if (text) return capMemoryExtractionText(text); } } const tailChunks: string[] = []; let tailLength = 0; for (let i = input.length - 1; i >= 0 && tailLength < MEMORY_EXTRACTION_TEXT_LIMIT; i -= 1) { const item = input[i] as Record; const text = (() => { const role = typeof item?.role === "string" ? item.role.trim().toLowerCase() : ""; const itemType = typeof item?.type === "string" ? item.type.trim().toLowerCase() : ""; if (role && role !== "user") return ""; if (itemType && itemType !== "message") return ""; if (typeof item?.content === "string") return item.content.trim(); if (Array.isArray(item?.content)) { return item.content .map((part: Record) => { if (typeof part?.text === "string") return part.text.trim(); if (part?.type === "input_text" && typeof part?.text === "string") return part.text.trim(); return ""; }) .filter(Boolean) .join("\n") .trim(); } return ""; })(); if (!text) continue; tailChunks.unshift(text); tailLength += text.length + 1; } const chunks = tailChunks.join("\n").trim(); if (chunks) return capMemoryExtractionText(chunks); } return ""; } async function maybeSyncClaudeExtraUsageState({ provider, connectionId, providerSpecificData, log, }: { provider: string | null | undefined; connectionId: string | null | undefined; providerSpecificData: unknown; log?: { debug?: (...args: unknown[]) => void; warn?: (...args: unknown[]) => void } | null; }) { if (!connectionId || !isClaudeExtraUsageBlockEnabled(provider, providerSpecificData)) { return; } try { await fetchLiveProviderLimits(connectionId); } catch (error) { const message = error instanceof Error ? error.message : String(error); log?.debug?.("CLAUDE_USAGE", `Failed to sync Claude extra-usage state: ${message}`); } } function resolveMemoryOwnerId(apiKeyInfo: Record | null): string | null { const rawId = apiKeyInfo?.id; if (typeof rawId === "string" && rawId.trim().length > 0) { return rawId; } return null; } export function shouldUseNativeCodexPassthrough({ provider, sourceFormat, endpointPath, }: { provider?: string | null; sourceFormat?: string | null; endpointPath?: string | null; }): boolean { if (provider !== "codex") return false; if (sourceFormat !== FORMATS.OPENAI_RESPONSES) return false; let normalizedEndpoint = String(endpointPath || ""); while (normalizedEndpoint.endsWith("/")) normalizedEndpoint = normalizedEndpoint.slice(0, -1); const segments = normalizedEndpoint.split("/"); return segments.includes("responses"); } /** * Convert all historical `thinking` / `redacted_thinking` blocks in assistant * messages to `redacted_thinking` carrying a synthetic default signature. * * A thinking block's `signature` is cryptographically bound to the auth token * that generated it. In Anthropic-native Claude OAuth passthrough, when a session * starts on one model (token A) and then switches model or falls over (token B), * Anthropic rejects every historical signature with 400 "Invalid signature in * thinking block" (issue #2454). `redacted_thinking` bypasses signature validation. * * ALL assistant turns are converted, including the last — under a different token * every signature is invalid, so there is no "preserve latest" exception. Returns a * new messages array (original is not mutated) only touching messages that changed. */ export function redactPassthroughThinkingSignatures(messages: unknown, signature: string): unknown { if (!Array.isArray(messages)) return messages; return (messages as Record[]).map((msg) => { if (!msg || msg.role !== "assistant" || !Array.isArray(msg.content)) return msg; let modified = false; const newContent = (msg.content as Record[]).map((block) => { if (block && (block.type === "thinking" || block.type === "redacted_thinking")) { modified = true; return { type: "redacted_thinking", data: signature }; } return block; }); return modified ? { ...msg, content: newContent } : msg; }); } export function isClaudeCodeSemanticPassthroughRequest({ provider, sourceFormat, targetFormat, headers, userAgent, }: { provider?: string | null; sourceFormat?: string | null; targetFormat?: string | null; headers?: Record | Headers | null; userAgent?: string | null; }): boolean { const isDirectClaudeCodeProvider = provider === "claude" || isClaudeCodeCompatibleProvider(provider); if (!isDirectClaudeCodeProvider) return false; if (sourceFormat !== FORMATS.CLAUDE) return false; if (targetFormat !== FORMATS.CLAUDE) return false; const headerUserAgent = getHeaderValueCaseInsensitive(headers, "user-agent"); const ua = `${userAgent || ""} ${headerUserAgent || ""}`.toLowerCase(); if (ua.includes("claude-code") || ua.includes("claude-cli")) return true; const appHeader = getHeaderValueCaseInsensitive(headers, "x-app"); if (typeof appHeader === "string" && appHeader.trim().toLowerCase() === "cli") return true; const sessionId = getHeaderValueCaseInsensitive(headers, "x-claude-code-session-id"); return typeof sessionId === "string" && sessionId.trim().length > 0; } function buildClaudePassthroughToolNameMap(body: Record | null | undefined) { if (!body || !Array.isArray(body.tools)) return null; const toolNameMap = new Map(); for (const tool of body.tools) { const toolRecord = tool as Record; const toolData = toolRecord?.type === "function" && toolRecord.function && typeof toolRecord.function === "object" ? (toolRecord.function as Record) : toolRecord; const originalName = typeof toolData?.name === "string" ? toolData.name.trim() : ""; if (!originalName) continue; toolNameMap.set(`${CLAUDE_OAUTH_TOOL_PREFIX}${originalName}`, originalName); } return toolNameMap.size > 0 ? toolNameMap : null; } function restoreClaudePassthroughToolNames( responseBody: Record, toolNameMap: Map | null ) { if (!toolNameMap || !Array.isArray(responseBody?.content)) return responseBody; let changed = false; const content = responseBody.content.map((block: Record) => { if (block?.type !== "tool_use" || typeof block?.name !== "string") return block; const restoredName = toolNameMap.get(block.name) ?? block.name; if (restoredName === block.name) return block; changed = true; return { ...block, name: restoredName, }; }); if (!changed) return responseBody; return { ...responseBody, content, }; } function mergeResponseToolNameMap( baseToolNameMap: Map | null, transformedBody: Record | null | undefined ) { const executorToolNameMap = transformedBody && transformedBody._toolNameMap instanceof Map ? (transformedBody._toolNameMap as Map) : null; if (!executorToolNameMap?.size) return baseToolNameMap; if (!baseToolNameMap?.size) return executorToolNameMap; const merged = new Map(baseToolNameMap); for (const [toolName, originalName] of executorToolNameMap.entries()) { merged.set(toolName, originalName); } return merged; } const STREAMING_RESPONSE_HEADER_DENYLIST = new Set([ "content-type", "content-encoding", "content-length", "transfer-encoding", ]); export function buildStreamingResponseHeaders( providerHeaders: Headers, meta: Parameters[0] ): Record { const forwardedHeaders: [string, string][] = []; providerHeaders.forEach((value, key) => { if (!STREAMING_RESPONSE_HEADER_DENYLIST.has(key.toLowerCase())) { forwardedHeaders.push([key, value]); } }); return { ...Object.fromEntries(forwardedHeaders), "Content-Type": "text/event-stream", "Cache-Control": "no-cache, no-transform", Connection: "keep-alive", "X-Accel-Buffering": "no", [OMNIROUTE_RESPONSE_HEADERS.cache]: "MISS", ...buildOmniRouteResponseMetaHeaders(meta), }; } function materializeDeduplicatedExecutionResult>(result: T): T { const snapshot = result && typeof result === "object" ? ((result as Record)._dedupSnapshot as | { status: number; statusText: string; headers: [string, string][]; payload: string; } | undefined) : undefined; if (!snapshot) return result; return { ...result, response: new Response(snapshot.payload, { status: snapshot.status, statusText: snapshot.statusText, headers: snapshot.headers, }), } as T; } function getSkillsProviderForFormat(format: string): "openai" | "anthropic" | "google" | "other" { switch (format) { case FORMATS.CLAUDE: return "anthropic"; case FORMATS.GEMINI: return "google"; default: return "openai"; } } function getSkillsModelIdForFormat(format: string): string { switch (format) { case FORMATS.CLAUDE: return "claude"; case FORMATS.GEMINI: return "gemini"; default: return "openai"; } } function parseNonStreamingSSEPayload( rawBody: string, preferredFormat: string, fallbackModel: string ): { body: Record; format: string } | null { const formatsToTry: string[] = []; const seen = new Set(); const queueFormat = (format: string) => { if (!format || seen.has(format)) return; seen.add(format); formatsToTry.push(format); }; queueFormat(preferredFormat); queueFormat(FORMATS.OPENAI_RESPONSES); queueFormat(FORMATS.CLAUDE); queueFormat(FORMATS.OPENAI); for (const format of formatsToTry) { const parsed = format === FORMATS.OPENAI_RESPONSES ? parseSSEToResponsesOutput(rawBody, fallbackModel) : format === FORMATS.CLAUDE ? parseSSEToClaudeResponse(rawBody, fallbackModel) : parseSSEToOpenAIResponse(rawBody, fallbackModel); if (parsed && typeof parsed === "object") { return { body: parsed as Record, format, }; } } return null; } function convertNDJSONToSSE(rawBody: string): string { const chunks = String(rawBody || "") .split(/\r?\n/) .map((line) => line.trim()) .filter((line) => line.length > 0); if (chunks.length === 0) return rawBody; return `${chunks.map((chunk) => `data: ${chunk}\n`).join("\n")}\n`; } function normalizeNonStreamingEventPayload(rawBody: string, contentType: string): string { if (contentType.includes("application/x-ndjson")) { return convertNDJSONToSSE(rawBody); } return rawBody; } function isTruthyStreamBody(body: unknown): boolean { return !!body && typeof body === "object" && (body as { stream?: unknown }).stream === true; } function isEventStreamAccepted(headers: Record | Headers | null | undefined) { return (getHeaderValueCaseInsensitive(headers, "accept") || "") .toLowerCase() .includes("text/event-stream"); } function shouldTreatBufferedEventResponseAsExpected( upstreamStream: boolean, providerHeaders: Record | Headers | null | undefined, finalBody: unknown ): boolean { return upstreamStream || isEventStreamAccepted(providerHeaders) || isTruthyStreamBody(finalBody); } const NON_STREAMING_SSE_TERMINAL_TYPES = new Set([ "message_stop", "response.completed", "response.done", "response.cancelled", "response.canceled", "response.failed", "response.incomplete", ]); type NonStreamingSseTerminalState = { currentEvent: string; pendingLine: string; }; function processNonStreamingSseTerminalLine( state: NonStreamingSseTerminalState, rawLine: string ): boolean { const trimmed = rawLine.trim(); if (!trimmed || trimmed.startsWith(":")) { if (!trimmed) state.currentEvent = ""; return false; } if (trimmed.startsWith("event:")) { state.currentEvent = trimmed.slice(6).trim(); return false; } if (!trimmed.startsWith("data:")) return false; const data = trimmed.slice(5).trim(); if (data === "[DONE]") return true; if (!data) return false; try { const parsed = JSON.parse(data); const eventType = parsed && typeof parsed === "object" && typeof parsed.type === "string" ? parsed.type : state.currentEvent; return NON_STREAMING_SSE_TERMINAL_TYPES.has(eventType); } catch { // Keep reading malformed data so the parser can report a useful upstream error. return false; } } function appendNonStreamingSseTerminalSignal( state: NonStreamingSseTerminalState, chunk: string ): boolean { const lines = `${state.pendingLine}${chunk}`.split(/\r?\n/); state.pendingLine = lines.pop() ?? ""; for (const rawLine of lines) { if (processNonStreamingSseTerminalLine(state, rawLine)) return true; } return false; } function createBodyTimeoutError(timeoutMs: number): Error { const err = new Error(`Response body read timeout after ${timeoutMs}ms`); err.name = "BodyTimeoutError"; return err; } function readStreamChunkWithTimeout( reader: ReadableStreamDefaultReader, timeoutMs: number ): Promise<{ done: boolean; value?: Uint8Array }> { if (timeoutMs <= 0) return reader.read(); return new Promise((resolve, reject) => { const timeout = setTimeout(() => reject(createBodyTimeoutError(timeoutMs)), timeoutMs); reader.read().then( (value) => { clearTimeout(timeout); resolve(value); }, (error) => { clearTimeout(timeout); reject(error); } ); }); } function createUpstreamStartTimeoutError( timeoutMs: number, provider: string, model: string ): Error { const err = new Error( `Upstream request did not return response headers after ${timeoutMs}ms (${provider}/${model})` ); err.name = "TimeoutError"; return err; } function createAbortError(signal: AbortSignal): Error { const reason = signal.reason; if (reason instanceof Error) return reason; const err = new Error(typeof reason === "string" ? reason : "The operation was aborted"); err.name = "AbortError"; return err; } /** Billable token total — mirrors the columns persisted by saveRequestUsage so the * live token-limit counter stays consistent with usage_history seed-on-miss. */ function computeBillableTokens(usage: unknown): number { // Cache read/creation tokens are a BREAKDOWN already contained inside // getLoggedInputTokens (prompt_tokens / input_tokens). Adding them here would // double-count. Canonical billable total = input + output + reasoning, matching // the columns persisted by saveRequestUsage and seedWindowUsageFromHistory. return getLoggedInputTokens(usage) + getLoggedOutputTokens(usage) + getReasoningTokens(usage); } function getExecutorTimeoutMs(executor: unknown): number { const getTimeoutMs = (executor as { getTimeoutMs?: () => unknown } | null)?.getTimeoutMs; if (typeof getTimeoutMs !== "function") return FETCH_TIMEOUT_MS; try { const timeoutMs = getTimeoutMs.call(executor); if (typeof timeoutMs !== "number" || !Number.isFinite(timeoutMs)) return FETCH_TIMEOUT_MS; return Math.max(0, Math.floor(timeoutMs)); } catch { return FETCH_TIMEOUT_MS; } } async function executeWithUpstreamStartTimeout({ executor, provider, model, signal, log, execute, }: { executor: unknown; provider: string; model: string; signal: AbortSignal; log?: { warn?: (tag: string, message: string) => void } | null; execute: (signal: AbortSignal) => Promise; }): Promise { const timeoutMs = getExecutorTimeoutMs(executor); if (timeoutMs <= 0) return execute(signal); if (signal.aborted) throw createAbortError(signal); const timeoutController = new AbortController(); const combinedController = new AbortController(); const timeoutError = createUpstreamStartTimeoutError(timeoutMs, provider, model); let timeoutId: ReturnType | null = null; let abortListener: (() => void) | null = null; let timeoutAbortListener: (() => void) | null = null; const abortCombined = (source: AbortSignal) => { if (combinedController.signal.aborted) return; const reason = source.reason instanceof Error ? source.reason : createAbortError(source); combinedController.abort(reason); }; abortListener = () => abortCombined(signal); timeoutAbortListener = () => abortCombined(timeoutController.signal); signal.addEventListener("abort", abortListener, { once: true }); timeoutController.signal.addEventListener("abort", timeoutAbortListener, { once: true }); const timeoutPromise = new Promise((_, reject) => { timeoutId = setTimeout(() => { log?.warn?.("TIMEOUT", timeoutError.message); timeoutController.abort(timeoutError); reject(timeoutError); }, timeoutMs); }); const abortPromise = new Promise((_, reject) => { signal.addEventListener("abort", () => reject(createAbortError(signal)), { once: true }); }); try { return await Promise.race([execute(combinedController.signal), timeoutPromise, abortPromise]); } finally { if (timeoutId) clearTimeout(timeoutId); if (abortListener) signal.removeEventListener("abort", abortListener); if (timeoutAbortListener) { timeoutController.signal.removeEventListener("abort", timeoutAbortListener); } } } /** * Strip hop-by-hop headers that describe the upstream wire encoding. * * `readNonStreamingResponseBody` reads (and, for compressed responses, also * decompresses via fetch's auto-decoder) the full upstream body into a JS * string before we re-emit it to the client. Once that happens, the original * `Content-Encoding`, `Content-Length`, and `Transfer-Encoding` all describe * a payload that no longer exists: * * - `Content-Length` is the *compressed* byte count, so clients honoring it * read only the first N bytes of the decompressed JSON and surface * "Unterminated string in JSON at position …" parse failures (observed * on gzipped Gemini responses). * - `Content-Encoding` advertises a compression we have already undone. * - `Transfer-Encoding` is hop-by-hop per RFC 7230 §6.1 and must not be * forwarded across a buffering proxy — its presence alongside a * re-emitted body is undefined behavior. * * Deleting all three lets the response framework set a fresh, correct * `Content-Length` (or fall back to `Transfer-Encoding: chunked`) for the * payload we are actually sending. */ export function stripStaleForwardingHeaders(headers: Headers): void { headers.delete("content-encoding"); headers.delete("content-length"); headers.delete("transfer-encoding"); } async function readNonStreamingResponseBody( response: Response, contentType: string, upstreamStream: boolean ): Promise { if ( !upstreamStream || !response.body || (!contentType.includes("text/event-stream") && !contentType.includes("application/x-ndjson")) ) { return withBodyTimeout(response.text()); } const reader = response.body.getReader(); const decoder = new TextDecoder(); const terminalState: NonStreamingSseTerminalState = { currentEvent: "", pendingLine: "", }; let rawBody = ""; const deadline = FETCH_BODY_TIMEOUT_MS > 0 ? Date.now() + FETCH_BODY_TIMEOUT_MS : 0; try { while (true) { const timeoutMs = deadline > 0 ? deadline - Date.now() : 0; if (deadline > 0 && timeoutMs <= 0) { throw createBodyTimeoutError(FETCH_BODY_TIMEOUT_MS); } const { done, value } = await readStreamChunkWithTimeout(reader, timeoutMs); if (done) break; if (!value) continue; const decodedChunk = decoder.decode(value, { stream: true }); rawBody += decodedChunk; if (appendNonStreamingSseTerminalSignal(terminalState, decodedChunk)) { await reader.cancel("non-streaming bridge consumed terminal SSE event").catch(() => {}); break; } } } catch (error) { await reader.cancel(error).catch(() => {}); throw error; } finally { rawBody += decoder.decode(); reader.releaseLock(); } return rawBody; } function getHeaderValueCaseInsensitive( headers: Record | Headers | null | undefined, targetName: string ) { if (!headers || typeof headers !== "object") return null; if (headers instanceof Headers) { return headers.get(targetName); } const lowered = targetName.toLowerCase(); for (const [key, value] of Object.entries(headers)) { if (key.toLowerCase() === lowered && typeof value === "string" && value.trim()) { return value.trim(); } } return null; } function toFiniteNumberOrNull(value: unknown): number | null { if (typeof value === "number" && Number.isFinite(value)) { return value; } if (typeof value === "string" && value.trim().length > 0) { const parsed = Number(value); return Number.isFinite(parsed) ? parsed : null; } return null; } function isSemaphoreCapacityError(error: unknown): error is Error & { code: string } { return ( !!error && typeof error === "object" && ((error as { code?: unknown }).code === "SEMAPHORE_TIMEOUT" || (error as { code?: unknown }).code === "SEMAPHORE_QUEUE_FULL") ); } function createStreamingErrorResult( statusCode: number, message: string, code?: string, type?: string ) { const errorBody = buildErrorBody(statusCode, message); if (code) { errorBody.error.code = code; } if (type) { errorBody.error.type = type; } const body = `data: ${JSON.stringify(errorBody)}\n\ndata: [DONE]\n\n`; return { success: false as const, status: statusCode, error: message, response: new Response(body, { status: statusCode, headers: { "Content-Type": "text/event-stream", "Cache-Control": "no-cache, no-transform", Connection: "keep-alive", "X-Accel-Buffering": "no", }, }), }; } function getUpstreamErrorIdentifier(error: unknown): string | undefined { if (!error || typeof error !== "object") return undefined; const value = (error as { code?: unknown }).code; return typeof value === "string" && value.length > 0 ? value : undefined; } function wrapReadableStreamWithFinalize( readable: ReadableStream, finalize: () => void ): ReadableStream { const reader = readable.getReader(); let finalized = false; const runFinalize = () => { if (finalized) return; finalized = true; finalize(); }; return new ReadableStream({ async pull(controller) { try { const { done, value } = await reader.read(); if (done) { runFinalize(); controller.close(); return; } controller.enqueue(value); } catch (error) { runFinalize(); controller.error(error); } }, async cancel(reason) { runFinalize(); try { await reader.cancel(reason); } catch (error) { // Ignored } }, }); } function resolveAccountSemaphoreAccountKey( connectionId: string | null | undefined, credentials: Record | null | undefined ): string | null { if (typeof connectionId === "string" && connectionId.trim().length > 0) { return connectionId; } const candidateKeys = [ credentials?.connectionId, credentials?.id, credentials?.email, credentials?.name, credentials?.displayName, ]; for (const candidate of candidateKeys) { if (typeof candidate === "string" && candidate.trim().length > 0) { return candidate.trim(); } } return null; } function resolveAccountSemaphoreMaxConcurrency( credentials: Record | null | undefined ): number | null { return toFiniteNumberOrNull(credentials?.maxConcurrent); } function resolveAccountSemaphoreKey({ provider, model, connectionId, credentials, }: { provider: string | null | undefined; model: string; connectionId: string | null | undefined; credentials: Record | null | undefined; }): string | null { const accountKey = resolveAccountSemaphoreAccountKey(connectionId, credentials); if (!accountKey || !provider) return null; return buildAccountSemaphoreKey({ provider, accountKey }); } function buildClaudePromptCacheLogMeta( targetFormat: string, finalBody: Record | null | undefined, providerHeaders: Record | Headers | null | undefined, clientHeaders?: Headers | Record | null | undefined ) { if (targetFormat !== FORMATS.CLAUDE || !finalBody || typeof finalBody !== "object") return null; const describeCacheControl = (cacheControl: Record | undefined, extra = {}) => ({ type: cacheControl && typeof cacheControl.type === "string" && cacheControl.type.trim() ? cacheControl.type.trim() : "ephemeral", ttl: cacheControl && typeof cacheControl.ttl === "string" && cacheControl.ttl.trim() ? cacheControl.ttl.trim() : null, ...extra, }); const systemBreakpoints = Array.isArray(finalBody.system) ? finalBody.system.flatMap((block, index) => { if (!block || typeof block !== "object") return []; const text = typeof block.text === "string" && block.text.trim().length > 0 ? block.text.trim() : ""; if (text.startsWith("x-anthropic-billing-header:")) { return []; } const cacheControl = block.cache_control && typeof block.cache_control === "object" ? block.cache_control : null; return cacheControl ? [describeCacheControl(cacheControl, { index })] : []; }) : []; const toolBreakpoints = Array.isArray(finalBody.tools) ? finalBody.tools.flatMap((tool, index) => { if (!tool || typeof tool !== "object") return []; const cacheControl = tool.cache_control && typeof tool.cache_control === "object" ? tool.cache_control : null; const name = typeof tool.name === "string" && tool.name.trim() ? tool.name.trim() : null; return cacheControl ? [describeCacheControl(cacheControl, { index, name })] : []; }) : []; const messageBreakpoints = Array.isArray(finalBody.messages) ? finalBody.messages.flatMap((message, messageIndex) => { if (!message || typeof message !== "object" || !Array.isArray(message.content)) return []; const role = typeof message.role === "string" && message.role.trim() ? message.role.trim() : "unknown"; return message.content.flatMap((block, contentIndex) => { if (!block || typeof block !== "object") return []; const cacheControl = block.cache_control && typeof block.cache_control === "object" ? block.cache_control : null; if (!cacheControl) return []; return [ describeCacheControl(cacheControl, { messageIndex, contentIndex, role, blockType: typeof block.type === "string" && block.type.trim() ? block.type.trim() : "unknown", }), ]; }); }) : []; const totalBreakpoints = systemBreakpoints.length + toolBreakpoints.length + messageBreakpoints.length; let anthropicBeta = getHeaderValueCaseInsensitive(providerHeaders, "Anthropic-Beta"); if (!anthropicBeta) { anthropicBeta = getHeaderValueCaseInsensitive(clientHeaders, "Anthropic-Beta"); } if (totalBreakpoints === 0 && !anthropicBeta) return null; return { applied: totalBreakpoints > 0, totalBreakpoints, anthropicBeta, systemBreakpoints, toolBreakpoints, messageBreakpoints, }; } function toPositiveNumber(value: unknown) { return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : 0; } function buildCacheUsageLogMeta(usage: Record | null | undefined) { if (!usage || typeof usage !== "object") return null; const promptTokenDetails = usage.prompt_tokens_details && typeof usage.prompt_tokens_details === "object" ? (usage.prompt_tokens_details as Record) : undefined; const hasCacheFields = "cache_read_input_tokens" in usage || "cached_tokens" in usage || "cache_creation_input_tokens" in usage || (!!promptTokenDetails && ("cached_tokens" in promptTokenDetails || "cache_creation_tokens" in promptTokenDetails)); const cacheReadTokens = toPositiveNumber( usage.cache_read_input_tokens ?? usage.cached_tokens ?? promptTokenDetails?.cached_tokens ); const cacheCreationTokens = toPositiveNumber( usage.cache_creation_input_tokens ?? promptTokenDetails?.cache_creation_tokens ); if (!hasCacheFields) return null; return { cacheReadTokens, cacheCreationTokens, }; } function attachLogMeta( payload: Record | null | undefined, meta: Record | null | undefined ) { if (!meta || typeof meta !== "object") return payload; const compactMeta = Object.fromEntries( Object.entries(meta).filter(([, value]) => value !== null && value !== undefined) ); if (Object.keys(compactMeta).length === 0) return payload; if (!payload || typeof payload !== "object" || Array.isArray(payload)) { return { _omniroute: compactMeta, _payload: payload ?? null }; } const existing = payload._omniroute && typeof payload._omniroute === "object" && !Array.isArray(payload._omniroute) ? payload._omniroute : {}; return { ...payload, _omniroute: { ...existing, ...compactMeta, }, }; } /** * Core chat handler - shared between SSE and Worker * Returns { success, response, status, error } for caller to handle fallback * @param {object} options * @param {object} options.body - Request body * @param {object} options.modelInfo - { provider, model } * @param {object} options.credentials - Provider credentials * @param {object} options.log - Logger instance (optional) * @param {function} options.onCredentialsRefreshed - Callback when credentials are refreshed * @param {function} options.onRequestSuccess - Callback when request succeeds (to clear error status) * @param {function} options.onDisconnect - Callback when client disconnects * @param {string} options.connectionId - Connection ID for usage tracking * @param {object} options.apiKeyInfo - API key metadata for usage attribution * @param {string} options.userAgent - Client user agent for caching decisions * @param {string} options.comboName - Combo name if this is a combo request * @param {string} options.comboStrategy - Combo routing strategy (e.g., 'priority', 'cost-optimized') * @param {boolean} options.isCombo - Whether this request is from a combo * @param {string} options.connectionId - Connection ID for settings lookup */ /** * Module-level cache for upstream proxy config (shared across all requests). * 10s TTL prevents per-request DB lookups while staying fresh enough for setting changes. */ const _proxyConfigCache = new Map(); const PROXY_CONFIG_CACHE_TTL = 10_000; /** * Module-level cache for all combos data (shared across all requests). * Uses cached promises to prevent thundering herd — all concurrent callers * wait for the same underlying DB query while it's in flight. */ let _combosPromise: Promise | null = null; let _combosCacheTs = 0; const COMBOS_CACHE_TTL = 10_000; async function getCombosCached(): Promise { const now = Date.now(); if (_combosPromise && now - _combosCacheTs < COMBOS_CACHE_TTL) { return _combosPromise; } _combosCacheTs = now; _combosPromise = (async () => { const { getCombos } = await import("@/lib/localDb"); return getCombos(); })(); return _combosPromise; } export function clearCombosCache() { _combosPromise = null; _combosCacheTs = 0; } export function clearUpstreamProxyConfigCache(providerId?: string) { if (providerId) { _proxyConfigCache.delete(providerId); return; } _proxyConfigCache.clear(); } async function getUpstreamProxyConfigCached(providerId: string) { const cached = _proxyConfigCache.get(providerId); if (cached && Date.now() - cached.ts < PROXY_CONFIG_CACHE_TTL) return cached; const cfg = await getUpstreamProxyConfig(providerId).catch(() => null); const result = cfg ? { mode: cfg.mode, enabled: cfg.enabled, ts: Date.now() } : { mode: "native" as const, enabled: false, ts: Date.now() }; _proxyConfigCache.set(providerId, result); return result; } function buildExecutorClientHeaders( headers: Headers | Record | null | undefined, userAgent?: string | null ) { const normalized: Record = {}; if (headers instanceof Headers) { headers.forEach((value, key) => { normalized[key] = value; }); } else if (headers && typeof headers === "object") { for (const [key, value] of Object.entries(headers)) { if (typeof value === "string") { normalized[key] = value; } } } const normalizedUserAgent = typeof userAgent === "string" ? userAgent.trim() : ""; if (normalizedUserAgent && !normalized["user-agent"] && !normalized["User-Agent"]) { normalized["user-agent"] = normalizedUserAgent; normalized["User-Agent"] = normalizedUserAgent; } return Object.keys(normalized).length > 0 ? normalized : null; } function isCopilotClient( headers: Headers | Record | null | undefined, userAgent?: string | null ) { const isMatch = (value: unknown) => typeof value === "string" && value.toLowerCase().includes("copilot"); if (isMatch(userAgent)) return true; if (headers instanceof Headers) { for (const [key, value] of headers as unknown as Iterable<[string, string]>) { if (isMatch(key) || isMatch(value)) return true; } } else if (headers && typeof headers === "object") { for (const [key, value] of Object.entries(headers)) { if (isMatch(key) || isMatch(value)) return true; } } return false; } export function extractSystemRoleMessages(payload: Record): void { if (!Array.isArray(payload.messages)) return; const messages = payload.messages as Array<{ role?: unknown; content?: unknown }>; // Treat both `system` and `developer` as system-equivalent (OpenAI's Responses // API renamed system → developer). Anthropic rejects either as a chat role, so // both must be lifted into the top-level `system` field — parity with the // normal-path extractSystemMessagesToBody closure. const isSystemRole = (role: unknown): boolean => typeof role === "string" && (role.toLowerCase() === "system" || role.toLowerCase() === "developer"); const systemMessages = messages.filter((m) => isSystemRole(m.role)); if (systemMessages.length === 0) return; const extraBlocks: Array> = []; for (const sm of systemMessages) { if (typeof sm.content === "string" && sm.content.length > 0) { extraBlocks.push({ type: "text", text: sm.content }); } else if (Array.isArray(sm.content)) { for (const block of sm.content as Array>) { if (block?.type === "text" && typeof block.text === "string" && block.text.length > 0) { extraBlocks.push({ ...block }); } } } } if (extraBlocks.length > 0) { const existingSystem = payload.system; if (typeof existingSystem === "string" && existingSystem.length > 0) { payload.system = [{ type: "text", text: existingSystem }, ...extraBlocks]; } else if (Array.isArray(existingSystem)) { payload.system = [...(existingSystem as Array>), ...extraBlocks]; } else { payload.system = extraBlocks; } } payload.messages = messages.filter((m) => !isSystemRole(m.role)); } export async function handleChatCore({ body, modelInfo, credentials, log, onCredentialsRefreshed, onRequestSuccess, onStreamFailure, onDisconnect, clientRawRequest, connectionId, apiKeyInfo = null, userAgent, comboName, comboStrategy = null, isCombo = false, comboStepId = null, comboExecutionKey = null, disableEmergencyFallback = false, cachedSettings = null, skipUpstreamRetry = false, }) { let { provider, model, extendedContext } = modelInfo; // apiFormat is an optional custom-model marker injected by getModelInfo for // providers whose models can route to /chat/completions or /responses // (Azure AI Foundry, OCI generic OpenAI). It's not on the base ModelInfo // shape, so we read it via a structural narrowing without `as any`. const apiFormat: string | undefined = modelInfo && typeof modelInfo === "object" && "apiFormat" in modelInfo ? typeof (modelInfo as { apiFormat?: unknown }).apiFormat === "string" ? ((modelInfo as { apiFormat?: string }).apiFormat as string) : undefined : undefined; const requestedModel = typeof body?.model === "string" && body.model.trim().length > 0 ? body.model : model; const isModelScope = () => isModelScopeProvider(provider, credentials?.providerSpecificData); const startTime = Date.now(); // Per-request trace id + checkpoint helper. Lets us see exactly which await // a hung request was sitting on in `[STAGE_TRACE]` log lines. const traceId = Math.random().toString(36).slice(2, 8); // Emit request.started event for real-time dashboard setImmediate(() => { emit("request.started", { id: traceId, model: model || "unknown", provider: provider || "unknown", timestamp: startTime, comboName: comboName || undefined, }); }); const trace = (label: string, extra?: Record) => { const elapsed = Date.now() - startTime; const suffix = extra ? ` ${JSON.stringify(extra)}` : ""; log?.info?.("STAGE_TRACE", `${traceId} ${label} t=${elapsed}ms${suffix}`); }; let tokensCompressed: number | null = null; body = injectSystemPrompt(body); // ── Plugin onRequest hook ── // Dynamic import cached by Node.js after first call — minimal overhead try { const { runOnRequest } = await import("@/lib/plugins/index"); const pluginCtx = { requestId: traceId, body, model, provider, apiKeyInfo, metadata: {}, }; const pluginResult = await runOnRequest(pluginCtx); if (pluginResult?.blocked) { log?.info?.("PLUGIN", `Request blocked by plugin`); return { success: false, status: 403, error: "Request blocked by plugin", response: pluginResult.response ? new Response(JSON.stringify(pluginResult.response), { status: 403, headers: { "Content-Type": "application/json" }, }) : new Response( JSON.stringify({ error: { message: "Request blocked by plugin", type: "plugin_block" }, }), { status: 403, headers: { "Content-Type": "application/json" }, } ), }; } if (pluginResult?.ctx && "body" in pluginResult.ctx) { body = (pluginResult.ctx as unknown as Record).body; } } catch (pluginErr) { log?.debug?.( "PLUGIN", `onRequest hook error (non-fatal): ${pluginErr instanceof Error ? pluginErr.message : String(pluginErr)}` ); } type EffectiveServiceTier = "standard" | CodexServiceTier; let effectiveServiceTier: EffectiveServiceTier = "standard"; const resolveEffectiveServiceTier = (requestBody?: unknown): EffectiveServiceTier => { if (provider !== "codex") return "standard"; const requestRecord = requestBody && typeof requestBody === "object" && !Array.isArray(requestBody) ? (requestBody as Record) : {}; const rawServiceTier = requestRecord.service_tier; if (typeof rawServiceTier === "string" && rawServiceTier.trim().length > 0) { const normalizedServiceTier = normalizeCodexServiceTier(rawServiceTier); if (normalizedServiceTier) return normalizedServiceTier; } return getCodexRequestDefaults(credentials?.providerSpecificData).serviceTier ?? "standard"; }; const resolveReportedServiceTier = ( payload?: unknown, maxDepth = 3 ): EffectiveServiceTier | null => { if ( maxDepth <= 0 || provider !== "codex" || !payload || typeof payload !== "object" || Array.isArray(payload) ) { return null; } const record = payload as Record; const rawServiceTier = record.service_tier; if (typeof rawServiceTier === "string" && rawServiceTier.trim().length > 0) { const normalizedServiceTier = normalizeCodexServiceTier(rawServiceTier); if (normalizedServiceTier) return normalizedServiceTier; } return resolveReportedServiceTier(record.response, maxDepth - 1); }; const persistFailureUsage = (statusCode: number, errorCode?: string | null) => { saveRequestUsage({ provider: provider || "unknown", model: model || "unknown", tokens: { input: 0, output: 0, cacheRead: 0, cacheCreation: 0, reasoning: 0 }, status: String(statusCode), success: false, latencyMs: Date.now() - startTime, timeToFirstTokenMs: 0, errorCode: errorCode || String(statusCode), timestamp: new Date().toISOString(), connectionId: connectionId || undefined, apiKeyId: apiKeyInfo?.id || undefined, apiKeyName: apiKeyInfo?.name || undefined, serviceTier: effectiveServiceTier, comboStrategy: isCombo ? comboStrategy || undefined : undefined, }).catch(() => {}); }; const recordKeyHealthStatus = ( status: number, creds: Record | null | undefined ): void => { const connId = creds?.connectionId as string | undefined; if (!connId) return; const psd = creds.providerSpecificData as Record | undefined; const extraKeys = (psd?.extraApiKeys as string[] | undefined) ?? []; const health = psd?.apiKeyHealth as Record | undefined; const currentKeyId = (psd?.selectedKeyId as string | undefined) ?? "primary"; trackConnectionExtraKeys(connId, extraKeys); if (status === 401) { const updatedHealth = recordKeyFailure(connId, currentKeyId); log?.warn?.( "AUTH", `401 on connection ${connId.slice(0, 8)} - key marked as failed (failure #${updatedHealth.failures})` ); // Persist health status to DB on every failure (not just invalid transitions) // This ensures in-memory state survives process restarts const prevStatus = health?.[currentKeyId]?.status; const prevFailures = health?.[currentKeyId]?.failures ?? 0; if (updatedHealth.status !== prevStatus || updatedHealth.failures !== prevFailures) { updateProviderConnection(connId, { providerSpecificData: { ...psd, apiKeyHealth: { ...health, [currentKeyId]: updatedHealth }, }, }).catch((err: unknown) => { log?.error?.( "DB", `Failed to persist apiKeyHealth: ${err instanceof Error ? err.message : String(err)}` ); }); } } else if (status >= 200 && status < 300) { const updatedHealth = recordKeySuccess(connId, currentKeyId); const prevStatus = health?.[currentKeyId]?.status; if (prevStatus === "warning" || prevStatus === "invalid") { updateProviderConnection(connId, { providerSpecificData: { ...psd, apiKeyHealth: { ...health, [currentKeyId]: updatedHealth }, }, }).catch((err: unknown) => { log?.error?.( "DB", `Failed to persist apiKeyHealth: ${err instanceof Error ? err.message : String(err)}` ); }); } } }; const persistCodexQuotaState = async ( headers: Record | null, status = 0 ) => { if (provider !== "codex" || !connectionId || !headers) return; try { const quota = parseCodexQuotaHeaders(headers); if (!quota) return; const existingProviderData = credentials?.providerSpecificData && typeof credentials.providerSpecificData === "object" ? credentials.providerSpecificData : {}; const scope = getCodexModelScope(model || requestedModel || ""); const quotaState = { usage5h: quota.usage5h, limit5h: quota.limit5h, resetAt5h: quota.resetAt5h, usage7d: quota.usage7d, limit7d: quota.limit7d, resetAt7d: quota.resetAt7d, scope, updatedAt: new Date().toISOString(), }; const nextProviderData: Record = { ...existingProviderData, codexQuotaState: quotaState, }; // T03/T09: on 429, persist exact reset time per scope to avoid global over-blocking. // Use dual-window cooldown to distinguish short-term and weekly Codex exhaustion. if (status === 429) { const { cooldownMs, window: exhaustedWindow } = getCodexDualWindowCooldownMs(quota); if (cooldownMs > 0) { const scopeUntil = new Date(Date.now() + cooldownMs).toISOString(); const scopeMapRaw = existingProviderData && typeof existingProviderData === "object" && existingProviderData.codexScopeRateLimitedUntil && typeof existingProviderData.codexScopeRateLimitedUntil === "object" ? existingProviderData.codexScopeRateLimitedUntil : {}; nextProviderData.codexScopeRateLimitedUntil = { ...(scopeMapRaw as Record), [scope]: scopeUntil, }; nextProviderData.codexExhaustedWindow = exhaustedWindow; log?.debug?.( "CODEX", `Quota exhaustion on ${exhaustedWindow} window, cooldown until ${scopeUntil}` ); } // Invalidate the preflight cache for this connection so the next // isModelAvailable check fetches fresh quota data. if (connectionId) { invalidateCodexQuotaCache(connectionId); } } await updateProviderConnection(connectionId, { providerSpecificData: nextProviderData, }); credentials.providerSpecificData = nextProviderData; } catch (err) { const errMessage = err instanceof Error ? err.message : String(err); log?.debug?.("CODEX", `Failed to persist codex quota state: ${errMessage}`); } }; // ── Phase 9.2: Idempotency check ── const idempotencyKey = getIdempotencyKey(clientRawRequest?.headers); const cachedIdemp = checkIdempotency(idempotencyKey); if (cachedIdemp) { log?.debug?.("IDEMPOTENCY", `Hit for key=${idempotencyKey?.slice(0, 12)}...`); const idempotentUsage = cachedIdemp.response && typeof cachedIdemp.response === "object" ? ((cachedIdemp.response as Record).usage as | Record | undefined) : undefined; const idempotentCost = idempotentUsage ? await calculateCost(provider, model, idempotentUsage as Record, { serviceTier: effectiveServiceTier, }) : 0; return { success: true, response: new Response(JSON.stringify(cachedIdemp.response), { status: cachedIdemp.status, headers: { "Content-Type": "application/json", "X-OmniRoute-Idempotent": "true", ...buildOmniRouteResponseMetaHeaders({ provider, model, cacheHit: false, latencyMs: Date.now() - startTime, usage: idempotentUsage, costUsd: idempotentCost, }), }, }), }; } // T07: Inject connectionId into credentials so executors can rotate API keys // using providerSpecificData.extraApiKeys (API Key Round-Robin feature) if (connectionId && credentials && !credentials.connectionId) { credentials.connectionId = connectionId; } const endpointPath = String(clientRawRequest?.endpoint || ""); const sourceFormat = detectFormatFromEndpoint(body, endpointPath); const isResponsesEndpoint = /\/responses(?=\/|$)/i.test(endpointPath) || /^responses(?=\/|$)/i.test(endpointPath); const nativeCodexPassthrough = shouldUseNativeCodexPassthrough({ provider, sourceFormat, endpointPath, }); const isDroidCLI = userAgent?.toLowerCase().includes("droid") || userAgent?.toLowerCase().includes("codex-cli"); const copilotCompatibleReasoning = isCopilotClient(clientRawRequest?.headers, userAgent); const clientResponseFormat = sourceFormat === FORMATS.OPENAI_RESPONSES && !isResponsesEndpoint && !isDroidCLI ? FORMATS.OPENAI : sourceFormat; // Check for bypass patterns (warmup, skip) - return fake response const bypassResponse = handleBypassRequest(body, model, userAgent); if (bypassResponse) { return bypassResponse; } // Detect source format and get target format // Model-specific targetFormat takes priority over provider default // ── Background Task Redirection (T41) ── const bgConfig = getBackgroundDegradationConfig(); const backgroundReason = bgConfig.enabled ? getBackgroundTaskReason(body, clientRawRequest?.headers) : null; if (backgroundReason) { const degradedModel = getDegradedModel(model); if (degradedModel !== model) { const originalModel = model; log?.info?.( "BACKGROUND", `Background task redirect (${backgroundReason}): ${originalModel} → ${degradedModel}` ); model = degradedModel; if (body && typeof body === "object") { body.model = model; } logAuditEvent({ action: "routing.background_task_redirect", actor: apiKeyInfo?.name || "system", target: connectionId || provider || "chat", details: { original_model: originalModel, redirected_to: degradedModel, reason: backgroundReason, }, }); } } // Apply custom model aliases (Settings → Model Aliases → Pattern→Target) before routing (#315, #472) // Custom aliases take priority over built-in and must be resolved here so the // downstream getModelTargetFormat() lookup AND the actual provider request use // the correct, aliased model ID. Without this, aliases only affect format detection. const resolvedModel = resolveModelAlias(model); // Use resolvedModel for all downstream operations (routing, provider requests, logging) const effectiveModel = resolvedModel === model ? model : resolvedModel; if (resolvedModel !== model) { log?.info?.("ALIAS", `Model alias applied: ${model} → ${resolvedModel}`); } const alias = PROVIDER_ID_TO_ALIAS[provider] || provider; const modelTargetFormat = getModelTargetFormat(alias, resolvedModel); const targetFormat = apiFormat === "responses" ? FORMATS.OPENAI_RESPONSES : modelTargetFormat || getTargetFormat(provider, credentials?.providerSpecificData); const initialProviderRequest = body && typeof body === "object" && !Array.isArray(body) ? { ...(body as Record), model: typeof (body as Record).model === "string" ? (body as Record).model : effectiveModel, } : body; // Track pending requests before slower optional enrichment (settings, logging, // compression) so active-request callers can observe the provider payload even // when upstream never returns response headers. trackPendingRequest(model, provider, connectionId, true, { clientEndpoint: clientRawRequest?.endpoint || "/v1/chat/completions", clientRequest: clientRawRequest?.body ?? body, providerRequest: initialProviderRequest, stage: "registered", }); // Initialize rate limit settings from persisted DB (once, lazy) await initializeRateLimits(); const { body: bodyWithWebSearchFallback, fallback: webSearchFallbackPlan } = prepareWebSearchFallbackBody(body as Record, { provider, sourceFormat, targetFormat, nativeCodexPassthrough, }); if (webSearchFallbackPlan.enabled) { body = bodyWithWebSearchFallback as typeof body; log?.info?.( "TOOLS", `Converted ${webSearchFallbackPlan.convertedToolCount} web_search tool(s) to OmniRoute fallback for ${provider}` ); } const noLogEnabled = apiKeyInfo?.noLog === true; const detailedLoggingEnabled = !noLogEnabled && (await isDetailedLoggingEnabled()); const capturePipelineStreamChunks = detailedLoggingEnabled && getCallLogPipelineCaptureStreamChunks(); const skillRequestId = generateRequestId(); let compressionAnalyticsWritePromise: Promise | null = null; const attachCompressionUsageReceiptAfterAnalytics = ( usage: Record, source: "provider" | "estimated" | "stream" ) => { const pendingWrite = compressionAnalyticsWritePromise; void (async () => { try { if (pendingWrite) await pendingWrite; const { attachCompressionUsageReceipt } = await import("../../src/lib/db/compressionAnalytics.ts"); attachCompressionUsageReceipt(skillRequestId, usage, source); } catch { // Compression analytics are best-effort and must never affect responses. } })(); }; const pipelineSessionId = (clientRawRequest?.headers && typeof clientRawRequest.headers.get === "function" ? clientRawRequest.headers.get("x-omniroute-session-id") : getHeaderValueCaseInsensitive( clientRawRequest?.headers ?? null, "x-omniroute-session-id" )) || skillRequestId; const persistAttemptLogs = ({ status, tokens, responseBody, error, providerRequest, providerResponse, clientResponse, claudeCacheMeta, claudeCacheUsageMeta, cacheSource, }: { status: number; tokens?: unknown; responseBody?: unknown; error?: string | null; providerRequest?: unknown; providerResponse?: unknown; clientResponse?: unknown; claudeCacheMeta?: Record; claudeCacheUsageMeta?: Record; cacheSource?: "upstream" | "semantic"; }) => { const providerWarnings = extractProviderWarnings( providerResponse, clientResponse, responseBody ); if (providerWarnings.length > 0) { logAuditEvent({ action: "provider.warning", actor: "system", target: [provider, connectionId].filter(Boolean).join(":") || provider || model, resourceType: "provider_warning", status: "warning", requestId: skillRequestId, details: { provider, model, connectionId, httpStatus: status, warnings: providerWarnings, }, }); } const callLogId = generateRequestId(); const pipelinePayloads = detailedLoggingEnabled ? reqLogger?.getPipelinePayloads?.() : null; if (pipelinePayloads) { if (providerResponse !== undefined && !pipelinePayloads.providerResponse) { pipelinePayloads.providerResponse = providerResponse as Record; } if (clientResponse !== undefined) { pipelinePayloads.clientResponse = clientResponse as Record; } if (error) { pipelinePayloads.error = { ...(typeof pipelinePayloads.error === "object" && pipelinePayloads.error ? (pipelinePayloads.error as Record) : {}), message: error, }; } } saveCallLog({ id: callLogId, method: "POST", path: clientRawRequest?.endpoint || "/v1/chat/completions", status, model, requestedModel, provider, connectionId, duration: Date.now() - startTime, tokens: tokens || {}, requestBody: cloneBoundedChatLogPayload( attachLogMeta((body as Record) ?? undefined, { claudePromptCache: claudeCacheMeta, }) ), responseBody: cloneBoundedChatLogPayload( attachLogMeta((responseBody as Record) ?? undefined, { claudePromptCache: claudeCacheMeta ? { applied: claudeCacheMeta.applied, totalBreakpoints: claudeCacheMeta.totalBreakpoints, anthropicBeta: claudeCacheMeta.anthropicBeta, } : null, claudePromptCacheUsage: claudeCacheUsageMeta, }) ), error: error || null, sourceFormat, targetFormat, comboName, comboStepId, comboExecutionKey, tokensCompressed, cacheSource: cacheSource === "semantic" ? "semantic" : "upstream", apiKeyId: apiKeyInfo?.id || null, apiKeyName: apiKeyInfo?.name || null, noLog: noLogEnabled, pipelinePayloads, }).catch(() => {}); }; // Primary path: merge client model id + alias target so config on either key applies; resolved // id wins on same header name. T5 family fallback uses only (nextModel, resolveModelAlias(next)) // so A-model headers are not sent to B — see buildUpstreamHeadersForExecute. const connectionCustomUserAgent = credentials?.providerSpecificData && typeof credentials.providerSpecificData === "object" && typeof credentials.providerSpecificData.customUserAgent === "string" ? credentials.providerSpecificData.customUserAgent.trim() : ""; const buildUpstreamHeadersForExecute = (modelToCall: string): Record => { const upstreamHeaders = modelToCall === effectiveModel ? { ...getModelUpstreamExtraHeaders(provider || "", model || "", sourceFormat), ...getModelUpstreamExtraHeaders(provider || "", resolvedModel || "", sourceFormat), } : (() => { const r = resolveModelAlias(modelToCall); return { ...getModelUpstreamExtraHeaders(provider || "", modelToCall || "", sourceFormat), ...getModelUpstreamExtraHeaders(provider || "", r || "", sourceFormat), }; })(); if (connectionCustomUserAgent) { upstreamHeaders["User-Agent"] = connectionCustomUserAgent; if ("user-agent" in upstreamHeaders) { upstreamHeaders["user-agent"] = connectionCustomUserAgent; } } // Claude Fast Mode opt-in. When the user has enabled this in // Settings > AI AND the target provider is the canonical Anthropic // `claude` provider (Claude Code-compatible CPA bridges are excluded // since they already select their own entrypoint) AND the model id // matches the configured list, signal to a paired CLIProxyAPI build to // rewrite the cc_entrypoint so the request can reach Anthropic Fast // Mode (speed:"fast"). CPA builds that do not understand the header // forward it harmlessly. if ( provider === "claude" && typeof settings !== "undefined" && shouldRequestClaudeFastMode(settings, modelToCall) ) { upstreamHeaders[CPA_FORCE_FAST_MODE_HEADER] = "1"; } return upstreamHeaders; }; // Default to false unless client explicitly sets stream: true (OpenAI spec compliant) const acceptHeader = clientRawRequest?.headers && typeof clientRawRequest.headers.get === "function" ? clientRawRequest.headers.get("accept") || clientRawRequest.headers.get("Accept") : clientRawRequest?.headers?.["accept"] || clientRawRequest?.headers?.["Accept"]; const explicitStreamAlias = resolveExplicitStreamAlias(body); // Remove non-standard non-stream aliases before provider translation/execution. // They are accepted for compatibility at the OmniRoute API boundary only. if (body && typeof body === "object") { const b = body as Record; if (explicitStreamAlias !== undefined) { b.stream = explicitStreamAlias; } delete b.non_stream; delete b.disable_stream; delete b.disable_streaming; delete b.streaming; } // Codex /responses/compact is JSON-only: Codex CLI does not send stream=false, // so route shape must override the usual Accept/header fallback. // sourceFormat="claude" applies the Anthropic Messages spec default (stream=false // when body omits stream), preventing STREAM_EARLY_EOF on /v1/messages when // clients send Accept: */* without an explicit stream flag. const stream = nativeCodexPassthrough && isCompactResponsesEndpoint(endpointPath) ? false : resolveStreamFlag(body?.stream, acceptHeader, sourceFormat); const settings = cachedSettings ?? (await getCachedSettings()); credentials = applyCodexGlobalFastServiceTier(provider, credentials, settings, { model: requestedModel, body: body && typeof body === "object" ? (body as Record) : null, }); effectiveServiceTier = resolveEffectiveServiceTier(body); setGeminiThoughtSignatureMode(settings.antigravitySignatureCacheMode); const semanticCacheEnabled = settings.semanticCacheEnabled !== false; // Create request logger for this session: sourceFormat_targetFormat_model const reqLogger = await createRequestLogger(sourceFormat, targetFormat, model, { enabled: detailedLoggingEnabled, captureStreamChunks: capturePipelineStreamChunks, }); // 0. Log client raw request (before format conversion) if (clientRawRequest) { reqLogger.logClientRawRequest( clientRawRequest.endpoint, clientRawRequest.body, clientRawRequest.headers ); } log?.debug?.("FORMAT", `${sourceFormat} → ${targetFormat} | stream=${stream}`); // ── Phase 9.1: Semantic cache check (temp=0, any streaming mode) ── // Streaming responses are cached after assembly; cache hits return JSON regardless of stream flag. if (semanticCacheEnabled && isCacheableForRead(body, clientRawRequest?.headers)) { const signature = generateSignature( model, body.messages ?? body.input, body.temperature, body.top_p ); const cached = getCachedResponse(signature); if (cached) { log?.debug?.("CACHE", `Semantic cache HIT for ${model} (stream=${stream})`); reqLogger.logConvertedResponse(cached as Record); const cachedUsage = extractUsageFromResponse(cached as Record, provider) || ((cached as Record)?.usage as Record | undefined); const cachedCost = cachedUsage ? await calculateCost(provider, model, cachedUsage as Record, { serviceTier: effectiveServiceTier, }) : 0; persistAttemptLogs({ status: 200, tokens: (cached as Record)?.usage, responseBody: cached, providerRequest: null, providerResponse: null, clientResponse: cached, cacheSource: "semantic", }); trackPendingRequest(model, provider, connectionId, false); return { success: true, response: new Response(JSON.stringify(cached), { headers: { "Content-Type": "application/json", [OMNIROUTE_RESPONSE_HEADERS.cache]: "HIT", ...buildOmniRouteResponseMetaHeaders({ provider, model, cacheHit: true, latencyMs: Date.now() - startTime, usage: cachedUsage, costUsd: cachedCost, }), }, }), }; } } // ── Common input sanitization (runs for ALL paths including passthrough) ── // #994: Normalize between max_output_tokens and max_tokens for universal compatibility. // For Responses API targets, max_output_tokens is the canonical field. For others, // max_tokens is preferred. We handle normalization here to support passthrough // paths where the translator is skipped. const prefersResponsesTokenField = sourceFormat === FORMATS.OPENAI_RESPONSES || targetFormat === FORMATS.OPENAI_RESPONSES; if (prefersResponsesTokenField) { if (body.max_output_tokens === undefined) { if (body.max_completion_tokens !== undefined) { body.max_output_tokens = body.max_completion_tokens; delete body.max_completion_tokens; } else if (body.max_tokens !== undefined) { body.max_output_tokens = body.max_tokens; delete body.max_tokens; } } } else if (body.max_output_tokens !== undefined) { if (body.max_tokens === undefined) { body.max_tokens = body.max_output_tokens; } delete body.max_output_tokens; } // #291: Strip empty name fields from messages/input items // Upstream providers (OpenAI, Codex) reject name:"" with 400 errors. if (Array.isArray(body.messages)) { body.messages = body.messages.map((msg: Record) => { if (msg.name === "") { const { name: _n, ...rest } = msg; return rest; } return msg; }); } if (Array.isArray(body.input)) { body.input = body.input.map((item: Record) => { if (item.name === "") { const { name: _n, ...rest } = item; return rest; } return item; }); } // #346/#637: Strip tools with empty name // Clients sometimes forward tool definitions with empty names, causing // upstream providers to reject with 400 "Invalid 'tools[0].name': empty string." if (Array.isArray(body.tools)) { body.tools = body.tools.filter((tool: Record) => { // Built-in Responses API tool types (web_search, file_search, computer, etc.) // are identified solely by their `type` field and carry no name — preserve them. const toolType = typeof tool.type === "string" ? tool.type : ""; if (toolType && toolType !== "function" && !tool.function && tool.name === undefined) { return true; } const fn = tool.function as Record | undefined; const name = fn?.name ?? tool.name; return name && String(name).trim().length > 0; }); // Sanitize OpenAI-format function tool schemas before they reach strict // upstream JSON Schema validators (e.g. Moonshot AI behind // opencode-go/kimi-k2.6). See toolSchemaSanitizer.ts for the specific bug. // sanitizeOpenAITool is safe to call on any input — it no-ops non-function // tools (e.g. Responses API built-ins) and non-object values. body.tools = body.tools.map((tool) => sanitizeOpenAITool(tool) as (typeof body.tools)[number]); } const memoryOwnerId = resolveMemoryOwnerId(apiKeyInfo as Record | null); const memorySettings = memoryOwnerId ? await getMemorySettings().catch(() => DEFAULT_MEMORY_SETTINGS) : null; if ( memoryOwnerId && memorySettings && shouldInjectMemory(body as Parameters[0], { enabled: memorySettings.enabled && memorySettings.maxTokens > 0, }) ) { try { // Plan 21 FAIL #1 fix: extract the last user message and pass it as // `query`. Without this, `config.query` is undefined in retrieveMemories // and the semantic/hybrid branches (sqlite-vec + RRF, and Qdrant // tier-2) never fire from the chat hot path — they only fire in the // Playground (retrievePreview, which gets `query` as a positional arg). const lastUserQuery = ((): string => { // Responses API item types that are NOT user input — never accept // their text as the retrieval query (e.g. function_call_output is the // tool's reply, reasoning is the model's chain of thought). const NON_USER_TYPES = new Set([ "function_call", "function_call_output", "tool_call", "tool_call_output", "reasoning", "computer_call", "computer_call_output", "web_search_call", "file_search_call", ]); function pickFrom(arr: unknown[]): string { for (let i = arr.length - 1; i >= 0; i--) { const item = arr[i] as Record | undefined; if (!item) continue; // Chat API: only role==="user" items. Responses API items often // have type instead of role — skip non-user types like // function_call_output so the tool's reply doesn't leak into the // memory query. if (item.role !== undefined && item.role !== "user") continue; if (item.role === undefined && typeof item.type === "string") { if (NON_USER_TYPES.has(item.type)) continue; } const content = item.content ?? item.text; if (typeof content === "string" && content.trim().length > 0) { return content; } if (Array.isArray(content)) { const parts: string[] = []; for (const p of content) { if (typeof p === "string") { parts.push(p); } else if (p && typeof p === "object") { const pp = p as Record; // Skip non-text content parts (image_url, tool_use, etc.) const ptype = typeof pp.type === "string" ? pp.type : ""; if ( ptype && ptype !== "text" && ptype !== "input_text" && ptype !== "output_text" ) { continue; } const t = pp.text ?? pp.input_text; if (typeof t === "string") parts.push(t); } } if (parts.length > 0) return parts.join(" ").trim(); } } return ""; } const b = body as Record; if (Array.isArray(b.messages)) { const r = pickFrom(b.messages); if (r) return r; } if (Array.isArray(b.input)) { const r = pickFrom(b.input); if (r) return r; } return ""; })(); const memories = await retrieveMemories( memoryOwnerId, toMemoryRetrievalConfig(memorySettings, { query: lastUserQuery }) ); if (memories.length > 0) { const injected = injectMemory( body as Parameters[0], memories, provider ); body = injected as typeof body; log?.debug?.("MEMORY", `Injected ${memories.length} memories for key=${memoryOwnerId}`); } } catch (memErr) { log?.debug?.( "MEMORY", `Memory injection skipped: ${memErr instanceof Error ? memErr.message : String(memErr)}` ); } } if (memoryOwnerId && memorySettings?.skillsEnabled) { const existingTools = Array.isArray(body.tools) ? body.tools : []; const mergedTools = injectSkills({ provider: getSkillsProviderForFormat(sourceFormat), existingTools, apiKeyId: memoryOwnerId, model: typeof effectiveModel === "string" ? effectiveModel : undefined, sourceFormat, targetFormat, backgroundReason, messages: Array.isArray(body.messages) ? body.messages : Array.isArray(body.input) ? body.input : undefined, }); if (mergedTools.length > existingTools.length) { body = { ...body, tools: mergedTools, }; log?.debug?.("SKILLS", `Injected ${mergedTools.length - existingTools.length} skills`); } } trace("post_injection", { provider, model }); // Translate request (pass reqLogger for intermediate logging) // ── Proactive Context Compression (Phase 4) ── // Check if context exceeds 70% of limit and compress proactively before sending to provider. // This prevents "prompt too long" errors for large-but-not-full contexts. const compressionBody = body ? adaptBodyForCompression(body as Record).body : null; const allMessages = compressionBody?.messages || body?.contents || body?.request?.contents || []; let cavemanOutputModeApplied = false; let cavemanOutputModeIntensity: string | null = null; let preCompressionBody: typeof body | null = null; if (body && Array.isArray(allMessages) && allMessages.length > 0) { let estimatedTokens = estimateTokens(JSON.stringify(allMessages)); let promptCompressionEnabled = false; let compressionSettings: CompressionConfig | null = null; try { const { getCompressionSettings } = await import("../../src/lib/db/compression.ts"); compressionSettings = await getCompressionSettings(); promptCompressionEnabled = compressionSettings.enabled; } catch (err) { log?.warn?.( "COMPRESSION", "Compression settings lookup skipped: " + (err instanceof Error ? err.message : String(err)) ); } // --- Modular Compression Pipeline (Phase 1 Lite + Phase 2 Standard/Caveman + Phase 3 Aggressive) --- // Runs BEFORE the existing reactive compressContext() to proactively reduce tokens. try { const { selectCompressionStrategy, applyCompression } = await import("../services/compression/strategySelector.ts"); const { trackCompressionStats } = await import("../services/compression/stats.ts"); let config: CompressionConfig = compressionSettings ?? { enabled: false, defaultMode: "off", autoTriggerTokens: 0, cacheMinutes: 5, preserveSystemPrompt: true, comboOverrides: {}, }; if (!promptCompressionEnabled || !compressionSettings) { log?.debug?.("COMPRESSION", "Prompt compression disabled or unavailable"); } let compressionComboKey = comboName ?? null; let compressionComboApplied = false; type RuntimeCompressionCombo = { id: string; pipeline: NonNullable; languagePacks: string[]; outputMode: boolean; outputModeIntensity: string; }; const isBuiltinStackedPipeline = ( pipeline: CompressionConfig["stackedPipeline"] | undefined ): boolean => { if (!Array.isArray(pipeline) || pipeline.length !== 2) return false; const [first, second] = pipeline; return ( first?.engine === "rtk" && (first.intensity === undefined || first.intensity === "standard") && !first.config && second?.engine === "caveman" && (second.intensity === undefined || second.intensity === "full") && !second.config ); }; const applyCompressionComboConfig = ( compressionCombo: RuntimeCompressionCombo | null, routingOverrideIds: string[] = [] ): boolean => { if (!compressionCombo || compressionCombo.pipeline.length === 0) return false; const comboLanguagePacks = [ ...new Set( compressionCombo.languagePacks .map((pack) => pack.trim()) .filter((pack) => pack.length > 0) ), ]; const comboOutputIntensity = ( ["lite", "full", "ultra"].includes(compressionCombo.outputModeIntensity) ? compressionCombo.outputModeIntensity : (config.cavemanOutputMode?.intensity ?? "full") ) as "lite" | "full" | "ultra"; const comboDefaultLanguage = comboLanguagePacks.find((pack) => pack === config.languageConfig?.defaultLanguage) ?? comboLanguagePacks[0] ?? config.languageConfig?.defaultLanguage ?? "en"; const comboOverrides = { ...(config.comboOverrides ?? {}) }; for (const id of routingOverrideIds) { if (id) comboOverrides[id] = "stacked"; } config = { ...config, compressionComboId: compressionCombo.id, stackedPipeline: compressionCombo.pipeline, languageConfig: { ...(config.languageConfig ?? { enabled: false, defaultLanguage: "en", autoDetect: true, enabledPacks: ["en"], }), enabled: true, defaultLanguage: comboDefaultLanguage, enabledPacks: comboLanguagePacks.length > 0 ? comboLanguagePacks : (config.languageConfig?.enabledPacks ?? ["en"]), }, cavemanOutputMode: { ...(config.cavemanOutputMode ?? { enabled: false, intensity: "full", autoClarity: true, }), enabled: compressionCombo.outputMode, intensity: comboOutputIntensity, }, comboOverrides, }; compressionComboApplied = true; return true; }; const isStackedCompressionCombo = ( compressionCombo: RuntimeCompressionCombo | null ): compressionCombo is RuntimeCompressionCombo => { return Boolean(compressionCombo && compressionCombo.pipeline.length > 1); }; if (isCombo && comboName) { try { const { getComboByName } = await import("../../src/lib/localDb"); let comboConfig = await getComboByName(comboName); if (!comboConfig && comboName.startsWith("combo/")) { comboConfig = await getComboByName(comboName.substring(6)); } const comboRuntimeConfig = comboConfig?.config && typeof comboConfig.config === "object" ? (comboConfig.config as Record) : {}; const comboMode = typeof comboRuntimeConfig.compressionMode === "string" ? comboRuntimeConfig.compressionMode : typeof comboConfig?.compressionOverride === "string" ? comboConfig.compressionOverride : null; if ( comboMode === "off" || comboMode === "lite" || comboMode === "standard" || comboMode === "aggressive" || comboMode === "ultra" || comboMode === "rtk" || comboMode === "stacked" ) { config = { ...config, comboOverrides: { ...(config.comboOverrides ?? {}), ...(comboName ? { [comboName]: comboMode } : {}), ...(comboConfig?.id ? { [String(comboConfig.id)]: comboMode } : {}), }, }; compressionComboKey = comboName; } const routingComboIds = [ comboConfig?.id, comboName, comboName.startsWith("combo/") ? comboName.substring(6) : null, ].filter((id): id is string => typeof id === "string" && id.length > 0); if (routingComboIds.length > 0) { const { getCompressionComboForRoutingCombo } = await import("../../src/lib/db/compressionCombos.ts"); const assignedCompressionCombo = routingComboIds .map((id) => getCompressionComboForRoutingCombo(id)) .find((combo) => combo !== null) ?? null; if ( applyCompressionComboConfig( assignedCompressionCombo as RuntimeCompressionCombo | null, routingComboIds ) ) { compressionComboKey = comboName; } } } catch (err) { log?.debug?.( "COMPRESSION", "Combo compression override lookup skipped: " + (err instanceof Error ? err.message : String(err)) ); } } const modeBeforeOutputTransform = selectCompressionStrategy( config, compressionComboKey, estimatedTokens, body as Record, { provider, targetFormat, model: effectiveModel } ); if ( modeBeforeOutputTransform === "stacked" && !compressionComboApplied && !config.compressionComboId && isBuiltinStackedPipeline(config.stackedPipeline) ) { try { const { getDefaultCompressionCombo } = await import("../../src/lib/db/compressionCombos.ts"); const defaultCompressionCombo = getDefaultCompressionCombo(); if ( isStackedCompressionCombo(defaultCompressionCombo as RuntimeCompressionCombo | null) && applyCompressionComboConfig(defaultCompressionCombo as RuntimeCompressionCombo | null) ) { log?.debug?.( "COMPRESSION", `Default compression combo applied: ${defaultCompressionCombo?.id}` ); } } catch (err) { log?.debug?.( "COMPRESSION", "Default compression combo lookup skipped: " + (err instanceof Error ? err.message : String(err)) ); } } if (config.cavemanOutputMode?.enabled) { try { const { applyCavemanOutputMode } = await import("../services/compression/outputMode.ts"); const outputModeLanguage = config.languageConfig?.enabled === true ? config.languageConfig.defaultLanguage : "en"; const outputMode = applyCavemanOutputMode( body as Parameters[0], config.cavemanOutputMode, outputModeLanguage ); if (outputMode.applied) { body = outputMode.body as typeof body; cavemanOutputModeApplied = true; cavemanOutputModeIntensity = config.cavemanOutputMode.intensity; estimatedTokens = estimateTokens(JSON.stringify(body?.messages ?? body?.input ?? [])); log?.debug?.("COMPRESSION", "Caveman output mode instruction applied"); } else if (outputMode.skippedReason && outputMode.skippedReason !== "disabled") { log?.debug?.("COMPRESSION", `Caveman output mode skipped: ${outputMode.skippedReason}`); } } catch (err) { log?.debug?.( "COMPRESSION", "Caveman output mode skipped: " + (err instanceof Error ? err.message : String(err)) ); } } const compressionInputBody = body as Record; const mode = selectCompressionStrategy( config, compressionComboKey, estimatedTokens, compressionInputBody, { provider, targetFormat, model: effectiveModel } ); let compressionAnalyticsRecorded = false; if (mode !== "off") { const result = applyCompression(compressionInputBody, mode, { model: effectiveModel, config, }); if (result.stats) { if (result.compressed) { body = result.body as typeof body; estimatedTokens = result.stats.compressedTokens; tokensCompressed = Math.max( 0, result.stats.originalTokens - result.stats.compressedTokens ); } if (result.compressed || result.stats.fallbackApplied || cavemanOutputModeApplied) { trackCompressionStats(result.stats); compressionAnalyticsRecorded = true; compressionAnalyticsWritePromise = (async () => { try { const { insertCompressionAnalyticsRow } = await import("../../src/lib/db/compressionAnalytics.ts"); const { calculateCost } = await import("../../src/lib/usage/costCalculator.ts"); const tokensSaved = Math.max( 0, result.stats.originalTokens - result.stats.compressedTokens ); const rtkPointers = result.stats.rtkRawOutputPointers ?? []; const estimatedUsdSaved = await calculateCost( provider ?? "", effectiveModel ?? "", { input: tokensSaved, }, { serviceTier: effectiveServiceTier } ); insertCompressionAnalyticsRow({ timestamp: new Date().toISOString(), combo_id: comboName ?? null, provider: provider ?? null, mode, engine: result.stats.engine ?? mode, compression_combo_id: result.stats.compressionComboId ?? config.compressionComboId ?? null, original_tokens: result.stats.originalTokens, compressed_tokens: result.stats.compressedTokens, tokens_saved: tokensSaved, duration_ms: result.stats.durationMs ?? null, request_id: skillRequestId, estimated_usd_saved: estimatedUsdSaved || null, validation_fallback: result.stats.fallbackApplied ? 1 : 0, output_mode: cavemanOutputModeApplied ? cavemanOutputModeIntensity : null, rtk_raw_output_pointer: rtkPointers[0]?.id ?? null, rtk_raw_output_bytes: rtkPointers[0]?.bytes ?? null, rtk_raw_output_pointers: rtkPointers.length ? JSON.stringify(rtkPointers.map((pointer) => pointer.id)) : null, rtk_raw_output_total_bytes: rtkPointers.length ? rtkPointers.reduce((total, pointer) => total + pointer.bytes, 0) : null, }); } catch (err) { log?.debug?.( "COMPRESSION", "Compression analytics write skipped: " + (err instanceof Error ? err.message : String(err)) ); } })(); } if (result.compressed) { void (async () => { try { const { detectCachingContext } = await import("../services/compression/cachingAware.ts"); const { recordCacheStats } = await import("../../src/lib/db/compressionCacheStats.ts"); const cacheContext = detectCachingContext(compressionInputBody, { provider, targetFormat, model: effectiveModel, }); const tokensSavedCompression = Math.max( 0, result.stats.originalTokens - result.stats.compressedTokens ); recordCacheStats({ provider: cacheContext.provider ?? provider ?? "unknown", model: effectiveModel ?? "", compressionMode: mode, cacheControlPresent: cacheContext.hasCacheControl, estimatedCacheHit: cacheContext.hasCacheControl && cacheContext.isCachingProvider, tokensSavedCompression, tokensSavedCaching: 0, netSavings: tokensSavedCompression, }); } catch (err) { log?.debug?.( "COMPRESSION", "Compression cache stats write skipped: " + (err instanceof Error ? err.message : String(err)) ); } })(); log?.info?.( "COMPRESSION", `Prompt compressed (${mode}): ${result.stats.originalTokens} -> ${result.stats.compressedTokens} tokens (${result.stats.savingsPercent}% saved, techniques: ${result.stats.techniquesUsed.join(",")})` ); } } } if (cavemanOutputModeApplied && !compressionAnalyticsRecorded) { compressionAnalyticsWritePromise = (async () => { try { const { insertCompressionAnalyticsRow } = await import("../../src/lib/db/compressionAnalytics.ts"); insertCompressionAnalyticsRow({ timestamp: new Date().toISOString(), combo_id: comboName ?? null, provider: provider ?? null, mode: "output-caveman", engine: "caveman-output", compression_combo_id: config.compressionComboId ?? null, original_tokens: estimatedTokens, compressed_tokens: estimatedTokens, tokens_saved: 0, request_id: skillRequestId, output_mode: cavemanOutputModeIntensity, }); } catch (err) { log?.debug?.( "COMPRESSION", "Caveman output analytics write skipped: " + (err instanceof Error ? err.message : String(err)) ); } })(); } } catch (err) { log?.warn?.( "COMPRESSION", "Compression pipeline error (non-fatal): " + (err instanceof Error ? err.message : String(err)) ); } // --- End Modular Compression Pipeline --- if (!promptCompressionEnabled) { log?.debug?.( "CONTEXT", "Skipping proactive context compression: Prompt Compression disabled" ); } let contextLimit = getTokenLimit(provider, effectiveModel); if (isCombo && comboName) { log?.info?.("CONTEXT", `Attempting to resolve combo limits for comboName=${comboName}`); try { const { getComboByName } = await import("../../src/lib/localDb"); const { parseModel } = await import("../services/model.ts"); const { resolveComboTargets } = await import("../services/combo.ts"); let comboConfig = await getComboByName(comboName); if (!comboConfig && comboName.startsWith("combo/")) { comboConfig = await getComboByName(comboName.substring(6)); } if (comboConfig) { const allCombosData = await getCombosCached(); const targets = resolveComboTargets( comboConfig as unknown as { name: string; models: unknown[] }, allCombosData as unknown as { name: string; models: unknown[] }[] ); const limits = targets.map((t: { modelStr?: string }) => { const parsed = parseModel(t.modelStr); return getTokenLimit(parsed.provider, parsed.model); }); if (limits.length > 0) { contextLimit = Math.min(...limits); log?.info?.("CONTEXT", `Combo min limit: ${contextLimit}`); } } } catch (err) { log?.warn?.("CONTEXT", "Failed to resolve combo limits for compression: " + err); } } const COMPRESSION_THRESHOLD = 0.7; let reservedTokens = 0; if (Array.isArray(body.tools)) { reservedTokens = estimateTokens(JSON.stringify(body.tools)); } const threshold = Math.max( 1, Math.floor((Math.max(1, contextLimit) - reservedTokens) * COMPRESSION_THRESHOLD) ); log?.debug?.( "CONTEXT", `Checking compression: ${estimatedTokens} tokens vs ${threshold} threshold (${contextLimit} limit, ${reservedTokens} reserved)` ); // Capture pre-compression body so translators can access original message // content even after compression alters it (e.g. stable Kiro conversationId). preCompressionBody = body; if (promptCompressionEnabled && estimatedTokens > threshold) { log?.info?.( "CONTEXT", `Proactive compression triggered: ${estimatedTokens} tokens > ${threshold} threshold (${contextLimit} limit)` ); const compressionResult = compressContext(body, { provider, model: effectiveModel, maxTokens: threshold, reserveTokens: 0, }); if (compressionResult.compressed) { body = compressionResult.body; const stats = compressionResult.stats; tokensCompressed = Math.max(0, (stats?.original ?? 0) - (stats?.final ?? 0)); const layersInfo = stats && "layers" in stats && Array.isArray(stats.layers) ? ` (layers: ${stats.layers.map((l: { name: string }) => l.name).join(", ")})` : ""; log?.info?.( "CONTEXT", `Context compressed: ${stats.original} → ${stats.final} tokens${layersInfo}` ); logAuditEvent({ action: "context.proactive_compression", actor: apiKeyInfo?.name || "system", target: connectionId || provider || "chat", details: { provider, model: effectiveModel, original_tokens: stats.original, final_tokens: stats.final, layers: "layers" in stats ? stats.layers : undefined, }, }); } else { log?.debug?.("CONTEXT", `Compression not applied: context already fits within target`); } } } else { log?.debug?.( "CONTEXT", `Skipping compression check: body=${!!body}, hasMessages=${Array.isArray(allMessages)}` ); } let translatedBody = body; const isClaudePassthrough = sourceFormat === FORMATS.CLAUDE && targetFormat === FORMATS.CLAUDE; const isClaudeCodeCompatible = isClaudeCodeCompatibleProvider(provider); const isClaudeCodeSemanticPassthrough = isClaudeCodeSemanticPassthroughRequest({ provider, sourceFormat, targetFormat, headers: clientRawRequest?.headers, userAgent, }); const upstreamStream = stream || isClaudeCodeCompatible; let ccSessionId: string | null = null; const stripTypes = getStripTypesForProviderModel(provider || "", model || ""); if (Array.isArray(translatedBody?.messages) && stripTypes.length > 0) { const stripResult = stripIncompatibleMessageContent(translatedBody.messages, stripTypes); if (stripResult.removedParts > 0) { translatedBody = { ...translatedBody, messages: stripResult.messages, }; log?.warn?.( "CONTENT", `Stripped ${stripResult.removedParts} incompatible content part(s) for ${provider}/${model}` ); } } // Determine if we should preserve client-side cache_control headers // Fetch settings from DB to get user preference const cacheControlMode = await getCacheControlSettings().catch(() => "auto" as const); const preserveCacheControl = shouldPreserveCacheControl({ userAgent, isCombo, comboStrategy, targetProvider: provider, targetFormat, settings: { alwaysPreserveClientCache: cacheControlMode }, }); if (preserveCacheControl) { log?.debug?.( "CACHE", `Preserving client cache_control (client=${userAgent?.substring(0, 20)}, combo=${isCombo}, strategy=${comboStrategy}, provider=${provider})` ); } type ClaudeContentBlock = Record; type ClaudeMessage = { role?: unknown; content?: unknown; }; // Shared helper: lift any system/developer role messages out of the messages // array into the top-level system parameter. Anthropic's Messages API rejects // system/developer roles inside messages[]. Case-insensitive to be defensive. const extractSystemMessagesToBody = (payload: Record) => { if (!Array.isArray(payload.messages)) return; const messages = payload.messages as ClaudeMessage[]; const systemMessages = messages.filter((m) => { const role = String(m.role || "").toLowerCase(); return role === "system" || role === "developer"; }); if (systemMessages.length === 0) return; const extraBlocks: ClaudeContentBlock[] = []; for (const sm of systemMessages) { if (typeof sm.content === "string" && sm.content.length > 0) { extraBlocks.push({ type: "text", text: sm.content }); } else if (Array.isArray(sm.content)) { for (const block of sm.content as ClaudeContentBlock[]) { if (block?.type === "text" && typeof block.text === "string" && block.text.length > 0) { extraBlocks.push(block); } } } } if (extraBlocks.length > 0) { const existingSystem = payload.system; if (typeof existingSystem === "string" && existingSystem.length > 0) { payload.system = [{ type: "text", text: existingSystem }, ...extraBlocks]; } else if (Array.isArray(existingSystem)) { payload.system = [...(existingSystem as ClaudeContentBlock[]), ...extraBlocks]; } else { payload.system = extraBlocks; } } payload.messages = messages.filter((m) => { const role = String(m.role || "").toLowerCase(); return role !== "system" && role !== "developer"; }); }; const normalizeClaudeUpstreamMessages = ( payload: Record, options?: { preserveToolResultBlocks?: boolean } ) => { const preserveToolResultBlocks = options?.preserveToolResultBlocks === true; if (!Array.isArray(payload.messages)) return; let messages = payload.messages as ClaudeMessage[]; // Extract system/developer role messages into top-level system parameter. extractSystemRoleMessages(payload); messages = payload.messages as ClaudeMessage[]; // Anthropic rejects empty text blocks in native Messages payloads. for (const msg of messages) { if (Array.isArray(msg.content)) { msg.content = msg.content.filter( (block: ClaudeContentBlock) => block.type !== "text" || (typeof block.text === "string" && block.text.length > 0) ); } } // Normalize unsupported content types without reintroducing the Claude -> OpenAI round-trip. for (const msg of messages) { if (msg.role !== "user" || !Array.isArray(msg.content)) continue; msg.content = (msg.content as ClaudeContentBlock[]).flatMap((block: ClaudeContentBlock) => { if ( block.type === "text" || block.type === "image_url" || block.type === "image" || block.type === "file_url" || block.type === "file" || block.type === "document" ) { const fileData = (block.file_url ?? block.file ?? block.document) as | Record | undefined; if ( (block.type === "file" || block.type === "document") && !fileData?.url && !fileData?.data ) { const fileContent = (block.file as ClaudeContentBlock)?.content ?? (block.file as ClaudeContentBlock)?.text ?? block.content ?? block.text; const fileName = (block.file as Record)?.name ?? block.name ?? "attachment"; if (typeof fileContent === "string" && fileContent.length > 0) { return [{ type: "text", text: `[${fileName}]\n${fileContent}` }]; } } return [block]; } if (block.type === "tool_result") { if (preserveToolResultBlocks) { return [block]; } const toolId = block.tool_use_id ?? block.id ?? "unknown"; const resultContent = block.content ?? block.text ?? block.output ?? ""; const resultText = typeof resultContent === "string" ? resultContent : Array.isArray(resultContent) ? resultContent .filter((c: Record) => c.type === "text") .map((c: Record) => c.text) .join("\n") : JSON.stringify(resultContent); if (resultText.length > 0) { return [{ type: "text", text: `[Tool Result: ${toolId}]\n${resultText}` }]; } return []; } log?.debug?.("CONTENT", `Dropped unsupported content part type="${block.type}"`); return []; }); } // #2815: move stray tool_result blocks out of assistant messages. payload.messages = splitMisplacedToolResults( payload.messages as ClaudeMessage[] ) as unknown as Record[]; }; try { if (nativeCodexPassthrough) { translatedBody = { ...body, _nativeCodexPassthrough: true }; log?.debug?.("FORMAT", "native codex passthrough enabled"); } else if (isClaudeCodeCompatible) { let normalizedForCc = { ...body }; // Claude Code-compatible providers expect Anthropic Messages-shaped payloads, // but we extract only role/text/max_tokens/effort from an OpenAI-like view first. if (sourceFormat === FORMATS.CLAUDE && isClaudeCodeSemanticPassthrough) { log?.debug?.("FORMAT", "claude-code semantic passthrough enabled for compatible bridge"); } else if (sourceFormat !== FORMATS.OPENAI) { const normalizeToolCallId = getModelNormalizeToolCallId( provider || "", model || "", sourceFormat ); const preserveDeveloperRole = getModelPreserveOpenAIDeveloperRole( provider || "", model || "", sourceFormat ); normalizedForCc = translateRequest( sourceFormat, FORMATS.OPENAI, model, { ...body }, stream, credentials, provider, reqLogger, { normalizeToolCallId, preserveDeveloperRole, preserveCacheControl, copilotClient: copilotCompatibleReasoning, } ); } ccSessionId = resolveClaudeCodeCompatibleSessionId(clientRawRequest?.headers); translatedBody = buildClaudeCodeCompatibleRequest({ sourceBody: body, normalizedBody: normalizedForCc, claudeBody: sourceFormat === FORMATS.CLAUDE ? body : null, model, stream: upstreamStream, sessionId: ccSessionId, cwd: process.cwd(), now: new Date(), preserveCacheControl, preserveClaudeMessages: sourceFormat === FORMATS.CLAUDE && isClaudeCodeSemanticPassthrough, }); log?.debug?.("FORMAT", "claude-code-compatible bridge enabled"); if (isClaudeCodeSemanticPassthrough) { // Semantic passthrough: only lift system/developer role messages // without converting file/document blocks, tool history, etc. extractSystemRoleMessages(translatedBody); } else { // Non-CC path: full normalization including content type conversion. normalizeClaudeUpstreamMessages(translatedBody, { preserveToolResultBlocks: true }); } } else if (isClaudePassthrough) { // Pure passthrough: forward the body as-is without OpenAI round-trip. // The Claude→OpenAI→Claude double translation was lossy and corrupted // payloads at high context (150+ msgs, 100+ tools). Fix: #1359. // Claude Code sends well-formed Messages API payloads — trust them // regardless of combo strategy or cache_control settings. translatedBody = { ...body }; translatedBody._disableToolPrefix = true; // Sanitize historical thinking-block signatures for Anthropic-native Claude OAuth. // Only Anthropic's first-party API validates these signatures (token-bound); third-party // Claude-shape providers do not. See redactPassthroughThinkingSignatures + issue #2454. if (provider === "claude") { translatedBody.messages = redactPassthroughThinkingSignatures( translatedBody.messages, DEFAULT_THINKING_CLAUDE_SIGNATURE ) as typeof translatedBody.messages; } // Fix #2468: always extract role:"system" → top-level system. // The semantic passthrough correctly skips the Claude→OpenAI→Claude // round-trip, but even pure Claude bodies may carry system content as // role:"system" messages rather than the top-level system field, which // Anthropic's Messages API now rejects with a 400. if (isClaudeCodeSemanticPassthrough) { // Only lift system/developer messages — preserves Claude Code's // native payload structure (documents, tool chains, thinking, etc.) extractSystemRoleMessages(translatedBody); if (Array.isArray(translatedBody.messages)) { translatedBody.messages = splitMisplacedToolResults( translatedBody.messages as ClaudeMessage[] ) as typeof translatedBody.messages; } } else { normalizeClaudeUpstreamMessages(translatedBody, { preserveToolResultBlocks: true }); } log?.debug?.("FORMAT", `claude passthrough (preserveCache=${preserveCacheControl})`); // Migrate deprecated top-level `output_format` → `output_config.format`. // Anthropic returns a 400 on the legacy field; some clients (e.g. ForgeCode) // still emit it. Preserves an existing output_config.format if present. if (translatedBody.output_format !== undefined) { const oc = translatedBody.output_config && typeof translatedBody.output_config === "object" ? (translatedBody.output_config as Record) : {}; if (oc.format === undefined) oc.format = translatedBody.output_format; translatedBody.output_config = oc; delete translatedBody.output_format; } // Fix #1719: Strip output_config.format for non-Anthropic Claude-compatible providers. // Third-party Claude endpoints (MiniMax, DeepSeek via aggregators) reject this field // with 400 errors since they don't support Anthropic's structured output / json_schema. if ( provider !== "claude" && translatedBody.output_config && typeof translatedBody.output_config === "object" ) { const oc = translatedBody.output_config as Record; delete oc.format; if (Object.keys(oc).length === 0) { delete translatedBody.output_config; } } } else { translatedBody = { ...body }; // Issue #199 + #618: Always disable tool name prefix in Claude passthrough. // The proxy_ prefix was designed for OpenAI→Claude translation to avoid // conflicts with Claude OAuth tools, but in the passthrough path the tools // are already in Claude format. Applying the prefix turns "Bash" into // "proxy_Bash", which Claude rejects ("No such tool available: proxy_Bash"). if (targetFormat === FORMATS.CLAUDE) { translatedBody._disableToolPrefix = true; normalizeClaudeUpstreamMessages(translatedBody); } // OpenAI-compatible providers only support function tools. // Non-function tool types (computer, mcp, web_search, custom, etc.) are handled: // - tools with a name → converted to function format in-place before translation // - tools without a name AND without .function → dropped (unconvertible) // This must happen before translateRequest, which validates and throws on unknown types. if (provider?.startsWith("openai-compatible-") && Array.isArray(translatedBody.tools)) { const before = (translatedBody.tools as unknown[]).length; translatedBody.tools = (translatedBody.tools as Record[]) .filter((t) => !t.type || t.type === "function" || !!t.function || !!t.name) .map((t) => { if (!t.type || t.type === "function" || t.function) return t; // Named non-function tool: normalise to function format so the translator // does not throw on the unknown type. return { type: "function", function: { name: t.name, ...(t.description === undefined ? {} : { description: t.description }), ...(t.parameters !== undefined || t.input_schema !== undefined ? { parameters: t.parameters ?? t.input_schema ?? {} } : {}), ...(t.strict === undefined ? {} : { strict: t.strict }), }, }; }); const dropped = before - (translatedBody.tools as unknown[]).length; if (dropped > 0) { log?.debug?.( "TOOLS", `Dropped ${dropped} unconvertible tool(s) for openai-compatible provider` ); } } const normalizeToolCallId = getModelNormalizeToolCallId( provider || "", model || "", sourceFormat ); const preserveDeveloperRole = getModelPreserveOpenAIDeveloperRole( provider || "", model || "", sourceFormat ); translatedBody = translateRequest( sourceFormat, targetFormat, model, translatedBody, stream, credentials, provider, reqLogger, { normalizeToolCallId, preserveDeveloperRole, preserveCacheControl, signatureNamespace: connectionId, copilotClient: copilotCompatibleReasoning, ...(preCompressionBody ? { preCompressionBody } : {}), } ); } } catch (error) { // ── Plugin onError hook ── try { const { runOnError } = await import("@/lib/plugins/index"); await runOnError( { requestId: traceId, body, model, provider, apiKeyInfo, metadata: {} }, error instanceof Error ? error : new Error(String(error)) ); } catch (pluginErr) { log?.debug?.( "PLUGIN", `onError hook error (non-fatal): ${pluginErr instanceof Error ? pluginErr.message : String(pluginErr)}` ); } const parsedStatus = Number(error?.statusCode); const statusCode = Number.isInteger(parsedStatus) && parsedStatus >= 400 && parsedStatus <= 599 ? parsedStatus : HTTP_STATUS.SERVER_ERROR; const message = error?.message || "Invalid request"; const errorType = typeof error?.errorType === "string" ? error.errorType : null; log?.warn?.("TRANSLATE", `Request translation failed: ${message}`); if (errorType) { trackPendingRequest(model, provider, connectionId, false); return { success: false, status: statusCode, error: message, response: new Response( JSON.stringify({ error: { message, type: errorType, code: errorType, }, }), { status: statusCode, headers: { "Content-Type": "application/json", }, } ), }; } trackPendingRequest(model, provider, connectionId, false); return createErrorResult(statusCode, message); } trace("post_translation"); // Extract toolNameMap for response translation (Claude OAuth) const translatedToolNameMap = translatedBody._toolNameMap; const nativeClaudeToolNameMap = isClaudePassthrough ? buildClaudePassthroughToolNameMap(body) : null; const toolNameMap = translatedToolNameMap instanceof Map && translatedToolNameMap.size > 0 ? translatedToolNameMap : nativeClaudeToolNameMap; delete translatedBody._toolNameMap; delete translatedBody._disableToolPrefix; // Update model in body — use resolved alias so the provider gets the correct model ID (#472) // Strip provider/alias prefix if it exactly matches the routing prefix so upstream receives the raw model name (#1261) let finalModelToUpstream = effectiveModel; // Defense-in-depth: only string-strip when effectiveModel is actually a string. // The API guards `model` via Zod (z.string()), but internal callers could pass a // non-string and a bare `.startsWith` would crash with `startsWith is not a // function` (same class as #2359 / #2463). Mirrors 9router's `?.startsWith?.()`. if (typeof finalModelToUpstream === "string") { if (finalModelToUpstream.startsWith(`${provider}/`)) { finalModelToUpstream = finalModelToUpstream.slice(provider.length + 1); } else if (alias && finalModelToUpstream.startsWith(`${alias}/`)) { finalModelToUpstream = finalModelToUpstream.slice(alias.length + 1); } } translatedBody.model = finalModelToUpstream; // #1789: Prevent output_config.effort from overriding effort encoded in model name (Codex) if (provider === "codex" || provider?.startsWith("codex")) { const hasEffortSuffix = finalModelToUpstream.match(/-(low|medium|high|xhigh)$/i); if ( hasEffortSuffix && translatedBody.output_config && typeof translatedBody.output_config === "object" ) { const oc = translatedBody.output_config as Record; if (oc.effort) { log?.warn?.( "PARAMS", `Stripped output_config.effort="${oc.effort}" because model "${finalModelToUpstream}" already encodes effort` ); delete oc.effort; if (Object.keys(oc).length === 0) { delete translatedBody.output_config; } } } } // Strip unsupported parameters for reasoning models (o1, o3, etc.) const unsupported = getUnsupportedParams(provider, model); if (unsupported.length > 0) { const stripped: string[] = []; for (const param of unsupported) { if (Object.hasOwn(translatedBody, param)) { stripped.push(param); delete translatedBody[param]; } } if (stripped.length > 0) { log?.warn?.("PARAMS", `Stripped unsupported params for ${model}: ${stripped.join(", ")}`); } } // Rename max_tokens to max_completion_tokens if not supported (#1961) if (!supportsMaxTokens({ provider, model })) { if (translatedBody.max_tokens !== undefined) { if (translatedBody.max_completion_tokens === undefined) { translatedBody.max_completion_tokens = translatedBody.max_tokens; } delete translatedBody.max_tokens; log?.debug?.("PARAMS", `Renamed max_tokens to max_completion_tokens for ${model}`); } } // OpenAI's `store` parameter is not supported by most compatible providers and breaks them if (provider !== "openai" && "store" in translatedBody) { delete translatedBody.store; } // Chat clients may send stream_options.include_usage, but OpenAI Responses // upstreams (including Azure AI Foundry /responses) reject stream_options. if (targetFormat === FORMATS.OPENAI_RESPONSES && "stream_options" in translatedBody) { delete translatedBody.stream_options; } // Provider-specific max_tokens caps (#711) // Some providers reject requests when max_tokens exceeds their API limit. // Cap before sending to avoid upstream HTTP 400 errors. const providerCap = PROVIDER_MAX_TOKENS[provider]; if (providerCap) { for (const field of ["max_tokens", "max_completion_tokens"] as const) { if (typeof translatedBody[field] === "number" && translatedBody[field] > providerCap) { log?.debug?.( "PARAMS", `Capping ${field} from ${translatedBody[field]} to ${providerCap} for ${provider}` ); translatedBody[field] = providerCap; } } } // Resolve executor with optional upstream proxy (CLIProxyAPI) routing. // mode="native" (default): returns the native executor unchanged. // mode="cliproxyapi": returns the CLIProxyAPI executor instead. // mode="fallback": returns a wrapper that tries native first, falls back to CLIProxyAPI on 5xx/network errors. const resolveExecutorWithProxy = async (prov: string) => { const cfg = await getUpstreamProxyConfigCached(prov); if (!cfg.enabled || cfg.mode === "native") return getExecutor(prov); if (cfg.mode === "cliproxyapi") { log?.info?.("UPSTREAM_PROXY", `${prov} routed through CLIProxyAPI (passthrough)`); return getExecutor("cliproxyapi"); } // mode === "fallback": try native first, retry via CLIProxyAPI on specific failures const nativeExec = getExecutor(prov); const proxyExec = getExecutor("cliproxyapi"); // Read custom fallback codes from settings. Default: 5xx + 429 + network errors. let fallbackCodes: number[] = [429, 500, 502, 503, 504]; try { const allSettings = await getCachedSettings(); if (typeof allSettings.cliproxyapi_fallback_codes === "string" && allSettings.cliproxyapi_fallback_codes.trim()) { const parsed = allSettings.cliproxyapi_fallback_codes .split(",") .map((s: string) => parseInt(s.trim(), 10)) .filter((n: number) => !isNaN(n)); if (parsed.length > 0) fallbackCodes = parsed; } } catch { /* use defaults */ } const isRetryableStatus = (s: number) => fallbackCodes.includes(s) || s === 0; const wrapper = Object.create(nativeExec); wrapper.execute = async (input: { model: string; body: unknown; stream: boolean; credentials: unknown; signal?: AbortSignal | null; log?: unknown; upstreamExtraHeaders?: Record | null; }) => { let result; try { result = await nativeExec.execute(input); } catch (err) { const errMsg = err instanceof Error ? err.message : String(err); log?.info?.("UPSTREAM_PROXY", `${prov} native error (${errMsg}), retrying via CLIProxyAPI`); try { return await proxyExec.execute(input); } catch (proxyErr) { const proxyMsg = proxyErr instanceof Error ? proxyErr.message : String(proxyErr); log?.error?.("UPSTREAM_PROXY", `${prov} CLIProxyAPI fallback also failed: ${proxyMsg}`); throw proxyErr; } } if (!isRetryableStatus(result.response.status)) { return result; } log?.info?.( "UPSTREAM_PROXY", `${prov} native failed (${result.response.status}), retrying via CLIProxyAPI` ); try { return await proxyExec.execute(input); } catch (proxyErr) { const proxyMsg = proxyErr instanceof Error ? proxyErr.message : String(proxyErr); log?.error?.("UPSTREAM_PROXY", `${prov} CLIProxyAPI fallback also failed: ${proxyMsg}`); throw proxyErr; } }; return wrapper; }; // === Quota Share enforcement PRE-hook (B/F7) === // Runs after provider/model/credentials/apiKeyInfo are fully resolved, // before dispatcher. Fail-open per B16: errors → allow. let quotaSoftDeprioritize = false; if (apiKeyInfo?.id && credentials?.connectionId) { try { const { enforceQuotaShare } = await import("@/lib/quota/enforce"); const decision = await enforceQuotaShare({ apiKeyId: apiKeyInfo.id, connectionId: credentials.connectionId, provider: provider ?? "unknown", estimatedCost: {}, }).catch((err: unknown) => { log?.warn?.( "QUOTA_SHARE", `enforceQuotaShare failed; fail-open: ${err instanceof Error ? err.message : String(err)}` ); return { kind: "allow" as const }; }); if (decision.kind === "block") { const { buildErrorBody } = await import("../utils/error.ts"); log?.warn?.( "QUOTA_SHARE", `[quotaShare] blocked apiKeyId=${apiKeyInfo.id} provider=${provider ?? "unknown"}: ${decision.reason}` ); const headers: Record = { "Content-Type": "application/json" }; if (decision.retryAfterSeconds) { headers["Retry-After"] = String(decision.retryAfterSeconds); } return new Response( JSON.stringify(buildErrorBody(429, decision.reason)), { status: 429, headers } ); } if (decision.kind === "allow" && decision.deprioritize) { quotaSoftDeprioritize = true; log?.info?.( "QUOTA_SHARE", `[quotaShare] soft deprioritize active for apiKeyId=${apiKeyInfo.id} provider=${provider ?? "unknown"}` ); } } catch (err) { // Outer fail-open guard — should not be reached (inner .catch covers it) log?.warn?.( "QUOTA_SHARE", `[quotaShare] enforceQuotaShare unexpected error; fail-open: ${err instanceof Error ? err.message : String(err)}` ); } } // G2: Propagate soft penalty to the current candidate so combo scoring can deprioritize. if (quotaSoftDeprioritize && isCombo && comboStepId) { try { const { setCandidateQuotaSoftPenalty } = await import("../services/combo"); setCandidateQuotaSoftPenalty(comboExecutionKey, comboStepId, true); } catch (err) { log?.warn?.( "QUOTA_SHARE", `[quotaShare] could not set soft penalty on candidate: ${err instanceof Error ? err.message : String(err)}` ); } } // === /Quota Share enforcement PRE-hook === // Get executor for this provider (with optional upstream proxy routing) const executor = await resolveExecutorWithProxy(provider); const getExecutionCredentials = () => { const nextCredentials = nativeCodexPassthrough ? { ...credentials, requestEndpointPath: endpointPath } : credentials; const providerSpecificData = nextCredentials?.providerSpecificData && typeof nextCredentials.providerSpecificData === "object" ? { ...nextCredentials.providerSpecificData } : {}; // Some providers (Azure AI Foundry, OCI OpenAI-compatible) choose upstream // endpoint path from providerSpecificData.apiType. When a model routes to // OpenAI Responses format, force apiType=responses unless explicitly set. if ( targetFormat === FORMATS.OPENAI_RESPONSES && (provider === "azure-ai" || provider === "oci") && providerSpecificData.apiType !== "responses" ) { providerSpecificData.apiType = "responses"; } if ( targetFormat === FORMATS.OPENAI_RESPONSES && (provider === "azure-ai" || provider === "oci") ) { providerSpecificData._omnirouteForceResponsesUpstream = true; } const withApiType = { ...nextCredentials, providerSpecificData, }; if (!ccSessionId) return withApiType; return { ...withApiType, providerSpecificData: { ...(withApiType?.providerSpecificData || {}), ccSessionId, }, }; }; // Create stream controller for disconnect detection const streamController = createStreamController({ onDisconnect, provider, model, connectionId, clientResponseFormat, }); const dedupRequestBody = { ...translatedBody, model: `${provider}/${model}`, stream }; const dedupEnabled = shouldDeduplicate(dedupRequestBody); const dedupHash = dedupEnabled ? computeRequestHash(dedupRequestBody) : null; const executeProviderRequest = async (modelToCall = effectiveModel, allowDedup = false) => { const execute = async () => { const executionCredentials = getExecutionCredentials(); // Track execution credentials for key health recording (to capture selectedKeyId) let lastExecCreds = executionCredentials; const accountSemaphoreMaxConcurrency = resolveAccountSemaphoreMaxConcurrency(executionCredentials); const accountSemaphoreKey = resolveAccountSemaphoreKey({ provider, model: modelToCall, connectionId, credentials: executionCredentials, }); let bodyToSend = translatedBody.model === modelToCall ? translatedBody : { ...translatedBody, model: modelToCall }; const payloadRuleModel = typeof bodyToSend.model === "string" && bodyToSend.model.length > 0 ? bodyToSend.model : modelToCall; const payloadRuleProtocols = resolvePayloadRuleProtocols({ provider, targetFormat, }); const payloadRuleResult = await applyConfiguredPayloadRules( bodyToSend, payloadRuleModel, payloadRuleProtocols ); bodyToSend = payloadRuleResult.payload; if (payloadRuleResult.applied.length > 0) { const appliedSummary = payloadRuleResult.applied .map((rule) => { if (rule.type === "filter") return `${rule.type}:${rule.path}`; const serializedValue = JSON.stringify(rule.value); const safeValue = typeof serializedValue === "string" && serializedValue.length > 80 ? `${serializedValue.slice(0, 77)}...` : serializedValue; return `${rule.type}:${rule.path}=${safeValue}`; }) .join(", "); log?.debug?.( "PAYLOAD_RULES", `Applied ${payloadRuleResult.applied.length} rule(s) for ${payloadRuleModel} (${payloadRuleProtocols.join(", ")}): ${appliedSummary}` ); } const effectiveToolLimit = getEffectiveToolLimit(provider); if ( effectiveToolLimit < MAX_TOOLS_LIMIT && Array.isArray(bodyToSend.tools) && bodyToSend.tools.length > effectiveToolLimit ) { const truncatedTools = bodyToSend.tools.slice(0, effectiveToolLimit); bodyToSend = { ...bodyToSend, tools: truncatedTools }; log?.debug?.( "TOOL_LIMIT", `Truncated ${bodyToSend.tools.length} tools to ${effectiveToolLimit} for ${provider}` ); } // Qwen OAuth rejects requests without a non-empty `user` field. // Some minimal OpenAI-compatible clients omit it, so we backfill a // stable default only for OAuth mode (API key mode is unaffected). const hasValidQwenUser = typeof bodyToSend.user === "string" && bodyToSend.user.trim().length > 0; const isQwenOAuthRequest = provider === "qwen" && !credentials?.apiKey && typeof credentials?.accessToken === "string" && credentials.accessToken.trim().length > 0; if (isQwenOAuthRequest && !hasValidQwenUser) { bodyToSend = { ...bodyToSend, user: "omniroute-qwen-oauth" }; log?.debug?.("QWEN", "Injected fallback user for OAuth request"); } // Inject prompt_cache_key only for providers that support it if ( targetFormat === FORMATS.OPENAI && providerSupportsCaching(provider) && !bodyToSend.prompt_cache_key && Array.isArray(bodyToSend.messages) && !["nvidia", "codex", "xai"].includes(provider) ) { const { generatePromptCacheKey } = await import("@/lib/promptCache"); const cacheKey = generatePromptCacheKey(bodyToSend.messages); if (cacheKey) { bodyToSend = { ...bodyToSend, prompt_cache_key: cacheKey }; } } updatePendingRequest(model, provider, connectionId, { providerRequest: bodyToSend, stage: "payload_prepared", }); trace("pre_semaphore", { semaphoreKey: accountSemaphoreKey, max: accountSemaphoreMaxConcurrency, }); if (accountSemaphoreKey && accountSemaphoreMaxConcurrency != null) { updatePendingRequest(model, provider, connectionId, { stage: "waiting_account_slot", }); } const acquireAccountSemaphoreRelease = accountSemaphoreKey && accountSemaphoreMaxConcurrency != null ? await acquireAccountSemaphore(accountSemaphoreKey, { maxConcurrency: accountSemaphoreMaxConcurrency, signal: streamController.signal, }) : () => {}; trace("post_semaphore"); updatePendingRequest(model, provider, connectionId, { stage: "waiting_rate_limit", }); try { trace("pre_rate_limit"); const rawResult = await withRateLimit( provider, connectionId, modelToCall, async () => { trace("inside_rate_limit"); updatePendingRequest(model, provider, connectionId, { stage: "rate_limit_slot_acquired", }); let attempts = 0; const isModelScopeForRequest = isModelScope(); const maxAttempts = isModelScopeForRequest ? 3 : provider === "qwen" ? 3 : provider === "codex" ? 3 : 1; // ── Codex 429 account-rotation state ───────────────────────────────── // Track excluded connection IDs for codex failover across attempts. const codexExcludedIds: string[] = []; // Derive session affinity key once for codex failover (used to clear affinity on 429). const codexSessionAffinityKey = provider === "codex" ? (extractSessionAffinityKey(body, clientRawRequest?.headers) ?? null) : null; while (attempts < maxAttempts) { trace("pre_executor", { attempt: attempts }); updatePendingRequest(model, provider, connectionId, { stage: "sending_to_provider", }); const execCreds = getExecutionCredentials(); const res = await executeWithUpstreamStartTimeout<{ response: Response; url: string; headers: Record; transformedBody: unknown; _executionCredentials?: unknown; }>({ executor, provider, model: modelToCall, signal: streamController.signal, log, execute: (signal) => executor.execute({ model: modelToCall, body: bodyToSend, stream: upstreamStream, credentials: execCreds, signal, log, extendedContext, upstreamExtraHeaders: buildUpstreamHeadersForExecute(modelToCall), clientHeaders: buildExecutorClientHeaders(clientRawRequest?.headers, userAgent), onCredentialsRefreshed, skipUpstreamRetry, }), }); trace("post_executor", { status: res?.response?.status }); updatePendingRequest(model, provider, connectionId, { stage: "provider_response_started", }); if (res.response.status === 401 && execCreds?.connectionId) { recordKeyHealthStatus(401, execCreds); } // Qwen 429 strict quota backoff (wait 1.5s, 3s and retry) if ( provider === "qwen" && res.response.status === 429 && attempts < maxAttempts - 1 ) { const bodyPeek = await res.response .clone() .text() .catch(() => ""); if (bodyPeek.toLowerCase().includes("exceeded your current quota")) { const delay = 1500 * (attempts + 1); log?.warn?.("QWEN_RETRY", `Quota 429 hit. Retrying in ${delay}ms...`); await new Promise((r) => setTimeout(r, delay)); attempts++; continue; } } if (isModelScope() && res.response.status === 429 && attempts < maxAttempts - 1) { const bodyPeek = await res.response .clone() .text() .catch(() => ""); const normalizedHeaders = normalizeHeaders(res.response.headers); const decision = classifyModelScope429(bodyPeek, normalizedHeaders); if (decision.retryable) { const delay = getModelScopeRetryDelayMs(normalizedHeaders, attempts); log?.warn?.( "MODELSCOPE_RETRY", `429 ${decision.kind}; retrying in ${delay}ms (model remaining: ${decision.snapshot.modelRemaining ?? "unknown"})` ); await new Promise((r) => setTimeout(r, delay)); attempts++; continue; } } // Codex 429 account-rotation failover (disabled for context-relay so combo.ts can inject handoff) if ( provider === "codex" && comboStrategy !== "context-relay" && res.response.status === 429 && attempts < maxAttempts - 1 ) { const failedConnectionId = credentials?.connectionId || connectionId; const normalizedHeaders = normalizeHeaders(res.response.headers); const retryAfterHeader = normalizedHeaders["retry-after"] ?? null; const retryAfterMs = retryAfterHeader ? Number.parseFloat(retryAfterHeader) * 1000 : null; log?.warn?.( "CODEX_FAILOVER", `429 on connection ${String(failedConnectionId).slice(0, 8)} (attempt ${attempts + 1}/${maxAttempts}), rotating account` ); // Mark current connection as rate-limited in the DB if (failedConnectionId) { const rateLimitedUntil = new Date( Date.now() + (retryAfterMs || 60_000) ).toISOString(); updateProviderConnection(String(failedConnectionId), { rateLimitedUntil, testStatus: "unavailable", lastError: "429 rate limited — codex account rotation", errorCode: 429, }).catch(() => {}); if (!codexExcludedIds.includes(String(failedConnectionId))) { codexExcludedIds.push(String(failedConnectionId)); } } // Clear session affinity so next request won't be pinned to the failing account if (codexSessionAffinityKey) { try { deleteSessionAccountAffinity(codexSessionAffinityKey, "codex"); } catch { // best-effort } } // Fetch next available codex connection (excluding all previously failed ones) const nextCreds = await getProviderCredentials("codex", null, null, null, { excludeConnectionIds: [...codexExcludedIds], }).catch(() => null); if (!nextCreds || nextCreds.allRateLimited) { log?.warn?.("CODEX_FAILOVER", "No more codex accounts available — returning 429"); return res; } const newConnectionId = nextCreds.connectionId; log?.info?.( "CODEX_FAILOVER", `Rotating codex account: ${String(failedConnectionId).slice(0, 8)} → ${newConnectionId.slice(0, 8)} (attempt ${attempts + 2}/${maxAttempts})` ); logAuditEvent({ action: "codex.account_rotation", actor: apiKeyInfo?.name || "system", target: newConnectionId, details: { failed_connection_id: failedConnectionId, new_connection_id: newConnectionId, attempt: attempts + 1, retry_after_ms: retryAfterMs, }, }); // Update credentials in-place so getExecutionCredentials() picks up the new account Object.assign(credentials, nextCreds); attempts++; continue; } // For streaming: release the semaphore when the client drains or cancels the stream. if (stream) { const originalBody = res.response.body; if (!originalBody) { acquireAccountSemaphoreRelease(); return res; } return { ...res, _executionCredentials: execCreds, response: new Response( wrapReadableStreamWithFinalize(originalBody, acquireAccountSemaphoreRelease), { status: res.response.status, statusText: res.response.statusText, headers: res.response.headers, } ), headers: res.response.headers, }; } return { ...res, _executionCredentials: execCreds, }; } }, streamController.signal ); if (stream) { return rawResult; } // Non-stream: release semaphore immediately after reading full response body. const status = rawResult.response.status; // Use execution credentials captured during request processing if ( rawResult._executionCredentials?.connectionId && rawResult._executionCredentials?.apiKey ) { recordKeyHealthStatus(status, rawResult._executionCredentials); } const statusText = rawResult.response.statusText; const headersObj = normalizeHeaders(rawResult.response.headers); const headers = new Headers(headersObj); stripStaleForwardingHeaders(headers); const contentType = (headers.get("content-type") || "").toLowerCase(); const payload = await readNonStreamingResponseBody( rawResult.response, contentType, upstreamStream ); acquireAccountSemaphoreRelease(); return { ...rawResult, response: new Response(payload, { status, statusText, headers }), headers, _dedupSnapshot: { status, statusText, headers: (() => { const arr: [string, string][] = []; headers.forEach((v, k) => arr.push([k, v])); return arr; })(), payload, }, }; } catch (error) { acquireAccountSemaphoreRelease(); throw error; } }; if (allowDedup && dedupEnabled && dedupHash) { const dedupResult = await deduplicate(dedupHash, execute); if (dedupResult.wasDeduplicated) { log?.debug?.("DEDUP", `Joined in-flight request hash=${dedupHash}`); } return materializeDeduplicatedExecutionResult(dedupResult.result); } return execute(); }; const registeredProviderRequest = translatedBody && typeof translatedBody === "object" && !Array.isArray(translatedBody) ? { ...(translatedBody as Record), model: typeof (translatedBody as Record).model === "string" ? (translatedBody as Record).model : effectiveModel, ...(!Array.isArray((translatedBody as Record).messages) && Array.isArray((body as Record).messages) ? { messages: (body as Record).messages } : {}), } : translatedBody; updatePendingRequest(model, provider, connectionId, { providerRequest: registeredProviderRequest, }); // T5: track which models we've tried for intra-family fallback const triedModels = new Set([effectiveModel]); let currentModel = effectiveModel; // Log start appendRequestLog({ model, provider, connectionId, status: "PENDING" }).catch(() => {}); const msgCount = translatedBody.messages?.length || translatedBody.contents?.length || translatedBody.request?.contents?.length || (translatedBody.conversationState?.history?.length ?? 0) + (translatedBody.conversationState?.currentMessage ? 1 : 0) || 0; log?.debug?.("REQUEST", `${provider.toUpperCase()} | ${model} | ${msgCount} msgs`); // ── Tier 2: Authoritative per-model/provider token-limit check (provider now resolved) ── if (apiKeyInfo?.id) { try { const tokenBreach = checkTokenLimits(apiKeyInfo.id, provider || undefined, model || undefined); if (tokenBreach) { const scopeLabel = tokenBreach.scopeType === "global" ? "account" : `${tokenBreach.scopeType} "${tokenBreach.scopeValue}"`; // FIX 6: clear the pending request marker before the early return so we do // not leak a phantom pending request (start was tracked at line ~1847). trackPendingRequest(model, provider, connectionId, false); // FIX 5: tag this as a per-API-key token-limit breach (errorCode // TOKEN_LIMIT_EXCEEDED) so the combo loop can distinguish it from an // upstream 429 and NOT cool shared accounts / retry it transiently. return createErrorResult( HTTP_STATUS.RATE_LIMITED, `Token limit exceeded for ${scopeLabel}: ${tokenBreach.tokensUsed}/${tokenBreach.limitValue} tokens used in the current window. Please try again later.`, null, "TOKEN_LIMIT_EXCEEDED" ); } } catch (err) { // Fail-open at Tier 2: Tier 1 already enforced the model/global limit pre-dispatch. // A transient counter read error here must not break an otherwise-valid request. log?.warn?.("TOKEN_LIMIT", "Tier 2 token-limit check failed; allowing request", { err }); } } // Execute request using executor (handles URL building, headers, fallback, transform) let providerResponse; let providerUrl; let providerHeaders; let finalBody; let claudePromptCacheLogMeta = null; try { const result = await executeProviderRequest(effectiveModel, true); providerResponse = result.response; providerUrl = result.url; providerHeaders = result.headers; finalBody = result.transformedBody; effectiveServiceTier = resolveEffectiveServiceTier(finalBody); claudePromptCacheLogMeta = buildClaudePromptCacheLogMeta( targetFormat, finalBody, providerHeaders, clientRawRequest?.headers ); // Log target request (final request to provider) reqLogger.logTargetRequest(providerUrl, providerHeaders, finalBody); updatePendingRequest(model, provider, connectionId, { providerRequest: finalBody, providerUrl, stage: "provider_response_started", }); // Update rate limiter from response headers (learn limits dynamically) updateFromHeaders( provider, connectionId, providerResponse.headers, providerResponse.status, model ); } catch (error) { trackPendingRequest(model, provider, connectionId, false); if (isSemaphoreCapacityError(error)) { appendRequestLog({ model, provider, connectionId, status: `FAILED ${error.code}`, }).catch(() => {}); const failureMessage = error.message || "Semaphore timeout"; persistAttemptLogs({ status: HTTP_STATUS.RATE_LIMITED, error: failureMessage, providerRequest: finalBody || translatedBody, clientResponse: buildErrorBody(HTTP_STATUS.RATE_LIMITED, failureMessage), claudeCacheMeta: claudePromptCacheLogMeta, cacheSource: "upstream", }); persistFailureUsage(HTTP_STATUS.RATE_LIMITED, error.code); const result = stream ? createStreamingErrorResult(HTTP_STATUS.RATE_LIMITED, failureMessage, error.code) : createErrorResult(HTTP_STATUS.RATE_LIMITED, failureMessage); return { ...result, errorType: "account_semaphore_capacity", errorCode: error.code, }; } const failureStatus = error.name === "AbortError" ? 499 : error.name === "TimeoutError" || error.name === "BodyTimeoutError" ? HTTP_STATUS.GATEWAY_TIMEOUT : HTTP_STATUS.BAD_GATEWAY; const failureMessage = error.name === "AbortError" ? "Request aborted" : formatProviderError(error, provider, model, failureStatus); const upstreamErrorCode = getUpstreamErrorIdentifier(error); const upstreamErrorType = upstreamErrorCode === ANTIGRAVITY_PRE_RESPONSE_TIMEOUT_CODE ? "upstream_timeout" : undefined; appendRequestLog({ model, provider, connectionId, status: `FAILED ${failureStatus}`, }).catch(() => {}); persistAttemptLogs({ status: failureStatus, error: failureMessage, providerRequest: finalBody || translatedBody, clientResponse: buildErrorBody(failureStatus, failureMessage), claudeCacheMeta: claudePromptCacheLogMeta, cacheSource: "upstream", }); if (error.name === "AbortError") { streamController.handleError(error); return createErrorResult(499, "Request aborted"); } persistFailureUsage( failureStatus, upstreamErrorCode || (error instanceof Error && error.name ? error.name : "upstream_error") ); console.log(`${COLORS.red}[ERROR] ${failureMessage}${COLORS.reset}`); if (stream && upstreamErrorCode) { const result = createStreamingErrorResult( failureStatus, failureMessage, upstreamErrorCode, upstreamErrorType ); return { ...result, errorType: upstreamErrorType, errorCode: upstreamErrorCode, }; } return createErrorResult( failureStatus, failureMessage, null, upstreamErrorCode, upstreamErrorType ); } // We need to peek at the error text if it's 400 for Qwen let upstreamErrorParsed = false; let parsedStatusCode = providerResponse.status; let parsedMessage = ""; let parsedRetryAfterMs: number | null = null; let upstreamErrorBody: unknown = null; if (provider === "qwen" && providerResponse.status === HTTP_STATUS.BAD_REQUEST) { const errorDetails = await parseUpstreamError(providerResponse, provider); parsedStatusCode = errorDetails.statusCode; parsedMessage = errorDetails.message; parsedRetryAfterMs = errorDetails.retryAfterMs; upstreamErrorBody = errorDetails.responseBody; upstreamErrorParsed = true; } const errorMessageForToolDetection = typeof upstreamErrorBody === "string" ? upstreamErrorBody : JSON.stringify(upstreamErrorBody ?? {}); if (shouldDetectLimit(errorMessageForToolDetection, parsedStatusCode)) { const detectedLimit = parseToolLimitFromError(errorMessageForToolDetection); if (detectedLimit) { setDetectedToolLimit(provider, detectedLimit); log?.info?.("TOOL_LIMIT", `Detected tool limit ${detectedLimit} for ${provider}`); } } const isQwenExpiredError = provider === "qwen" && parsedStatusCode === HTTP_STATUS.BAD_REQUEST && parsedMessage?.toLowerCase().includes("session has expired"); // Track whether stream_options was present and stripped — if so, 401/403 after // that may be from the modification rather than a genuine auth failure, so we // skip the credential refresh attempt in that case. const hadStreamOptions = targetFormat === FORMATS.OPENAI_RESPONSES && "stream_options" in translatedBody; if (hadStreamOptions) { delete translatedBody.stream_options; } // Handle 401/403 (and Qwen explicit expiration) - try token refresh using executor if ( (providerResponse.status === HTTP_STATUS.UNAUTHORIZED || providerResponse.status === HTTP_STATUS.FORBIDDEN || isQwenExpiredError) && !hadStreamOptions // Skip refresh if failure may be from stream_options removal, not auth ) { // Fix A: wrap refreshCredentials in runWithOnPersist so the persist callback // executes INSIDE the per-connection mutex held by getAccessToken. This makes // [network refresh + DB write + outer-state mutation] one atomic step and // prevents concurrent requests from reading a stale refreshToken before the // DB has been updated (refresh_token_reused on Codex/OpenAI). // // Not every executor routes refresh through getAccessToken (e.g. github.ts // calls refreshCopilotToken directly). When the persistFn doesn't fire from // inside getAccessToken, we still need to do the credentials mutation + user // callback after refreshCredentials returns. The `persistFnRan` flag tracks // which path executed so we don't double-fire (race-prone) or skip (regression). let persistFnRan = false; const persistFn = onCredentialsRefreshed ? async (refreshResult: Record) => { persistFnRan = true; // Mutate the shared credentials object so subsequent executor calls // in this request see the new tokens. Runs INSIDE the mutex. Object.assign(credentials, refreshResult); await onCredentialsRefreshed(refreshResult); } : undefined; const newCredentials = (await refreshWithRetry( () => runWithOnPersist(persistFn, () => executor.refreshCredentials(credentials, log)), 3, log, provider // Explicitly pass the provider to avoid universally tripping the "unknown" circuit breaker )) as null | { accessToken?: string; copilotToken?: string; }; if (newCredentials?.accessToken || newCredentials?.copilotToken) { log?.info?.("TOKEN", `${provider.toUpperCase()} | refreshed`); // Fall back to post-mutex mutation only for executors that don't route // through getAccessToken (and therefore never fire onPersist). For // executors that DO route through it (Codex, Claude, Gemini, etc.) the // mutation already happened atomically inside the mutex. if (!persistFnRan) { Object.assign(credentials, newCredentials); if (onCredentialsRefreshed) { await onCredentialsRefreshed(newCredentials); } } // Retry with new credentials — model + extra headers follow translatedBody.model so they // stay aligned if this block ever runs after a path that mutates body.model (e.g. fallback). try { const retryModelId = String(translatedBody.model || effectiveModel); const retryResult = await executor.execute({ model: retryModelId, body: translatedBody, stream: upstreamStream, credentials: getExecutionCredentials(), signal: streamController.signal, log, extendedContext, upstreamExtraHeaders: buildUpstreamHeadersForExecute(retryModelId), clientHeaders: buildExecutorClientHeaders(clientRawRequest?.headers, userAgent), onCredentialsRefreshed, skipUpstreamRetry: isCombo, }); if (retryResult.response.ok) { providerResponse = retryResult.response; providerUrl = retryResult.url; providerHeaders = new Headers(retryResult.headers || {}); finalBody = retryResult.transformedBody; reqLogger.logTargetRequest(providerUrl, providerHeaders, finalBody); updatePendingRequest(model, provider, connectionId, { providerRequest: finalBody, providerUrl, stage: "provider_response_started", }); upstreamErrorParsed = false; // Reset since new response is OK } else { providerResponse = retryResult.response; upstreamErrorParsed = false; // Let it be parsed downstream } } catch (retryErr) { // Refresh succeeded but the retry leg failed (network blip, AbortError, // executor throw). Don't swallow — the operator-visible signal "the user // saw 401 even though auth was actually fixed" is much more confusing // than the original 401 alone. Surface at error level with sanitization. log?.error?.( "TOKEN", `${provider.toUpperCase()} | retry after refresh failed: ${sanitizeErrorMessage(retryErr)}` ); } } else { log?.warn?.("TOKEN", `${provider.toUpperCase()} | refresh failed`); if (isUnrecoverableRefreshError(newCredentials) && onCredentialsRefreshed) { await onCredentialsRefreshed({ testStatus: "expired", isActive: false }); } } } await persistCodexQuotaState(normalizeHeaders(providerResponse.headers), providerResponse.status); // Check provider response - return error info for fallback handling if (!providerResponse.ok) { trackPendingRequest(model, provider, connectionId, false); let statusCode = providerResponse.status; let message = ""; let retryAfterMs: number | null = null; let upstreamErrorCode: string | undefined; let upstreamErrorType: string | undefined; if (upstreamErrorParsed) { statusCode = parsedStatusCode; message = parsedMessage; retryAfterMs = parsedRetryAfterMs; } else { const details = await parseUpstreamError(providerResponse, provider); statusCode = details.statusCode; message = details.message; retryAfterMs = details.retryAfterMs; upstreamErrorBody = details.responseBody; upstreamErrorCode = details.errorCode as string | undefined; upstreamErrorType = details.errorType as string | undefined; } // T06/T10/T36: classify provider errors and persist terminal account states. let errorType = classifyProviderError(statusCode, message, provider); if (statusCode === 429 && isModelScope()) { const decision = classifyModelScope429(message, normalizeHeaders(providerResponse.headers)); errorType = decision.kind === "quota_exhausted" ? PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED : PROVIDER_ERROR_TYPES.RATE_LIMITED; log?.warn?.( "MODELSCOPE_429", `${decision.kind} (model remaining: ${decision.snapshot.modelRemaining ?? "unknown"}, total remaining: ${decision.snapshot.totalRemaining ?? "unknown"})` ); } if (connectionId && errorType) { try { if (errorType === PROVIDER_ERROR_TYPES.FORBIDDEN) { await updateProviderConnection(connectionId, { isActive: false, testStatus: "banned", lastErrorType: errorType, lastError: message, errorCode: statusCode, }); console.warn( `[provider] Node ${connectionId} banned (${statusCode}) — disabling permanently` ); } else if (errorType === PROVIDER_ERROR_TYPES.ACCOUNT_DEACTIVATED) { // Plan A: if connection has extra API keys, don't disable — only the failing key is affected. // Single-key connections still get disabled as before. if ( connectionHasExtraKeys( connectionId, (credentials?.providerSpecificData as Record | undefined) ?.extraApiKeys as string[] | undefined ) ) { await updateProviderConnection(connectionId, { lastErrorType: errorType, lastError: message, errorCode: statusCode, }); console.warn( `[provider] Node ${connectionId} account deactivated (${statusCode}) — has extra keys, keeping connection active` ); } else { await updateProviderConnection(connectionId, { isActive: false, testStatus: "deactivated", lastErrorType: errorType, lastError: message, errorCode: statusCode, }); console.warn( `[provider] Node ${connectionId} account deactivated (${statusCode}) — disabling permanently` ); } } else if (errorType === PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED) { // Providers with per-model quotas — lock the model only, not the connection const quotaCooldownMs = retryAfterMs || COOLDOWN_MS.rateLimit; const accountSemaphoreKey = resolveAccountSemaphoreKey({ provider, model: currentModel, connectionId, credentials, }); if (accountSemaphoreKey) { markAccountSemaphoreBlocked(accountSemaphoreKey, quotaCooldownMs); } if (isModelScope() && connectionId) { lockModel(provider, connectionId, model, "quota_exhausted", quotaCooldownMs); console.warn( `[provider] Node ${connectionId} ModelScope model quota exhausted (${statusCode}) for ${model} - ${Math.ceil(quotaCooldownMs / 1000)}s (connection stays active)` ); } else if ( lockModelIfPerModelQuota( provider, connectionId, model, "quota_exhausted", quotaCooldownMs ) ) { console.warn( `[provider] Node ${connectionId} model-only quota exhausted (${statusCode}) for ${model} - ${Math.ceil(quotaCooldownMs / 1000)}s (connection stays active)` ); } else { await updateProviderConnection(connectionId, { testStatus: "credits_exhausted", lastErrorType: errorType, lastError: message, errorCode: statusCode, }); console.warn(`[provider] Node ${connectionId} exhausted quota (${statusCode})`); } } else if (errorType === PROVIDER_ERROR_TYPES.ACCOUNT_DEACTIVATED) { await updateProviderConnection(connectionId, { isActive: false, testStatus: "expired", lastErrorType: errorType, lastError: message, errorCode: statusCode, }); console.warn( `[provider] Node ${connectionId} account deactivated (${statusCode}) — marked expired` ); } else if (errorType === PROVIDER_ERROR_TYPES.UNAUTHORIZED) { // Normal 401 (token/session auth issue): keep account active for refresh/re-auth. await updateProviderConnection(connectionId, { lastErrorType: errorType, lastError: message, errorCode: statusCode, }); } else if (errorType === PROVIDER_ERROR_TYPES.OAUTH_INVALID_TOKEN) { // OAuth 401 with invalid credentials - token refresh can recover await updateProviderConnection(connectionId, { lastErrorType: errorType, lastError: message, errorCode: statusCode, }); console.warn( `[provider] Node ${connectionId} OAuth token invalid (${statusCode}) — token refresh available` ); } else if (errorType === PROVIDER_ERROR_TYPES.PROJECT_ROUTE_ERROR) { // Cloud Code 403 with stale project: not a ban, keep account active. await updateProviderConnection(connectionId, { lastErrorType: errorType, lastError: message, errorCode: statusCode, }); console.warn( `[provider] Node ${connectionId} project routing error (${statusCode}) — not banning` ); } } catch { // Best-effort state update; request flow should continue with fallback handling. } } appendRequestLog({ model, provider, connectionId, status: `FAILED ${statusCode}` }).catch( () => {} ); const errMsg = formatProviderError(new Error(message), provider, model, statusCode); console.log(`${COLORS.red}[ERROR] ${errMsg}${COLORS.reset}`); // Log Antigravity retry time if available if (retryAfterMs && provider === "antigravity") { const retrySeconds = Math.ceil(retryAfterMs / 1000); log?.debug?.("RETRY", `Antigravity quota reset in ${retrySeconds}s (${retryAfterMs}ms)`); } // Log error with full request body for debugging reqLogger.logError(new Error(message), finalBody || translatedBody); reqLogger.logProviderResponse( providerResponse.status, providerResponse.statusText, providerResponse.headers, upstreamErrorBody ); // Update rate limiter from error response headers updateFromHeaders(provider, connectionId, providerResponse.headers, statusCode, model); if (connectionId && upstreamErrorBody !== null && upstreamErrorBody !== undefined) { updateFromResponseBody(provider, connectionId, upstreamErrorBody, statusCode, model); } // ── T5: Intra-family model fallback ────────────────────────────────────── // Before returning a model-unavailable error upstream, try sibling models // from the same family. This keeps the request alive on the same account // instead of failing the entire combo. if (isModelUnavailableError(statusCode, message)) { const nextModel = getNextFamilyFallback(currentModel, triedModels); if (nextModel) { triedModels.add(nextModel); currentModel = nextModel; translatedBody.model = nextModel; log?.info?.("MODEL_FALLBACK", `${model} unavailable (${statusCode}) → trying ${nextModel}`); // Re-execute with the fallback model try { const fallbackResult = await executeProviderRequest(nextModel, false); if (fallbackResult.response.ok) { providerResponse = fallbackResult.response; providerUrl = fallbackResult.url; providerHeaders = fallbackResult.headers; finalBody = fallbackResult.transformedBody; reqLogger.logTargetRequest(providerUrl, providerHeaders, finalBody); updatePendingRequest(model, provider, connectionId, { providerRequest: finalBody, providerUrl, stage: "provider_response_started", }); // Continue processing with the fallback response — skip error return log?.info?.("MODEL_FALLBACK", `Serving ${nextModel} as fallback for ${model}`); // Jump to streaming/non-streaming handling below // We fall through by NOT returning here } else { // Fallback also failed — return original error persistAttemptLogs({ status: statusCode, error: errMsg, providerRequest: finalBody || translatedBody, providerResponse: upstreamErrorBody, clientResponse: buildErrorBody(statusCode, errMsg), cacheSource: "upstream", }); persistFailureUsage(statusCode, "model_unavailable"); return createErrorResult( statusCode, errMsg, retryAfterMs, upstreamErrorCode, upstreamErrorType, upstreamErrorBody ); } } catch { persistAttemptLogs({ status: statusCode, error: errMsg, providerRequest: finalBody || translatedBody, providerResponse: upstreamErrorBody, clientResponse: buildErrorBody(statusCode, errMsg), cacheSource: "upstream", }); persistFailureUsage(statusCode, "model_unavailable"); return createErrorResult( statusCode, errMsg, retryAfterMs, upstreamErrorCode, upstreamErrorType, upstreamErrorBody ); } } else { persistAttemptLogs({ status: statusCode, error: errMsg, providerRequest: finalBody || translatedBody, providerResponse: upstreamErrorBody, clientResponse: buildErrorBody(statusCode, errMsg), cacheSource: "upstream", }); persistFailureUsage(statusCode, "model_unavailable"); return createErrorResult( statusCode, errMsg, retryAfterMs, upstreamErrorCode, upstreamErrorType, upstreamErrorBody ); } } else if (isContextOverflowError(statusCode, message)) { const familyCandidates = getModelFamily(currentModel).filter( (m) => m !== currentModel && !triedModels.has(m) ); const nextModel = findLargerContextModel(currentModel, familyCandidates) ?? getNextFamilyFallback(currentModel, triedModels); if (nextModel) { triedModels.add(nextModel); currentModel = nextModel; translatedBody.model = nextModel; log?.info?.("CONTEXT_OVERFLOW_FALLBACK", `${model} context overflow → trying ${nextModel}`); try { const fallbackResult = await executeProviderRequest(nextModel, false); if (fallbackResult.response.ok) { providerResponse = fallbackResult.response; providerUrl = fallbackResult.url; providerHeaders = fallbackResult.headers; finalBody = fallbackResult.transformedBody; reqLogger.logTargetRequest(providerUrl, providerHeaders, finalBody); updatePendingRequest(model, provider, connectionId, { providerRequest: finalBody, providerUrl, stage: "provider_response_started", }); log?.info?.( "CONTEXT_OVERFLOW_FALLBACK", `Serving ${nextModel} as fallback for ${model}` ); } else { persistAttemptLogs({ status: statusCode, error: errMsg, providerRequest: finalBody || translatedBody, providerResponse: upstreamErrorBody, clientResponse: buildErrorBody(statusCode, errMsg), cacheSource: "upstream", }); persistFailureUsage(statusCode, "context_overflow"); return createErrorResult( statusCode, errMsg, retryAfterMs, upstreamErrorCode, upstreamErrorType, upstreamErrorBody ); } } catch { persistAttemptLogs({ status: statusCode, error: errMsg, providerRequest: finalBody || translatedBody, providerResponse: upstreamErrorBody, clientResponse: buildErrorBody(statusCode, errMsg), cacheSource: "upstream", }); persistFailureUsage(statusCode, "context_overflow"); return createErrorResult( statusCode, errMsg, retryAfterMs, upstreamErrorCode, upstreamErrorType, upstreamErrorBody ); } } else { persistAttemptLogs({ status: statusCode, error: errMsg, providerRequest: finalBody || translatedBody, providerResponse: upstreamErrorBody, clientResponse: buildErrorBody(statusCode, errMsg), cacheSource: "upstream", }); persistFailureUsage(statusCode, "context_overflow"); return createErrorResult( statusCode, errMsg, retryAfterMs, upstreamErrorCode, upstreamErrorType, upstreamErrorBody ); } } else { persistAttemptLogs({ status: statusCode, error: errMsg, providerRequest: finalBody || translatedBody, providerResponse: upstreamErrorBody, clientResponse: buildErrorBody(statusCode, errMsg), cacheSource: "upstream", }); persistFailureUsage(statusCode, `upstream_${statusCode}`); const requestHasTools = Array.isArray(translatedBody.tools) && translatedBody.tools.length > 0; let emergencyFallbackServed = false; if (!disableEmergencyFallback && !stream) { const fbDecision = shouldUseFallback( statusCode, message, requestHasTools, EMERGENCY_FALLBACK_CONFIG ); if (isFallbackDecision(fbDecision)) { log?.info?.("EMERGENCY_FALLBACK", fbDecision.reason); try { const originalProvider = provider; const fbExecutor = getExecutor(fbDecision.provider); const fbResult = await fbExecutor.execute({ model: fbDecision.model, body: { ...translatedBody, model: fbDecision.model, max_tokens: Math.min( typeof translatedBody.max_tokens === "number" ? translatedBody.max_tokens : fbDecision.maxOutputTokens, fbDecision.maxOutputTokens ), max_completion_tokens: Math.min( typeof translatedBody.max_completion_tokens === "number" ? translatedBody.max_completion_tokens : typeof translatedBody.max_tokens === "number" ? translatedBody.max_tokens : fbDecision.maxOutputTokens, fbDecision.maxOutputTokens ), }, stream: false, credentials: credentials, signal: streamController.signal, log, extendedContext, }); if (fbResult.response.ok) { provider = fbDecision.provider; model = fbDecision.model; translatedBody.model = fbDecision.model; providerResponse = fbResult.response; providerUrl = fbResult.url; providerHeaders = new Headers(fbResult.headers || {}); finalBody = fbResult.transformedBody; reqLogger.logTargetRequest(providerUrl, providerHeaders, finalBody); log?.info?.( "EMERGENCY_FALLBACK", `Serving ${fbDecision.provider}/${fbDecision.model} as budget fallback for ${originalProvider}/${requestedModel}` ); emergencyFallbackServed = true; } else { log?.warn?.( "EMERGENCY_FALLBACK", `Emergency fallback also failed (${fbResult.response.status})` ); } } catch (fbErr) { const errMessage = fbErr instanceof Error ? fbErr.message : String(fbErr); log?.warn?.("EMERGENCY_FALLBACK", `Emergency fallback error: ${errMessage}`); } } } if (!emergencyFallbackServed) { return createErrorResult( statusCode, errMsg, retryAfterMs, upstreamErrorCode, upstreamErrorType, upstreamErrorBody ); } } // ── End T5 ─────────────────────────────────────────────────────────────── } // Non-streaming response if (!stream) { trackPendingRequest(model, provider, connectionId, false); const contentType = (providerResponse.headers.get("content-type") || "").toLowerCase(); let responseBody; let responsePayloadFormat = targetFormat; const rawBody = await readNonStreamingResponseBody( providerResponse, contentType, upstreamStream ); const normalizedProviderPayload = normalizePayloadForLog(rawBody); const looksLikeSSE = contentType.includes("text/event-stream") || contentType.includes("application/x-ndjson") || /(^|\n)\s*(event|data):/m.test(rawBody); if (looksLikeSSE) { const streamPayload = normalizeNonStreamingEventPayload(rawBody, contentType); const streamKind = contentType.includes("application/x-ndjson") ? "NDJSON" : "SSE"; if (shouldTreatBufferedEventResponseAsExpected(upstreamStream, providerHeaders, finalBody)) { log?.debug?.( "STREAM", `Buffering upstream ${streamKind} response for non-streaming client request` ); } else { log?.warn?.( "STREAM", `Unexpected ${streamKind} response for non-streaming request — buffering` ); } // Upstream returned an event stream for a non-streaming client; convert best-effort to JSON. const parsedFromSSE = parseNonStreamingSSEPayload(streamPayload, targetFormat, model); if (!parsedFromSSE) { appendRequestLog({ model, provider, connectionId, status: `FAILED ${HTTP_STATUS.BAD_GATEWAY}`, }).catch(() => {}); const invalidSseMessage = "Invalid SSE response for non-streaming request"; persistAttemptLogs({ status: HTTP_STATUS.BAD_GATEWAY, error: invalidSseMessage, providerRequest: finalBody || translatedBody, providerResponse: normalizedProviderPayload, clientResponse: buildErrorBody(HTTP_STATUS.BAD_GATEWAY, invalidSseMessage), cacheSource: "upstream", }); persistFailureUsage(HTTP_STATUS.BAD_GATEWAY, "invalid_sse_payload"); return createErrorResult(HTTP_STATUS.BAD_GATEWAY, invalidSseMessage); } responseBody = parsedFromSSE.body; responsePayloadFormat = parsedFromSSE.format; } else { try { responseBody = rawBody ? JSON.parse(rawBody) : {}; } catch (err) { appendRequestLog({ model, provider, connectionId, status: `FAILED ${HTTP_STATUS.BAD_GATEWAY}`, }).catch(() => {}); const detailedError = `Invalid JSON response from provider (error: ${err instanceof Error ? err.message : String(err)}): ${rawBody.substring(0, 1000)}`; const invalidJsonMessage = "Invalid JSON response from provider"; persistAttemptLogs({ status: HTTP_STATUS.BAD_GATEWAY, error: detailedError, providerRequest: finalBody || translatedBody, providerResponse: normalizedProviderPayload, clientResponse: buildErrorBody(HTTP_STATUS.BAD_GATEWAY, invalidJsonMessage), cacheSource: "upstream", }); persistFailureUsage(HTTP_STATUS.BAD_GATEWAY, "invalid_json_payload"); return createErrorResult(HTTP_STATUS.BAD_GATEWAY, invalidJsonMessage); } } // Check for empty content response (fake success) - trigger fallback if (isEmptyContentResponse(responseBody)) { appendRequestLog({ model, provider, connectionId, status: `FAILED ${HTTP_STATUS.BAD_GATEWAY}`, }).catch(() => {}); const emptyContentMessage = "Provider returned empty content"; persistAttemptLogs({ status: HTTP_STATUS.BAD_GATEWAY, error: emptyContentMessage, providerRequest: finalBody || translatedBody, providerResponse: normalizedProviderPayload, clientResponse: buildErrorBody(HTTP_STATUS.BAD_GATEWAY, emptyContentMessage), cacheSource: "upstream", }); persistFailureUsage(HTTP_STATUS.BAD_GATEWAY, "empty_content"); // Trigger non-recursive fallback for empty content const nextModel = getNextFamilyFallback(currentModel, triedModels); if (nextModel) { triedModels.add(nextModel); currentModel = nextModel; translatedBody.model = nextModel; log?.info?.( "EMPTY_CONTENT_FALLBACK", `${model} returned empty content → trying ${nextModel}` ); try { const fallbackResult = await executeProviderRequest(nextModel, false); if (fallbackResult.response.ok) { const fallbackRaw = await withBodyTimeout(fallbackResult.response.text()); try { responseBody = fallbackRaw ? JSON.parse(fallbackRaw) : {}; providerUrl = fallbackResult.url; providerHeaders = fallbackResult.headers; finalBody = fallbackResult.transformedBody; reqLogger.logTargetRequest(providerUrl, providerHeaders, finalBody); log?.info?.( "EMPTY_CONTENT_FALLBACK", `Serving ${nextModel} as fallback for ${model}` ); // Fall through — continue processing with the new responseBody } catch { return createErrorResult(HTTP_STATUS.BAD_GATEWAY, emptyContentMessage); } } else { return createErrorResult(HTTP_STATUS.BAD_GATEWAY, emptyContentMessage); } } catch { return createErrorResult(HTTP_STATUS.BAD_GATEWAY, emptyContentMessage); } } else { return createErrorResult(HTTP_STATUS.BAD_GATEWAY, emptyContentMessage); } } const responseToolNameMap = mergeResponseToolNameMap( toolNameMap, (finalBody as Record | null | undefined) ?? null ); if (sourceFormat === FORMATS.CLAUDE && targetFormat === FORMATS.CLAUDE) { responseBody = restoreClaudePassthroughToolNames(responseBody, responseToolNameMap); } reqLogger.logProviderResponse( providerResponse.status, providerResponse.statusText, providerResponse.headers, looksLikeSSE ? { _streamed: true, _format: "sse-json", summary: responseBody, } : responseBody ); effectiveServiceTier = resolveReportedServiceTier(responseBody) ?? effectiveServiceTier; // Notify success - caller can clear error status if needed if (onRequestSuccess) { await onRequestSuccess(); } await maybeSyncClaudeExtraUsageState({ provider, connectionId, providerSpecificData: credentials?.providerSpecificData, log, }); // Log usage for non-streaming responses const usage = extractUsageFromResponse(responseBody, provider); if (usage && typeof usage === "object") { attachCompressionUsageReceiptAfterAnalytics(usage as Record, "provider"); } appendRequestLog({ model, provider, connectionId, tokens: usage, status: "200 OK" }).catch( () => {} ); // Save structured call log with full payloads const cacheUsageLogMeta = buildCacheUsageLogMeta(usage); if (usage && typeof usage === "object") { const msg = `[${new Date().toLocaleTimeString("en-US", { hour12: false, hour: "2-digit", minute: "2-digit" })}] 📊 [USAGE] ${provider.toUpperCase()} | ${formatUsageLog(usage)}${connectionId ? ` | account=${connectionId.slice(0, 8)}...` : ""}`; console.log(`${COLORS.green}${msg}${COLORS.reset}`); saveRequestUsage({ provider: provider || "unknown", model: model || "unknown", tokens: usage, status: "200", success: true, latencyMs: Date.now() - startTime, timeToFirstTokenMs: Date.now() - startTime, errorCode: null, timestamp: new Date().toISOString(), connectionId: connectionId || undefined, apiKeyId: apiKeyInfo?.id || undefined, apiKeyName: apiKeyInfo?.name || undefined, serviceTier: effectiveServiceTier, comboStrategy: isCombo ? comboStrategy || undefined : undefined, }).catch((err) => { console.error("Failed to save usage stats:", err.message); }); if (apiKeyInfo?.id) { try { const billable = computeBillableTokens(usage); if (billable > 0) recordTokenUsage(apiKeyInfo.id, provider || "unknown", model || "unknown", billable); } catch { // never block the response on counter recording } } } // Translate response to client's expected format (usually OpenAI) // Pass toolNameMap so Claude OAuth proxy_ prefix is stripped in tool_use blocks (#605) let translatedResponse = needsTranslation(responsePayloadFormat, clientResponseFormat) ? translateNonStreamingResponse( responseBody, responsePayloadFormat, clientResponseFormat, responseToolNameMap ) : responseBody; const memoryExtractionResponse = translatedResponse; // T26: Strip markdown code blocks if provider format is Claude if (sourceFormat === "claude" && !stream) { if (typeof translatedResponse?.choices?.[0]?.message?.content === "string") { translatedResponse.choices[0].message.content = stripMarkdownCodeFence( translatedResponse.choices[0].message.content ) as string; } } // T18: Normalize finish_reason to 'tool_calls' if tool calls are present if (translatedResponse?.choices) { for (const choice of translatedResponse.choices) { if ( choice.message?.tool_calls && choice.message.tool_calls.length > 0 && choice.finish_reason !== "tool_calls" ) { choice.finish_reason = "tool_calls"; } } } // Reasoning Replay Cache (#1628): Capture reasoning_content from non-streaming responses // with tool_calls so it can be replayed on subsequent turns (DeepSeek V4, Kimi K2, etc.) try { const firstChoice = translatedResponse?.choices?.[0]; const msg = firstChoice?.message; cacheReasoningFromAssistantMessage(msg, provider, model, { requestId: skillRequestId, messageIndex: 0, }); } catch { // Cache capture is non-critical — never block the response } // Sanitize response for OpenAI SDK compatibility // Strips non-standard fields (x_groq, usage_breakdown, service_tier, etc.) // Extracts and tags into reasoning_content // Source format determines output shape. If we are outputting OpenAI shape or pseudo-OpenAI shape, sanitize. if (clientResponseFormat === FORMATS.OPENAI_RESPONSES) { translatedResponse = sanitizeResponsesApiResponse(translatedResponse); } else if (clientResponseFormat === FORMATS.OPENAI) { translatedResponse = sanitizeOpenAIResponse(translatedResponse); } // Add buffer and filter usage for client (to prevent CLI context errors) if (translatedResponse?.usage) { const buffered = addBufferToUsage(translatedResponse.usage); translatedResponse.usage = filterUsageForFormat(buffered, clientResponseFormat); } else { // Fallback: estimate usage when provider returned no usage block const contentLength = JSON.stringify( translatedResponse?.choices?.[0]?.message?.content || "" ).length; if (contentLength > 0) { const estimated = estimateUsage(body, contentLength, clientResponseFormat); translatedResponse.usage = filterUsageForFormat(estimated, clientResponseFormat); } } if (memoryOwnerId && memorySettings?.enabled && memorySettings.maxTokens > 0) { const requestMemoryText = extractMemoryTextFromRequestBody(body as Record); if (requestMemoryText) { extractFacts(requestMemoryText, memoryOwnerId, pipelineSessionId); } const memoryText = extractMemoryTextFromResponse(memoryExtractionResponse); if (memoryText) { extractFacts(memoryText, memoryOwnerId, pipelineSessionId); } } const customSkillExecutionEnabled = Boolean(memoryOwnerId) && memorySettings?.skillsEnabled === true; const builtinToolNames = webSearchFallbackPlan.toolName ? [webSearchFallbackPlan.toolName] : []; if (customSkillExecutionEnabled || builtinToolNames.length > 0) { const skillSessionId = pipelineSessionId; translatedResponse = await handleToolCallExecution( translatedResponse, getSkillsModelIdForFormat(sourceFormat), { apiKeyId: memoryOwnerId || "local", sessionId: skillSessionId, requestId: skillRequestId, builtinToolNames, customSkillExecutionEnabled, } ); } const guardrailContext = { apiKeyInfo, disabledGuardrails: resolveDisabledGuardrails({ apiKeyInfo: (apiKeyInfo as Record | null) ?? null, body, headers: (clientRawRequest?.headers as Headers | Record | null) ?? null, }), endpoint: clientRawRequest?.endpoint || null, headers: (clientRawRequest?.headers as Headers | Record | null) ?? null, log, method: "POST", model, provider, sourceFormat: responsePayloadFormat, stream: false, targetFormat: clientResponseFormat, } as const; const postCallGuardrails = await guardrailRegistry.runPostCallHooks( translatedResponse, guardrailContext ); translatedResponse = postCallGuardrails.response; const responseUsage = (usage && typeof usage === "object" ? usage : null) || (translatedResponse?.usage && typeof translatedResponse.usage === "object" ? translatedResponse.usage : null); const estimatedCost = responseUsage ? await calculateCost(provider, model, responseUsage, { serviceTier: effectiveServiceTier }) : 0; if (postCallGuardrails.blocked) { const guardrailMessage = postCallGuardrails.message || "Response blocked by guardrail"; persistAttemptLogs({ status: HTTP_STATUS.BAD_REQUEST, tokens: usage, responseBody, providerRequest: finalBody || translatedBody, providerResponse: looksLikeSSE ? { _streamed: true, _format: "sse-json", summary: responseBody, } : responseBody, clientResponse: buildErrorBody(HTTP_STATUS.BAD_REQUEST, guardrailMessage), claudeCacheMeta: claudePromptCacheLogMeta, claudeCacheUsageMeta: cacheUsageLogMeta, cacheSource: "upstream", }); if (apiKeyInfo?.id && estimatedCost > 0) { recordCost(apiKeyInfo.id, estimatedCost); } log?.warn?.( "GUARDRAIL", `Response blocked by ${postCallGuardrails.guardrail || "guardrail"}: ${guardrailMessage}` ); return createErrorResult(HTTP_STATUS.BAD_REQUEST, guardrailMessage); } // ── Phase 9.1: Cache store (non-streaming, temp=0) ── if ( semanticCacheEnabled && isCacheableForWrite(body, clientRawRequest?.headers) && isSmallEnoughForSemanticCache(translatedResponse) ) { const signature = generateSignature( model, body.messages ?? body.input, body.temperature, body.top_p ); const tokensSaved = usage?.prompt_tokens + usage?.completion_tokens || 0; setCachedResponse(signature, model, translatedResponse, tokensSaved); log?.debug?.("CACHE", `Stored response for ${model} (${tokensSaved} tokens)`); } // ── Phase 9.2: Save for idempotency ── saveIdempotency(idempotencyKey, translatedResponse, 200); reqLogger.logConvertedResponse(translatedResponse); persistAttemptLogs({ status: 200, tokens: usage, responseBody, providerRequest: finalBody || translatedBody, providerResponse: looksLikeSSE ? { _streamed: true, _format: "sse-json", summary: responseBody, } : responseBody, clientResponse: translatedResponse, claudeCacheMeta: claudePromptCacheLogMeta, claudeCacheUsageMeta: cacheUsageLogMeta, cacheSource: "upstream", }); if (apiKeyInfo?.id && estimatedCost > 0) { recordCost(apiKeyInfo.id, estimatedCost); } // === Quota Share POST-hook (B/F7) — fire-and-forget, fail-open === if (apiKeyInfo?.id && credentials?.connectionId) { try { const { scheduleRecordConsumption } = await import("@/lib/quota/spendRecorder"); scheduleRecordConsumption( { apiKeyId: apiKeyInfo.id, connectionId: credentials.connectionId, provider: provider ?? "unknown", cost: { tokens: usage && typeof usage === "object" ? ((usage as Record).prompt_tokens as number ?? 0) + ((usage as Record).completion_tokens as number ?? 0) : 0, usd: estimatedCost > 0 ? estimatedCost : 0, requests: 1, }, }, log ); } catch (_) { // Outer fail-open — never throws to caller } } // === /Quota Share POST-hook === // ── Gamification event (fire-and-forget) ── if (apiKeyInfo?.id) { try { const { emitGamificationEvent } = await import("@/lib/gamification/events"); emitGamificationEvent({ apiKeyId: apiKeyInfo.id, action: "request", metadata: { model, provider }, }); } catch (_) { /* gamification optional */ } } return { success: true, response: new Response(JSON.stringify(translatedResponse), { headers: { "Content-Type": "application/json", [OMNIROUTE_RESPONSE_HEADERS.cache]: "MISS", ...buildOmniRouteResponseMetaHeaders({ provider, model, cacheHit: false, latencyMs: Date.now() - startTime, usage: responseUsage, costUsd: estimatedCost, }), }, }), }; } // Streaming response const streamReadinessPolicy = resolveStreamReadinessTimeout({ baseTimeoutMs: STREAM_READINESS_TIMEOUT_MS, provider, model, body: (finalBody || translatedBody) as Record | null | undefined, }); if (streamReadinessPolicy.timeoutMs !== streamReadinessPolicy.baseTimeoutMs) { log?.debug?.( "STREAM", `adaptive readiness timeout=${streamReadinessPolicy.timeoutMs}ms base=${streamReadinessPolicy.baseTimeoutMs}ms reason=${streamReadinessPolicy.reasons.join(",")}` ); } const streamReadiness = await ensureStreamReadiness(providerResponse, { timeoutMs: streamReadinessPolicy.timeoutMs, provider, model, log, }); if (streamReadiness.ok === false) { const { response: failureResponse, reason } = streamReadiness; const failure = { status: failureResponse.status, message: reason, code: streamReadiness.code, type: streamReadiness.type, }; trackPendingRequest(model, provider, connectionId, false); appendRequestLog({ model, provider, connectionId, status: `FAILED ${failureResponse.status}`, }).catch(() => {}); persistAttemptLogs({ status: failureResponse.status, error: reason, providerRequest: finalBody || translatedBody, clientResponse: buildErrorBody(failureResponse.status, reason), claudeCacheMeta: claudePromptCacheLogMeta, cacheSource: "upstream", }); persistFailureUsage(failureResponse.status, streamReadiness.code); // Do NOT call onStreamFailure — a stream stall is an upstream issue, // not an account/quota failure. Marking the account unavailable here // would lock out legitimate accounts when the upstream hangs. return { success: false, status: failureResponse.status, error: reason, errorType: streamReadiness.type, errorCode: streamReadiness.code, response: failureResponse, }; } providerResponse = streamReadiness.response; // Notify success - caller can clear error status if needed if (onRequestSuccess) { await onRequestSuccess(); } const responseHeaders: Record = buildStreamingResponseHeaders( providerResponse.headers, { provider, model, cacheHit: false, latencyMs: 0, usage: null, costUsd: 0, } ); // Create transform stream with logger for streaming response let transformStream; const responseToolNameMap = mergeResponseToolNameMap( toolNameMap, (finalBody as Record | null | undefined) ?? null ); // Callback to save call log when stream completes (include responseBody when provided by stream) const onStreamComplete = ({ status: streamStatus, usage: streamUsage, responseBody: streamResponseBody, providerPayload, clientPayload, ttft, }) => { const cacheUsageLogMeta = buildCacheUsageLogMeta(streamUsage); if (streamStatus === 200) { void maybeSyncClaudeExtraUsageState({ provider, connectionId, providerSpecificData: credentials?.providerSpecificData, log, }); } // Reasoning Replay Cache (#1628): Capture reasoning_content from streaming responses // with tool_calls so it can be replayed on subsequent turns (DeepSeek V4, Kimi K2, etc.) if (streamStatus === 200 && streamResponseBody) { try { const body = streamResponseBody as Record; const choices = body.choices as { message?: Record }[] | undefined; const msg = choices?.[0]?.message; cacheReasoningFromAssistantMessage(msg, provider, model, { requestId: skillRequestId, messageIndex: 0, }); } catch { // Cache capture is non-critical — never block the stream } } effectiveServiceTier = resolveReportedServiceTier(streamResponseBody) ?? effectiveServiceTier; // Track cache token metrics for streaming responses if (streamUsage && typeof streamUsage === "object") { attachCompressionUsageReceiptAfterAnalytics(streamUsage as Record, "stream"); saveRequestUsage({ provider: provider || "unknown", model: model || "unknown", tokens: streamUsage, status: String(streamStatus || 200), success: streamStatus === 200, latencyMs: Date.now() - startTime, timeToFirstTokenMs: ttft, errorCode: null, timestamp: new Date().toISOString(), connectionId: connectionId || undefined, apiKeyId: apiKeyInfo?.id || undefined, apiKeyName: apiKeyInfo?.name || undefined, serviceTier: effectiveServiceTier, comboStrategy: isCombo ? comboStrategy || undefined : undefined, }).catch((err) => { console.error("Failed to save usage stats:", err.message); }); if (apiKeyInfo?.id && streamStatus === 200) { try { const billable = computeBillableTokens(streamUsage); if (billable > 0) recordTokenUsage(apiKeyInfo.id, provider || "unknown", model || "unknown", billable); } catch { // never block the stream on counter recording } } } persistAttemptLogs({ status: streamStatus || 200, tokens: streamUsage || {}, responseBody: streamResponseBody ?? undefined, providerRequest: finalBody || translatedBody, providerResponse: providerPayload, clientResponse: clientPayload ?? streamResponseBody ?? undefined, claudeCacheMeta: claudePromptCacheLogMeta, claudeCacheUsageMeta: cacheUsageLogMeta, cacheSource: "upstream", }); if (apiKeyInfo?.id && streamUsage) { calculateCost(provider, model, streamUsage, { serviceTier: effectiveServiceTier }) .then((estimatedCost) => { if (estimatedCost > 0) recordCost(apiKeyInfo.id, estimatedCost); }) .catch(() => {}); } // === Quota Share POST-hook streaming (B/F7) — fire-and-forget, fail-open === if (apiKeyInfo?.id && credentials?.connectionId && streamStatus === 200) { const su = streamUsage as Record | null; const quotaApiKeyId = apiKeyInfo.id; const quotaConnectionId = credentials.connectionId; // onStreamComplete is sync — use .then() (fire-and-forget, fail-open) instead of await import("@/lib/quota/spendRecorder") .then(({ scheduleRecordConsumption }) => { scheduleRecordConsumption( { apiKeyId: quotaApiKeyId, connectionId: quotaConnectionId, provider: provider ?? "unknown", cost: { tokens: su ? (Number(su.prompt_tokens ?? 0) || 0) + (Number(su.completion_tokens ?? 0) || 0) : 0, usd: 0, // estimatedCost resolved async above; omit to avoid dependency requests: 1, }, }, log ); }) .catch(() => { // Outer fail-open — never throws to caller }); } // === /Quota Share POST-hook streaming === if ( memoryOwnerId && memorySettings?.enabled && memorySettings.maxTokens > 0 && streamStatus === 200 ) { const requestMemoryText = extractMemoryTextFromRequestBody(body as Record); if (requestMemoryText) { extractFacts(requestMemoryText, memoryOwnerId, pipelineSessionId); } const streamedMemoryText = extractMemoryTextFromResponse( (streamResponseBody ?? null) as Record | null ); if (streamedMemoryText) { extractFacts(streamedMemoryText, memoryOwnerId, pipelineSessionId); } } // Semantic cache: store assembled streaming response for future cache hits if ( semanticCacheEnabled && streamStatus === 200 && streamResponseBody && isCacheableForWrite(body, clientRawRequest?.headers) ) { try { const cleanBody = { ...streamResponseBody }; delete cleanBody._streamed; if (!isSmallEnoughForSemanticCache(cleanBody)) return; const sig = generateSignature( model, body.messages ?? body.input, body.temperature, body.top_p ); const u = streamUsage as Record | null; const tokensSaved = (Number(u?.prompt_tokens ?? 0) || 0) + (Number(u?.completion_tokens ?? 0) || 0); setCachedResponse(sig, model, cleanBody, tokensSaved); log?.debug?.("CACHE", `Stored streaming response for ${model} (${tokensSaved} tokens)`); } catch { // Cache write failed — non-critical } } }; const handleStreamFailure = (failure: { status: number; message: string; code?: string; type?: string; }) => { persistFailureUsage(failure.status || HTTP_STATUS.BAD_GATEWAY, failure.code || failure.type); try { onStreamFailure?.(failure); } catch { // Best-effort fallback state update only. } }; // For providers using Responses API format, translate stream back to openai (Chat Completions) format // UNLESS client is Droid CLI which expects openai-responses format back const needsResponsesTranslation = targetFormat === FORMATS.OPENAI_RESPONSES && clientResponseFormat === FORMATS.OPENAI && !isResponsesEndpoint && !isDroidCLI; const streamStateBody = finalBody || body; if (needsResponsesTranslation) { // Provider returns openai-responses, translate to openai (Chat Completions) that clients expect log?.debug?.("STREAM", `Responses translation mode: openai-responses → openai`); transformStream = createSSETransformStreamWithLogger( "openai-responses", "openai", provider, reqLogger, responseToolNameMap, model, connectionId, streamStateBody, onStreamComplete, apiKeyInfo, handleStreamFailure, copilotCompatibleReasoning ); } else if (needsTranslation(targetFormat, clientResponseFormat)) { // Standard translation for other providers log?.debug?.("STREAM", `Translation mode: ${targetFormat} → ${clientResponseFormat}`); transformStream = createSSETransformStreamWithLogger( targetFormat, clientResponseFormat, provider, reqLogger, responseToolNameMap, model, connectionId, streamStateBody, onStreamComplete, apiKeyInfo, handleStreamFailure, copilotCompatibleReasoning ); } else { log?.debug?.("STREAM", `Standard passthrough mode`); transformStream = createPassthroughStreamWithLogger( provider, reqLogger, responseToolNameMap, model, connectionId, streamStateBody, onStreamComplete, apiKeyInfo, handleStreamFailure, clientResponseFormat ); } // ── Phase 9.3: Progress tracking (opt-in) ── const progressEnabled = wantsProgress(clientRawRequest?.headers); let finalStream; if (progressEnabled) { const progressTransform = createProgressTransform({ signal: streamController.signal }); // Chain: provider → transform → progress → client const transformedBody = pipeWithDisconnect(providerResponse, transformStream, streamController); finalStream = transformedBody.pipeThrough(progressTransform); responseHeaders[OMNIROUTE_RESPONSE_HEADERS.progress] = "enabled"; } else { finalStream = pipeWithDisconnect(providerResponse, transformStream, streamController); } finalStream = finalStream.pipeThrough( createSseHeartbeatTransform({ signal: streamController.signal, intervalMs: SSE_HEARTBEAT_INTERVAL_MS, shape: shapeForClientFormat(clientResponseFormat), }) ); // ── Gamification event (fire-and-forget) ── if (apiKeyInfo?.id) { try { const { emitGamificationEvent } = await import("@/lib/gamification/events"); emitGamificationEvent({ apiKeyId: apiKeyInfo.id, action: "request", metadata: { model, provider }, }); } catch (_) { /* gamification optional */ } } return { success: true, response: new Response(finalStream, { headers: responseHeaders, }), }; } /** * Check if token is expired or about to expire */ export function isTokenExpiringSoon(expiresAt, bufferMs = 5 * 60 * 1000) { if (!expiresAt) return false; const expiresAtMs = new Date(expiresAt).getTime(); return expiresAtMs - Date.now() < bufferMs; }