diff --git a/docs/security/ERROR_SANITIZATION.md b/docs/security/ERROR_SANITIZATION.md index 0edc466b04..e3ffe04c77 100644 --- a/docs/security/ERROR_SANITIZATION.md +++ b/docs/security/ERROR_SANITIZATION.md @@ -169,8 +169,8 @@ Every field is projected onto the bounded public-identifier vocabulary. Unsafe, control-character, or overlong values fall back to the status-derived type/code; an unsafe optional reason is omitted. Three-digit HTTP status identifiers (`100` through `599`) remain valid for provider contracts that expose the numeric upstream status as a machine-readable code. The same -bounded range is accepted in the locally generated `HTTP_NNN` form; arbitrary provider numbers and -names remain outside the vocabulary. +bounded range is accepted in the locally generated HTTP-status placeholder form; arbitrary provider +numbers and names remain outside the vocabulary. Pass every explicit classification in that fourth argument. Never overwrite `body.error.code`, `body.error.type`, or `body.error.reason` after `buildErrorBody()` returns; diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index d0eb32fe56..9e0f71e132 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -6,6 +6,7 @@ import { injectMemoryAndSkills } from "./chatCore/memorySkillsInjection.ts"; import { resolveChatCoreRequestSetup } from "./chatCore/requestSetup.ts"; import { normalizeOpenAICompatibleTools } from "./chatCore/openAICompatibleTools.ts"; import { buildFailureUsageRecord, projectFailureUsageErrorCode } from "./chatCore/failureUsage.ts"; +import { createTranslationFailureResult } from "./chatCore/translationFailure.ts"; import { estimateFinalInputTokens } from "./chatCore/contextEstimation.ts"; import { extractSystemRoleMessages, @@ -2489,33 +2490,11 @@ export async function handleChatCore({ : HTTP_STATUS.SERVER_ERROR; const message = error?.message || "Invalid request"; const errorType = typeof error?.errorType === "string" ? error.errorType : null; - const errorBody = buildErrorBody( - statusCode, - message, - undefined, - errorType ? { type: errorType, code: errorType } : undefined - ); - const safeMessage = errorBody.error.message; - - log?.warn?.("TRANSLATE", `Request translation failed: ${safeMessage}`); - - if (errorType) { - trackPendingRequest(model, provider, connectionId, false); - return { - success: false, - status: statusCode, - error: safeMessage, - response: new Response(JSON.stringify(errorBody), { - status: statusCode, - headers: { - "Content-Type": "application/json", - }, - }), - }; - } + const result = createTranslationFailureResult(statusCode, message, errorType); + log?.warn?.("TRANSLATE", `Request translation failed: ${result.error}`); trackPendingRequest(model, provider, connectionId, false); - return createErrorResult(statusCode, safeMessage); + return result; } // The latest OmniGlyph release has protocol-native OpenAI transforms. Run diff --git a/open-sse/handlers/chatCore/translationFailure.ts b/open-sse/handlers/chatCore/translationFailure.ts new file mode 100644 index 0000000000..622b5c8c47 --- /dev/null +++ b/open-sse/handlers/chatCore/translationFailure.ts @@ -0,0 +1,24 @@ +import { buildErrorBody, createErrorResult } from "../../utils/error.ts"; + +export function createTranslationFailureResult( + status: number, + message: string, + errorType: string | null +) { + if (!errorType) return createErrorResult(status, message); + const body = buildErrorBody( + status, + message, + undefined, + { type: errorType, code: errorType } + ); + return { + success: false as const, + status, + error: body.error.message, + response: new Response(JSON.stringify(body), { + status, + headers: { "Content-Type": "application/json" }, + }), + }; +} diff --git a/open-sse/mcp-server/errorMessage.ts b/open-sse/mcp-server/errorMessage.ts new file mode 100644 index 0000000000..f90c570166 --- /dev/null +++ b/open-sse/mcp-server/errorMessage.ts @@ -0,0 +1,13 @@ +import { sanitizeErrorMessage } from "../utils/error.ts"; + +export function toSafeMcpErrorMessage( + value: unknown, + fallback = "MCP tool execution failed" +): string { + try { + const raw = value instanceof Error ? value.message : value; + return sanitizeErrorMessage(raw) || fallback; + } catch { + return fallback; + } +} diff --git a/open-sse/mcp-server/server.ts b/open-sse/mcp-server/server.ts index 4d2f3e3742..4c30e608f9 100644 --- a/open-sse/mcp-server/server.ts +++ b/open-sse/mcp-server/server.ts @@ -93,7 +93,7 @@ import { import { getDbInstance, ensureDbInitialized } from "../../src/lib/db/core.ts"; import { normalizeQuotaResponse } from "../../src/shared/contracts/quota.ts"; import { resolveOmniRouteBaseUrl } from "../../src/shared/utils/resolveOmniRouteBaseUrl.ts"; -import { sanitizeErrorMessage } from "../utils/error.ts"; +import { toSafeMcpErrorMessage } from "./errorMessage.ts"; import { mcpFetchTimeoutSignal } from "./fetchTimeout.ts"; import { getMcpModelsCatalog } from "./catalog.ts"; import { registerRadarCatalogTool } from "./radarCatalog.ts"; @@ -123,15 +123,6 @@ const TOTAL_MCP_TOOL_COUNT = countUniqueMcpTools({ compressionTools, }); -function toSafeMcpErrorMessage(value: unknown, fallback = "MCP tool execution failed"): string { - try { - const raw = value instanceof Error ? value.message : value; - return sanitizeErrorMessage(raw) || fallback; - } catch { - return fallback; - } -} - type JsonRecord = Record; function readMcpDescriptionCompressionEnabled(): boolean { @@ -337,9 +328,7 @@ async function handleGetHealth() { .filter(({ settled }) => settled.status === "rejected") .map(({ source, settled }) => ({ source, - error: sanitizeErrorMessage( - settled.status === "rejected" ? (settled as PromiseRejectedResult).reason : undefined - ), + error: toSafeMcpErrorMessage((settled as PromiseRejectedResult).reason, ""), })); const result = { diff --git a/open-sse/translator/response/openai-responses.ts b/open-sse/translator/response/openai-responses.ts index 36a22072f9..622d8a5996 100644 --- a/open-sse/translator/response/openai-responses.ts +++ b/open-sse/translator/response/openai-responses.ts @@ -5,7 +5,7 @@ import { register } from "../registry.ts"; import { FORMATS } from "../formats.ts"; import { appendToolCallArgumentDelta } from "../../utils/toolCallArguments.ts"; -import { buildErrorBody } from "../../utils/error.ts"; +import { projectCompletedStreamError } from "../../utils/streamErrorFormat.ts"; import { fallbackToolCallId } from "../helpers/toolCallHelper.ts"; import { shouldParseTextualReasoningTags } from "../../handlers/responseSanitizer.ts"; import { getReadableReasoningValue } from "../../utils/reasoningFields.ts"; @@ -747,17 +747,7 @@ function sendCompleted(state, emit) { // translator or the OpenAI-Responses translator itself when the upstream // SSE stream emits a JSON error object after partial content. const upstreamErr = state.upstreamError; - const publicUpstreamError = upstreamErr - ? buildErrorBody( - Number.isInteger(upstreamErr.status) ? upstreamErr.status : 502, - upstreamErr.message, - undefined, - { - type: upstreamErr.type ?? "server_error", - code: String(upstreamErr.status ?? 502), - } - ).error - : null; + const publicUpstreamError = projectCompletedStreamError(upstreamErr); const response: Record = { id: state.responseId, diff --git a/open-sse/utils/responsesFailureOutput.ts b/open-sse/utils/responsesFailureOutput.ts new file mode 100644 index 0000000000..c08938333f --- /dev/null +++ b/open-sse/utils/responsesFailureOutput.ts @@ -0,0 +1,67 @@ +type JsonRecord = Record; + +export type ResponsesFailureOutputStringField = "id" | "text" | "refusal"; + +export type ResponsesFailureOutputStringProjector = ( + field: ResponsesFailureOutputStringField, + value: string +) => string; + +function asRecord(value: unknown): JsonRecord { + return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {}; +} + +/** + * Retain only public assistant text/refusal output from a failed Responses payload. + * Failure envelopes may contain reasoning, tool arguments, annotations, commentary, + * or provider diagnostics, so every retained field is reconstructed explicitly. + */ +export function projectResponsesFailureOutput( + value: unknown, + projectString: ResponsesFailureOutputStringProjector +): JsonRecord[] { + if (!Array.isArray(value)) return []; + + const output: JsonRecord[] = []; + for (const item of value) { + const record = asRecord(item); + if (record.type !== "message" || record.role !== "assistant" || record.phase === "commentary") { + continue; + } + + const content: JsonRecord[] = []; + if (Array.isArray(record.content)) { + for (const part of record.content) { + const contentPart = asRecord(part); + if (contentPart.phase === "commentary") continue; + if (contentPart.type === "output_text" && typeof contentPart.text === "string") { + content.push({ + type: "output_text", + text: projectString("text", contentPart.text), + }); + } else if (contentPart.type === "refusal" && typeof contentPart.refusal === "string") { + content.push({ + type: "refusal", + refusal: projectString("refusal", contentPart.refusal), + }); + } + } + } + + const projected: JsonRecord = { + type: "message", + role: "assistant", + content, + }; + if (typeof record.id === "string") projected.id = projectString("id", record.id); + if ( + record.status === "in_progress" || + record.status === "completed" || + record.status === "incomplete" + ) { + projected.status = record.status; + } + output.push(projected); + } + return output; +} diff --git a/open-sse/utils/stream.ts b/open-sse/utils/stream.ts index 29338e6b1c..f7b01b1464 100644 --- a/open-sse/utils/stream.ts +++ b/open-sse/utils/stream.ts @@ -50,10 +50,11 @@ import { parseTextualToolCallCandidate, isValidToolCallHeaderPrefix } from "./te import { stripObfuscationZeroWidth } from "./zeroWidth.ts"; import { formatTranslatedStreamError, - normalizeStreamFailurePayload, + prepareTranslatedStreamFailure, projectStreamFailureEvent, type StreamFailurePayload, } from "./streamErrorFormat.ts"; +import { createStreamFailureAborter } from "./streamFailureBoundary.ts"; import { recordToolLatency } from "../services/toolLatencyTracker.ts"; import { extractToolSchemaMap } from "../translator/response/openai-responses/toolSchemas.ts"; import { @@ -1174,89 +1175,37 @@ export function createSSEStream(options: StreamOptions = {}) { } }; - const abortStreamFailure = ( - controller: TransformStreamDefaultController, - failurePayload: StreamFailurePayload, - publicMessage: string, - options: { notifyComplete?: boolean } = {} - ): void => { - let failureHandled = false; - timing.markInterrupted(); - if (onFailure) { - try { - // Keep the raw provider wording internal for quota/reset classification. The - // persistence seam sanitizes it after classification; public output uses the - // separately projected payload and message. - failureHandled = onFailure(failurePayload) === true; - } catch (e) { - console.debug(`[STREAM] onFailure callback error:`, e); - } - } - let publicErrorMessage = publicMessage || "Upstream failure"; - if (options.notifyComplete && onComplete) { - const errorBody = buildErrorBody(failurePayload.status, failurePayload.message); - publicErrorMessage = errorBody.error.message; - try { - onComplete({ - status: failurePayload.status, - usage: state?.usage, - responseBody: errorBody, - ttft: timing.ttftMs(), - itlMs: timing.avgItlMs(), - interrupted: timing.interrupted, - error: publicErrorMessage, - errorCode: failurePayload.code, - providerPayload: providerPayloadCollector.build(providerPayloadCollector.getSummary(), { - includeEvents: false, - }), - clientPayload: clientPayloadCollector.build(errorBody, { includeEvents: false }), - }); - failureHandled = true; - } catch (e) { - console.debug( - `[STREAM] onComplete callback error in error path (${model || "unknown"}):`, - e - ); - } - } - clearIdleTimer(); - if (!failureHandled) { - clearPendingRequestFromStream(); - } - controller.error(markPendingRequestCleared(new Error(publicErrorMessage))); - }; + const abortStreamFailure = createStreamFailureAborter({ + onFailure, + onComplete, + getUsage: () => state?.usage, + timing, + buildProviderPayload: () => + providerPayloadCollector.build(providerPayloadCollector.getSummary(), { + includeEvents: false, + }), + buildClientPayload: (body) => clientPayloadCollector.build(body, { includeEvents: false }), + clearIdleTimer, + clearPendingRequest: clearPendingRequestFromStream, + markPendingRequestCleared, + model, + }); const emitTranslatedFailureAndAbort = ( controller: TransformStreamDefaultController, payload: unknown ): boolean => { - const record = - payload && typeof payload === "object" && !Array.isArray(payload) - ? (payload as JsonRecord) - : {}; - const projectedFailure = projectStreamFailureEvent(record); - if (!projectedFailure && !record.error) return false; - - providerPayloadCollector.push(projectedFailure?.publicPayload ?? record); - - const internalFailure = projectedFailure?.internalFailure ?? - normalizeStreamFailurePayload(record) ?? { - status: 502, - message: "Upstream failure", - code: "stream_error", - type: "server_error", - }; - const output = formatTranslatedStreamError(record, sourceFormat); + const failure = prepareTranslatedStreamFailure(payload); + if (!failure) return false; + providerPayloadCollector.push(failure.providerPayload); + const output = formatTranslatedStreamError(failure.record, sourceFormat); reqLogger?.appendConvertedChunk?.(output); forward(controller, encoder.encode(output)); upstreamErrorForwarded = true; doneSent = true; - abortStreamFailure( - controller, - internalFailure, - projectedFailure?.publicMessage || "Upstream failure", - { notifyComplete: true } - ); + abortStreamFailure(controller, failure.internalFailure, failure.publicMessage, { + notifyComplete: true, + }); return true; }; diff --git a/open-sse/utils/streamErrorFormat.ts b/open-sse/utils/streamErrorFormat.ts index 7c23f8ada4..05a065a864 100644 --- a/open-sse/utils/streamErrorFormat.ts +++ b/open-sse/utils/streamErrorFormat.ts @@ -1,5 +1,6 @@ import { FORMATS } from "../translator/formats.ts"; import { buildErrorBody, sanitizeErrorMessage } from "./error.ts"; +import { projectResponsesFailureOutput } from "./responsesFailureOutput.ts"; /** * Upstream stream-failure normalization + client-format error framing. @@ -23,6 +24,24 @@ export type ProjectedStreamFailureEvent = { publicPayload: JsonRecord; }; +export type PreparedTranslatedStreamFailure = { + record: JsonRecord; + providerPayload: JsonRecord; + internalFailure: StreamFailurePayload; + publicMessage: string; +}; + +export function projectCompletedStreamError( + failure: StreamFailurePayload | null | undefined +): JsonRecord | null { + if (!failure) return null; + const status = Number.isInteger(failure.status) ? failure.status : 502; + return buildErrorBody(status, failure.message, undefined, { + type: failure.type ?? "server_error", + code: String(failure.status ?? 502), + }).error; +} + function asRecord(value: unknown): JsonRecord { return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {}; } @@ -106,7 +125,12 @@ function projectResponsesFailureObject(response: JsonRecord, publicError: JsonRe else if (value === null || typeof value === "number" || typeof value === "boolean") projected[key] = value; } - if (Array.isArray(response.output)) projected.output = response.output; + if (Array.isArray(response.output)) { + projected.output = projectResponsesFailureOutput( + response.output, + projectResponsesFailureString + ); + } const usage = projectResponsesFailureUsage(response.usage); if (usage) projected.usage = usage; if ("last_error" in response) projected.last_error = publicError; @@ -181,6 +205,26 @@ export function normalizeStreamFailurePayload(payload: unknown): StreamFailurePa }; } +export function prepareTranslatedStreamFailure( + payload: unknown +): PreparedTranslatedStreamFailure | null { + const record = asRecord(payload); + const projected = projectStreamFailureEvent(record); + if (!projected && !record.error) return null; + return { + record, + providerPayload: projected?.publicPayload ?? record, + internalFailure: projected?.internalFailure ?? + normalizeStreamFailurePayload(record) ?? { + status: 502, + message: "Upstream failure", + code: "stream_error", + type: "server_error", + }, + publicMessage: projected?.publicMessage || "Upstream failure", + }; +} + /** * Project same-format upstream failure events before they cross the client/log boundary. * diff --git a/open-sse/utils/streamFailureBoundary.ts b/open-sse/utils/streamFailureBoundary.ts new file mode 100644 index 0000000000..bb3da7c850 --- /dev/null +++ b/open-sse/utils/streamFailureBoundary.ts @@ -0,0 +1,76 @@ +import { buildErrorBody } from "./error.ts"; +import type { StreamFailurePayload } from "./streamErrorFormat.ts"; +import type { StreamTiming } from "./streamTiming.ts"; + +type CompletePayload = { + status: number; + usage: unknown; + responseBody: unknown; + providerPayload: unknown; + clientPayload: unknown; + error: string; + errorCode?: string; + ttft: number | null; + itlMs: number | null; + interrupted: boolean; +}; + +type AborterContext = { + onFailure?: ((payload: StreamFailurePayload) => boolean | void | Promise) | null; + onComplete?: ((payload: CompletePayload) => void) | null; + getUsage: () => unknown; + timing: StreamTiming; + buildProviderPayload: () => unknown; + buildClientPayload: (body: unknown) => unknown; + clearIdleTimer: () => void; + clearPendingRequest: () => void; + markPendingRequestCleared: (error: Error) => Error; + model?: string | null; +}; + +export function createStreamFailureAborter(context: AborterContext) { + return ( + controller: TransformStreamDefaultController, + failure: StreamFailurePayload, + publicMessage: string, + options: { notifyComplete?: boolean } = {} + ): void => { + let handled = false; + context.timing.markInterrupted(); + if (context.onFailure) { + try { + handled = context.onFailure(failure) === true; + } catch (error) { + console.debug("[STREAM] onFailure callback error:", error); + } + } + let safeMessage = publicMessage || "Upstream failure"; + if (options.notifyComplete && context.onComplete) { + const body = buildErrorBody(failure.status, failure.message); + safeMessage = body.error.message; + try { + context.onComplete({ + status: failure.status, + usage: context.getUsage(), + responseBody: body, + ttft: context.timing.ttftMs(), + itlMs: context.timing.avgItlMs(), + interrupted: context.timing.interrupted, + error: safeMessage, + errorCode: failure.code, + providerPayload: context.buildProviderPayload(), + clientPayload: context.buildClientPayload(body), + }); + handled = true; + } catch (error) { + console.debug( + `[STREAM] onComplete callback error in error path (${context.model || "unknown"}):`, + error + ); + } + } + context.clearIdleTimer(); + if (!handled) context.clearPendingRequest(); + controller.error(context.markPendingRequestCleared(new Error(safeMessage))); + }; +} diff --git a/src/app/api/providers/[id]/test/publicErrorBoundary.ts b/src/app/api/providers/[id]/test/publicErrorBoundary.ts new file mode 100644 index 0000000000..30addfeffb --- /dev/null +++ b/src/app/api/providers/[id]/test/publicErrorBoundary.ts @@ -0,0 +1,155 @@ +import { projectProviderValidationResultForPublicResponse } from "@/lib/providers/validation/transport"; +import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/errorSanitization.ts"; +import { makeDiagnosis } from "./codexAppServerHealth"; +import { classifyAmbiguousOrAuthError, type ClassifyFailureArgs } from "./mistralAmbiguousAuth"; + +export function toSafeMessage(value: unknown, fallback = "Unknown error"): string { + const safeMessage = sanitizeErrorMessage(value).trim(); + return safeMessage || fallback; +} + +/** + * A provider/account that the upstream has deactivated (vs. a revoked/expired token). + * #1444: a Codex account can have a perfectly healthy OAuth refresh while its ChatGPT + * account is deactivated, in which case the API returns 401 — mislabeling that as + * "Token invalid or revoked" hides the real cause. Mirrors the deactivation phrases the + * account-fallback classifier already trusts. + */ +export function isAccountDeactivatedMessage(text: string): boolean { + const normalized = (text || "").toLowerCase(); + return ( + normalized.includes("account_deactivated") || + (normalized.includes("deactivat") && normalized.includes("account")) + ); +} + +export function classifyFailure({ + error, + statusCode = null, + refreshFailed = false, + unsupported = false, + provider, +}: ClassifyFailureArgs) { + const message = toSafeMessage(error, "Connection test failed"); + const normalized = message.toLowerCase(); + const numericStatus = Number.isFinite(statusCode) ? Number(statusCode) : null; + + if (unsupported) { + return makeDiagnosis("unsupported", "validation", message, "unsupported"); + } + + if (refreshFailed || normalized.includes("refresh failed")) { + return makeDiagnosis("token_refresh_failed", "oauth", message, "refresh_failed"); + } + + // #1444: a deactivated account is distinct from a revoked/expired token — surface it + // as account_deactivated (which the dashboard renders as "Account Deactivated") before + // the generic 401/403 branch below would mark it "upstream_auth_error". + if (isAccountDeactivatedMessage(normalized)) { + return makeDiagnosis("account_deactivated", "account", message, "account_deactivated"); + } + + if (numericStatus === 401 || numericStatus === 403) { + return classifyAmbiguousOrAuthError(provider, normalized, message, numericStatus); + } + + if (numericStatus === 429) { + return makeDiagnosis("upstream_rate_limited", "upstream", message, "429"); + } + + if (numericStatus && numericStatus >= 500) { + return makeDiagnosis("upstream_unavailable", "upstream", message, String(numericStatus)); + } + + if (normalized.includes("token expired") || normalized.includes("expired")) { + return makeDiagnosis("token_expired", "oauth", message, "token_expired"); + } + + if ( + normalized.includes("invalid api key") || + normalized.includes("token invalid") || + normalized.includes("revoked") || + normalized.includes("access denied") || + normalized.includes("unauthorized") || + normalized.includes("forbidden") + ) { + return makeDiagnosis( + "upstream_auth_error", + "upstream", + message, + numericStatus ? String(numericStatus) : "auth_failed" + ); + } + + if ( + normalized.includes("rate limit") || + normalized.includes("quota") || + normalized.includes("too many requests") + ) { + return makeDiagnosis( + "upstream_rate_limited", + "upstream", + message, + numericStatus ? String(numericStatus) : "rate_limited" + ); + } + + if ( + normalized.includes("fetch failed") || + normalized.includes("network") || + normalized.includes("timeout") || + normalized.includes("timed out") || + normalized.includes("econn") || + normalized.includes("enotfound") || + normalized.includes("socket") + ) { + return makeDiagnosis("network_error", "upstream", message, "network_error"); + } + + return makeDiagnosis( + "upstream_error", + "upstream", + message, + numericStatus ? String(numericStatus) : "upstream_error" + ); +} + +/** Allowlist the CLI health fields safe to expose outside the local runtime boundary. */ +export function projectProviderRuntimeForPublicResponse( + runtime: unknown +): Record | null { + if (!runtime || typeof runtime !== "object" || Array.isArray(runtime)) return null; + const record = runtime as Record; + const projected: Record = {}; + + for (const field of ["installed", "runnable", "requiresBinary"] as const) { + if (typeof record[field] === "boolean") projected[field] = record[field]; + } + for (const field of ["reason", "runtimeMode", "version", "command"] as const) { + if (typeof record[field] !== "string") continue; + const safeValue = sanitizeErrorMessage(record[field]).trim(); + if (safeValue) projected[field] = safeValue.slice(0, 512); + } + + return projected; +} + +/** Sanitize every connection-test result before health writes, logs, and HTTP responses. */ +export function projectConnectionTestResultForPublicResponse< + T extends { error?: unknown; warning?: unknown; diagnosis?: unknown }, +>(result: T) { + const projected = projectProviderValidationResultForPublicResponse(result); + if (!projected.diagnosis || typeof projected.diagnosis !== "object") return projected; + + const diagnosis = projected.diagnosis as Record; + return { + ...projected, + diagnosis: { + ...diagnosis, + message: + diagnosis.message === null || diagnosis.message === undefined + ? null + : toSafeMessage(diagnosis.message, "Connection test failed"), + }, + }; +} diff --git a/src/app/api/providers/[id]/test/route.ts b/src/app/api/providers/[id]/test/route.ts index de28fb96de..1a81180358 100644 --- a/src/app/api/providers/[id]/test/route.ts +++ b/src/app/api/providers/[id]/test/route.ts @@ -30,12 +30,19 @@ import { testCodexAppServerConnection, makeDiagnosis } from "./codexAppServerHea import { recoverKeyHealth } from "@omniroute/open-sse/services/apiKeyRotator.ts"; import { shouldClearErrorStateOnValidProbe } from "@/lib/usage/providerLimits"; import { isConnectionUnavailableToAuxiliaryActivity } from "@/lib/exclusiveLeaseIsolation"; -import { classifyAmbiguousOrAuthError, type ClassifyFailureArgs } from "./mistralAmbiguousAuth"; import { buildApiKeyConnectionTestResult } from "./apiKeyTestResult"; import { classifyOAuthProbeInconclusive, OAUTH_TEST_CONFIG } from "./oauthTestConfig"; import { isGeoBlockedError } from "@omniroute/open-sse/services/errorClassifier.ts"; -import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/errorSanitization.ts"; import * as retirement from "@/lib/providers/chatgptWebRetirementResponse"; +import { + classifyFailure, + isAccountDeactivatedMessage, + projectConnectionTestResultForPublicResponse, + projectProviderRuntimeForPublicResponse, + toSafeMessage, +} from "./publicErrorBoundary"; + +export { classifyFailure, projectProviderRuntimeForPublicResponse } from "./publicErrorBoundary"; // Match the API-key path's 30s timeout so a hung OAuth upstream cannot block the test queue. const OAUTH_TEST_TIMEOUT_MS = 30_000; @@ -47,114 +54,6 @@ const providerConnectionTestBodySchema = z.object({ validationModelId: z.string().max(500).optional(), }); -function toSafeMessage(value: unknown, fallback = "Unknown error"): string { - const safeMessage = sanitizeErrorMessage(value).trim(); - return safeMessage || fallback; -} - -/** - * A provider/account that the upstream has deactivated (vs. a revoked/expired token). - * #1444: a Codex account can have a perfectly healthy OAuth refresh while its ChatGPT - * account is deactivated, in which case the API returns 401 — mislabeling that as - * "Token invalid or revoked" hides the real cause. Mirrors the deactivation phrases the - * account-fallback classifier already trusts. - */ -function isAccountDeactivatedMessage(text: string): boolean { - const n = (text || "").toLowerCase(); - return n.includes("account_deactivated") || (n.includes("deactivat") && n.includes("account")); -} - -export function classifyFailure({ - error, - statusCode = null, - refreshFailed = false, - unsupported = false, - provider, -}: ClassifyFailureArgs) { - const message = toSafeMessage(error, "Connection test failed"); - const normalized = message.toLowerCase(); - const numericStatus = Number.isFinite(statusCode) ? Number(statusCode) : null; - - if (unsupported) { - return makeDiagnosis("unsupported", "validation", message, "unsupported"); - } - - if (refreshFailed || normalized.includes("refresh failed")) { - return makeDiagnosis("token_refresh_failed", "oauth", message, "refresh_failed"); - } - - // #1444: a deactivated account is distinct from a revoked/expired token — surface it - // as account_deactivated (which the dashboard renders as "Account Deactivated") before - // the generic 401/403 branch below would mark it "upstream_auth_error". - if (isAccountDeactivatedMessage(normalized)) { - return makeDiagnosis("account_deactivated", "account", message, "account_deactivated"); - } - - if (numericStatus === 401 || numericStatus === 403) { - return classifyAmbiguousOrAuthError(provider, normalized, message, numericStatus); - } - - if (numericStatus === 429) { - return makeDiagnosis("upstream_rate_limited", "upstream", message, "429"); - } - - if (numericStatus && numericStatus >= 500) { - return makeDiagnosis("upstream_unavailable", "upstream", message, String(numericStatus)); - } - - if (normalized.includes("token expired") || normalized.includes("expired")) { - return makeDiagnosis("token_expired", "oauth", message, "token_expired"); - } - - if ( - normalized.includes("invalid api key") || - normalized.includes("token invalid") || - normalized.includes("revoked") || - normalized.includes("access denied") || - normalized.includes("unauthorized") || - normalized.includes("forbidden") - ) { - return makeDiagnosis( - "upstream_auth_error", - "upstream", - message, - numericStatus ? String(numericStatus) : "auth_failed" - ); - } - - if ( - normalized.includes("rate limit") || - normalized.includes("quota") || - normalized.includes("too many requests") - ) { - return makeDiagnosis( - "upstream_rate_limited", - "upstream", - message, - numericStatus ? String(numericStatus) : "rate_limited" - ); - } - - if ( - normalized.includes("fetch failed") || - normalized.includes("network") || - normalized.includes("timeout") || - normalized.includes("timed out") || - normalized.includes("econn") || - normalized.includes("enotfound") || - normalized.includes("socket") - ) { - return makeDiagnosis("network_error", "upstream", message, "network_error"); - } - - return makeDiagnosis( - "upstream_error", - "upstream", - message, - numericStatus ? String(numericStatus) : "upstream_error" - ); -} - function hasQoderToken(connection: any): boolean { if (typeof connection?.apiKey === "string" && connection.apiKey.trim().length > 0) return true; const psd = connection?.providerSpecificData; @@ -233,26 +132,6 @@ async function getProviderRuntimeStatus(connection: any) { } } -/** Allowlist the CLI health fields safe to expose outside the local runtime boundary. */ -export function projectProviderRuntimeForPublicResponse( - runtime: unknown -): Record | null { - if (!runtime || typeof runtime !== "object" || Array.isArray(runtime)) return null; - const record = runtime as Record; - const projected: Record = {}; - - for (const field of ["installed", "runnable", "requiresBinary"] as const) { - if (typeof record[field] === "boolean") projected[field] = record[field]; - } - for (const field of ["reason", "runtimeMode", "version", "command"] as const) { - if (typeof record[field] !== "string") continue; - const safeValue = sanitizeErrorMessage(record[field]).trim(); - if (safeValue) projected[field] = safeValue.slice(0, 512); - } - - return projected; -} - /** * Refresh OAuth token using the shared open-sse getAccessToken. * This shares the in-flight promise cache with the SSE layer, @@ -1084,19 +963,7 @@ export async function testSingleConnection(connectionId: string, validationModel // Every runtime path converges here before any health-state write, diagnosis, // persistent log, or public response. API-key validation is projected at its // own seam above as well so future refactors cannot move it past this boundary. - result = projectProviderValidationResultForPublicResponse(result); - if (result.diagnosis && typeof result.diagnosis === "object") { - result = { - ...result, - diagnosis: { - ...result.diagnosis, - message: - result.diagnosis.message === null || result.diagnosis.message === undefined - ? null - : toSafeMessage(result.diagnosis.message, "Connection test failed"), - }, - }; - } + result = projectConnectionTestResultForPublicResponse(result); const publicRuntime = projectProviderRuntimeForPublicResponse(runtime); const latencyMs = Date.now() - startTime; diff --git a/src/lib/logPayloads.ts b/src/lib/logPayloads.ts index 42e744086d..5fdef8c675 100644 --- a/src/lib/logPayloads.ts +++ b/src/lib/logPayloads.ts @@ -2,6 +2,7 @@ import { sanitizeErrorMessage, sanitizeUpstreamDetails, } from "@omniroute/open-sse/utils/errorSanitization.ts"; +import { projectResponsesFailureOutput } from "@omniroute/open-sse/utils/responsesFailureOutput.ts"; import { sanitizePII } from "./piiSanitizer"; const SENSITIVE_KEYS = new Set([ @@ -182,12 +183,24 @@ function projectErrorSubtreesForLog( // as content rather than treating it as an error message. const normalizedKey = key.replace(/[-_]/g, "").toLowerCase(); const preservePartialOutput = - responsesFailure && protocolResponseObject && normalizedKey === "output"; - const childIsProtocolResponse = declaresResponsesFailure && normalizedKey === "response"; + responsesFailure && + normalizedKey === "output" && + (protocolResponseObject || declaresResponsesFailure); + if (preservePartialOutput) { + projected[key] = projectResponsesFailureOutput( + entryValue, + (_field, stringValue) => sanitizeErrorMessage(stringValue) || "[REDACTED]" + ); + found = true; + continue; + } + const childIsProtocolResponse = + normalizedKey === "response" && + (declaresResponsesFailure || (forceResponsesFailure && !protocolResponseObject)); const result = projectErrorSubtreesForLog( entryValue, seen, - responsesFailure && !preservePartialOutput, + responsesFailure, childIsProtocolResponse ); projected[key] = result.value; diff --git a/src/sse/services/auth.ts b/src/sse/services/auth.ts index bd815a7320..9e201c891e 100644 --- a/src/sse/services/auth.ts +++ b/src/sse/services/auth.ts @@ -2724,8 +2724,7 @@ export async function markAccountUnavailable( // the opt-in setting probeCanDisable restores the historical behavior. if (await shouldIsolateProbeFailures()) { await updateProviderConnection(connectionId, { - // Keep safe provider wording for probe visibility, but project it at - // this persistence seam after classification has consumed the raw text. + // Persist safe wording only after classification has consumed the raw provider text. // backoffLevel is deliberately NOT written: a positive backoff // triggers the selection-time auto-decay (resetConnectionBackoff, // auth.ts getProviderCredentials) which wipes lastError back to @@ -3146,7 +3145,6 @@ export async function markAccountUnavailable( ); return { shouldFallback: true, cooldownMs: lockout.cooldownMs }; } - const errorMsg = sanitizeErrorMessage(describeUpstreamFailure(errorText)) || "Provider request failed"; diff --git a/tests/unit/chatcore-translation-paths.test.ts b/tests/unit/chatcore-translation-paths.test.ts index 0b87c1c98d..c0125fff83 100644 --- a/tests/unit/chatcore-translation-paths.test.ts +++ b/tests/unit/chatcore-translation-paths.test.ts @@ -1945,38 +1945,12 @@ test("chatCore logs chat completions endpoint as OpenAI protocol", async () => { assert.equal(logEntry.sourceFormat, FORMATS.OPENAI); }); test("chatCore surfaces translation errors with explicit status codes", async () => { - register( - FORMATS.OPENAI_RESPONSES, - FORMATS.OPENAI, - () => { - const error = new Error("responses translator rejected the payload"); - error.statusCode = 409; - throw error; - }, - null - ); - - const { result } = await invokeChatCore({ - provider: "openai", - model: "gpt-4o-mini", - endpoint: "/v1/responses", - body: { - model: "gpt-4o-mini", - input: "hello", - }, - }); - - assert.equal(result.success, false); - assert.equal(result.status, 409); - assert.equal(result.error, "responses translator rejected the payload"); -}); -test("chatCore surfaces typed translation errors with the declared error type", async () => { register( FORMATS.OPENAI_RESPONSES, FORMATS.OPENAI, () => { const error = new Error( - "typed translator failure access_token=translation-secret at /srv/private/translator.ts\n" + + "translator rejected access_token=translation-secret at /srv/private/translator.ts\n" + " at translate (/srv/private/translator.ts:41:8)" ); error.statusCode = 422; @@ -1998,13 +1972,12 @@ test("chatCore surfaces typed translation errors with the declared error type", assert.equal(result.success, false); assert.equal(result.status, 422); - const payload = (await result.response.json()) as { error: { message: string; type: string; code: string }; }; assert.equal(payload.error.type, "invalid_request_error"); assert.equal(payload.error.code, ""); - assert.match(payload.error.message, /typed translator failure/); + assert.match(payload.error.message, /translator rejected/); assert.doesNotMatch( JSON.stringify({ payload, internalError: result.error }), /translation-secret|type-secret|srv\/private|translator\.ts|type\.ts|\bat translate\b/i @@ -2624,44 +2597,6 @@ test("chatCore 429 lets account fallback apply the configured resilience cooldow assert.equal((afterFallback as any).testStatus, "unavailable"); assert.ok(cooldownRemaining > 0 && cooldownRemaining <= 2_000); }); -test("chatCore sanitizes nonterminal provider lastError without changing raw classification", async () => { - const hostile = - "invalid credential access_token=chatcore-last-error-secret at /srv/private/chatcore.ts\n" + - " at request (/srv/private/chatcore.ts:14:3)"; - const connection = await providersDb.createProviderConnection({ - provider: "openai", - authType: "apikey", - name: "chatCore last-error boundary", - apiKey: "chatcore-last-error-test-key", - isActive: true, - testStatus: "active", - }); - - const { result } = await invokeChatCore({ - provider: "openai", - model: "gpt-4o-mini", - connectionId: connection.id, - body: { - model: "gpt-4o-mini", - stream: false, - messages: [{ role: "user", content: "trigger nonterminal auth failure" }], - }, - responseFactory() { - return Response.json({ error: { message: hostile } }, { status: 401 }); - }, - }); - - const updated = await providersDb.getProviderConnectionById(connection.id); - const publicBody = await result.response.json(); - const boundaries = JSON.stringify({ lastError: updated.lastError, publicBody }); - - assert.equal(result.status, 401); - assert.match(String(updated.lastError), /invalid credential/i); - assert.doesNotMatch( - boundaries, - /chatcore-last-error-secret|srv\/private|chatcore\.ts|\bat request\b/i - ); -}); test("chatCore does not substitute an OpenAI model after model-unavailable", async () => { const { calls, result } = await invokeChatCore({ provider: "openai", diff --git a/tests/unit/request-log-payloads.test.ts b/tests/unit/request-log-payloads.test.ts index 4d5557a6fb..999e302045 100644 --- a/tests/unit/request-log-payloads.test.ts +++ b/tests/unit/request-log-payloads.test.ts @@ -302,10 +302,19 @@ test("sanitizes response.failed messages without rewriting unrelated deep diagno output: { trace: hostile }, level1: { level2: { level3: { level4: { level5: { label: "legitimate diagnostic" } } } } }, }; + const output = [ + { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "safe direct partial output" }], + }, + { type: "reasoning", reasoning_content: "private direct reasoning" }, + ]; const objectPayload = protectPayloadForLog({ type: "response.failed", message: hostile, diagnostics, + output, }); const protectedPipeline = protectPipelinePayloads({ streamChunks: { @@ -322,13 +331,88 @@ test("sanitizes response.failed messages without rewriting unrelated deep diagno (objectPayload as { diagnostics: typeof diagnostics }).diagnostics.level1, diagnostics.level1 ); + assert.deepEqual((objectPayload as { output: unknown }).output, [ + { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "safe direct partial output" }], + }, + ]); + assert.doesNotMatch(serialized, /private direct reasoning/); assert.match(serialized, /response\.failed/); }); +test("projects nested output when the SSE event alone marks response.failed", () => { + const protectedPipeline = protectPipelinePayloads({ + streamChunks: { + provider: [ + `event: response.failed\ndata: ${JSON.stringify({ + response: { + output: [ + { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "safe event partial output" }], + }, + { type: "reasoning", reasoning_content: "private event reasoning" }, + ], + }, + })}\n\n`, + ], + }, + }); + const serialized = JSON.stringify(protectedPipeline); + + assert.match(serialized, /safe event partial output/); + assert.doesNotMatch(serialized, /private event reasoning|"reasoning"/); +}); + test("sanitizes response.completed failed siblings in objects, SSE, and NDJSON", () => { const hostile = "Bearer completed-failed-secret at /srv/private/completed-failed.ts:8:2"; const partialOutput = [ - { type: "message", content: [{ type: "output_text", text: "partial safe output" }] }, + { + id: "msg_partial", + type: "message", + role: "assistant", + status: "in_progress", + diagnostics: { trace: hostile }, + content: [ + { + type: "output_text", + text: "partial safe output", + annotations: [{ type: "url_citation", url: "file:///srv/private/citation" }], + }, + { type: "output_text", phase: "commentary", text: "private commentary" }, + { type: "refusal", refusal: "safe refusal" }, + ], + }, + { + id: "msg_roleless", + type: "message", + content: [{ type: "output_text", text: "private roleless output" }], + }, + { + type: "reasoning", + reasoning_content: "private chain of thought", + encrypted_content: "private encrypted reasoning", + }, + { + type: "function_call", + name: "read_private_file", + arguments: '{"api_key":"private tool argument"}', + }, + ]; + const projectedOutput = [ + { + id: "msg_partial", + type: "message", + role: "assistant", + status: "in_progress", + content: [ + { type: "output_text", text: "partial safe output" }, + { type: "refusal", refusal: "safe refusal" }, + ], + }, ]; const completedFailure = { type: "response.completed", @@ -352,11 +436,16 @@ test("sanitizes response.completed failed siblings in objects, SSE, and NDJSON", }); const serialized = JSON.stringify({ objectPayload, protectedPipeline }); - assert.doesNotMatch(serialized, /completed-failed-secret|srv\/private|completed-failed\.ts/i); + assert.doesNotMatch( + serialized, + /completed-failed-secret|srv\/private|completed-failed\.ts|private commentary|private roleless|private chain|private encrypted|private tool/i + ); + assert.doesNotMatch(serialized, /"annotations"|"diagnostics"|"function_call"|"reasoning"/); assert.match(serialized, /partial safe output/); + assert.match(serialized, /safe refusal/); assert.deepEqual( - (objectPayload as { response: { output: typeof partialOutput } }).response.output, - partialOutput + (objectPayload as { response: { output: typeof projectedOutput } }).response.output, + projectedOutput ); }); diff --git a/tests/unit/stream-passthrough-error-redaction.test.ts b/tests/unit/stream-passthrough-error-redaction.test.ts index f8efb988c2..4082bbd001 100644 --- a/tests/unit/stream-passthrough-error-redaction.test.ts +++ b/tests/unit/stream-passthrough-error-redaction.test.ts @@ -173,14 +173,80 @@ test("Responses response.failed is projected before forwarding, logging, and onF type: "message", role: "assistant", status: "in_progress", + diagnostics: { + stack: "at /srv/omniroute/private-runtime.ts:47:2", + api_key: "sk-stream-secret-output-diagnostics", + }, content: [ { type: "output_text", text: "safe partial output", + annotations: [ + { + type: "url_citation", + url: "https://example.invalid/?token=sk-stream-secret-annotation", + title: "at /srv/omniroute/private-runtime.ts:48:2", + }, + ], + }, + { + type: "output_text", + phase: "commentary", + text: "hidden nested commentary must not be public", + }, + { type: "refusal", refusal: "safe refusal" }, + ], + }, + { + id: "msg_commentary", + type: "message", + role: "assistant", + phase: "commentary", + content: [ + { + type: "output_text", + text: "hidden commentary at /srv/omniroute/private-runtime.ts:49:2", + }, + ], + }, + { + id: "msg_roleless", + type: "message", + content: [ + { + type: "output_text", + text: "roleless output must not be public", annotations: [], }, ], }, + { + id: "reasoning_private", + type: "reasoning", + encrypted_content: "sk-stream-secret-encrypted-reasoning", + summary: [ + { + type: "summary_text", + text: "at /srv/omniroute/private-runtime.ts:50:2", + }, + ], + }, + { + id: "call_private", + type: "function_call", + call_id: "call_private", + name: "read_private_file", + arguments: + '{"path":"/srv/omniroute/private-runtime.ts","api_key":"sk-stream-secret-tool"}', + }, + { + id: "provider_private", + type: "provider_diagnostics", + diagnostics: { + stack: "at /srv/omniroute/private-runtime.ts:51:2", + api_key: "sk-stream-secret-unknown-item", + }, + }, ], error: { type: "server_error", @@ -218,12 +284,22 @@ test("Responses response.failed is projected before forwarding, logging, and onF assert.match(result.output, /response\.failed/); assert.match(result.output, /"last_error":\{/); assert.match(result.output, /safe partial output/); + assert.match(result.output, /safe refusal/); + assert.doesNotMatch(result.output, /"annotations"/); + assert.doesNotMatch(result.output, /hidden nested commentary must not be public/); + assert.doesNotMatch(result.output, /roleless output must not be public/); assert.match(result.output, /"cached_tokens":1/); assert.doesNotMatch(result.output, /\[truncated\]/); assertNoHostileDetail(result.output); assertNoHostileDetail(convertedLog.join("\n")); - assert.doesNotMatch(result.output, /"diagnosis"|"settings"/); - assert.doesNotMatch(convertedLog.join("\n"), /"diagnosis"|"settings"/); + assert.doesNotMatch( + result.output, + /"diagnosis"|"diagnostics"|"settings"|"encrypted_content"|"function_call"|"provider_diagnostics"|"phase"|"url_citation"/ + ); + assert.doesNotMatch( + convertedLog.join("\n"), + /"diagnosis"|"diagnostics"|"settings"|"encrypted_content"|"function_call"|"provider_diagnostics"|"phase"|"url_citation"/ + ); assert.ok(result.failure); assert.match(result.failure.message, /private-runtime\.ts/); assertNoHostileDetail(String(result.error));