fix(security): finalize public failure projections

This commit is contained in:
diegosouzapw
2026-09-02 15:48:03 -03:00
parent dc66426462
commit c0cc6ee4bb
17 changed files with 614 additions and 350 deletions

View File

@@ -169,8 +169,8 @@ Every field is projected onto the bounded public-identifier vocabulary. Unsafe,
control-character, or overlong values fall back to the status-derived type/code; an unsafe optional
reason is omitted. Three-digit HTTP status identifiers (`100` through `599`) remain valid for
provider contracts that expose the numeric upstream status as a machine-readable code. The same
bounded range is accepted in the locally generated `HTTP_NNN` form; arbitrary provider numbers and
names remain outside the vocabulary.
bounded range is accepted in the locally generated HTTP-status placeholder form; arbitrary provider
numbers and names remain outside the vocabulary.
Pass every explicit classification in that fourth argument. Never overwrite
`body.error.code`, `body.error.type`, or `body.error.reason` after `buildErrorBody()` returns;

View File

@@ -6,6 +6,7 @@ import { injectMemoryAndSkills } from "./chatCore/memorySkillsInjection.ts";
import { resolveChatCoreRequestSetup } from "./chatCore/requestSetup.ts";
import { normalizeOpenAICompatibleTools } from "./chatCore/openAICompatibleTools.ts";
import { buildFailureUsageRecord, projectFailureUsageErrorCode } from "./chatCore/failureUsage.ts";
import { createTranslationFailureResult } from "./chatCore/translationFailure.ts";
import { estimateFinalInputTokens } from "./chatCore/contextEstimation.ts";
import {
extractSystemRoleMessages,
@@ -2489,33 +2490,11 @@ export async function handleChatCore({
: HTTP_STATUS.SERVER_ERROR;
const message = error?.message || "Invalid request";
const errorType = typeof error?.errorType === "string" ? error.errorType : null;
const errorBody = buildErrorBody(
statusCode,
message,
undefined,
errorType ? { type: errorType, code: errorType } : undefined
);
const safeMessage = errorBody.error.message;
log?.warn?.("TRANSLATE", `Request translation failed: ${safeMessage}`);
if (errorType) {
trackPendingRequest(model, provider, connectionId, false);
return {
success: false,
status: statusCode,
error: safeMessage,
response: new Response(JSON.stringify(errorBody), {
status: statusCode,
headers: {
"Content-Type": "application/json",
},
}),
};
}
const result = createTranslationFailureResult(statusCode, message, errorType);
log?.warn?.("TRANSLATE", `Request translation failed: ${result.error}`);
trackPendingRequest(model, provider, connectionId, false);
return createErrorResult(statusCode, safeMessage);
return result;
}
// The latest OmniGlyph release has protocol-native OpenAI transforms. Run

View File

@@ -0,0 +1,24 @@
import { buildErrorBody, createErrorResult } from "../../utils/error.ts";
export function createTranslationFailureResult(
status: number,
message: string,
errorType: string | null
) {
if (!errorType) return createErrorResult(status, message);
const body = buildErrorBody(
status,
message,
undefined,
{ type: errorType, code: errorType }
);
return {
success: false as const,
status,
error: body.error.message,
response: new Response(JSON.stringify(body), {
status,
headers: { "Content-Type": "application/json" },
}),
};
}

View File

@@ -0,0 +1,13 @@
import { sanitizeErrorMessage } from "../utils/error.ts";
export function toSafeMcpErrorMessage(
value: unknown,
fallback = "MCP tool execution failed"
): string {
try {
const raw = value instanceof Error ? value.message : value;
return sanitizeErrorMessage(raw) || fallback;
} catch {
return fallback;
}
}

View File

@@ -93,7 +93,7 @@ import {
import { getDbInstance, ensureDbInitialized } from "../../src/lib/db/core.ts";
import { normalizeQuotaResponse } from "../../src/shared/contracts/quota.ts";
import { resolveOmniRouteBaseUrl } from "../../src/shared/utils/resolveOmniRouteBaseUrl.ts";
import { sanitizeErrorMessage } from "../utils/error.ts";
import { toSafeMcpErrorMessage } from "./errorMessage.ts";
import { mcpFetchTimeoutSignal } from "./fetchTimeout.ts";
import { getMcpModelsCatalog } from "./catalog.ts";
import { registerRadarCatalogTool } from "./radarCatalog.ts";
@@ -123,15 +123,6 @@ const TOTAL_MCP_TOOL_COUNT = countUniqueMcpTools({
compressionTools,
});
function toSafeMcpErrorMessage(value: unknown, fallback = "MCP tool execution failed"): string {
try {
const raw = value instanceof Error ? value.message : value;
return sanitizeErrorMessage(raw) || fallback;
} catch {
return fallback;
}
}
type JsonRecord = Record<string, unknown>;
function readMcpDescriptionCompressionEnabled(): boolean {
@@ -337,9 +328,7 @@ async function handleGetHealth() {
.filter(({ settled }) => settled.status === "rejected")
.map(({ source, settled }) => ({
source,
error: sanitizeErrorMessage(
settled.status === "rejected" ? (settled as PromiseRejectedResult).reason : undefined
),
error: toSafeMcpErrorMessage((settled as PromiseRejectedResult).reason, ""),
}));
const result = {

View File

@@ -5,7 +5,7 @@
import { register } from "../registry.ts";
import { FORMATS } from "../formats.ts";
import { appendToolCallArgumentDelta } from "../../utils/toolCallArguments.ts";
import { buildErrorBody } from "../../utils/error.ts";
import { projectCompletedStreamError } from "../../utils/streamErrorFormat.ts";
import { fallbackToolCallId } from "../helpers/toolCallHelper.ts";
import { shouldParseTextualReasoningTags } from "../../handlers/responseSanitizer.ts";
import { getReadableReasoningValue } from "../../utils/reasoningFields.ts";
@@ -747,17 +747,7 @@ function sendCompleted(state, emit) {
// translator or the OpenAI-Responses translator itself when the upstream
// SSE stream emits a JSON error object after partial content.
const upstreamErr = state.upstreamError;
const publicUpstreamError = upstreamErr
? buildErrorBody(
Number.isInteger(upstreamErr.status) ? upstreamErr.status : 502,
upstreamErr.message,
undefined,
{
type: upstreamErr.type ?? "server_error",
code: String(upstreamErr.status ?? 502),
}
).error
: null;
const publicUpstreamError = projectCompletedStreamError(upstreamErr);
const response: Record<string, unknown> = {
id: state.responseId,

View File

@@ -0,0 +1,67 @@
type JsonRecord = Record<string, unknown>;
export type ResponsesFailureOutputStringField = "id" | "text" | "refusal";
export type ResponsesFailureOutputStringProjector = (
field: ResponsesFailureOutputStringField,
value: string
) => string;
function asRecord(value: unknown): JsonRecord {
return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {};
}
/**
* Retain only public assistant text/refusal output from a failed Responses payload.
* Failure envelopes may contain reasoning, tool arguments, annotations, commentary,
* or provider diagnostics, so every retained field is reconstructed explicitly.
*/
export function projectResponsesFailureOutput(
value: unknown,
projectString: ResponsesFailureOutputStringProjector
): JsonRecord[] {
if (!Array.isArray(value)) return [];
const output: JsonRecord[] = [];
for (const item of value) {
const record = asRecord(item);
if (record.type !== "message" || record.role !== "assistant" || record.phase === "commentary") {
continue;
}
const content: JsonRecord[] = [];
if (Array.isArray(record.content)) {
for (const part of record.content) {
const contentPart = asRecord(part);
if (contentPart.phase === "commentary") continue;
if (contentPart.type === "output_text" && typeof contentPart.text === "string") {
content.push({
type: "output_text",
text: projectString("text", contentPart.text),
});
} else if (contentPart.type === "refusal" && typeof contentPart.refusal === "string") {
content.push({
type: "refusal",
refusal: projectString("refusal", contentPart.refusal),
});
}
}
}
const projected: JsonRecord = {
type: "message",
role: "assistant",
content,
};
if (typeof record.id === "string") projected.id = projectString("id", record.id);
if (
record.status === "in_progress" ||
record.status === "completed" ||
record.status === "incomplete"
) {
projected.status = record.status;
}
output.push(projected);
}
return output;
}

View File

@@ -50,10 +50,11 @@ import { parseTextualToolCallCandidate, isValidToolCallHeaderPrefix } from "./te
import { stripObfuscationZeroWidth } from "./zeroWidth.ts";
import {
formatTranslatedStreamError,
normalizeStreamFailurePayload,
prepareTranslatedStreamFailure,
projectStreamFailureEvent,
type StreamFailurePayload,
} from "./streamErrorFormat.ts";
import { createStreamFailureAborter } from "./streamFailureBoundary.ts";
import { recordToolLatency } from "../services/toolLatencyTracker.ts";
import { extractToolSchemaMap } from "../translator/response/openai-responses/toolSchemas.ts";
import {
@@ -1174,89 +1175,37 @@ export function createSSEStream(options: StreamOptions = {}) {
}
};
const abortStreamFailure = (
controller: TransformStreamDefaultController<Uint8Array>,
failurePayload: StreamFailurePayload,
publicMessage: string,
options: { notifyComplete?: boolean } = {}
): void => {
let failureHandled = false;
timing.markInterrupted();
if (onFailure) {
try {
// Keep the raw provider wording internal for quota/reset classification. The
// persistence seam sanitizes it after classification; public output uses the
// separately projected payload and message.
failureHandled = onFailure(failurePayload) === true;
} catch (e) {
console.debug(`[STREAM] onFailure callback error:`, e);
}
}
let publicErrorMessage = publicMessage || "Upstream failure";
if (options.notifyComplete && onComplete) {
const errorBody = buildErrorBody(failurePayload.status, failurePayload.message);
publicErrorMessage = errorBody.error.message;
try {
onComplete({
status: failurePayload.status,
usage: state?.usage,
responseBody: errorBody,
ttft: timing.ttftMs(),
itlMs: timing.avgItlMs(),
interrupted: timing.interrupted,
error: publicErrorMessage,
errorCode: failurePayload.code,
providerPayload: providerPayloadCollector.build(providerPayloadCollector.getSummary(), {
includeEvents: false,
}),
clientPayload: clientPayloadCollector.build(errorBody, { includeEvents: false }),
});
failureHandled = true;
} catch (e) {
console.debug(
`[STREAM] onComplete callback error in error path (${model || "unknown"}):`,
e
);
}
}
clearIdleTimer();
if (!failureHandled) {
clearPendingRequestFromStream();
}
controller.error(markPendingRequestCleared(new Error(publicErrorMessage)));
};
const abortStreamFailure = createStreamFailureAborter({
onFailure,
onComplete,
getUsage: () => state?.usage,
timing,
buildProviderPayload: () =>
providerPayloadCollector.build(providerPayloadCollector.getSummary(), {
includeEvents: false,
}),
buildClientPayload: (body) => clientPayloadCollector.build(body, { includeEvents: false }),
clearIdleTimer,
clearPendingRequest: clearPendingRequestFromStream,
markPendingRequestCleared,
model,
});
const emitTranslatedFailureAndAbort = (
controller: TransformStreamDefaultController<Uint8Array>,
payload: unknown
): boolean => {
const record =
payload && typeof payload === "object" && !Array.isArray(payload)
? (payload as JsonRecord)
: {};
const projectedFailure = projectStreamFailureEvent(record);
if (!projectedFailure && !record.error) return false;
providerPayloadCollector.push(projectedFailure?.publicPayload ?? record);
const internalFailure = projectedFailure?.internalFailure ??
normalizeStreamFailurePayload(record) ?? {
status: 502,
message: "Upstream failure",
code: "stream_error",
type: "server_error",
};
const output = formatTranslatedStreamError(record, sourceFormat);
const failure = prepareTranslatedStreamFailure(payload);
if (!failure) return false;
providerPayloadCollector.push(failure.providerPayload);
const output = formatTranslatedStreamError(failure.record, sourceFormat);
reqLogger?.appendConvertedChunk?.(output);
forward(controller, encoder.encode(output));
upstreamErrorForwarded = true;
doneSent = true;
abortStreamFailure(
controller,
internalFailure,
projectedFailure?.publicMessage || "Upstream failure",
{ notifyComplete: true }
);
abortStreamFailure(controller, failure.internalFailure, failure.publicMessage, {
notifyComplete: true,
});
return true;
};

View File

@@ -1,5 +1,6 @@
import { FORMATS } from "../translator/formats.ts";
import { buildErrorBody, sanitizeErrorMessage } from "./error.ts";
import { projectResponsesFailureOutput } from "./responsesFailureOutput.ts";
/**
* Upstream stream-failure normalization + client-format error framing.
@@ -23,6 +24,24 @@ export type ProjectedStreamFailureEvent = {
publicPayload: JsonRecord;
};
export type PreparedTranslatedStreamFailure = {
record: JsonRecord;
providerPayload: JsonRecord;
internalFailure: StreamFailurePayload;
publicMessage: string;
};
export function projectCompletedStreamError(
failure: StreamFailurePayload | null | undefined
): JsonRecord | null {
if (!failure) return null;
const status = Number.isInteger(failure.status) ? failure.status : 502;
return buildErrorBody(status, failure.message, undefined, {
type: failure.type ?? "server_error",
code: String(failure.status ?? 502),
}).error;
}
function asRecord(value: unknown): JsonRecord {
return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {};
}
@@ -106,7 +125,12 @@ function projectResponsesFailureObject(response: JsonRecord, publicError: JsonRe
else if (value === null || typeof value === "number" || typeof value === "boolean")
projected[key] = value;
}
if (Array.isArray(response.output)) projected.output = response.output;
if (Array.isArray(response.output)) {
projected.output = projectResponsesFailureOutput(
response.output,
projectResponsesFailureString
);
}
const usage = projectResponsesFailureUsage(response.usage);
if (usage) projected.usage = usage;
if ("last_error" in response) projected.last_error = publicError;
@@ -181,6 +205,26 @@ export function normalizeStreamFailurePayload(payload: unknown): StreamFailurePa
};
}
export function prepareTranslatedStreamFailure(
payload: unknown
): PreparedTranslatedStreamFailure | null {
const record = asRecord(payload);
const projected = projectStreamFailureEvent(record);
if (!projected && !record.error) return null;
return {
record,
providerPayload: projected?.publicPayload ?? record,
internalFailure: projected?.internalFailure ??
normalizeStreamFailurePayload(record) ?? {
status: 502,
message: "Upstream failure",
code: "stream_error",
type: "server_error",
},
publicMessage: projected?.publicMessage || "Upstream failure",
};
}
/**
* Project same-format upstream failure events before they cross the client/log boundary.
*

View File

@@ -0,0 +1,76 @@
import { buildErrorBody } from "./error.ts";
import type { StreamFailurePayload } from "./streamErrorFormat.ts";
import type { StreamTiming } from "./streamTiming.ts";
type CompletePayload = {
status: number;
usage: unknown;
responseBody: unknown;
providerPayload: unknown;
clientPayload: unknown;
error: string;
errorCode?: string;
ttft: number | null;
itlMs: number | null;
interrupted: boolean;
};
type AborterContext = {
onFailure?: ((payload: StreamFailurePayload) => boolean | void | Promise<void>) | null;
onComplete?: ((payload: CompletePayload) => void) | null;
getUsage: () => unknown;
timing: StreamTiming;
buildProviderPayload: () => unknown;
buildClientPayload: (body: unknown) => unknown;
clearIdleTimer: () => void;
clearPendingRequest: () => void;
markPendingRequestCleared: (error: Error) => Error;
model?: string | null;
};
export function createStreamFailureAborter(context: AborterContext) {
return (
controller: TransformStreamDefaultController<Uint8Array>,
failure: StreamFailurePayload,
publicMessage: string,
options: { notifyComplete?: boolean } = {}
): void => {
let handled = false;
context.timing.markInterrupted();
if (context.onFailure) {
try {
handled = context.onFailure(failure) === true;
} catch (error) {
console.debug("[STREAM] onFailure callback error:", error);
}
}
let safeMessage = publicMessage || "Upstream failure";
if (options.notifyComplete && context.onComplete) {
const body = buildErrorBody(failure.status, failure.message);
safeMessage = body.error.message;
try {
context.onComplete({
status: failure.status,
usage: context.getUsage(),
responseBody: body,
ttft: context.timing.ttftMs(),
itlMs: context.timing.avgItlMs(),
interrupted: context.timing.interrupted,
error: safeMessage,
errorCode: failure.code,
providerPayload: context.buildProviderPayload(),
clientPayload: context.buildClientPayload(body),
});
handled = true;
} catch (error) {
console.debug(
`[STREAM] onComplete callback error in error path (${context.model || "unknown"}):`,
error
);
}
}
context.clearIdleTimer();
if (!handled) context.clearPendingRequest();
controller.error(context.markPendingRequestCleared(new Error(safeMessage)));
};
}

View File

@@ -0,0 +1,155 @@
import { projectProviderValidationResultForPublicResponse } from "@/lib/providers/validation/transport";
import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/errorSanitization.ts";
import { makeDiagnosis } from "./codexAppServerHealth";
import { classifyAmbiguousOrAuthError, type ClassifyFailureArgs } from "./mistralAmbiguousAuth";
export function toSafeMessage(value: unknown, fallback = "Unknown error"): string {
const safeMessage = sanitizeErrorMessage(value).trim();
return safeMessage || fallback;
}
/**
* A provider/account that the upstream has deactivated (vs. a revoked/expired token).
* #1444: a Codex account can have a perfectly healthy OAuth refresh while its ChatGPT
* account is deactivated, in which case the API returns 401 — mislabeling that as
* "Token invalid or revoked" hides the real cause. Mirrors the deactivation phrases the
* account-fallback classifier already trusts.
*/
export function isAccountDeactivatedMessage(text: string): boolean {
const normalized = (text || "").toLowerCase();
return (
normalized.includes("account_deactivated") ||
(normalized.includes("deactivat") && normalized.includes("account"))
);
}
export function classifyFailure({
error,
statusCode = null,
refreshFailed = false,
unsupported = false,
provider,
}: ClassifyFailureArgs) {
const message = toSafeMessage(error, "Connection test failed");
const normalized = message.toLowerCase();
const numericStatus = Number.isFinite(statusCode) ? Number(statusCode) : null;
if (unsupported) {
return makeDiagnosis("unsupported", "validation", message, "unsupported");
}
if (refreshFailed || normalized.includes("refresh failed")) {
return makeDiagnosis("token_refresh_failed", "oauth", message, "refresh_failed");
}
// #1444: a deactivated account is distinct from a revoked/expired token — surface it
// as account_deactivated (which the dashboard renders as "Account Deactivated") before
// the generic 401/403 branch below would mark it "upstream_auth_error".
if (isAccountDeactivatedMessage(normalized)) {
return makeDiagnosis("account_deactivated", "account", message, "account_deactivated");
}
if (numericStatus === 401 || numericStatus === 403) {
return classifyAmbiguousOrAuthError(provider, normalized, message, numericStatus);
}
if (numericStatus === 429) {
return makeDiagnosis("upstream_rate_limited", "upstream", message, "429");
}
if (numericStatus && numericStatus >= 500) {
return makeDiagnosis("upstream_unavailable", "upstream", message, String(numericStatus));
}
if (normalized.includes("token expired") || normalized.includes("expired")) {
return makeDiagnosis("token_expired", "oauth", message, "token_expired");
}
if (
normalized.includes("invalid api key") ||
normalized.includes("token invalid") ||
normalized.includes("revoked") ||
normalized.includes("access denied") ||
normalized.includes("unauthorized") ||
normalized.includes("forbidden")
) {
return makeDiagnosis(
"upstream_auth_error",
"upstream",
message,
numericStatus ? String(numericStatus) : "auth_failed"
);
}
if (
normalized.includes("rate limit") ||
normalized.includes("quota") ||
normalized.includes("too many requests")
) {
return makeDiagnosis(
"upstream_rate_limited",
"upstream",
message,
numericStatus ? String(numericStatus) : "rate_limited"
);
}
if (
normalized.includes("fetch failed") ||
normalized.includes("network") ||
normalized.includes("timeout") ||
normalized.includes("timed out") ||
normalized.includes("econn") ||
normalized.includes("enotfound") ||
normalized.includes("socket")
) {
return makeDiagnosis("network_error", "upstream", message, "network_error");
}
return makeDiagnosis(
"upstream_error",
"upstream",
message,
numericStatus ? String(numericStatus) : "upstream_error"
);
}
/** Allowlist the CLI health fields safe to expose outside the local runtime boundary. */
export function projectProviderRuntimeForPublicResponse(
runtime: unknown
): Record<string, unknown> | null {
if (!runtime || typeof runtime !== "object" || Array.isArray(runtime)) return null;
const record = runtime as Record<string, unknown>;
const projected: Record<string, unknown> = {};
for (const field of ["installed", "runnable", "requiresBinary"] as const) {
if (typeof record[field] === "boolean") projected[field] = record[field];
}
for (const field of ["reason", "runtimeMode", "version", "command"] as const) {
if (typeof record[field] !== "string") continue;
const safeValue = sanitizeErrorMessage(record[field]).trim();
if (safeValue) projected[field] = safeValue.slice(0, 512);
}
return projected;
}
/** Sanitize every connection-test result before health writes, logs, and HTTP responses. */
export function projectConnectionTestResultForPublicResponse<
T extends { error?: unknown; warning?: unknown; diagnosis?: unknown },
>(result: T) {
const projected = projectProviderValidationResultForPublicResponse(result);
if (!projected.diagnosis || typeof projected.diagnosis !== "object") return projected;
const diagnosis = projected.diagnosis as Record<string, unknown>;
return {
...projected,
diagnosis: {
...diagnosis,
message:
diagnosis.message === null || diagnosis.message === undefined
? null
: toSafeMessage(diagnosis.message, "Connection test failed"),
},
};
}

View File

@@ -30,12 +30,19 @@ import { testCodexAppServerConnection, makeDiagnosis } from "./codexAppServerHea
import { recoverKeyHealth } from "@omniroute/open-sse/services/apiKeyRotator.ts";
import { shouldClearErrorStateOnValidProbe } from "@/lib/usage/providerLimits";
import { isConnectionUnavailableToAuxiliaryActivity } from "@/lib/exclusiveLeaseIsolation";
import { classifyAmbiguousOrAuthError, type ClassifyFailureArgs } from "./mistralAmbiguousAuth";
import { buildApiKeyConnectionTestResult } from "./apiKeyTestResult";
import { classifyOAuthProbeInconclusive, OAUTH_TEST_CONFIG } from "./oauthTestConfig";
import { isGeoBlockedError } from "@omniroute/open-sse/services/errorClassifier.ts";
import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/errorSanitization.ts";
import * as retirement from "@/lib/providers/chatgptWebRetirementResponse";
import {
classifyFailure,
isAccountDeactivatedMessage,
projectConnectionTestResultForPublicResponse,
projectProviderRuntimeForPublicResponse,
toSafeMessage,
} from "./publicErrorBoundary";
export { classifyFailure, projectProviderRuntimeForPublicResponse } from "./publicErrorBoundary";
// Match the API-key path's 30s timeout so a hung OAuth upstream cannot block the test queue.
const OAUTH_TEST_TIMEOUT_MS = 30_000;
@@ -47,114 +54,6 @@ const providerConnectionTestBodySchema = z.object({
validationModelId: z.string().max(500).optional(),
});
function toSafeMessage(value: unknown, fallback = "Unknown error"): string {
const safeMessage = sanitizeErrorMessage(value).trim();
return safeMessage || fallback;
}
/**
* A provider/account that the upstream has deactivated (vs. a revoked/expired token).
* #1444: a Codex account can have a perfectly healthy OAuth refresh while its ChatGPT
* account is deactivated, in which case the API returns 401 — mislabeling that as
* "Token invalid or revoked" hides the real cause. Mirrors the deactivation phrases the
* account-fallback classifier already trusts.
*/
function isAccountDeactivatedMessage(text: string): boolean {
const n = (text || "").toLowerCase();
return n.includes("account_deactivated") || (n.includes("deactivat") && n.includes("account"));
}
export function classifyFailure({
error,
statusCode = null,
refreshFailed = false,
unsupported = false,
provider,
}: ClassifyFailureArgs) {
const message = toSafeMessage(error, "Connection test failed");
const normalized = message.toLowerCase();
const numericStatus = Number.isFinite(statusCode) ? Number(statusCode) : null;
if (unsupported) {
return makeDiagnosis("unsupported", "validation", message, "unsupported");
}
if (refreshFailed || normalized.includes("refresh failed")) {
return makeDiagnosis("token_refresh_failed", "oauth", message, "refresh_failed");
}
// #1444: a deactivated account is distinct from a revoked/expired token — surface it
// as account_deactivated (which the dashboard renders as "Account Deactivated") before
// the generic 401/403 branch below would mark it "upstream_auth_error".
if (isAccountDeactivatedMessage(normalized)) {
return makeDiagnosis("account_deactivated", "account", message, "account_deactivated");
}
if (numericStatus === 401 || numericStatus === 403) {
return classifyAmbiguousOrAuthError(provider, normalized, message, numericStatus);
}
if (numericStatus === 429) {
return makeDiagnosis("upstream_rate_limited", "upstream", message, "429");
}
if (numericStatus && numericStatus >= 500) {
return makeDiagnosis("upstream_unavailable", "upstream", message, String(numericStatus));
}
if (normalized.includes("token expired") || normalized.includes("expired")) {
return makeDiagnosis("token_expired", "oauth", message, "token_expired");
}
if (
normalized.includes("invalid api key") ||
normalized.includes("token invalid") ||
normalized.includes("revoked") ||
normalized.includes("access denied") ||
normalized.includes("unauthorized") ||
normalized.includes("forbidden")
) {
return makeDiagnosis(
"upstream_auth_error",
"upstream",
message,
numericStatus ? String(numericStatus) : "auth_failed"
);
}
if (
normalized.includes("rate limit") ||
normalized.includes("quota") ||
normalized.includes("too many requests")
) {
return makeDiagnosis(
"upstream_rate_limited",
"upstream",
message,
numericStatus ? String(numericStatus) : "rate_limited"
);
}
if (
normalized.includes("fetch failed") ||
normalized.includes("network") ||
normalized.includes("timeout") ||
normalized.includes("timed out") ||
normalized.includes("econn") ||
normalized.includes("enotfound") ||
normalized.includes("socket")
) {
return makeDiagnosis("network_error", "upstream", message, "network_error");
}
return makeDiagnosis(
"upstream_error",
"upstream",
message,
numericStatus ? String(numericStatus) : "upstream_error"
);
}
function hasQoderToken(connection: any): boolean {
if (typeof connection?.apiKey === "string" && connection.apiKey.trim().length > 0) return true;
const psd = connection?.providerSpecificData;
@@ -233,26 +132,6 @@ async function getProviderRuntimeStatus(connection: any) {
}
}
/** Allowlist the CLI health fields safe to expose outside the local runtime boundary. */
export function projectProviderRuntimeForPublicResponse(
runtime: unknown
): Record<string, unknown> | null {
if (!runtime || typeof runtime !== "object" || Array.isArray(runtime)) return null;
const record = runtime as Record<string, unknown>;
const projected: Record<string, unknown> = {};
for (const field of ["installed", "runnable", "requiresBinary"] as const) {
if (typeof record[field] === "boolean") projected[field] = record[field];
}
for (const field of ["reason", "runtimeMode", "version", "command"] as const) {
if (typeof record[field] !== "string") continue;
const safeValue = sanitizeErrorMessage(record[field]).trim();
if (safeValue) projected[field] = safeValue.slice(0, 512);
}
return projected;
}
/**
* Refresh OAuth token using the shared open-sse getAccessToken.
* This shares the in-flight promise cache with the SSE layer,
@@ -1084,19 +963,7 @@ export async function testSingleConnection(connectionId: string, validationModel
// Every runtime path converges here before any health-state write, diagnosis,
// persistent log, or public response. API-key validation is projected at its
// own seam above as well so future refactors cannot move it past this boundary.
result = projectProviderValidationResultForPublicResponse(result);
if (result.diagnosis && typeof result.diagnosis === "object") {
result = {
...result,
diagnosis: {
...result.diagnosis,
message:
result.diagnosis.message === null || result.diagnosis.message === undefined
? null
: toSafeMessage(result.diagnosis.message, "Connection test failed"),
},
};
}
result = projectConnectionTestResultForPublicResponse(result);
const publicRuntime = projectProviderRuntimeForPublicResponse(runtime);
const latencyMs = Date.now() - startTime;

View File

@@ -2,6 +2,7 @@ import {
sanitizeErrorMessage,
sanitizeUpstreamDetails,
} from "@omniroute/open-sse/utils/errorSanitization.ts";
import { projectResponsesFailureOutput } from "@omniroute/open-sse/utils/responsesFailureOutput.ts";
import { sanitizePII } from "./piiSanitizer";
const SENSITIVE_KEYS = new Set([
@@ -182,12 +183,24 @@ function projectErrorSubtreesForLog(
// as content rather than treating it as an error message.
const normalizedKey = key.replace(/[-_]/g, "").toLowerCase();
const preservePartialOutput =
responsesFailure && protocolResponseObject && normalizedKey === "output";
const childIsProtocolResponse = declaresResponsesFailure && normalizedKey === "response";
responsesFailure &&
normalizedKey === "output" &&
(protocolResponseObject || declaresResponsesFailure);
if (preservePartialOutput) {
projected[key] = projectResponsesFailureOutput(
entryValue,
(_field, stringValue) => sanitizeErrorMessage(stringValue) || "[REDACTED]"
);
found = true;
continue;
}
const childIsProtocolResponse =
normalizedKey === "response" &&
(declaresResponsesFailure || (forceResponsesFailure && !protocolResponseObject));
const result = projectErrorSubtreesForLog(
entryValue,
seen,
responsesFailure && !preservePartialOutput,
responsesFailure,
childIsProtocolResponse
);
projected[key] = result.value;

View File

@@ -2724,8 +2724,7 @@ export async function markAccountUnavailable(
// the opt-in setting probeCanDisable restores the historical behavior.
if (await shouldIsolateProbeFailures()) {
await updateProviderConnection(connectionId, {
// Keep safe provider wording for probe visibility, but project it at
// this persistence seam after classification has consumed the raw text.
// Persist safe wording only after classification has consumed the raw provider text.
// backoffLevel is deliberately NOT written: a positive backoff
// triggers the selection-time auto-decay (resetConnectionBackoff,
// auth.ts getProviderCredentials) which wipes lastError back to
@@ -3146,7 +3145,6 @@ export async function markAccountUnavailable(
);
return { shouldFallback: true, cooldownMs: lockout.cooldownMs };
}
const errorMsg =
sanitizeErrorMessage(describeUpstreamFailure(errorText)) || "Provider request failed";

View File

@@ -1945,38 +1945,12 @@ test("chatCore logs chat completions endpoint as OpenAI protocol", async () => {
assert.equal(logEntry.sourceFormat, FORMATS.OPENAI);
});
test("chatCore surfaces translation errors with explicit status codes", async () => {
register(
FORMATS.OPENAI_RESPONSES,
FORMATS.OPENAI,
() => {
const error = new Error("responses translator rejected the payload");
error.statusCode = 409;
throw error;
},
null
);
const { result } = await invokeChatCore({
provider: "openai",
model: "gpt-4o-mini",
endpoint: "/v1/responses",
body: {
model: "gpt-4o-mini",
input: "hello",
},
});
assert.equal(result.success, false);
assert.equal(result.status, 409);
assert.equal(result.error, "responses translator rejected the payload");
});
test("chatCore surfaces typed translation errors with the declared error type", async () => {
register(
FORMATS.OPENAI_RESPONSES,
FORMATS.OPENAI,
() => {
const error = new Error(
"typed translator failure access_token=translation-secret at /srv/private/translator.ts\n" +
"translator rejected access_token=translation-secret at /srv/private/translator.ts\n" +
" at translate (/srv/private/translator.ts:41:8)"
);
error.statusCode = 422;
@@ -1998,13 +1972,12 @@ test("chatCore surfaces typed translation errors with the declared error type",
assert.equal(result.success, false);
assert.equal(result.status, 422);
const payload = (await result.response.json()) as {
error: { message: string; type: string; code: string };
};
assert.equal(payload.error.type, "invalid_request_error");
assert.equal(payload.error.code, "");
assert.match(payload.error.message, /typed translator failure/);
assert.match(payload.error.message, /translator rejected/);
assert.doesNotMatch(
JSON.stringify({ payload, internalError: result.error }),
/translation-secret|type-secret|srv\/private|translator\.ts|type\.ts|\bat translate\b/i
@@ -2624,44 +2597,6 @@ test("chatCore 429 lets account fallback apply the configured resilience cooldow
assert.equal((afterFallback as any).testStatus, "unavailable");
assert.ok(cooldownRemaining > 0 && cooldownRemaining <= 2_000);
});
test("chatCore sanitizes nonterminal provider lastError without changing raw classification", async () => {
const hostile =
"invalid credential access_token=chatcore-last-error-secret at /srv/private/chatcore.ts\n" +
" at request (/srv/private/chatcore.ts:14:3)";
const connection = await providersDb.createProviderConnection({
provider: "openai",
authType: "apikey",
name: "chatCore last-error boundary",
apiKey: "chatcore-last-error-test-key",
isActive: true,
testStatus: "active",
});
const { result } = await invokeChatCore({
provider: "openai",
model: "gpt-4o-mini",
connectionId: connection.id,
body: {
model: "gpt-4o-mini",
stream: false,
messages: [{ role: "user", content: "trigger nonterminal auth failure" }],
},
responseFactory() {
return Response.json({ error: { message: hostile } }, { status: 401 });
},
});
const updated = await providersDb.getProviderConnectionById(connection.id);
const publicBody = await result.response.json();
const boundaries = JSON.stringify({ lastError: updated.lastError, publicBody });
assert.equal(result.status, 401);
assert.match(String(updated.lastError), /invalid credential/i);
assert.doesNotMatch(
boundaries,
/chatcore-last-error-secret|srv\/private|chatcore\.ts|\bat request\b/i
);
});
test("chatCore does not substitute an OpenAI model after model-unavailable", async () => {
const { calls, result } = await invokeChatCore({
provider: "openai",

View File

@@ -302,10 +302,19 @@ test("sanitizes response.failed messages without rewriting unrelated deep diagno
output: { trace: hostile },
level1: { level2: { level3: { level4: { level5: { label: "legitimate diagnostic" } } } } },
};
const output = [
{
type: "message",
role: "assistant",
content: [{ type: "output_text", text: "safe direct partial output" }],
},
{ type: "reasoning", reasoning_content: "private direct reasoning" },
];
const objectPayload = protectPayloadForLog({
type: "response.failed",
message: hostile,
diagnostics,
output,
});
const protectedPipeline = protectPipelinePayloads({
streamChunks: {
@@ -322,13 +331,88 @@ test("sanitizes response.failed messages without rewriting unrelated deep diagno
(objectPayload as { diagnostics: typeof diagnostics }).diagnostics.level1,
diagnostics.level1
);
assert.deepEqual((objectPayload as { output: unknown }).output, [
{
type: "message",
role: "assistant",
content: [{ type: "output_text", text: "safe direct partial output" }],
},
]);
assert.doesNotMatch(serialized, /private direct reasoning/);
assert.match(serialized, /response\.failed/);
});
test("projects nested output when the SSE event alone marks response.failed", () => {
const protectedPipeline = protectPipelinePayloads({
streamChunks: {
provider: [
`event: response.failed\ndata: ${JSON.stringify({
response: {
output: [
{
type: "message",
role: "assistant",
content: [{ type: "output_text", text: "safe event partial output" }],
},
{ type: "reasoning", reasoning_content: "private event reasoning" },
],
},
})}\n\n`,
],
},
});
const serialized = JSON.stringify(protectedPipeline);
assert.match(serialized, /safe event partial output/);
assert.doesNotMatch(serialized, /private event reasoning|"reasoning"/);
});
test("sanitizes response.completed failed siblings in objects, SSE, and NDJSON", () => {
const hostile = "Bearer completed-failed-secret at /srv/private/completed-failed.ts:8:2";
const partialOutput = [
{ type: "message", content: [{ type: "output_text", text: "partial safe output" }] },
{
id: "msg_partial",
type: "message",
role: "assistant",
status: "in_progress",
diagnostics: { trace: hostile },
content: [
{
type: "output_text",
text: "partial safe output",
annotations: [{ type: "url_citation", url: "file:///srv/private/citation" }],
},
{ type: "output_text", phase: "commentary", text: "private commentary" },
{ type: "refusal", refusal: "safe refusal" },
],
},
{
id: "msg_roleless",
type: "message",
content: [{ type: "output_text", text: "private roleless output" }],
},
{
type: "reasoning",
reasoning_content: "private chain of thought",
encrypted_content: "private encrypted reasoning",
},
{
type: "function_call",
name: "read_private_file",
arguments: '{"api_key":"private tool argument"}',
},
];
const projectedOutput = [
{
id: "msg_partial",
type: "message",
role: "assistant",
status: "in_progress",
content: [
{ type: "output_text", text: "partial safe output" },
{ type: "refusal", refusal: "safe refusal" },
],
},
];
const completedFailure = {
type: "response.completed",
@@ -352,11 +436,16 @@ test("sanitizes response.completed failed siblings in objects, SSE, and NDJSON",
});
const serialized = JSON.stringify({ objectPayload, protectedPipeline });
assert.doesNotMatch(serialized, /completed-failed-secret|srv\/private|completed-failed\.ts/i);
assert.doesNotMatch(
serialized,
/completed-failed-secret|srv\/private|completed-failed\.ts|private commentary|private roleless|private chain|private encrypted|private tool/i
);
assert.doesNotMatch(serialized, /"annotations"|"diagnostics"|"function_call"|"reasoning"/);
assert.match(serialized, /partial safe output/);
assert.match(serialized, /safe refusal/);
assert.deepEqual(
(objectPayload as { response: { output: typeof partialOutput } }).response.output,
partialOutput
(objectPayload as { response: { output: typeof projectedOutput } }).response.output,
projectedOutput
);
});

View File

@@ -173,14 +173,80 @@ test("Responses response.failed is projected before forwarding, logging, and onF
type: "message",
role: "assistant",
status: "in_progress",
diagnostics: {
stack: "at /srv/omniroute/private-runtime.ts:47:2",
api_key: "sk-stream-secret-output-diagnostics",
},
content: [
{
type: "output_text",
text: "safe partial output",
annotations: [
{
type: "url_citation",
url: "https://example.invalid/?token=sk-stream-secret-annotation",
title: "at /srv/omniroute/private-runtime.ts:48:2",
},
],
},
{
type: "output_text",
phase: "commentary",
text: "hidden nested commentary must not be public",
},
{ type: "refusal", refusal: "safe refusal" },
],
},
{
id: "msg_commentary",
type: "message",
role: "assistant",
phase: "commentary",
content: [
{
type: "output_text",
text: "hidden commentary at /srv/omniroute/private-runtime.ts:49:2",
},
],
},
{
id: "msg_roleless",
type: "message",
content: [
{
type: "output_text",
text: "roleless output must not be public",
annotations: [],
},
],
},
{
id: "reasoning_private",
type: "reasoning",
encrypted_content: "sk-stream-secret-encrypted-reasoning",
summary: [
{
type: "summary_text",
text: "at /srv/omniroute/private-runtime.ts:50:2",
},
],
},
{
id: "call_private",
type: "function_call",
call_id: "call_private",
name: "read_private_file",
arguments:
'{"path":"/srv/omniroute/private-runtime.ts","api_key":"sk-stream-secret-tool"}',
},
{
id: "provider_private",
type: "provider_diagnostics",
diagnostics: {
stack: "at /srv/omniroute/private-runtime.ts:51:2",
api_key: "sk-stream-secret-unknown-item",
},
},
],
error: {
type: "server_error",
@@ -218,12 +284,22 @@ test("Responses response.failed is projected before forwarding, logging, and onF
assert.match(result.output, /response\.failed/);
assert.match(result.output, /"last_error":\{/);
assert.match(result.output, /safe partial output/);
assert.match(result.output, /safe refusal/);
assert.doesNotMatch(result.output, /"annotations"/);
assert.doesNotMatch(result.output, /hidden nested commentary must not be public/);
assert.doesNotMatch(result.output, /roleless output must not be public/);
assert.match(result.output, /"cached_tokens":1/);
assert.doesNotMatch(result.output, /\[truncated\]/);
assertNoHostileDetail(result.output);
assertNoHostileDetail(convertedLog.join("\n"));
assert.doesNotMatch(result.output, /"diagnosis"|"settings"/);
assert.doesNotMatch(convertedLog.join("\n"), /"diagnosis"|"settings"/);
assert.doesNotMatch(
result.output,
/"diagnosis"|"diagnostics"|"settings"|"encrypted_content"|"function_call"|"provider_diagnostics"|"phase"|"url_citation"/
);
assert.doesNotMatch(
convertedLog.join("\n"),
/"diagnosis"|"diagnostics"|"settings"|"encrypted_content"|"function_call"|"provider_diagnostics"|"phase"|"url_citation"/
);
assert.ok(result.failure);
assert.match(result.failure.message, /private-runtime\.ts/);
assertNoHostileDetail(String(result.error));