mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-09-18 04:42:30 +03:00
security(runtime): harden TLS provenance, lifecycle, and public error boundaries (#11742)
* security(deps): pin and verify tls-client native artifacts * docs(changelog): link tls-client provenance PR * security(runtime): harden TLS and public error boundaries * security(runtime): resolve CodeQL error-boundary findings * security(lmarena): close public stream error boundary * fix(lmarena): normalize public error statuses * chore(quality): rebaseline chatCore.ts for the surviving log-boundary hardening open-sse/handlers/chatCore.ts 6219 -> 6287. This is the one part of #11742 that survived the rebase: sanitizeErrorMessage on the plugin onError hook, on the semaphore-timeout path and on failureMessage before it reaches console.log and the call log, sanitizeUpstreamDetails on the malformed-response log, and getSafeErrorMetadata + try/catch where hostile (Proxy) metadata could throw. That is the LOG boundary, which is broader than Hard Rule #12 (responses). The rest of the PR was dropped as already landed on the tip.
This commit is contained in:
committed by
GitHub
parent
57729db54d
commit
2fa6ef0bdd
@@ -0,0 +1 @@
|
||||
- **security(runtime):** fail closed on hostile thrown values and keep upstream text out of public error surfaces — the chat pipeline now reads rejection metadata through a safe accessor, sanitizes the message before it reaches call logs and console, and projects the failure-usage code onto the bounded public vocabulary; Perplexity's non-streaming quota/upstream error body sanitizes the upstream message and projects the provider-supplied error code; Arena (lmarena) maps every public failure onto a fixed vocabulary instead of echoing the upstream error; Notion's TLS transport failure sanitizes the transport error before it reaches the response body ([#11742](https://github.com/diegosouzapw/OmniRoute/pull/11742)).
|
||||
@@ -69,11 +69,6 @@
|
||||
"count": 1
|
||||
}
|
||||
},
|
||||
"open-sse/executors/chipotle.ts": {
|
||||
"@typescript-eslint/no-unused-vars": {
|
||||
"count": 1
|
||||
}
|
||||
},
|
||||
"open-sse/executors/claude-web.ts": {
|
||||
"@typescript-eslint/no-unused-vars": {
|
||||
"count": 1
|
||||
@@ -164,11 +159,6 @@
|
||||
"count": 2
|
||||
}
|
||||
},
|
||||
"open-sse/executors/notion-web.ts": {
|
||||
"@typescript-eslint/no-unused-vars": {
|
||||
"count": 1
|
||||
}
|
||||
},
|
||||
"open-sse/executors/opencode.ts": {
|
||||
"@typescript-eslint/no-explicit-any": {
|
||||
"count": 2
|
||||
@@ -2807,11 +2797,6 @@
|
||||
"count": 1
|
||||
}
|
||||
},
|
||||
"tests/unit/chipotle-executor.test.ts": {
|
||||
"@typescript-eslint/no-explicit-any": {
|
||||
"count": 3
|
||||
}
|
||||
},
|
||||
"tests/unit/claude-code-compatible-helpers.test.ts": {
|
||||
"@typescript-eslint/no-explicit-any": {
|
||||
"count": 2
|
||||
|
||||
@@ -466,7 +466,7 @@
|
||||
"open-sse/executors/codex.ts": 1528,
|
||||
"open-sse/executors/cursor.ts": 1847,
|
||||
"open-sse/executors/muse-spark-web.ts": 1405,
|
||||
"open-sse/handlers/chatCore.ts": 6219,
|
||||
"open-sse/handlers/chatCore.ts": 6287,
|
||||
"open-sse/handlers/imageGeneration.ts": 3304,
|
||||
"open-sse/handlers/search.ts": 1789,
|
||||
"open-sse/mcp-server/schemas/tools.ts": 1621,
|
||||
@@ -703,5 +703,6 @@
|
||||
"_rebaseline_2026_09_07_roundrobin_crosses_new_file_cap": "open-sse/services/combo/roundRobinCombo.ts 1198->1205, crossing the 1200 new-file cap. #12884 wires the quota-skip diagnostics into the round-robin attempt path so an ALL_TARGETS_SKIPPED 503 names which windows were exhausted instead of returning an opaque skip. The file was already at 1198 when #12811 lifted it out of combo.ts, so seven lines cross it; the diagnostics themselves live in quotaSkipDiagnostics.ts, under cap. Frozen rather than split: the natural next extraction is the attempt-loop body, which #12746/#12811 just moved and should settle before being cut again.",
|
||||
"_rebaseline_2026_09_08_13033_responses_websearch_sse": "Own growth after rebase onto v3.8.51 tip af49d4972: open-sse/handlers/chatCore.ts 6036->6035 (-1, check-file-size split-newline). Branch stamps clientRequestedResponsesStream before web_search fallback forces stream:false, then wraps JSON via synthesizeOpenAiSseFromJson. Call-site wiring next to the existing web_search non-stream fallback; no new god-file. Covered tests/unit/responses-websearch-sse-13033.test.ts.",
|
||||
"_rebaseline_pr12723_kimi_narration": "PR #12723 Kimi narration recovery integration: cursor.ts 1764->1798 (+34 = +52 feature integration net, absorbed -18 of base growth headroom since the branch's own rebaseline). Every added line is functional PR wiring (scrubber creation with onToolCall emit closure, StreamCtx.narrationScrubber field, per-delta feed + finalizeKimiTurn hooks); the narration logic itself lives in open-sse/utils/kimiToolCallNarration.ts (593 lines). Covered by tests/unit/kimi-tool-call-narration.test.ts.",
|
||||
"_rebaseline_2026_09_17_merge_wave_growth": "Crescimento da leva de merges de 2026-09-17 (lote /merge-batch + uplift). Cada teto foi atribuido a sua causa antes de subir, nao rebaselinado em bloco: src/sse/handlers/chatHelpers.ts 1245->1246 (#13551, combo scope no guard fail-closed do proxy); open-sse/handlers/chatCore.ts 6203->6219 (#12905 DSML/preamble, #13910 classificacao de 2xx disfarcado, #12904 injecao unica do system prompt pos-traducao); open-sse/utils/stream.ts 3123->3140 (#12905 e #12906 retry de empty_response 502 + timeout reasoning-aware); tests/integration/chat-pipeline.test.ts 1736->1740 (#13419, as duas assercoes de header exatas atualizadas para o charset=utf-8). Todos cobertos pelos testes das proprias PRs, verificados verdes no tip apos o merge."
|
||||
"_rebaseline_2026_09_17_merge_wave_growth": "Crescimento da leva de merges de 2026-09-17 (lote /merge-batch + uplift). Cada teto foi atribuido a sua causa antes de subir, nao rebaselinado em bloco: src/sse/handlers/chatHelpers.ts 1245->1246 (#13551, combo scope no guard fail-closed do proxy); open-sse/handlers/chatCore.ts 6203->6219 (#12905 DSML/preamble, #13910 classificacao de 2xx disfarcado, #12904 injecao unica do system prompt pos-traducao); open-sse/utils/stream.ts 3123->3140 (#12905 e #12906 retry de empty_response 502 + timeout reasoning-aware); tests/integration/chat-pipeline.test.ts 1736->1740 (#13419, as duas assercoes de header exatas atualizadas para o charset=utf-8). Todos cobertos pelos testes das proprias PRs, verificados verdes no tip apos o merge.",
|
||||
"_rebaseline_2026_09_17_11742_log_boundary_hardening": "PR #11742 (rebase para release/v3.8.51): open-sse/handlers/chatCore.ts 6219->6287. O crescimento e a unica parte da PR que sobreviveu ao tip: endurecimento da fronteira de LOG (mais amplo que a Hard Rule #12, que cobre respostas). Sao 136 linhas adicionadas, das quais ~40 sao chamadas diretas de sanitizacao — sanitizeErrorMessage em erro de plugin (onError), em timeout de semaforo e na failureMessage antes de ela chegar ao console.log e ao call-log; sanitizeUpstreamDetails no log de resposta malformada; getSafeErrorMetadata + try/catch nos pontos onde metadata hostil (Proxy) podia lançar. O resto da PR foi descartado por ja estar no tip (#12506/#12945/#13635 error boundaries, #12429 wreq-js, #11754 aposentadoria do ChatGPT Web) — open-sse/utils/ difere do tip por UMA linha (registro do identificador publico lmarena_stream_error)."
|
||||
}
|
||||
|
||||
File diff suppressed because one or more lines are too long
|
Before Width: | Height: | Size: 12 KiB After Width: | Height: | Size: 12 KiB |
@@ -113,6 +113,7 @@ export async function buildToolModeResponse(
|
||||
stream: boolean,
|
||||
meta: { cid: string; created: number; model: string; idSeed?: string }
|
||||
): Promise<Response> {
|
||||
if (!bufferedJson.ok) return bufferedJson;
|
||||
const jsonResponse = await applyToolCallsToJsonResponse(
|
||||
bufferedJson,
|
||||
requestedTools,
|
||||
|
||||
@@ -39,6 +39,9 @@ const CLAUDE_WEB_API_BASE = "https://claude.ai/api";
|
||||
const CLAUDE_WEB_ORGS_URL = `${CLAUDE_WEB_API_BASE}/organizations`;
|
||||
const CLAUDE_SESSION_COOKIE_NAME = "sessionKey";
|
||||
const MAX_ERROR_BODY_BYTES = 64 * 1024;
|
||||
const MAX_FORWARDED_RETRY_AFTER_SECONDS = 24 * 60 * 60;
|
||||
const HTTP_DATE_PATTERN =
|
||||
/^(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun), \d{2} (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d{4} \d{2}:\d{2}:\d{2} GMT$/;
|
||||
const CLAUDE_USER_AGENT = CLAUDE_WEB_FINGERPRINT.userAgent;
|
||||
|
||||
type SendClaudeWebTransport = (
|
||||
@@ -291,7 +294,9 @@ async function readTransportErrorText(result: ClaudeWebTransportResult): Promise
|
||||
output += decoder.decode();
|
||||
return output;
|
||||
} finally {
|
||||
await reader.cancel().catch(() => {});
|
||||
await reader.cancel().catch(() => {
|
||||
// The bounded error body is already captured; cancel can race an upstream close.
|
||||
});
|
||||
try {
|
||||
reader.releaseLock();
|
||||
} catch {
|
||||
@@ -300,6 +305,26 @@ async function readTransportErrorText(result: ClaudeWebTransportResult): Promise
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeForwardedRetryAfter(value: string | null): string | null {
|
||||
const trimmed = value?.trim();
|
||||
if (!trimmed) return null;
|
||||
|
||||
if (/^\d{1,9}$/.test(trimmed)) {
|
||||
const seconds = Number(trimmed);
|
||||
return Number.isSafeInteger(seconds) && seconds <= MAX_FORWARDED_RETRY_AFTER_SECONDS
|
||||
? String(seconds)
|
||||
: null;
|
||||
}
|
||||
|
||||
if (!HTTP_DATE_PATTERN.test(trimmed)) return null;
|
||||
const timestamp = Date.parse(trimmed);
|
||||
if (!Number.isFinite(timestamp)) return null;
|
||||
const normalized = new Date(timestamp).toUTCString();
|
||||
if (normalized !== trimmed) return null;
|
||||
if (Math.abs(timestamp - Date.now()) > MAX_FORWARDED_RETRY_AFTER_SECONDS * 1000) return null;
|
||||
return normalized;
|
||||
}
|
||||
|
||||
async function errorResponseForTransport(
|
||||
result: ClaudeWebTransportResult,
|
||||
turn: PreparedClaudeWebTurn
|
||||
@@ -311,7 +336,7 @@ async function errorResponseForTransport(
|
||||
}
|
||||
if (result.status === 429) {
|
||||
const extraHeaders: Record<string, string> = {};
|
||||
const upstreamRetryAfter = result.headers.get("retry-after");
|
||||
const upstreamRetryAfter = normalizeForwardedRetryAfter(result.headers.get("retry-after"));
|
||||
if (upstreamRetryAfter) {
|
||||
extraHeaders["Retry-After"] = upstreamRetryAfter;
|
||||
}
|
||||
@@ -349,6 +374,7 @@ export class ClaudeWebExecutor extends BaseExecutor {
|
||||
const cookieHeader = normalizeClaudeSessionCookie(rawCookie);
|
||||
return verifyCookieValidity(cookieHeader, readClaudeWebDeviceId(credentials), signal);
|
||||
} catch {
|
||||
// Connection checks deliberately collapse malformed credentials and transport failures to false.
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,6 +55,23 @@ import {
|
||||
const GROK_CHAT_API = "https://grok.com/rest/app-chat/conversations/new";
|
||||
const GROK_USER_AGENT =
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36";
|
||||
const GROK_PUBLIC_UPSTREAM_ERROR = "Grok upstream error";
|
||||
|
||||
function sanitizeGrokUpstreamError(message: unknown): string {
|
||||
const sanitized = sanitizeErrorMessage(message);
|
||||
return sanitized.trim() && !/^(?:[A-Za-z_$][\w$]*)?Error:\s*$/.test(sanitized)
|
||||
? sanitized
|
||||
: GROK_PUBLIC_UPSTREAM_ERROR;
|
||||
}
|
||||
|
||||
function isTlsClientUnavailableError(error: unknown): error is TlsClientUnavailableError {
|
||||
try {
|
||||
return error instanceof TlsClientUnavailableError;
|
||||
} catch {
|
||||
// A rejected Proxy may throw while instanceof walks its prototype chain.
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Model mappings ─────────────────────────────────────────────────────────
|
||||
// Grok Web exposes UI modes, not stable public model IDs. Keep OmniRoute model
|
||||
@@ -637,7 +654,11 @@ async function buildNonStreamingResponse(
|
||||
if (chunk.error) {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
error: { message: chunk.error, type: "upstream_error", code: "GROK_ERROR" },
|
||||
error: {
|
||||
message: sanitizeGrokUpstreamError(chunk.error),
|
||||
type: "upstream_error",
|
||||
code: "GROK_ERROR",
|
||||
},
|
||||
}),
|
||||
{ status: 502, headers: { "Content-Type": "application/json" } }
|
||||
);
|
||||
@@ -1030,12 +1051,13 @@ export class GrokWebExecutor extends BaseExecutor {
|
||||
streamEofSymbol: "[DONE]",
|
||||
});
|
||||
} catch (err) {
|
||||
if (err instanceof TlsClientUnavailableError) {
|
||||
log?.error?.("GROK-WEB", `TLS client unavailable: ${err.message}`);
|
||||
const publicError = sanitizeGrokUpstreamError(err);
|
||||
if (isTlsClientUnavailableError(err)) {
|
||||
log?.error?.("GROK-WEB", `TLS client unavailable: ${publicError}`);
|
||||
const errResp = new Response(
|
||||
JSON.stringify({
|
||||
error: {
|
||||
message: sanitizeErrorMessage(`Grok TLS client unavailable: ${err.message}`),
|
||||
message: `Grok TLS client unavailable: ${publicError}`,
|
||||
type: "upstream_error",
|
||||
code: "TLS_CLIENT_UNAVAILABLE",
|
||||
},
|
||||
@@ -1044,13 +1066,11 @@ export class GrokWebExecutor extends BaseExecutor {
|
||||
);
|
||||
return { response: errResp, url: GROK_CHAT_API, headers, transformedBody: grokPayload };
|
||||
}
|
||||
log?.error?.("GROK-WEB", `Fetch failed: ${err instanceof Error ? err.message : String(err)}`);
|
||||
log?.error?.("GROK-WEB", `Fetch failed: ${publicError}`);
|
||||
const errResp = new Response(
|
||||
JSON.stringify({
|
||||
error: {
|
||||
message: sanitizeErrorMessage(
|
||||
`Grok connection failed: ${err instanceof Error ? err.message : String(err)}`
|
||||
),
|
||||
message: `Grok connection failed: ${publicError}`,
|
||||
type: "upstream_error",
|
||||
},
|
||||
}),
|
||||
|
||||
@@ -11,6 +11,7 @@ import { v7 as uuidv7 } from "uuid";
|
||||
import { BaseExecutor, type ExecuteInput } from "./base.ts";
|
||||
import { tlsFetchLMArena, TlsClientUnavailableError } from "../services/lmarenaTlsClient.ts";
|
||||
import { readLMArenaCookie, reconstructLMArenaCookie } from "./lmarena/cookie.ts";
|
||||
import { sanitizeLMArenaError } from "./lmarena/error.ts";
|
||||
import {
|
||||
LMARENA_STREAM_URL,
|
||||
LMARENA_USER_AGENT,
|
||||
@@ -50,6 +51,15 @@ interface OpenAIMessage {
|
||||
content?: unknown;
|
||||
}
|
||||
|
||||
function isTlsClientUnavailableError(error: unknown): error is TlsClientUnavailableError {
|
||||
try {
|
||||
return error instanceof TlsClientUnavailableError;
|
||||
} catch {
|
||||
// A rejected Proxy may throw while instanceof walks its prototype chain.
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** Optional browser-issued reCAPTCHA v3 token (operator-supplied). */
|
||||
function readRecaptchaToken(credentials: unknown, body: unknown): string | null {
|
||||
const fromObj = (v: unknown): string | null => {
|
||||
@@ -146,13 +156,13 @@ export class LMArenaExecutor extends BaseExecutor {
|
||||
log,
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof TlsClientUnavailableError) {
|
||||
log?.error?.("LMArenaExecutor", `TLS client unavailable: ${error.message}`);
|
||||
return mapTlsUnavailable(error, url, headers, transformedBody);
|
||||
if (isTlsClientUnavailableError(error)) {
|
||||
log?.error?.("LMArenaExecutor", `TLS client unavailable: ${sanitizeLMArenaError(error)}`);
|
||||
return mapTlsUnavailable(url, headers, transformedBody);
|
||||
}
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
log?.error?.("LMArenaExecutor", `Request failed: ${message}`);
|
||||
return mapNetworkError(message, url, headers, transformedBody);
|
||||
const logMessage = sanitizeLMArenaError(error);
|
||||
log?.error?.("LMArenaExecutor", `Request failed: ${logMessage}`);
|
||||
return mapNetworkError(url, headers, transformedBody);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
17
open-sse/executors/lmarena/error.ts
Normal file
17
open-sse/executors/lmarena/error.ts
Normal file
@@ -0,0 +1,17 @@
|
||||
import { sanitizeErrorMessage } from "../../utils/error.ts";
|
||||
|
||||
const ERROR_NAME_ONLY_RE = /^[A-Za-z]*Error:?$/;
|
||||
|
||||
/** Convert an unknown Arena failure into a stable, public-safe message. */
|
||||
export function sanitizeLMArenaError(value: unknown, fallback = "Arena upstream error"): string {
|
||||
let candidate = value;
|
||||
try {
|
||||
if (value instanceof Error) candidate = value.message;
|
||||
} catch {
|
||||
// Hostile thrown values can expose coercing prototype/message accessors.
|
||||
}
|
||||
|
||||
const sanitized = sanitizeErrorMessage(candidate).trim();
|
||||
if (!sanitized || ERROR_NAME_ONLY_RE.test(sanitized)) return fallback;
|
||||
return sanitized;
|
||||
}
|
||||
@@ -2,6 +2,8 @@
|
||||
* LMArena live model list parsing, catalog normalization, and name→UUID resolution.
|
||||
*/
|
||||
|
||||
import { sanitizeLMArenaError } from "./error.ts";
|
||||
|
||||
export const LMARENA_API_BASE = "https://arena.ai";
|
||||
export const LMARENA_STREAM_URL = `${LMARENA_API_BASE}/nextjs-api/stream/create-evaluation`;
|
||||
/**
|
||||
@@ -297,10 +299,9 @@ export async function resolveLMArenaModelId(model: string, log?: LogFn): Promise
|
||||
if (fromSeed) return fromSeed;
|
||||
return pickLMArenaModelId(requested, await getLMArenaModels(log));
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
log?.warn?.(
|
||||
"LMArenaExecutor",
|
||||
`Using raw model id after static catalog lookup failed: ${message}`
|
||||
`Using raw model id after static catalog lookup failed: ${sanitizeLMArenaError(error, "Arena catalog lookup error")}`
|
||||
);
|
||||
return requested;
|
||||
}
|
||||
|
||||
@@ -2,25 +2,186 @@
|
||||
* Response mapping helpers for the Arena (lmarena) executor — kept small so
|
||||
* the executor methods stay under complexity / max-lines gates.
|
||||
*/
|
||||
import { sanitizeErrorMessage } from "../../utils/error.ts";
|
||||
import { isCloudflareChallenge } from "../../services/lmarenaTlsClient.ts";
|
||||
import { buildErrorBody } from "../../utils/error.ts";
|
||||
import { sanitizeLMArenaError } from "./error.ts";
|
||||
import { markLMArenaCatalogModelDead } from "./models.ts";
|
||||
import { parseArenaSSE } from "./stream.ts";
|
||||
|
||||
const encoder = new TextEncoder();
|
||||
const SAFE_ARENA_STREAM_ERROR_NAMES = new Set([
|
||||
"AbortError",
|
||||
"ResponseAborted",
|
||||
"TimeoutError",
|
||||
"BodyTimeoutError",
|
||||
]);
|
||||
|
||||
export function errorResponse(
|
||||
status: number,
|
||||
message: string,
|
||||
type: string,
|
||||
code: string
|
||||
): Response {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
error: { message: sanitizeErrorMessage(message), type, code },
|
||||
type ArenaPublicFailure =
|
||||
| { kind: "missing-cookie" }
|
||||
| {
|
||||
kind: "bot-block";
|
||||
status: number;
|
||||
reason: "cloudflare" | "token-present" | "token-needed";
|
||||
}
|
||||
| { kind: "http-status"; status: number }
|
||||
| { kind: "tls-unavailable" }
|
||||
| { kind: "network" }
|
||||
| { kind: "upstream-event" }
|
||||
| { kind: "stream-transport" };
|
||||
|
||||
interface ArenaProjectedPublicFailure {
|
||||
status: number;
|
||||
message: string;
|
||||
type: string;
|
||||
code: string;
|
||||
}
|
||||
|
||||
type ArenaProjectedStreamError = Error & {
|
||||
statusCode?: number;
|
||||
type?: string;
|
||||
code?: string;
|
||||
};
|
||||
|
||||
const ARENA_PUBLIC_MESSAGES = {
|
||||
missingCookie:
|
||||
"Arena requires a session cookie. Paste the full Cookie header from arena.ai (include arena-auth-prod-v1.* chunks and ideally cf_clearance).",
|
||||
cloudflareBot:
|
||||
"Arena blocked by Cloudflare bot management. Use a residential/browser-grade network if needed, paste a fresh full Cookie header (include cf_clearance / __cf_bm when present), and optionally set providerSpecificData.recaptchaV3Token from a live browser session.",
|
||||
botTokenNeeded:
|
||||
"If this persists, supply a browser reCAPTCHA v3 token via credentials.providerSpecificData.recaptchaV3Token (in addition to the session cookie).",
|
||||
tlsUnavailable:
|
||||
"Arena TLS impersonation unavailable: Arena upstream error. Verify the wreq-js 3.2 native binding.",
|
||||
upstream: "Arena upstream error",
|
||||
upstreamStream: "Arena upstream stream error",
|
||||
} as const;
|
||||
|
||||
function normalizeArenaErrorStatus(status: number, fallback: number): number {
|
||||
return Number.isInteger(status) && status >= 400 && status <= 599 ? status : fallback;
|
||||
}
|
||||
|
||||
function projectArenaPublicFailure(failure: ArenaPublicFailure): ArenaProjectedPublicFailure {
|
||||
switch (failure.kind) {
|
||||
case "missing-cookie":
|
||||
return {
|
||||
status: 401,
|
||||
message: ARENA_PUBLIC_MESSAGES.missingCookie,
|
||||
type: "authentication_error",
|
||||
code: "missing_cookie",
|
||||
};
|
||||
case "bot-block": {
|
||||
const status = normalizeArenaErrorStatus(failure.status, 403);
|
||||
let message: string;
|
||||
switch (failure.reason) {
|
||||
case "cloudflare":
|
||||
message = ARENA_PUBLIC_MESSAGES.cloudflareBot;
|
||||
break;
|
||||
case "token-present":
|
||||
message = `Arena API error: ${status}`;
|
||||
break;
|
||||
case "token-needed":
|
||||
message = `Arena API error: ${status}. ${ARENA_PUBLIC_MESSAGES.botTokenNeeded}`;
|
||||
break;
|
||||
default: {
|
||||
const exhaustiveReason: never = failure.reason;
|
||||
return exhaustiveReason;
|
||||
}
|
||||
}
|
||||
return {
|
||||
status,
|
||||
message,
|
||||
type: "api_error",
|
||||
code: "cloudflare_or_bot",
|
||||
};
|
||||
}
|
||||
case "http-status": {
|
||||
const status = normalizeArenaErrorStatus(failure.status, 502);
|
||||
return {
|
||||
status,
|
||||
message: `Arena API error: ${status}`,
|
||||
type: "api_error",
|
||||
code: String(status),
|
||||
};
|
||||
}
|
||||
case "tls-unavailable":
|
||||
return {
|
||||
status: 502,
|
||||
message: ARENA_PUBLIC_MESSAGES.tlsUnavailable,
|
||||
type: "upstream_error",
|
||||
code: "TLS_CLIENT_UNAVAILABLE",
|
||||
};
|
||||
case "network":
|
||||
return {
|
||||
status: 502,
|
||||
message: ARENA_PUBLIC_MESSAGES.upstream,
|
||||
type: "network_error",
|
||||
code: "request_failed",
|
||||
};
|
||||
case "upstream-event":
|
||||
return {
|
||||
status: 502,
|
||||
message: ARENA_PUBLIC_MESSAGES.upstream,
|
||||
type: "api_error",
|
||||
code: "lmarena_error",
|
||||
};
|
||||
case "stream-transport":
|
||||
return {
|
||||
status: 502,
|
||||
message: ARENA_PUBLIC_MESSAGES.upstreamStream,
|
||||
type: "upstream_error",
|
||||
code: "lmarena_stream_error",
|
||||
};
|
||||
default: {
|
||||
const exhaustiveFailure: never = failure;
|
||||
return exhaustiveFailure;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function projectArenaStreamError(error: unknown): Error {
|
||||
const publicError = buildArenaPublicError({ kind: "stream-transport" }).body.error;
|
||||
const projected = new Error(publicError.message) as ArenaProjectedStreamError;
|
||||
projected.stack = undefined;
|
||||
projected.type = publicError.type;
|
||||
projected.code = publicError.code;
|
||||
if (!error || typeof error !== "object") return projected;
|
||||
|
||||
try {
|
||||
const name = (error as { name?: unknown }).name;
|
||||
if (typeof name === "string" && SAFE_ARENA_STREAM_ERROR_NAMES.has(name)) {
|
||||
projected.name = name;
|
||||
}
|
||||
const rawStatusCode = (error as { statusCode?: unknown }).statusCode;
|
||||
if (
|
||||
typeof rawStatusCode === "number" &&
|
||||
Number.isInteger(rawStatusCode) &&
|
||||
rawStatusCode >= 400 &&
|
||||
rawStatusCode <= 599
|
||||
) {
|
||||
projected.statusCode = rawStatusCode;
|
||||
}
|
||||
} catch {
|
||||
// Hostile thrown values must not escape through coercing metadata accessors.
|
||||
}
|
||||
return projected;
|
||||
}
|
||||
|
||||
function buildArenaPublicError(failure: ArenaPublicFailure) {
|
||||
const projected = projectArenaPublicFailure(failure);
|
||||
return {
|
||||
status: projected.status,
|
||||
body: buildErrorBody(projected.status, projected.message, undefined, {
|
||||
type: projected.type,
|
||||
code: projected.code,
|
||||
}),
|
||||
{ status, headers: { "Content-Type": "application/json" } }
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
function errorResponse(failure: ArenaPublicFailure): Response {
|
||||
const projected = buildArenaPublicError(failure);
|
||||
return new Response(JSON.stringify(projected.body), {
|
||||
status: projected.status,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
}
|
||||
|
||||
export function missingCookieResult(
|
||||
@@ -29,41 +190,30 @@ export function missingCookieResult(
|
||||
transformedBody: unknown
|
||||
) {
|
||||
return {
|
||||
response: errorResponse(
|
||||
401,
|
||||
"Arena requires a session cookie. Paste the full Cookie header from arena.ai (include arena-auth-prod-v1.* chunks and ideally cf_clearance).",
|
||||
"authentication_error",
|
||||
"missing_cookie"
|
||||
),
|
||||
response: errorResponse({ kind: "missing-cookie" }),
|
||||
url,
|
||||
headers,
|
||||
transformedBody,
|
||||
};
|
||||
}
|
||||
|
||||
function parseArenaErrorBody(text: string | null | undefined, status: number): string {
|
||||
const fallback = `Arena API error: ${status}`;
|
||||
if (!text) return fallback;
|
||||
try {
|
||||
const errorJson = JSON.parse(text) as { error?: { message?: string }; message?: string };
|
||||
return errorJson.error?.message || errorJson.message || fallback;
|
||||
} catch {
|
||||
return text.slice(0, 500) || fallback;
|
||||
}
|
||||
}
|
||||
|
||||
function isBotOrChallenge(status: number, text: string | null | undefined): boolean {
|
||||
if (status === 403) return true;
|
||||
if (isCloudflareChallenge(text)) return true;
|
||||
return Boolean(text && text.trimStart().startsWith("<!DOCTYPE"));
|
||||
}
|
||||
|
||||
function botBlockMessage(text: string | null | undefined, hasRecaptcha: boolean, status: number) {
|
||||
if (isCloudflareChallenge(text)) {
|
||||
return "Arena blocked by Cloudflare bot management. Use a residential/browser-grade network if needed, paste a fresh full Cookie header (include cf_clearance / __cf_bm when present), and optionally set providerSpecificData.recaptchaV3Token from a live browser session.";
|
||||
}
|
||||
if (hasRecaptcha) return `Arena API error: ${status}`;
|
||||
return `Arena API error: ${status}. If this persists, supply a browser reCAPTCHA v3 token via credentials.providerSpecificData.recaptchaV3Token (in addition to the session cookie).`;
|
||||
function botBlockFailure(
|
||||
text: string | null | undefined,
|
||||
hasRecaptcha: boolean,
|
||||
status: number
|
||||
): ArenaPublicFailure {
|
||||
const reason = isCloudflareChallenge(text)
|
||||
? "cloudflare"
|
||||
: hasRecaptcha
|
||||
? "token-present"
|
||||
: "token-needed";
|
||||
return { kind: "bot-block", status, reason };
|
||||
}
|
||||
|
||||
/** Map non-2xx / CF TLS results to an executor failure payload, or null if OK. */
|
||||
@@ -80,12 +230,7 @@ export function mapFailedTlsResult(opts: {
|
||||
const { status, text, hasRecaptcha, model, arenaModelId, url, headers, transformedBody } = opts;
|
||||
if (isBotOrChallenge(status, text)) {
|
||||
return {
|
||||
response: errorResponse(
|
||||
status || 403,
|
||||
botBlockMessage(text, hasRecaptcha, status),
|
||||
"api_error",
|
||||
"cloudflare_or_bot"
|
||||
),
|
||||
response: errorResponse(botBlockFailure(text, hasRecaptcha, status)),
|
||||
url,
|
||||
headers,
|
||||
transformedBody,
|
||||
@@ -97,8 +242,10 @@ export function mapFailedTlsResult(opts: {
|
||||
markLMArenaCatalogModelDead(model);
|
||||
markLMArenaCatalogModelDead(arenaModelId);
|
||||
}
|
||||
// Fail closed: TLS error bodies can contain upstream stacks, causes, or internal identifiers.
|
||||
// Preserve the HTTP classification without projecting any body-derived text to the caller.
|
||||
return {
|
||||
response: errorResponse(status, parseArenaErrorBody(text, status), "api_error", String(status)),
|
||||
response: errorResponse({ kind: "http-status", status }),
|
||||
url,
|
||||
headers,
|
||||
transformedBody,
|
||||
@@ -106,18 +253,12 @@ export function mapFailedTlsResult(opts: {
|
||||
}
|
||||
|
||||
export function mapTlsUnavailable(
|
||||
error: Error,
|
||||
url: string,
|
||||
headers: Record<string, string>,
|
||||
transformedBody: unknown
|
||||
) {
|
||||
return {
|
||||
response: errorResponse(
|
||||
502,
|
||||
`Arena TLS impersonation unavailable: ${error.message}. Verify the wreq-js 3.2 native binding.`,
|
||||
"upstream_error",
|
||||
"TLS_CLIENT_UNAVAILABLE"
|
||||
),
|
||||
response: errorResponse({ kind: "tls-unavailable" }),
|
||||
url,
|
||||
headers,
|
||||
transformedBody,
|
||||
@@ -125,13 +266,12 @@ export function mapTlsUnavailable(
|
||||
}
|
||||
|
||||
export function mapNetworkError(
|
||||
message: string,
|
||||
url: string,
|
||||
headers: Record<string, string>,
|
||||
transformedBody: unknown
|
||||
) {
|
||||
return {
|
||||
response: errorResponse(502, message, "network_error", "request_failed"),
|
||||
response: errorResponse({ kind: "network" }),
|
||||
url,
|
||||
headers,
|
||||
transformedBody,
|
||||
@@ -196,10 +336,11 @@ function handleArenaEventLine(
|
||||
return false;
|
||||
}
|
||||
if (event.type === "error") {
|
||||
const upstreamError = buildArenaPublicError({ kind: "upstream-event" });
|
||||
enqueueSse(controller, {
|
||||
...baseChunk(model),
|
||||
choices: [{ index: 0, delta: {}, finish_reason: "stop" }],
|
||||
error: { message: sanitizeErrorMessage(event.content || "Unknown error") },
|
||||
error: upstreamError.body.error,
|
||||
});
|
||||
controller.close();
|
||||
return true;
|
||||
@@ -220,9 +361,28 @@ export function createOpenAIArenaStream(opts: {
|
||||
const { reader, model, signal, log } = opts;
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = "";
|
||||
let readerCleanup: Promise<void> | null = null;
|
||||
|
||||
const cleanupReader = (): Promise<void> => {
|
||||
if (readerCleanup) return readerCleanup;
|
||||
readerCleanup = (async () => {
|
||||
try {
|
||||
await reader.cancel();
|
||||
} catch {
|
||||
// The upstream may already be closed or errored; still release its lock below.
|
||||
}
|
||||
try {
|
||||
reader.releaseLock();
|
||||
} catch {
|
||||
// A concurrent cleanup may already have released this reader.
|
||||
}
|
||||
})();
|
||||
return readerCleanup;
|
||||
};
|
||||
|
||||
const onAbort = () => {
|
||||
void reader.cancel().catch(() => undefined);
|
||||
// The upstream reader may already be closed; cleanup failure must not replace the abort outcome.
|
||||
void cleanupReader();
|
||||
};
|
||||
if (signal) {
|
||||
if (signal.aborted) onAbort();
|
||||
@@ -234,7 +394,8 @@ export function createOpenAIArenaStream(opts: {
|
||||
try {
|
||||
while (true) {
|
||||
if (signal?.aborted) {
|
||||
await reader.cancel().catch(() => undefined);
|
||||
// Cancellation is best-effort cleanup; the already-observed abort remains authoritative.
|
||||
await cleanupReader();
|
||||
controller.close();
|
||||
return;
|
||||
}
|
||||
@@ -252,15 +413,17 @@ export function createOpenAIArenaStream(opts: {
|
||||
}
|
||||
emitStopAndDone(controller, model);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
log?.error?.("LMArenaExecutor", `Streaming error: ${message}`);
|
||||
controller.error(error);
|
||||
const logMessage = sanitizeLMArenaError(error, "Arena upstream stream error");
|
||||
log?.error?.("LMArenaExecutor", `Streaming error: ${logMessage}`);
|
||||
controller.error(projectArenaStreamError(error));
|
||||
} finally {
|
||||
await cleanupReader();
|
||||
if (signal) signal.removeEventListener("abort", onAbort);
|
||||
}
|
||||
},
|
||||
cancel() {
|
||||
void reader.cancel().catch(() => undefined);
|
||||
async cancel() {
|
||||
// The consumer may cancel after the upstream reader closed; cleanup must not mask that outcome.
|
||||
await cleanupReader();
|
||||
if (signal) signal.removeEventListener("abort", onAbort);
|
||||
},
|
||||
});
|
||||
@@ -272,7 +435,7 @@ export async function handleNonStreamingArenaResponse(
|
||||
): Promise<Response> {
|
||||
const text = await response.text();
|
||||
let fullText = "";
|
||||
let error: string | null = null;
|
||||
let hasUpstreamError = false;
|
||||
|
||||
for (const line of text.split("\n")) {
|
||||
if (!line.trim()) continue;
|
||||
@@ -281,12 +444,14 @@ export async function handleNonStreamingArenaResponse(
|
||||
if (!event) continue;
|
||||
if (event.type === "text" && event.content) fullText += event.content;
|
||||
else if (event.type === "error") {
|
||||
error = event.content || "Unknown error";
|
||||
hasUpstreamError = true;
|
||||
break;
|
||||
} else if (event.type === "done") break;
|
||||
}
|
||||
|
||||
if (error) return errorResponse(502, error, "api_error", "lmarena_error");
|
||||
if (hasUpstreamError) {
|
||||
return errorResponse({ kind: "upstream-event" });
|
||||
}
|
||||
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
|
||||
@@ -29,7 +29,10 @@
|
||||
*/
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { BaseExecutor, type ExecuteInput } from "./base.ts";
|
||||
import { makeExecutorErrorResult as makeErrorResult } from "../utils/error.ts";
|
||||
import {
|
||||
makeExecutorErrorResult as makeErrorResult,
|
||||
sanitizeErrorMessage,
|
||||
} from "../utils/error.ts";
|
||||
import {
|
||||
BROWSER_HEADERS,
|
||||
extractNotionUserIdFromCookie,
|
||||
@@ -57,7 +60,6 @@ import {
|
||||
} from "../services/notionStreamParser.ts";
|
||||
import {
|
||||
buildNotionTranscript,
|
||||
messagesForNotionTranscript,
|
||||
type NotionAgentOptions,
|
||||
} from "../services/notionTranscriptBuilder.ts";
|
||||
import { tlsFetchNotion } from "../services/notionTlsClient.ts";
|
||||
@@ -311,6 +313,16 @@ function clientFacingModelId(model: unknown): string {
|
||||
return clientFacingModel;
|
||||
}
|
||||
|
||||
function sanitizeNotionTransportError(error: unknown): string {
|
||||
let candidate = error;
|
||||
try {
|
||||
if (error instanceof Error) candidate = error.message;
|
||||
} catch {
|
||||
// Keep the unknown value for the canonical fail-closed sanitizer.
|
||||
}
|
||||
return sanitizeErrorMessage(candidate).trim() || "unknown error";
|
||||
}
|
||||
|
||||
/** Resolves workspace + user (cached). Required for createThread payloads. */
|
||||
async function resolveExecuteWorkspace(
|
||||
cookie: string,
|
||||
@@ -508,7 +520,7 @@ async function sendNotionInferenceRequest(opts: {
|
||||
return {
|
||||
errorResult: makeErrorResult(
|
||||
502,
|
||||
`Notion fetch failed: ${err instanceof Error ? err.message : "unknown error"}`,
|
||||
`Notion fetch failed: ${sanitizeNotionTransportError(err)}`,
|
||||
reqBody,
|
||||
NOTION_URL
|
||||
),
|
||||
|
||||
@@ -15,7 +15,7 @@ import {
|
||||
} from "../services/perplexityTlsClient.ts";
|
||||
import { prepareToolMessages } from "../translator/webTools.ts";
|
||||
import { buildToolModeResponse } from "./chatgptWebTools.ts";
|
||||
import { sanitizeErrorMessage } from "../utils/error.ts";
|
||||
import { projectPublicErrorIdentifier, sanitizeErrorMessage } from "../utils/error.ts";
|
||||
import { buildSessionCookieHeader, mergeRefreshedCookie } from "../utils/nextAuthCookie.ts";
|
||||
import { formatTranslatedStreamError } from "../utils/streamErrorFormat.ts";
|
||||
import {
|
||||
@@ -32,6 +32,23 @@ import {
|
||||
sseChunk,
|
||||
} from "./perplexity-web/protocol.ts";
|
||||
|
||||
const PPLX_PUBLIC_UPSTREAM_ERROR = "Perplexity upstream error";
|
||||
|
||||
/** Project an unknown upstream failure onto a stable, public-safe message (Hard Rule #12). */
|
||||
function sanitizePerplexityUpstreamError(message: unknown): string {
|
||||
const sanitized = sanitizeErrorMessage(message);
|
||||
return sanitized.trim() && !/^(?:[A-Za-z_$][\w$]*)?Error:\s*$/.test(sanitized)
|
||||
? sanitized
|
||||
: PPLX_PUBLIC_UPSTREAM_ERROR;
|
||||
}
|
||||
|
||||
/** Project a provider-controlled error code onto the bounded public identifier vocabulary. */
|
||||
export function toPublicPerplexityErrorCode(errorCode: unknown, isQuota: boolean): string {
|
||||
if (isQuota) return "quota_exhausted";
|
||||
if (typeof errorCode !== "string" || errorCode.length > 64) return "PPLX_ERROR";
|
||||
return projectPublicErrorIdentifier(errorCode, "PPLX_ERROR");
|
||||
}
|
||||
|
||||
// ─── Session continuity ─────────────────────────────────────────────────────
|
||||
|
||||
const SESSION_MAX_AGE_MS = 3600_000;
|
||||
@@ -347,10 +364,10 @@ async function buildNonStreamingResponse(
|
||||
/quota exhausted/i.test(chunk.error) ||
|
||||
(typeof chunk.resetSeconds === "number" && chunk.resetSeconds > 0);
|
||||
const status = isQuota ? 429 : 502;
|
||||
const code = chunk.errorCode || (isQuota ? "quota_exhausted" : "PPLX_ERROR");
|
||||
const code = toPublicPerplexityErrorCode(chunk.errorCode, isQuota);
|
||||
const type = isQuota ? "quota_exhausted" : "upstream_error";
|
||||
const errBody: Record<string, unknown> = {
|
||||
message: chunk.error,
|
||||
message: sanitizePerplexityUpstreamError(chunk.error),
|
||||
type,
|
||||
code,
|
||||
};
|
||||
|
||||
@@ -439,9 +439,8 @@ export interface ContentChunk {
|
||||
/** Structured error code for quota / rate-limit surfaces (e.g. quota_exhausted). */
|
||||
errorCode?: string;
|
||||
/**
|
||||
* Suggested client/account cooldown in seconds when the stream failed due to
|
||||
* advanced-model weekly quota (or similar). Downstream marks the connection
|
||||
* rate_limited_until and VibeProxy limit badges parse this + "reset after Xs".
|
||||
* Suggested cooldown when quota is classified before the HTTP stream is committed.
|
||||
* Once SSE 200 starts, a late error cannot retroactively add status or Retry-After metadata.
|
||||
*/
|
||||
resetSeconds?: number;
|
||||
done?: boolean;
|
||||
@@ -797,6 +796,7 @@ export async function* extractContent(
|
||||
if (event.error_code || event.error_message) {
|
||||
yield {
|
||||
error: event.error_message || `Perplexity error: ${event.error_code}`,
|
||||
errorCode: event.error_code,
|
||||
done: true,
|
||||
};
|
||||
return;
|
||||
|
||||
@@ -216,7 +216,9 @@ import {
|
||||
createErrorResult,
|
||||
parseUpstreamError,
|
||||
formatProviderError,
|
||||
projectPublicErrorIdentifier,
|
||||
sanitizeErrorMessage,
|
||||
sanitizeUpstreamDetails,
|
||||
} from "../utils/error.ts";
|
||||
import {
|
||||
reportMalformed200,
|
||||
@@ -250,8 +252,11 @@ import { recordKeyHealthStatus as recordKeyHealthStatusFor } from "./chatCore/ke
|
||||
import { getSkillsModelIdForFormat } from "./chatCore/skillsFormat.ts";
|
||||
import { readNonStreamingResponseBody } from "./chatCore/nonStreamingResponseBody.ts";
|
||||
import {
|
||||
isSemaphoreCapacityError,
|
||||
createSafeAbortError,
|
||||
createStreamingErrorResult,
|
||||
isSemaphoreCapacityError,
|
||||
formatStreamRecoveryRetryWarning,
|
||||
getSafeErrorMetadata,
|
||||
getUpstreamErrorIdentifier,
|
||||
} from "./chatCore/streamErrorResult.ts";
|
||||
import { wrapReadableStreamWithFinalize } from "./chatCore/streamFinalize.ts";
|
||||
@@ -587,12 +592,11 @@ export async function handleChatCore({
|
||||
status: 409,
|
||||
});
|
||||
};
|
||||
const isManagedLeaseFenceError = (error: unknown): boolean =>
|
||||
managedLease !== null &&
|
||||
typeof (error as { code?: unknown })?.code === "string" &&
|
||||
String((error as { code: string }).code).startsWith("LEASE_");
|
||||
const managedLeaseFenceErrorResult = (error: unknown) => {
|
||||
const code = (error as { code: string }).code;
|
||||
const getManagedLeaseFenceErrorCode = (code: string | undefined): string | undefined => {
|
||||
if (managedLease === null) return undefined;
|
||||
return code?.startsWith("LEASE_") ? code : undefined;
|
||||
};
|
||||
const managedLeaseFenceErrorResult = (code: string) => {
|
||||
return {
|
||||
...createErrorResult(409, "Managed lease request fence rejected the dispatch", null, code),
|
||||
errorType: "lease_error",
|
||||
@@ -2544,19 +2548,35 @@ export async function handleChatCore({
|
||||
error instanceof Error ? error : new Error(String(error))
|
||||
);
|
||||
} catch (pluginErr) {
|
||||
log?.debug?.(
|
||||
"PLUGIN",
|
||||
`onError hook error (non-fatal): ${pluginErr instanceof Error ? pluginErr.message : String(pluginErr)}`
|
||||
);
|
||||
const pluginErrorMessage = sanitizeErrorMessage(pluginErr) || "Plugin onError hook failed";
|
||||
log?.debug?.("PLUGIN", `onError hook error (non-fatal): ${pluginErrorMessage}`);
|
||||
}
|
||||
|
||||
const parsedStatus = Number(error?.statusCode);
|
||||
let parsedStatus = Number.NaN;
|
||||
try {
|
||||
parsedStatus = Number(error?.statusCode);
|
||||
} catch {
|
||||
// Hostile thrown values may expose Symbols or throwing status accessors.
|
||||
}
|
||||
const statusCode =
|
||||
Number.isInteger(parsedStatus) && parsedStatus >= 400 && parsedStatus <= 599
|
||||
? parsedStatus
|
||||
: HTTP_STATUS.SERVER_ERROR;
|
||||
const message = error?.message || "Invalid request";
|
||||
const errorType = typeof error?.errorType === "string" ? error.errorType : null;
|
||||
let message = "Invalid request";
|
||||
try {
|
||||
const candidate = error?.message;
|
||||
message =
|
||||
(typeof candidate === "string" ? candidate : sanitizeErrorMessage(candidate)) || message;
|
||||
} catch {
|
||||
// Hostile thrown values may expose throwing property accessors.
|
||||
}
|
||||
let errorType: string | null = null;
|
||||
try {
|
||||
const candidate = error?.errorType;
|
||||
errorType = typeof candidate === "string" ? candidate : null;
|
||||
} catch {
|
||||
// Hostile thrown values may expose throwing classification accessors.
|
||||
}
|
||||
const result = createTranslationFailureResult(statusCode, message, errorType);
|
||||
log?.warn?.("TRANSLATE", `Request translation failed: ${result.error}`);
|
||||
|
||||
@@ -3461,9 +3481,11 @@ export async function handleChatCore({
|
||||
onRetry: (attempt, err) =>
|
||||
log?.warn?.(
|
||||
"STREAM_RECOVERY",
|
||||
`transparent early-retry ${attempt}/${STREAM_RECOVERY.EARLY_RETRY_MAX} after ${
|
||||
(err as { name?: string })?.name || "truncation"
|
||||
}`
|
||||
formatStreamRecoveryRetryWarning(
|
||||
attempt,
|
||||
STREAM_RECOVERY.EARLY_RETRY_MAX,
|
||||
err
|
||||
)
|
||||
),
|
||||
continueStream,
|
||||
...buildContinuationLogHooks(log),
|
||||
@@ -4247,15 +4269,20 @@ export async function handleChatCore({
|
||||
}
|
||||
} catch (error) {
|
||||
trackPendingRequest(model, provider, connectionId, false);
|
||||
if (isManagedLeaseFenceError(error)) return managedLeaseFenceErrorResult(error);
|
||||
const errorMetadata = getSafeErrorMetadata(error);
|
||||
const managedLeaseFenceCode = getManagedLeaseFenceErrorCode(errorMetadata.code);
|
||||
if (managedLeaseFenceCode) return managedLeaseFenceErrorResult(managedLeaseFenceCode);
|
||||
// isSemaphoreCapacityError already reads the code through getSafeErrorMetadata,
|
||||
// so a hostile rejection cannot escape this classification.
|
||||
if (isSemaphoreCapacityError(error)) {
|
||||
const semaphoreCode = errorMetadata.code as string;
|
||||
appendRequestLog({
|
||||
model,
|
||||
provider,
|
||||
connectionId,
|
||||
status: `FAILED ${error.code}`,
|
||||
status: `FAILED ${semaphoreCode}`,
|
||||
}).catch(() => {});
|
||||
const failureMessage = error.message || "Semaphore timeout";
|
||||
const failureMessage = sanitizeErrorMessage(errorMetadata.message) || "Semaphore timeout";
|
||||
persistAttemptLogs({
|
||||
status: HTTP_STATUS.RATE_LIMITED,
|
||||
error: failureMessage,
|
||||
@@ -4264,25 +4291,32 @@ export async function handleChatCore({
|
||||
claudeCacheMeta: claudePromptCacheLogMeta,
|
||||
cacheSource: "upstream",
|
||||
});
|
||||
persistFailureUsage(HTTP_STATUS.RATE_LIMITED, error.code);
|
||||
persistFailureUsage(HTTP_STATUS.RATE_LIMITED, semaphoreCode);
|
||||
const result = stream
|
||||
? createStreamingErrorResult(HTTP_STATUS.RATE_LIMITED, failureMessage, error.code)
|
||||
? createStreamingErrorResult(HTTP_STATUS.RATE_LIMITED, failureMessage, semaphoreCode)
|
||||
: createErrorResult(HTTP_STATUS.RATE_LIMITED, failureMessage);
|
||||
return {
|
||||
...result,
|
||||
errorType: "account_semaphore_capacity",
|
||||
errorCode: error.code,
|
||||
errorCode: semaphoreCode,
|
||||
};
|
||||
}
|
||||
// abort(reason) can reject with a raw string lacking `name`/`status`; classify
|
||||
// it through isLocalStreamLifecycleError so it maps to 499 rather than the
|
||||
// 502 provider-failure default.
|
||||
const isRequestAborted = isLocalStreamLifecycleError(error);
|
||||
let isRequestAborted = errorMetadata.name === "AbortError";
|
||||
if (!isRequestAborted) {
|
||||
try {
|
||||
isRequestAborted = isLocalStreamLifecycleError(error);
|
||||
} catch {
|
||||
// A hostile Proxy must not escape the provider-error boundary during classification.
|
||||
}
|
||||
}
|
||||
// #8376: proxyFetch tags unreachable transport failures so they remain
|
||||
// distinguishable from ordinary provider 5xx responses.
|
||||
const isProxyUnreachableFailure =
|
||||
!isRequestAborted && (error as { errorCode?: unknown })?.errorCode === "proxy_unreachable";
|
||||
const errorCode = getUpstreamErrorIdentifier(error);
|
||||
!isRequestAborted && errorMetadata.errorCode === "proxy_unreachable";
|
||||
const errorCode = errorMetadata.code;
|
||||
const localRateLimitFailure = localLimiterErrors.getClientSafeLocalRateLimitError(error);
|
||||
const failureStatus = isRequestAborted
|
||||
? 499
|
||||
@@ -4290,14 +4324,27 @@ export async function handleChatCore({
|
||||
? HTTP_STATUS.BAD_GATEWAY
|
||||
: localRateLimitFailure
|
||||
? localRateLimitFailure.status
|
||||
: error.name === "TimeoutError" || error.name === "BodyTimeoutError"
|
||||
: errorMetadata.name === "TimeoutError" || errorMetadata.name === "BodyTimeoutError"
|
||||
? HTTP_STATUS.GATEWAY_TIMEOUT
|
||||
: error.status && typeof error.status === "number"
|
||||
? error.status
|
||||
: errorMetadata.status
|
||||
? errorMetadata.status
|
||||
: HTTP_STATUS.BAD_GATEWAY;
|
||||
const failureMessage = isRequestAborted
|
||||
? "Request aborted"
|
||||
: formatProviderError(localRateLimitFailure ?? error, provider, model, failureStatus);
|
||||
: (() => {
|
||||
try {
|
||||
return formatProviderError(
|
||||
localRateLimitFailure ?? error,
|
||||
provider,
|
||||
model,
|
||||
failureStatus
|
||||
);
|
||||
} catch {
|
||||
// Formatting is diagnostic only; hostile rejection metadata falls back safely.
|
||||
return errorMetadata.message || "Upstream provider error";
|
||||
}
|
||||
})();
|
||||
const safeFailureMessage = sanitizeErrorMessage(failureMessage) || "Upstream provider error";
|
||||
const upstreamErrorCode =
|
||||
localRateLimitFailure?.code ??
|
||||
(isProxyUnreachableFailure ? "proxy_unreachable" : errorCode);
|
||||
@@ -4307,7 +4354,7 @@ export async function handleChatCore({
|
||||
// tags its pre-response timeout via the code below.)
|
||||
const isOwnDeadlineTimeout =
|
||||
failureStatus === HTTP_STATUS.GATEWAY_TIMEOUT &&
|
||||
(error.name === "TimeoutError" || error.name === "BodyTimeoutError");
|
||||
(errorMetadata.name === "TimeoutError" || errorMetadata.name === "BodyTimeoutError");
|
||||
const upstreamErrorType =
|
||||
upstreamErrorCode === ANTIGRAVITY_PRE_RESPONSE_TIMEOUT_CODE || isOwnDeadlineTimeout
|
||||
? "upstream_timeout"
|
||||
@@ -4322,7 +4369,7 @@ export async function handleChatCore({
|
||||
}).catch(() => {});
|
||||
persistAttemptLogs({
|
||||
status: failureStatus,
|
||||
error: failureMessage,
|
||||
error: safeFailureMessage,
|
||||
providerRequest: finalBody || translatedBody,
|
||||
// On a client-abort (AbortError), the client already disconnected before
|
||||
// we ever got here — this body is what we WOULD have sent, not what was
|
||||
@@ -4330,24 +4377,27 @@ export async function handleChatCore({
|
||||
// dashboard reads that field as "what the client received"), so omit it
|
||||
// for this case; `error` above already records the failure reason.
|
||||
clientResponse:
|
||||
error.name === "AbortError" ? undefined : buildErrorBody(failureStatus, failureMessage),
|
||||
errorMetadata.name === "AbortError"
|
||||
? undefined
|
||||
: buildErrorBody(failureStatus, failureMessage),
|
||||
claudeCacheMeta: claudePromptCacheLogMeta,
|
||||
cacheSource: "upstream",
|
||||
});
|
||||
if (isRequestAborted) {
|
||||
streamController.handleError(error);
|
||||
streamController.handleError(createSafeAbortError());
|
||||
return createErrorResult(499, "Request aborted");
|
||||
}
|
||||
const persistentErrorCode = projectFailureUsageErrorCode({
|
||||
statusCode: failureStatus,
|
||||
message: failureMessage,
|
||||
errorCode:
|
||||
upstreamErrorCode ||
|
||||
(error instanceof Error && error.name ? error.name : "upstream_error"),
|
||||
errorCode: projectPublicErrorIdentifier(
|
||||
upstreamErrorCode || errorMetadata.name,
|
||||
"upstream_error"
|
||||
),
|
||||
errorType: upstreamErrorType,
|
||||
});
|
||||
persistFailureUsage(failureStatus, persistentErrorCode);
|
||||
console.log(`${COLORS.red}[ERROR] ${failureMessage}${COLORS.reset}`);
|
||||
console.log(`${COLORS.red}[ERROR] ${safeFailureMessage}${COLORS.reset}`);
|
||||
if (stream && upstreamErrorCode) {
|
||||
const result = createStreamingErrorResult(
|
||||
failureStatus,
|
||||
@@ -4509,7 +4559,10 @@ export async function handleChatCore({
|
||||
upstreamErrorParsed = false; // Let it be parsed downstream
|
||||
}
|
||||
} catch (retryErr) {
|
||||
if (isManagedLeaseFenceError(retryErr)) return managedLeaseFenceErrorResult(retryErr);
|
||||
const retryLeaseFenceCode = getManagedLeaseFenceErrorCode(
|
||||
getUpstreamErrorIdentifier(retryErr)
|
||||
);
|
||||
if (retryLeaseFenceCode) return managedLeaseFenceErrorResult(retryLeaseFenceCode);
|
||||
// Refresh succeeded but the retry leg failed (network blip, AbortError,
|
||||
// executor throw). Don't swallow — the operator-visible signal "the user
|
||||
// saw 401 even though auth was actually fixed" is much more confusing
|
||||
@@ -4569,8 +4622,8 @@ export async function handleChatCore({
|
||||
message = details.message;
|
||||
retryAfterMs = details.retryAfterMs;
|
||||
upstreamErrorBody = details.responseBody;
|
||||
upstreamErrorCode = details.errorCode as string | undefined;
|
||||
upstreamErrorType = details.errorType as string | undefined;
|
||||
upstreamErrorCode = typeof details.errorCode === "string" ? details.errorCode : undefined;
|
||||
upstreamErrorType = typeof details.errorType === "string" ? details.errorType : undefined;
|
||||
}
|
||||
|
||||
// Gateways like agentrouter misstate temporary quota exhaustion as 403/400,
|
||||
@@ -4628,8 +4681,14 @@ export async function handleChatCore({
|
||||
message = signatureRecovery.error.message;
|
||||
retryAfterMs = signatureRecovery.error.retryAfterMs;
|
||||
upstreamErrorBody = signatureRecovery.error.responseBody;
|
||||
upstreamErrorCode = signatureRecovery.error.errorCode as string | undefined;
|
||||
upstreamErrorType = signatureRecovery.error.errorType as string | undefined;
|
||||
upstreamErrorCode =
|
||||
typeof signatureRecovery.error.errorCode === "string"
|
||||
? signatureRecovery.error.errorCode
|
||||
: undefined;
|
||||
upstreamErrorType =
|
||||
typeof signatureRecovery.error.errorType === "string"
|
||||
? signatureRecovery.error.errorType
|
||||
: undefined;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4680,7 +4739,9 @@ export async function handleChatCore({
|
||||
}).catch(() => {});
|
||||
|
||||
const errMsg = formatProviderError(new Error(message), provider, model, statusCode);
|
||||
console.log(`${COLORS.red}[ERROR] ${errMsg}${COLORS.reset}`);
|
||||
const safeErrMsg = sanitizeErrorMessage(errMsg) || "Upstream provider error";
|
||||
const safeUpstreamErrorBody = sanitizeUpstreamDetails(upstreamErrorBody);
|
||||
console.log(`${COLORS.red}[ERROR] ${safeErrMsg}${COLORS.reset}`);
|
||||
|
||||
// Log Antigravity retry time if available
|
||||
if (retryAfterMs && provider === "antigravity") {
|
||||
@@ -4694,7 +4755,7 @@ export async function handleChatCore({
|
||||
providerResponse.status,
|
||||
providerResponse.statusText,
|
||||
providerResponse.headers,
|
||||
upstreamErrorBody
|
||||
safeUpstreamErrorBody
|
||||
);
|
||||
|
||||
// Rate limiter updated in applyProviderFailureClassification
|
||||
@@ -4735,9 +4796,9 @@ export async function handleChatCore({
|
||||
// Fallback also failed — return original error
|
||||
persistAttemptLogs({
|
||||
status: statusCode,
|
||||
error: errMsg,
|
||||
error: safeErrMsg,
|
||||
providerRequest: finalBody || translatedBody,
|
||||
providerResponse: upstreamErrorBody,
|
||||
providerResponse: safeUpstreamErrorBody,
|
||||
clientResponse: buildErrorBody(statusCode, errMsg),
|
||||
cacheSource: "upstream",
|
||||
});
|
||||
@@ -4755,9 +4816,9 @@ export async function handleChatCore({
|
||||
} catch {
|
||||
persistAttemptLogs({
|
||||
status: statusCode,
|
||||
error: errMsg,
|
||||
error: safeErrMsg,
|
||||
providerRequest: finalBody || translatedBody,
|
||||
providerResponse: upstreamErrorBody,
|
||||
providerResponse: safeUpstreamErrorBody,
|
||||
clientResponse: buildErrorBody(statusCode, errMsg),
|
||||
cacheSource: "upstream",
|
||||
});
|
||||
@@ -4775,9 +4836,9 @@ export async function handleChatCore({
|
||||
} else {
|
||||
persistAttemptLogs({
|
||||
status: statusCode,
|
||||
error: errMsg,
|
||||
error: safeErrMsg,
|
||||
providerRequest: finalBody || translatedBody,
|
||||
providerResponse: upstreamErrorBody,
|
||||
providerResponse: safeUpstreamErrorBody,
|
||||
clientResponse: buildErrorBody(statusCode, errMsg),
|
||||
cacheSource: "upstream",
|
||||
});
|
||||
@@ -4827,9 +4888,9 @@ export async function handleChatCore({
|
||||
} else {
|
||||
persistAttemptLogs({
|
||||
status: statusCode,
|
||||
error: errMsg,
|
||||
error: safeErrMsg,
|
||||
providerRequest: finalBody || translatedBody,
|
||||
providerResponse: upstreamErrorBody,
|
||||
providerResponse: safeUpstreamErrorBody,
|
||||
clientResponse: buildErrorBody(statusCode, errMsg),
|
||||
cacheSource: "upstream",
|
||||
});
|
||||
@@ -4847,9 +4908,9 @@ export async function handleChatCore({
|
||||
} catch {
|
||||
persistAttemptLogs({
|
||||
status: statusCode,
|
||||
error: errMsg,
|
||||
error: safeErrMsg,
|
||||
providerRequest: finalBody || translatedBody,
|
||||
providerResponse: upstreamErrorBody,
|
||||
providerResponse: safeUpstreamErrorBody,
|
||||
clientResponse: buildErrorBody(statusCode, errMsg),
|
||||
cacheSource: "upstream",
|
||||
});
|
||||
@@ -4867,9 +4928,9 @@ export async function handleChatCore({
|
||||
} else {
|
||||
persistAttemptLogs({
|
||||
status: statusCode,
|
||||
error: errMsg,
|
||||
error: safeErrMsg,
|
||||
providerRequest: finalBody || translatedBody,
|
||||
providerResponse: upstreamErrorBody,
|
||||
providerResponse: safeUpstreamErrorBody,
|
||||
clientResponse: buildErrorBody(statusCode, errMsg),
|
||||
cacheSource: "upstream",
|
||||
});
|
||||
@@ -4887,9 +4948,9 @@ export async function handleChatCore({
|
||||
} else {
|
||||
persistAttemptLogs({
|
||||
status: statusCode,
|
||||
error: errMsg,
|
||||
error: safeErrMsg,
|
||||
providerRequest: finalBody || translatedBody,
|
||||
providerResponse: upstreamErrorBody,
|
||||
providerResponse: safeUpstreamErrorBody,
|
||||
clientResponse: buildErrorBody(statusCode, errMsg),
|
||||
cacheSource: "upstream",
|
||||
});
|
||||
@@ -5488,14 +5549,16 @@ export async function handleChatCore({
|
||||
undefined,
|
||||
{ code: malformed.code, type: malformed.type }
|
||||
);
|
||||
const sanitizedMalformedResponse = sanitizeUpstreamDetails(responseBody);
|
||||
const sanitizedMalformedProviderResponse = looksLikeSSE
|
||||
? { _streamed: true, _format: "sse-json", summary: sanitizedMalformedResponse }
|
||||
: sanitizedMalformedResponse;
|
||||
persistAttemptLogs({
|
||||
status: HTTP_STATUS.BAD_GATEWAY,
|
||||
tokens: usage,
|
||||
responseBody,
|
||||
responseBody: sanitizedMalformedResponse,
|
||||
providerRequest: finalBody || translatedBody,
|
||||
providerResponse: looksLikeSSE
|
||||
? { _streamed: true, _format: "sse-json", summary: responseBody }
|
||||
: responseBody,
|
||||
providerResponse: sanitizedMalformedProviderResponse,
|
||||
clientResponse: malformedClientBody,
|
||||
claudeCacheMeta: claudePromptCacheLogMeta,
|
||||
claudeCacheUsageMeta: cacheUsageLogMeta,
|
||||
@@ -5672,15 +5735,20 @@ export async function handleChatCore({
|
||||
};
|
||||
} catch (error) {
|
||||
trackPendingRequest(model, provider, connectionId, false);
|
||||
if (isManagedLeaseFenceError(error)) return managedLeaseFenceErrorResult(error);
|
||||
const errorMetadata = getSafeErrorMetadata(error);
|
||||
const managedLeaseFenceCode = getManagedLeaseFenceErrorCode(errorMetadata.code);
|
||||
if (managedLeaseFenceCode) return managedLeaseFenceErrorResult(managedLeaseFenceCode);
|
||||
// isSemaphoreCapacityError already reads the code through getSafeErrorMetadata,
|
||||
// so a hostile rejection cannot escape this classification.
|
||||
if (isSemaphoreCapacityError(error)) {
|
||||
const semaphoreCode = errorMetadata.code as string;
|
||||
appendRequestLog({
|
||||
model,
|
||||
provider,
|
||||
connectionId,
|
||||
status: `FAILED ${error.code}`,
|
||||
status: `FAILED ${semaphoreCode}`,
|
||||
}).catch(() => {});
|
||||
const failureMessage = error.message || "Semaphore timeout";
|
||||
const failureMessage = sanitizeErrorMessage(errorMetadata.message) || "Semaphore timeout";
|
||||
persistAttemptLogs({
|
||||
status: HTTP_STATUS.RATE_LIMITED,
|
||||
error: failureMessage,
|
||||
@@ -5689,12 +5757,12 @@ export async function handleChatCore({
|
||||
claudeCacheMeta: claudePromptCacheLogMeta,
|
||||
cacheSource: "upstream",
|
||||
});
|
||||
persistFailureUsage(HTTP_STATUS.RATE_LIMITED, error.code);
|
||||
persistFailureUsage(HTTP_STATUS.RATE_LIMITED, semaphoreCode);
|
||||
const result = createErrorResult(HTTP_STATUS.RATE_LIMITED, failureMessage);
|
||||
return {
|
||||
...result,
|
||||
errorType: "account_semaphore_capacity",
|
||||
errorCode: error.code,
|
||||
errorCode: semaphoreCode,
|
||||
};
|
||||
}
|
||||
throw error;
|
||||
|
||||
@@ -4,19 +4,65 @@
|
||||
*
|
||||
* Extracted from chatCore: identify semaphore capacity errors, build a sanitized SSE error result
|
||||
* (an `data: {...}\n\ndata: [DONE]\n\n` body wrapped in an event-stream Response), and pull a string
|
||||
* error code off an unknown error. Side-effect-free; behaviour is byte-identical to the previous
|
||||
* module-level functions.
|
||||
* error code off an unknown error. The status and SSE envelope remain stable while every public
|
||||
* message/code/type crosses the canonical sanitizer; raw internal fields stay outside this body.
|
||||
*/
|
||||
|
||||
import { buildErrorBody } from "../../utils/error.ts";
|
||||
import { buildErrorBody, sanitizeErrorMessage } from "../../utils/error.ts";
|
||||
|
||||
export interface SafeErrorMetadata {
|
||||
code?: string;
|
||||
errorCode?: string;
|
||||
message?: string;
|
||||
name?: string;
|
||||
status?: number;
|
||||
}
|
||||
|
||||
function readErrorProperty(error: object, property: string): unknown {
|
||||
try {
|
||||
return Reflect.get(error, property);
|
||||
} catch {
|
||||
// Provider rejections may be hostile Proxies; public error handling fails closed per field.
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
export function getSafeErrorMetadata(error: unknown): SafeErrorMetadata {
|
||||
if (error === null || (typeof error !== "object" && typeof error !== "function")) {
|
||||
return {};
|
||||
}
|
||||
const code = readErrorProperty(error, "code");
|
||||
const errorCode = readErrorProperty(error, "errorCode");
|
||||
const message = readErrorProperty(error, "message");
|
||||
const name = readErrorProperty(error, "name");
|
||||
const status = readErrorProperty(error, "status");
|
||||
return {
|
||||
code: typeof code === "string" && code.length > 0 ? code : undefined,
|
||||
errorCode: typeof errorCode === "string" && errorCode.length > 0 ? errorCode : undefined,
|
||||
message: typeof message === "string" && message.length > 0 ? message : undefined,
|
||||
name: typeof name === "string" && name.length > 0 ? name : undefined,
|
||||
status: typeof status === "number" && Number.isFinite(status) ? status : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
export function createSafeAbortError(): Error {
|
||||
const error = new Error("Request aborted");
|
||||
error.name = "AbortError";
|
||||
return error;
|
||||
}
|
||||
|
||||
export function formatStreamRecoveryRetryWarning(
|
||||
attempt: number,
|
||||
maxAttempts: number,
|
||||
error: unknown
|
||||
): string {
|
||||
const safeName = sanitizeErrorMessage(getSafeErrorMetadata(error).name) || "truncation";
|
||||
return `transparent early-retry ${attempt}/${maxAttempts} after ${safeName}`;
|
||||
}
|
||||
|
||||
export function isSemaphoreCapacityError(error: unknown): error is Error & { code: string } {
|
||||
return (
|
||||
!!error &&
|
||||
typeof error === "object" &&
|
||||
((error as { code?: unknown }).code === "SEMAPHORE_TIMEOUT" ||
|
||||
(error as { code?: unknown }).code === "SEMAPHORE_QUEUE_FULL")
|
||||
);
|
||||
const code = getSafeErrorMetadata(error).code;
|
||||
return code === "SEMAPHORE_TIMEOUT" || code === "SEMAPHORE_QUEUE_FULL";
|
||||
}
|
||||
|
||||
export function createStreamingErrorResult(
|
||||
@@ -46,7 +92,5 @@ export function createStreamingErrorResult(
|
||||
}
|
||||
|
||||
export function getUpstreamErrorIdentifier(error: unknown): string | undefined {
|
||||
if (!error || typeof error !== "object") return undefined;
|
||||
const value = (error as { code?: unknown }).code;
|
||||
return typeof value === "string" && value.length > 0 ? value : undefined;
|
||||
return getSafeErrorMetadata(error).code;
|
||||
}
|
||||
|
||||
@@ -172,6 +172,7 @@ const SAFE_PUBLIC_ERROR_IDENTIFIERS = new Set([
|
||||
"lease_unsupported_route",
|
||||
"lease_unsupported_transport",
|
||||
"lmarena_error",
|
||||
"lmarena_stream_error",
|
||||
"message_limit",
|
||||
"meta_ai_empty_response",
|
||||
"meta_ai_mode_switch_failed",
|
||||
|
||||
@@ -7,8 +7,11 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
createSafeAbortError,
|
||||
formatStreamRecoveryRetryWarning,
|
||||
isSemaphoreCapacityError,
|
||||
createStreamingErrorResult,
|
||||
getSafeErrorMetadata,
|
||||
getUpstreamErrorIdentifier,
|
||||
} from "../../open-sse/handlers/chatCore/streamErrorResult.ts";
|
||||
|
||||
@@ -20,6 +23,35 @@ test("isSemaphoreCapacityError matches the two semaphore codes only", () => {
|
||||
assert.equal(isSemaphoreCapacityError("SEMAPHORE_TIMEOUT"), false);
|
||||
});
|
||||
|
||||
test("formatStreamRecoveryRetryWarning sanitizes hostile error names", () => {
|
||||
const secret = "STREAM_RECOVERY_NAME_SECRET";
|
||||
const credentialName = new Proxy(
|
||||
{},
|
||||
{
|
||||
get(_target, key) {
|
||||
if (key === "name") return `password=${secret} /home/alice/recovery.ts`;
|
||||
throw new Error("hostile recovery metadata");
|
||||
},
|
||||
}
|
||||
);
|
||||
const credentialWarning = formatStreamRecoveryRetryWarning(1, 4, credentialName);
|
||||
assert.doesNotMatch(credentialWarning, /STREAM_RECOVERY_NAME_SECRET|\/home\/alice/);
|
||||
assert.match(credentialWarning, /\[REDACTED\]/);
|
||||
|
||||
const hostileGetter = new Proxy(
|
||||
{},
|
||||
{
|
||||
get() {
|
||||
throw new Error("hostile name getter");
|
||||
},
|
||||
}
|
||||
);
|
||||
assert.equal(
|
||||
formatStreamRecoveryRetryWarning(2, 4, hostileGetter),
|
||||
"transparent early-retry 2/4 after truncation"
|
||||
);
|
||||
});
|
||||
|
||||
test("createStreamingErrorResult builds an SSE error envelope with [DONE] terminator", async () => {
|
||||
const result = createStreamingErrorResult(503, "boom");
|
||||
assert.equal(result.success, false);
|
||||
@@ -60,12 +92,66 @@ test("createStreamingErrorResult sanitizes code and type at the SSE boundary", a
|
||||
assert.doesNotMatch(body, /sk-live-secret-value|X-Leak/);
|
||||
});
|
||||
|
||||
test("createStreamingErrorResult sanitizes message, code, and type at the SSE boundary", async () => {
|
||||
const secret = "STREAM_RESULT_SECRET";
|
||||
const result = createStreamingErrorResult(
|
||||
502,
|
||||
`upstream password=${secret} at /home/alice/stream.ts:10:2`,
|
||||
"password_hunter2",
|
||||
"authorization_BearerSecret"
|
||||
);
|
||||
const body = await result.response.text();
|
||||
const json = JSON.parse(body.slice("data: ".length, body.indexOf("\n\n")));
|
||||
|
||||
assert.equal(json.error.message, "upstream password=[REDACTED]");
|
||||
assert.equal(json.error.code, "bad_gateway");
|
||||
assert.equal(json.error.type, "server_error");
|
||||
assert.doesNotMatch(body, new RegExp(`${secret}|/home/alice|\\bat \\S`));
|
||||
assert.equal(result.error, `upstream password=${secret} at /home/alice/stream.ts:10:2`);
|
||||
});
|
||||
|
||||
test("getUpstreamErrorIdentifier returns a non-empty string code or undefined", () => {
|
||||
assert.equal(getUpstreamErrorIdentifier({ code: "ECONNRESET" }), "ECONNRESET");
|
||||
assert.equal(getUpstreamErrorIdentifier({ code: "" }), undefined);
|
||||
assert.equal(getUpstreamErrorIdentifier({ code: 123 }), undefined);
|
||||
assert.equal(getUpstreamErrorIdentifier(null), undefined);
|
||||
assert.equal(getUpstreamErrorIdentifier("ECONNRESET"), undefined);
|
||||
const hostile = new Proxy(
|
||||
{},
|
||||
{
|
||||
get() {
|
||||
throw new Error("hostile code getter");
|
||||
},
|
||||
}
|
||||
);
|
||||
assert.doesNotThrow(() => isSemaphoreCapacityError(hostile));
|
||||
assert.equal(getUpstreamErrorIdentifier(hostile), undefined);
|
||||
const hostileAbort = new Proxy(
|
||||
{},
|
||||
{
|
||||
get(_target, key) {
|
||||
if (key === "name") return "AbortError";
|
||||
throw new Error("hostile abort metadata");
|
||||
},
|
||||
}
|
||||
);
|
||||
assert.equal(getSafeErrorMetadata(hostileAbort).name, "AbortError");
|
||||
let codeReads = 0;
|
||||
const mutableCode = new Proxy(
|
||||
{},
|
||||
{
|
||||
get(_target, key) {
|
||||
if (key !== "code") return undefined;
|
||||
codeReads += 1;
|
||||
return codeReads === 1 ? "SEMAPHORE_TIMEOUT" : "OTHER";
|
||||
},
|
||||
}
|
||||
);
|
||||
assert.equal(isSemaphoreCapacityError(mutableCode), true);
|
||||
assert.equal(codeReads, 1);
|
||||
const safeAbort = createSafeAbortError();
|
||||
assert.equal(safeAbort.name, "AbortError");
|
||||
assert.equal(safeAbort.message, "Request aborted");
|
||||
});
|
||||
|
||||
test("non-streaming runNonStreamingProviderLeg is inside a try that maps semaphore errors", async () => {
|
||||
|
||||
@@ -37,6 +37,27 @@ function installNotionTlsMock(
|
||||
return () => __setTlsFetchOverrideForTesting(null);
|
||||
}
|
||||
|
||||
function hostilePrototypeFailure(label: string): unknown {
|
||||
return new Proxy(
|
||||
{},
|
||||
{
|
||||
getPrototypeOf() {
|
||||
throw new Error(`access_token=${label}-prototype-secret at /srv/private/${label}.ts:1:2`);
|
||||
},
|
||||
get(_target, property) {
|
||||
if (property === "toString") {
|
||||
return () => {
|
||||
throw new Error(
|
||||
`access_token=${label}-coercion-secret at /srv/private/${label}-coercion.ts:1:2`
|
||||
);
|
||||
};
|
||||
}
|
||||
return undefined;
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
describe("NotionWebExecutor — registry consistency", () => {
|
||||
it("is present in WEB_COOKIE_PROVIDERS with the expected shape", () => {
|
||||
const entry = (WEB_COOKIE_PROVIDERS as Record<string, Record<string, unknown>>)["notion-web"];
|
||||
@@ -427,6 +448,68 @@ describe("NotionWebExecutor — upstream translation (mocked TLS fetch)", () =>
|
||||
}
|
||||
});
|
||||
|
||||
it("sanitizes credentials, private paths, and stack frames from TLS transport errors", async () => {
|
||||
const executor = new mod.NotionWebExecutor();
|
||||
const restore = installNotionTlsMock(async () => {
|
||||
throw new Error(
|
||||
"TLS transport failed; access token: notion-tls-secret while reading " +
|
||||
"/srv/private/notion/request.ts:42:7\n at send (/srv/private/notion/request.ts:42:7)"
|
||||
);
|
||||
});
|
||||
try {
|
||||
const result = await executor.execute({
|
||||
model: "notion-ai",
|
||||
body: { messages: [{ role: "user", content: "hi" }] },
|
||||
stream: false,
|
||||
credentials: { apiKey: COOKIE_WITH_SPACE },
|
||||
signal: null,
|
||||
} as never);
|
||||
|
||||
assert.equal(result.response.status, 502);
|
||||
assert.equal(result.url, "https://app.notion.com/api/v3/runInferenceTranscript");
|
||||
assert.equal(
|
||||
(result.transformedBody as { spaceId?: string }).spaceId,
|
||||
"space-1",
|
||||
"the executor must preserve the upstream request body on transport failure"
|
||||
);
|
||||
const errBody = (await result.response.json()) as {
|
||||
error: { message: string; type: string; code: string };
|
||||
};
|
||||
assert.match(errBody.error.message, /Notion fetch failed: TLS transport failed/);
|
||||
assert.equal(errBody.error.type, "upstream_error");
|
||||
assert.equal(errBody.error.code, "HTTP_502");
|
||||
assert.ok(!errBody.error.message.includes("notion-tls-secret"));
|
||||
assert.ok(!errBody.error.message.includes("/srv/private"));
|
||||
assert.ok(!errBody.error.message.includes("at send"));
|
||||
} finally {
|
||||
restore();
|
||||
}
|
||||
});
|
||||
|
||||
it("fails closed when a TLS rejection has a hostile prototype", async () => {
|
||||
const executor = new mod.NotionWebExecutor();
|
||||
const restoreTls = installNotionTlsMock(async () => {
|
||||
throw hostilePrototypeFailure("notion-tls-proxy");
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await executor.execute({
|
||||
model: "notion-ai",
|
||||
body: { messages: [{ role: "user", content: "hi" }] },
|
||||
stream: false,
|
||||
credentials: { apiKey: COOKIE_WITH_SPACE },
|
||||
signal: null,
|
||||
} as never);
|
||||
|
||||
assert.equal(result.response.status, 502);
|
||||
const responseText = await result.response.text();
|
||||
assert.match(responseText, /Notion fetch failed: unknown error/);
|
||||
assert.doesNotMatch(responseText, /prototype-secret|coercion-secret|\/srv\/private/);
|
||||
} finally {
|
||||
restoreTls();
|
||||
}
|
||||
});
|
||||
|
||||
it("surfaces nested patch-start temporarily-unavailable as a typed error (not empty-body 502)", async () => {
|
||||
const executor = new mod.NotionWebExecutor();
|
||||
const restore = installNotionTlsMock(async () => ({
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { __setTlsFetchOverrideForTesting } from "../../open-sse/services/grokTlsClient.ts";
|
||||
import {
|
||||
__setTlsFetchOverrideForTesting,
|
||||
TlsClientUnavailableError,
|
||||
} from "../../open-sse/services/grokTlsClient.ts";
|
||||
|
||||
const { GrokWebExecutor } = await import("../../open-sse/executors/grok-web.ts");
|
||||
const { getExecutor, hasSpecializedExecutor } = await import("../../open-sse/executors/index.ts");
|
||||
@@ -40,6 +43,15 @@ function mockFetch(status: number, events: unknown[]) {
|
||||
};
|
||||
}
|
||||
|
||||
function mockFetchError(error: unknown) {
|
||||
__setTlsFetchOverrideForTesting(async () => {
|
||||
throw error;
|
||||
});
|
||||
return () => {
|
||||
__setTlsFetchOverrideForTesting(null);
|
||||
};
|
||||
}
|
||||
|
||||
function mockFetchCapture(events: unknown[]) {
|
||||
let capturedUrl: string | null = null;
|
||||
let capturedHeaders: Record<string, string> = {};
|
||||
@@ -77,6 +89,11 @@ const SIMPLE_RESPONSE = [
|
||||
{ result: { response: { modelResponse: { message: "Hello world!", responseId: "resp-123" } } } },
|
||||
];
|
||||
|
||||
const SENSITIVE_GROK_UPSTREAM_ERROR =
|
||||
"Grok event failed at /srv/private/grok-secret.ts:41:9; " +
|
||||
"access_token=grok-upstream-secret\n" +
|
||||
" at SecretGrokFrame (/srv/private/grok-stack.ts:3:4)";
|
||||
|
||||
test.afterEach(() => {
|
||||
__setTlsFetchOverrideForTesting(null);
|
||||
});
|
||||
@@ -2496,6 +2513,135 @@ test("Error: Grok stream error returns 502", async () => {
|
||||
}
|
||||
});
|
||||
|
||||
test("Error: non-streaming sanitizes upstream event errors before JSON output", async () => {
|
||||
const restore = mockFetch(200, [
|
||||
{ error: { message: SENSITIVE_GROK_UPSTREAM_ERROR, code: "500" } },
|
||||
]);
|
||||
try {
|
||||
const executor = new GrokWebExecutor();
|
||||
const result = await executor.execute({
|
||||
model: "grok-4",
|
||||
body: { messages: [{ role: "user", content: "test" }], stream: false },
|
||||
stream: false,
|
||||
credentials: { apiKey: "test" },
|
||||
signal: AbortSignal.timeout(10000),
|
||||
log: null,
|
||||
});
|
||||
|
||||
assert.equal(result.response.status, 502);
|
||||
const payloadText = await result.response.text();
|
||||
const json = JSON.parse(payloadText);
|
||||
assert.equal(json.error?.type, "upstream_error");
|
||||
assert.equal(json.error?.code, "GROK_ERROR");
|
||||
assert.match(String(json.error?.message || ""), /Grok event failed/);
|
||||
assert.doesNotMatch(payloadText, /\/srv\/private\/grok-(?:secret|stack)\.ts/);
|
||||
assert.doesNotMatch(payloadText, /grok-upstream-secret|SecretGrokFrame/);
|
||||
} finally {
|
||||
restore();
|
||||
}
|
||||
});
|
||||
|
||||
test("Error: fetch failures sanitize sensitive details in logs and JSON", async (t) => {
|
||||
const cases = [
|
||||
{
|
||||
name: "generic fetch rejection",
|
||||
error: new Proxy(new Error(SENSITIVE_GROK_UPSTREAM_ERROR), {
|
||||
getPrototypeOf() {
|
||||
throw new Error("grok-prototype-secret");
|
||||
},
|
||||
}),
|
||||
expectedMessage: /Grok connection failed/,
|
||||
expectedCode: undefined,
|
||||
},
|
||||
{
|
||||
name: "TLS client unavailable",
|
||||
error: new TlsClientUnavailableError(SENSITIVE_GROK_UPSTREAM_ERROR),
|
||||
expectedMessage: /Grok TLS client unavailable/,
|
||||
expectedCode: "TLS_CLIENT_UNAVAILABLE",
|
||||
},
|
||||
];
|
||||
|
||||
for (const testCase of cases) {
|
||||
await t.test(testCase.name, async () => {
|
||||
const errorLogs: string[] = [];
|
||||
const restore = mockFetchError(testCase.error);
|
||||
try {
|
||||
const executor = new GrokWebExecutor();
|
||||
const result = await executor.execute({
|
||||
model: "grok-4",
|
||||
body: { messages: [{ role: "user", content: "test" }] },
|
||||
stream: false,
|
||||
credentials: { apiKey: "test" },
|
||||
signal: AbortSignal.timeout(10000),
|
||||
log: {
|
||||
error: (_tag, message) => errorLogs.push(String(message)),
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(result.response.status, 502);
|
||||
assert.equal(errorLogs.length, 1);
|
||||
const responseText = await result.response.text();
|
||||
const json = JSON.parse(responseText);
|
||||
assert.equal(json.error?.type, "upstream_error");
|
||||
assert.match(String(json.error?.message || ""), testCase.expectedMessage);
|
||||
assert.equal(json.error?.code, testCase.expectedCode);
|
||||
const publicOutput = `${errorLogs.join("\n")}\n${responseText}`;
|
||||
assert.doesNotMatch(publicOutput, /\/srv\/private\/grok-(?:secret|stack)\.ts/);
|
||||
assert.doesNotMatch(publicOutput, /grok-(?:upstream|prototype)-secret|SecretGrokFrame/);
|
||||
assert.doesNotMatch(responseText, /"(?:stack|cause)"\s*:/i);
|
||||
} finally {
|
||||
restore();
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
test("Error: blank fetch failures use a stable fallback in logs and JSON", async (t) => {
|
||||
const cases = [
|
||||
{
|
||||
name: "generic fetch rejection",
|
||||
error: new Error("\n at SecretOnlyFrame (/srv/private/grok-fetch-stack-only.ts:2:3)"),
|
||||
expectedLog: "Fetch failed: Grok upstream error",
|
||||
expectedMessage: "Grok connection failed: Grok upstream error",
|
||||
},
|
||||
{
|
||||
name: "TLS client unavailable",
|
||||
error: new TlsClientUnavailableError(
|
||||
"\n at SecretOnlyFrame (/srv/private/grok-tls-stack-only.ts:2:3)"
|
||||
),
|
||||
expectedLog: "TLS client unavailable: Grok upstream error",
|
||||
expectedMessage: "Grok TLS client unavailable: Grok upstream error",
|
||||
},
|
||||
];
|
||||
|
||||
for (const testCase of cases) {
|
||||
await t.test(testCase.name, async () => {
|
||||
const errorLogs: string[] = [];
|
||||
const restore = mockFetchError(testCase.error);
|
||||
try {
|
||||
const executor = new GrokWebExecutor();
|
||||
const result = await executor.execute({
|
||||
model: "grok-4",
|
||||
body: { messages: [{ role: "user", content: "test" }] },
|
||||
stream: false,
|
||||
credentials: { apiKey: "test" },
|
||||
signal: AbortSignal.timeout(10000),
|
||||
log: {
|
||||
error: (_tag, message) => errorLogs.push(String(message)),
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(result.response.status, 502);
|
||||
assert.deepEqual(errorLogs, [testCase.expectedLog]);
|
||||
const json = JSON.parse(await result.response.text());
|
||||
assert.equal(json.error?.message, testCase.expectedMessage);
|
||||
} finally {
|
||||
restore();
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// ─── Auth headers ───────────────────────────────────────────────────────────
|
||||
|
||||
test("Auth: cookie sends sso= header", async () => {
|
||||
|
||||
@@ -20,7 +20,10 @@ import {
|
||||
parseLMArenaInitialModels,
|
||||
pickLMArenaModelId,
|
||||
} from "../../open-sse/executors/lmarena.ts";
|
||||
import { clearLMArenaDeadCatalogModels } from "../../open-sse/executors/lmarena/models.ts";
|
||||
import {
|
||||
clearLMArenaDeadCatalogModels,
|
||||
resolveLMArenaModelId,
|
||||
} from "../../open-sse/executors/lmarena/models.ts";
|
||||
import { __setTlsFetchOverrideForTesting } from "../../open-sse/services/lmarenaTlsClient.ts";
|
||||
|
||||
const TEST_ARENA_MODEL_ID = "019e080d-c29d-7d9a-aa54-faed41da0763";
|
||||
@@ -30,7 +33,11 @@ const UUID_V7_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9
|
||||
type LMArenaExecutorTestAccess = {
|
||||
provider: string;
|
||||
buildUrl: (model: string, credentials: unknown) => string;
|
||||
buildRequestHeaders: (model: string, credentials: unknown, body: unknown) => Record<string, string>;
|
||||
buildRequestHeaders: (
|
||||
model: string,
|
||||
credentials: unknown,
|
||||
body: unknown
|
||||
) => Record<string, string>;
|
||||
transformRequest: (
|
||||
body: unknown,
|
||||
model: string,
|
||||
@@ -159,7 +166,11 @@ describe("LMArena Executor", () => {
|
||||
assert.equal(headers.Cookie, "session=def");
|
||||
|
||||
// providerSpecificData.cookie
|
||||
headers = ex.buildRequestHeaders("gpt-4", { providerSpecificData: { cookie: "session=ghi" } }, {});
|
||||
headers = ex.buildRequestHeaders(
|
||||
"gpt-4",
|
||||
{ providerSpecificData: { cookie: "session=ghi" } },
|
||||
{}
|
||||
);
|
||||
assert.equal(headers.Cookie, "session=ghi");
|
||||
|
||||
// Priority: direct > apiKey > providerSpecificData
|
||||
@@ -474,6 +485,44 @@ describe("LMArena Executor", () => {
|
||||
assert.equal(pickLMArenaModelId(TEST_ARENA_MODEL_ID, []), TEST_ARENA_MODEL_ID);
|
||||
});
|
||||
|
||||
it("sanitizes static catalog lookup failures before warning", async () => {
|
||||
const warnings: string[] = [];
|
||||
const resolved = await resolveLMArenaModelId("unknown-model-for-log-test", {
|
||||
debug: () => {
|
||||
throw new Error(
|
||||
"Catalog lookup failed at /srv/private/lmarena-catalog.ts:17:5; " +
|
||||
"access_token=lmarena-catalog-secret\n" +
|
||||
" at SecretCatalogFrame (/srv/private/lmarena-catalog-stack.ts:2:3)"
|
||||
);
|
||||
},
|
||||
warn: (_scope, message) => warnings.push(String(message)),
|
||||
});
|
||||
|
||||
assert.equal(resolved, "unknown-model-for-log-test");
|
||||
assert.equal(warnings.length, 1);
|
||||
assert.match(warnings[0], /Using raw model id after static catalog lookup failed/);
|
||||
assert.match(warnings[0], /Catalog lookup failed/);
|
||||
assert.doesNotMatch(warnings[0], /\/srv\/private\/lmarena-catalog(?:-stack)?\.ts/);
|
||||
assert.doesNotMatch(warnings[0], /lmarena-catalog-secret|SecretCatalogFrame/);
|
||||
});
|
||||
|
||||
it("uses a stable fallback when the catalog failure sanitizes to blank", async () => {
|
||||
const warnings: string[] = [];
|
||||
const resolved = await resolveLMArenaModelId("unknown-model-for-blank-log-test", {
|
||||
debug: () => {
|
||||
throw new Error(
|
||||
"\n at SecretOnlyFrame (/srv/private/lmarena-catalog-stack-only.ts:2:3)"
|
||||
);
|
||||
},
|
||||
warn: (_scope, message) => warnings.push(String(message)),
|
||||
});
|
||||
|
||||
assert.equal(resolved, "unknown-model-for-blank-log-test");
|
||||
assert.deepEqual(warnings, [
|
||||
"Using raw model id after static catalog lookup failed: Arena catalog lookup error",
|
||||
]);
|
||||
});
|
||||
|
||||
it("resolves catalog public names via static Direct-chat allowlist (no arena.ai fetch)", async () => {
|
||||
const executor = new LMArenaExecutor();
|
||||
let arenaHomeFetches = 0;
|
||||
@@ -630,4 +679,33 @@ describe("LMArena Executor", () => {
|
||||
__setTlsFetchOverrideForTesting(null);
|
||||
}
|
||||
});
|
||||
|
||||
it("uses an error status for a Cloudflare challenge returned with HTTP 200", async () => {
|
||||
__setTlsFetchOverrideForTesting(async () => ({
|
||||
status: 200,
|
||||
headers: new Headers({ "Content-Type": "text/html" }),
|
||||
text: "<html>Just a moment... challenges.cloudflare.com</html>",
|
||||
body: null,
|
||||
}));
|
||||
|
||||
try {
|
||||
const result = await new LMArenaExecutor().execute({
|
||||
model: TEST_ARENA_MODEL_ID,
|
||||
body: { messages: [{ role: "user", content: "Hello" }] },
|
||||
credentials: { cookie: "session=test" },
|
||||
signal: new AbortController().signal,
|
||||
log: null,
|
||||
});
|
||||
|
||||
assert.equal(result.response.status, 403);
|
||||
assert.deepEqual((await result.response.json()).error, {
|
||||
message:
|
||||
"Arena blocked by Cloudflare bot management. Use a residential/browser-grade network if needed, paste a fresh full Cookie header (include cf_clearance / __cf_bm when present), and optionally set providerSpecificData.recaptchaV3Token from a live browser session.",
|
||||
type: "api_error",
|
||||
code: "cloudflare_or_bot",
|
||||
});
|
||||
} finally {
|
||||
__setTlsFetchOverrideForTesting(null);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
618
tests/unit/lmarena-public-error-boundary-11742.test.ts
Normal file
618
tests/unit/lmarena-public-error-boundary-11742.test.ts
Normal file
@@ -0,0 +1,618 @@
|
||||
/**
|
||||
* #11742 — Arena (lmarena) public error boundary.
|
||||
*
|
||||
* Split out of `lmarena-provider.test.ts` to keep that file under the test size cap.
|
||||
* Every public failure is projected onto Arena's fixed public vocabulary
|
||||
* (`open-sse/executors/lmarena/error.ts` + `lmarena/response.ts`); upstream bodies,
|
||||
* transport rejections and TLS failures never reach the client verbatim.
|
||||
*
|
||||
* Run: node --import tsx/esm --test tests/unit/lmarena-public-error-boundary-11742.test.ts
|
||||
*/
|
||||
|
||||
import { describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { LMArenaExecutor } from "../../open-sse/executors/lmarena.ts";
|
||||
import {
|
||||
__setTlsFetchOverrideForTesting,
|
||||
TlsClientUnavailableError,
|
||||
} from "../../open-sse/services/lmarenaTlsClient.ts";
|
||||
|
||||
const TEST_ARENA_MODEL_ID = "019e080d-c29d-7d9a-aa54-faed41da0763";
|
||||
|
||||
describe("LMArena Executor — public error boundary", () => {
|
||||
it("normalizes an invalid upstream status before building the public error", async () => {
|
||||
__setTlsFetchOverrideForTesting(async () => ({
|
||||
status: 700,
|
||||
headers: new Headers({ "Content-Type": "text/plain" }),
|
||||
text: "InvalidStatusInternalFailure secret-status-id",
|
||||
body: null,
|
||||
}));
|
||||
|
||||
try {
|
||||
const result = await new LMArenaExecutor().execute({
|
||||
model: TEST_ARENA_MODEL_ID,
|
||||
body: { messages: [{ role: "user", content: "Hello" }] },
|
||||
credentials: { cookie: "session=test" },
|
||||
signal: new AbortController().signal,
|
||||
log: null,
|
||||
});
|
||||
|
||||
assert.equal(result.response.status, 502);
|
||||
const responseText = await result.response.text();
|
||||
assert.deepEqual(JSON.parse(responseText).error, {
|
||||
message: "Arena API error: 502",
|
||||
type: "api_error",
|
||||
code: "502",
|
||||
});
|
||||
assert.doesNotMatch(responseText, /InvalidStatusInternalFailure|secret-status-id/i);
|
||||
} finally {
|
||||
__setTlsFetchOverrideForTesting(null);
|
||||
}
|
||||
});
|
||||
|
||||
it("normalizes an upstream redirect before building the public error", async () => {
|
||||
__setTlsFetchOverrideForTesting(async () => ({
|
||||
status: 302,
|
||||
headers: new Headers({ Location: "https://internal.arena.invalid/login" }),
|
||||
text: "RedirectInternalFailure secret-redirect-id",
|
||||
body: null,
|
||||
}));
|
||||
|
||||
try {
|
||||
const result = await new LMArenaExecutor().execute({
|
||||
model: TEST_ARENA_MODEL_ID,
|
||||
body: { messages: [{ role: "user", content: "Hello" }] },
|
||||
credentials: { cookie: "session=test" },
|
||||
signal: new AbortController().signal,
|
||||
log: null,
|
||||
});
|
||||
|
||||
assert.equal(result.response.status, 502);
|
||||
const responseText = await result.response.text();
|
||||
assert.deepEqual(JSON.parse(responseText).error, {
|
||||
message: "Arena API error: 502",
|
||||
type: "api_error",
|
||||
code: "502",
|
||||
});
|
||||
assert.doesNotMatch(
|
||||
responseText,
|
||||
/RedirectInternalFailure|secret-redirect-id|internal\.arena/i
|
||||
);
|
||||
} finally {
|
||||
__setTlsFetchOverrideForTesting(null);
|
||||
}
|
||||
});
|
||||
|
||||
it("does not expose structured upstream error details while preserving classification", async () => {
|
||||
const executor = new LMArenaExecutor();
|
||||
__setTlsFetchOverrideForTesting(async () => ({
|
||||
status: 500,
|
||||
headers: new Headers({ "Content-Type": "application/json" }),
|
||||
text: JSON.stringify({
|
||||
error: {
|
||||
message:
|
||||
"SensitiveDatabaseAdapter failed\n" +
|
||||
" at loadSecret (/srv/private/lmarena/database.ts:46:7)",
|
||||
stack: "Error: database failure at /srv/private/lmarena/database.ts:46:7",
|
||||
cause: "postgresql://private-user:private-password@internal-db/arena",
|
||||
},
|
||||
}),
|
||||
body: null,
|
||||
}));
|
||||
|
||||
try {
|
||||
const result = await executor.execute({
|
||||
model: TEST_ARENA_MODEL_ID,
|
||||
body: { messages: [{ role: "user", content: "Hello" }] },
|
||||
credentials: { cookie: "session=test" },
|
||||
signal: new AbortController().signal,
|
||||
log: console,
|
||||
});
|
||||
|
||||
assert.equal(result.response.status, 500);
|
||||
const responseText = await result.response.text();
|
||||
const errorBody = JSON.parse(responseText);
|
||||
assert.deepEqual(errorBody.error, {
|
||||
message: "Arena API error: 500",
|
||||
type: "api_error",
|
||||
code: "500",
|
||||
});
|
||||
assert.doesNotMatch(
|
||||
responseText,
|
||||
/SensitiveDatabaseAdapter|loadSecret|database\.ts|private-password|stack|cause/i
|
||||
);
|
||||
} finally {
|
||||
__setTlsFetchOverrideForTesting(null);
|
||||
}
|
||||
});
|
||||
|
||||
it("does not expose plaintext upstream error details while preserving classification", async () => {
|
||||
__setTlsFetchOverrideForTesting(async () => ({
|
||||
status: 500,
|
||||
headers: new Headers({ "Content-Type": "text/plain" }),
|
||||
text:
|
||||
"SensitivePlaintextFailure: internal adapter failed\n" +
|
||||
" at loadSecret (/srv/private/lmarena/plaintext.ts:71:9)",
|
||||
body: null,
|
||||
}));
|
||||
|
||||
try {
|
||||
const result = await new LMArenaExecutor().execute({
|
||||
model: TEST_ARENA_MODEL_ID,
|
||||
body: { messages: [{ role: "user", content: "Hello" }] },
|
||||
credentials: { cookie: "session=test" },
|
||||
signal: new AbortController().signal,
|
||||
log: console,
|
||||
});
|
||||
|
||||
assert.equal(result.response.status, 500);
|
||||
const responseText = await result.response.text();
|
||||
const errorBody = JSON.parse(responseText);
|
||||
assert.deepEqual(errorBody.error, {
|
||||
message: "Arena API error: 500",
|
||||
type: "api_error",
|
||||
code: "500",
|
||||
});
|
||||
assert.doesNotMatch(
|
||||
responseText,
|
||||
/SensitivePlaintextFailure|internal adapter|loadSecret|plaintext\.ts/i
|
||||
);
|
||||
} finally {
|
||||
__setTlsFetchOverrideForTesting(null);
|
||||
}
|
||||
});
|
||||
|
||||
it("sanitizes network failure details before logging or responding", async () => {
|
||||
const errorLogs: string[] = [];
|
||||
__setTlsFetchOverrideForTesting(async () => {
|
||||
throw new Error(
|
||||
"Arena request failed at /srv/private/lmarena-request.ts:17:5; " +
|
||||
"access_token=lmarena-network-secret\n" +
|
||||
" at SecretArenaFrame (/srv/private/lmarena-stack.ts:2:3)"
|
||||
);
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await new LMArenaExecutor().execute({
|
||||
model: TEST_ARENA_MODEL_ID,
|
||||
body: { messages: [{ role: "user", content: "Hello" }] },
|
||||
credentials: { cookie: "session=test" },
|
||||
signal: new AbortController().signal,
|
||||
log: { error: (_scope, message) => errorLogs.push(String(message)) },
|
||||
});
|
||||
|
||||
assert.equal(result.response.status, 502);
|
||||
assert.equal(errorLogs.length, 1);
|
||||
const responseText = await result.response.text();
|
||||
const json = JSON.parse(responseText);
|
||||
assert.equal(json.error?.message, "Arena upstream error");
|
||||
assert.equal(json.error?.type, "network_error");
|
||||
assert.equal(json.error?.code, "request_failed");
|
||||
const publicOutput = `${errorLogs.join("\n")}\n${responseText}`;
|
||||
assert.match(publicOutput, /Arena request failed/);
|
||||
assert.doesNotMatch(publicOutput, /\/srv\/private\/lmarena-(?:request|stack)\.ts/);
|
||||
assert.doesNotMatch(publicOutput, /lmarena-network-secret|SecretArenaFrame/);
|
||||
assert.doesNotMatch(responseText, /"(?:stack|cause)"\s*:/i);
|
||||
} finally {
|
||||
__setTlsFetchOverrideForTesting(null);
|
||||
}
|
||||
});
|
||||
|
||||
it("fails closed when a network rejection refuses string coercion", async () => {
|
||||
const errorLogs: string[] = [];
|
||||
__setTlsFetchOverrideForTesting(async () => {
|
||||
throw {
|
||||
toString() {
|
||||
throw new Error("access_token=hostile-secret at /srv/private/lmarena.ts:1:2");
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await new LMArenaExecutor().execute({
|
||||
model: TEST_ARENA_MODEL_ID,
|
||||
body: { messages: [{ role: "user", content: "Hello" }] },
|
||||
credentials: { cookie: "session=test" },
|
||||
signal: new AbortController().signal,
|
||||
log: { error: (_scope, message) => errorLogs.push(String(message)) },
|
||||
});
|
||||
|
||||
assert.equal(result.response.status, 502);
|
||||
assert.deepEqual(errorLogs, ["Request failed: Arena upstream error"]);
|
||||
const json = await result.response.json();
|
||||
assert.deepEqual(json.error, {
|
||||
message: "Arena upstream error",
|
||||
type: "network_error",
|
||||
code: "request_failed",
|
||||
});
|
||||
} finally {
|
||||
__setTlsFetchOverrideForTesting(null);
|
||||
}
|
||||
});
|
||||
|
||||
it("fails closed when network rejection prototype inspection throws", async () => {
|
||||
const hostileFailure = new Proxy(
|
||||
{},
|
||||
{
|
||||
getPrototypeOf() {
|
||||
throw new Error("access_token=prototype-secret at /srv/private/prototype.ts:1:2");
|
||||
},
|
||||
get(_target, property) {
|
||||
if (property === "toString") {
|
||||
return () => {
|
||||
throw new Error("access_token=coercion-secret at /srv/private/coercion.ts:1:2");
|
||||
};
|
||||
}
|
||||
return undefined;
|
||||
},
|
||||
}
|
||||
);
|
||||
const errorLogs: string[] = [];
|
||||
__setTlsFetchOverrideForTesting(async () => {
|
||||
throw hostileFailure;
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await new LMArenaExecutor().execute({
|
||||
model: TEST_ARENA_MODEL_ID,
|
||||
body: { messages: [{ role: "user", content: "Hello" }] },
|
||||
credentials: { cookie: "session=test" },
|
||||
signal: new AbortController().signal,
|
||||
log: { error: (_scope, message) => errorLogs.push(String(message)) },
|
||||
});
|
||||
|
||||
assert.equal(result.response.status, 502);
|
||||
assert.deepEqual(errorLogs, ["Request failed: Arena upstream error"]);
|
||||
const responseText = await result.response.text();
|
||||
assert.doesNotMatch(responseText, /prototype-secret|coercion-secret|\/srv\/private/);
|
||||
} finally {
|
||||
__setTlsFetchOverrideForTesting(null);
|
||||
}
|
||||
});
|
||||
|
||||
it("uses a stable fallback for blank TLS-unavailable errors", async () => {
|
||||
const errorLogs: string[] = [];
|
||||
__setTlsFetchOverrideForTesting(async () => {
|
||||
throw new TlsClientUnavailableError(
|
||||
"\n at SecretOnlyFrame (/srv/private/lmarena-tls-stack-only.ts:2:3)"
|
||||
);
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await new LMArenaExecutor().execute({
|
||||
model: TEST_ARENA_MODEL_ID,
|
||||
body: { messages: [{ role: "user", content: "Hello" }] },
|
||||
credentials: { cookie: "session=test" },
|
||||
signal: new AbortController().signal,
|
||||
log: { error: (_scope, message) => errorLogs.push(String(message)) },
|
||||
});
|
||||
|
||||
assert.equal(result.response.status, 502);
|
||||
assert.deepEqual(errorLogs, ["TLS client unavailable: Arena upstream error"]);
|
||||
const responseText = await result.response.text();
|
||||
const json = JSON.parse(responseText);
|
||||
assert.equal(json.error?.type, "upstream_error");
|
||||
assert.equal(json.error?.code, "TLS_CLIENT_UNAVAILABLE");
|
||||
assert.equal(
|
||||
json.error?.message,
|
||||
"Arena TLS impersonation unavailable: Arena upstream error. Verify the wreq-js 3.2 native binding."
|
||||
);
|
||||
assert.doesNotMatch(responseText, /SecretOnlyFrame|lmarena-tls-stack-only/);
|
||||
} finally {
|
||||
__setTlsFetchOverrideForTesting(null);
|
||||
}
|
||||
});
|
||||
|
||||
it("does not expose TLS-client failure details in the public response", async () => {
|
||||
const tlsFailure = new TlsClientUnavailableError(
|
||||
"NativeTlsBridgeFailure secret-adapter-id at /srv/private/lmarena-native.ts:22:4"
|
||||
) as TlsClientUnavailableError & { cause?: unknown };
|
||||
tlsFailure.cause = new Error("NativeTlsCause secret-cause-id");
|
||||
__setTlsFetchOverrideForTesting(async () => {
|
||||
throw tlsFailure;
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await new LMArenaExecutor().execute({
|
||||
model: TEST_ARENA_MODEL_ID,
|
||||
body: { messages: [{ role: "user", content: "Hello" }] },
|
||||
credentials: { cookie: "session=test" },
|
||||
signal: new AbortController().signal,
|
||||
log: null,
|
||||
});
|
||||
|
||||
assert.equal(result.response.status, 502);
|
||||
const responseText = await result.response.text();
|
||||
const json = JSON.parse(responseText);
|
||||
assert.deepEqual(json.error, {
|
||||
message:
|
||||
"Arena TLS impersonation unavailable: Arena upstream error. Verify the wreq-js 3.2 native binding.",
|
||||
type: "upstream_error",
|
||||
code: "TLS_CLIENT_UNAVAILABLE",
|
||||
});
|
||||
assert.doesNotMatch(
|
||||
responseText,
|
||||
/NativeTlsBridgeFailure|secret-adapter-id|lmarena-native|NativeTlsCause|secret-cause-id/i
|
||||
);
|
||||
} finally {
|
||||
__setTlsFetchOverrideForTesting(null);
|
||||
}
|
||||
});
|
||||
|
||||
it("uses a stable public fallback for blank network and upstream event errors", async (t) => {
|
||||
const stackOnly = "\n at SecretOnlyFrame (/srv/private/lmarena-stack-only.ts:2:3)";
|
||||
const cases = [
|
||||
{
|
||||
name: "network rejection",
|
||||
setup: () =>
|
||||
__setTlsFetchOverrideForTesting(async () => {
|
||||
throw stackOnly;
|
||||
}),
|
||||
expectedType: "network_error",
|
||||
expectedCode: "request_failed",
|
||||
},
|
||||
{
|
||||
name: "non-streaming upstream event",
|
||||
setup: () =>
|
||||
__setTlsFetchOverrideForTesting(async () => ({
|
||||
status: 200,
|
||||
headers: new Headers({ "Content-Type": "text/event-stream" }),
|
||||
text: `3:${JSON.stringify(stackOnly)}\n`,
|
||||
body: null,
|
||||
})),
|
||||
expectedType: "api_error",
|
||||
expectedCode: "lmarena_error",
|
||||
},
|
||||
];
|
||||
|
||||
for (const testCase of cases) {
|
||||
await t.test(testCase.name, async () => {
|
||||
testCase.setup();
|
||||
try {
|
||||
const result = await new LMArenaExecutor().execute({
|
||||
model: TEST_ARENA_MODEL_ID,
|
||||
body: { messages: [{ role: "user", content: "Hello" }] },
|
||||
credentials: { cookie: "session=test" },
|
||||
signal: new AbortController().signal,
|
||||
log: null,
|
||||
});
|
||||
|
||||
assert.equal(result.response.status, 502);
|
||||
const responseText = await result.response.text();
|
||||
const json = JSON.parse(responseText);
|
||||
assert.equal(json.error?.message, "Arena upstream error");
|
||||
assert.equal(json.error?.type, testCase.expectedType);
|
||||
assert.equal(json.error?.code, testCase.expectedCode);
|
||||
assert.doesNotMatch(responseText, /SecretOnlyFrame|lmarena-stack-only/);
|
||||
} finally {
|
||||
__setTlsFetchOverrideForTesting(null);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it("does not expose non-streaming upstream event details", async () => {
|
||||
const upstreamFailure =
|
||||
"ArenaEventInternalFailure secret-event-id\n" +
|
||||
" at ArenaEventAdapter (/srv/private/lmarena-event.ts:31:8)";
|
||||
__setTlsFetchOverrideForTesting(async () => ({
|
||||
status: 200,
|
||||
headers: new Headers({ "Content-Type": "text/event-stream" }),
|
||||
text: `3:${JSON.stringify(upstreamFailure)}\n`,
|
||||
body: null,
|
||||
}));
|
||||
|
||||
try {
|
||||
const result = await new LMArenaExecutor().execute({
|
||||
model: TEST_ARENA_MODEL_ID,
|
||||
body: { messages: [{ role: "user", content: "Hello" }] },
|
||||
credentials: { cookie: "session=test" },
|
||||
signal: new AbortController().signal,
|
||||
log: null,
|
||||
});
|
||||
|
||||
assert.equal(result.response.status, 502);
|
||||
const responseText = await result.response.text();
|
||||
const json = JSON.parse(responseText);
|
||||
assert.deepEqual(json.error, {
|
||||
message: "Arena upstream error",
|
||||
type: "api_error",
|
||||
code: "lmarena_error",
|
||||
});
|
||||
assert.doesNotMatch(
|
||||
responseText,
|
||||
/ArenaEventInternalFailure|secret-event-id|ArenaEventAdapter|lmarena-event/i
|
||||
);
|
||||
} finally {
|
||||
__setTlsFetchOverrideForTesting(null);
|
||||
}
|
||||
});
|
||||
|
||||
it("uses a stable public fallback for blank streaming event errors", async () => {
|
||||
const stackOnly = "\n at SecretOnlyFrame (/srv/private/lmarena-stream-stack-only.ts:2:3)";
|
||||
const encoded = new TextEncoder().encode(`data: 3:${JSON.stringify(stackOnly)}\n\n`);
|
||||
__setTlsFetchOverrideForTesting(async () => ({
|
||||
status: 200,
|
||||
headers: new Headers({ "Content-Type": "text/event-stream" }),
|
||||
text: null,
|
||||
body: new ReadableStream({
|
||||
start(controller) {
|
||||
controller.enqueue(encoded);
|
||||
controller.close();
|
||||
},
|
||||
}),
|
||||
}));
|
||||
|
||||
try {
|
||||
const result = await new LMArenaExecutor().execute({
|
||||
model: TEST_ARENA_MODEL_ID,
|
||||
body: { messages: [{ role: "user", content: "Hello" }], stream: true },
|
||||
stream: true,
|
||||
credentials: { cookie: "session=test" },
|
||||
signal: new AbortController().signal,
|
||||
log: null,
|
||||
});
|
||||
|
||||
assert.equal(result.response.status, 200);
|
||||
const responseText = await result.response.text();
|
||||
const payload = responseText
|
||||
.split("\n")
|
||||
.filter((line) => line.startsWith("data: ") && line !== "data: [DONE]")
|
||||
.map((line) => JSON.parse(line.slice(6)))
|
||||
.find((chunk) => chunk.error);
|
||||
assert.deepEqual(payload?.error, {
|
||||
message: "Arena upstream error",
|
||||
type: "api_error",
|
||||
code: "lmarena_error",
|
||||
});
|
||||
assert.doesNotMatch(responseText, /SecretOnlyFrame|lmarena-stream-stack-only/);
|
||||
} finally {
|
||||
__setTlsFetchOverrideForTesting(null);
|
||||
}
|
||||
});
|
||||
|
||||
it("does not expose streaming upstream event details", async () => {
|
||||
const upstreamFailure =
|
||||
"StreamingArenaInternalFailure secret-stream-id\n" +
|
||||
" at StreamingArenaAdapter (/srv/private/lmarena-stream-event.ts:44:9)";
|
||||
const encoded = new TextEncoder().encode(`data: 3:${JSON.stringify(upstreamFailure)}\n\n`);
|
||||
__setTlsFetchOverrideForTesting(async () => ({
|
||||
status: 200,
|
||||
headers: new Headers({ "Content-Type": "text/event-stream" }),
|
||||
text: null,
|
||||
body: new ReadableStream({
|
||||
start(controller) {
|
||||
controller.enqueue(encoded);
|
||||
controller.close();
|
||||
},
|
||||
}),
|
||||
}));
|
||||
|
||||
try {
|
||||
const result = await new LMArenaExecutor().execute({
|
||||
model: TEST_ARENA_MODEL_ID,
|
||||
body: { messages: [{ role: "user", content: "Hello" }], stream: true },
|
||||
stream: true,
|
||||
credentials: { cookie: "session=test" },
|
||||
signal: new AbortController().signal,
|
||||
log: null,
|
||||
});
|
||||
|
||||
assert.equal(result.response.status, 200);
|
||||
const responseText = await result.response.text();
|
||||
const payload = responseText
|
||||
.split("\n")
|
||||
.filter((line) => line.startsWith("data: ") && line !== "data: [DONE]")
|
||||
.map((line) => JSON.parse(line.slice(6)))
|
||||
.find((chunk) => chunk.error);
|
||||
assert.deepEqual(payload?.error, {
|
||||
message: "Arena upstream error",
|
||||
type: "api_error",
|
||||
code: "lmarena_error",
|
||||
});
|
||||
assert.doesNotMatch(
|
||||
responseText,
|
||||
/StreamingArenaInternalFailure|secret-stream-id|StreamingArenaAdapter|lmarena-stream-event/i
|
||||
);
|
||||
} finally {
|
||||
__setTlsFetchOverrideForTesting(null);
|
||||
}
|
||||
});
|
||||
|
||||
it("does not expose transport errors rejected by the upstream stream", async () => {
|
||||
const streamFailure = new Error(
|
||||
"ArenaStreamTransportFailure secret-transport-id at /srv/private/lmarena-reader.ts:52:6"
|
||||
) as Error & { cause?: unknown; statusCode?: number };
|
||||
streamFailure.cause = new Error("ArenaStreamCause secret-stream-cause-id");
|
||||
streamFailure.statusCode = 502;
|
||||
__setTlsFetchOverrideForTesting(async () => ({
|
||||
status: 200,
|
||||
headers: new Headers({ "Content-Type": "text/event-stream" }),
|
||||
text: null,
|
||||
body: new ReadableStream({
|
||||
start(controller) {
|
||||
controller.error(streamFailure);
|
||||
},
|
||||
}),
|
||||
}));
|
||||
|
||||
try {
|
||||
const result = await new LMArenaExecutor().execute({
|
||||
model: TEST_ARENA_MODEL_ID,
|
||||
body: { messages: [{ role: "user", content: "Hello" }], stream: true },
|
||||
stream: true,
|
||||
credentials: { cookie: "session=test" },
|
||||
signal: new AbortController().signal,
|
||||
log: null,
|
||||
});
|
||||
|
||||
assert.equal(result.response.status, 200);
|
||||
await assert.rejects(result.response.text(), (error: unknown) => {
|
||||
assert.ok(error instanceof Error);
|
||||
assert.equal(error.message, "Arena upstream stream error");
|
||||
assert.equal(error.stack, undefined);
|
||||
const projected = error as Error & {
|
||||
cause?: unknown;
|
||||
statusCode?: number;
|
||||
type?: string;
|
||||
code?: string;
|
||||
};
|
||||
assert.equal(projected.cause, undefined);
|
||||
assert.equal(projected.statusCode, 502);
|
||||
assert.equal(projected.type, "upstream_error");
|
||||
assert.equal(projected.code, "lmarena_stream_error");
|
||||
assert.doesNotMatch(
|
||||
error.message,
|
||||
/ArenaStreamTransportFailure|secret-transport-id|lmarena-reader|ArenaStreamCause|secret-stream-cause-id/i
|
||||
);
|
||||
return true;
|
||||
});
|
||||
} finally {
|
||||
__setTlsFetchOverrideForTesting(null);
|
||||
}
|
||||
});
|
||||
|
||||
it("does not coerce transport status metadata from the upstream stream", async () => {
|
||||
let coercionCalls = 0;
|
||||
const hostileStatus = {
|
||||
[Symbol.toPrimitive]() {
|
||||
coercionCalls += 1;
|
||||
throw new Error("StatusCoercionFailure secret-coercion-id");
|
||||
},
|
||||
};
|
||||
const streamFailure = Object.assign(new Error("upstream transport failure"), {
|
||||
statusCode: hostileStatus,
|
||||
});
|
||||
__setTlsFetchOverrideForTesting(async () => ({
|
||||
status: 200,
|
||||
headers: new Headers({ "Content-Type": "text/event-stream" }),
|
||||
text: null,
|
||||
body: new ReadableStream({
|
||||
start(controller) {
|
||||
controller.error(streamFailure);
|
||||
},
|
||||
}),
|
||||
}));
|
||||
|
||||
try {
|
||||
const result = await new LMArenaExecutor().execute({
|
||||
model: TEST_ARENA_MODEL_ID,
|
||||
body: { messages: [{ role: "user", content: "Hello" }], stream: true },
|
||||
stream: true,
|
||||
credentials: { cookie: "session=test" },
|
||||
signal: new AbortController().signal,
|
||||
log: null,
|
||||
});
|
||||
|
||||
await assert.rejects(result.response.text(), (error: unknown) => {
|
||||
assert.ok(error instanceof Error);
|
||||
assert.equal((error as Error & { statusCode?: unknown }).statusCode, undefined);
|
||||
assert.equal(error.message, "Arena upstream stream error");
|
||||
return true;
|
||||
});
|
||||
assert.equal(coercionCalls, 0);
|
||||
} finally {
|
||||
__setTlsFetchOverrideForTesting(null);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -17,9 +17,7 @@ describe("Arena AI stream readiness (#9306)", () => {
|
||||
const upstreamReader = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(new TextEncoder().encode('a0:{"text":"Hello"}\n'));
|
||||
controller.enqueue(
|
||||
new TextEncoder().encode('a0:{"text":", world!"}\nad:{}\n')
|
||||
);
|
||||
controller.enqueue(new TextEncoder().encode('a0:{"text":", world!"}\nad:{}\n'));
|
||||
controller.close();
|
||||
},
|
||||
}).getReader();
|
||||
@@ -91,4 +89,162 @@ describe("Arena AI stream readiness (#9306)", () => {
|
||||
assert.ok(chunk instanceof Uint8Array, "Each chunk must be Uint8Array, not string");
|
||||
}
|
||||
});
|
||||
|
||||
it("cancels and unlocks the upstream reader after an early done event", async () => {
|
||||
let upstreamCancelled = false;
|
||||
const upstream = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(new TextEncoder().encode("ad:{}\n"));
|
||||
},
|
||||
cancel() {
|
||||
upstreamCancelled = true;
|
||||
},
|
||||
});
|
||||
const upstreamReader = upstream.getReader();
|
||||
const stream = createOpenAIArenaStream({ reader: upstreamReader, model: "test-model" });
|
||||
|
||||
const responseText = await new Response(stream).text();
|
||||
|
||||
assert.match(responseText, /\[DONE\]/);
|
||||
assert.equal(upstreamCancelled, true);
|
||||
assert.equal(upstream.locked, false);
|
||||
});
|
||||
|
||||
it("projects reader failures as sanitized stackless stream errors", async () => {
|
||||
const upstreamError = new Error(
|
||||
"Arena reader failed at /srv/private/lmarena-reader.ts:17:5; " +
|
||||
"access_token=lmarena-reader-secret\n" +
|
||||
" at SecretReaderFrame (/srv/private/lmarena-reader-stack.ts:2:3)"
|
||||
);
|
||||
const upstreamReader = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.error(upstreamError);
|
||||
},
|
||||
}).getReader();
|
||||
const errorLogs: string[] = [];
|
||||
const stream = createOpenAIArenaStream({
|
||||
reader: upstreamReader,
|
||||
model: "test-model",
|
||||
log: { error: (_scope, message) => errorLogs.push(String(message)) },
|
||||
});
|
||||
|
||||
let projectedError: unknown;
|
||||
try {
|
||||
await stream.getReader().read();
|
||||
assert.fail("reader must reject when the Arena reader fails");
|
||||
} catch (error) {
|
||||
projectedError = error;
|
||||
}
|
||||
|
||||
assert.ok(projectedError instanceof Error);
|
||||
assert.equal(projectedError.stack, undefined, "public stream error must not carry a stack");
|
||||
assert.equal(
|
||||
(projectedError as Error & { cause?: unknown }).cause,
|
||||
undefined,
|
||||
"public stream error must not carry the upstream cause"
|
||||
);
|
||||
assert.equal(errorLogs.length, 1);
|
||||
const publicOutput = `${errorLogs.join("\n")}\n${projectedError.message}`;
|
||||
assert.match(publicOutput, /Arena reader failed/);
|
||||
assert.doesNotMatch(publicOutput, /\/srv\/private\/lmarena-reader(?:-stack)?\.ts/);
|
||||
assert.doesNotMatch(publicOutput, /lmarena-reader-secret|SecretReaderFrame/);
|
||||
});
|
||||
|
||||
it("uses a stable fallback for stack-only reader failures", async () => {
|
||||
const upstreamReader = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.error(
|
||||
new Error("\n at SecretOnlyFrame (/srv/private/lmarena-reader-stack-only.ts:2:3)")
|
||||
);
|
||||
},
|
||||
}).getReader();
|
||||
const errorLogs: string[] = [];
|
||||
const stream = createOpenAIArenaStream({
|
||||
reader: upstreamReader,
|
||||
model: "test-model",
|
||||
log: { error: (_scope, message) => errorLogs.push(String(message)) },
|
||||
});
|
||||
|
||||
let projectedError: unknown;
|
||||
try {
|
||||
await stream.getReader().read();
|
||||
assert.fail("reader must reject when the Arena reader fails");
|
||||
} catch (error) {
|
||||
projectedError = error;
|
||||
}
|
||||
|
||||
assert.ok(projectedError instanceof Error);
|
||||
assert.equal(projectedError.message, "Arena upstream stream error");
|
||||
assert.equal(projectedError.stack, undefined);
|
||||
assert.equal((projectedError as Error & { cause?: unknown }).cause, undefined);
|
||||
assert.deepEqual(errorLogs, ["Streaming error: Arena upstream stream error"]);
|
||||
});
|
||||
|
||||
it("fails closed when reader failure coercion and metadata access throw", async () => {
|
||||
const hostileFailure = {
|
||||
get name() {
|
||||
throw new Error("access_token=name-secret at /srv/private/name.ts:1:2");
|
||||
},
|
||||
get statusCode() {
|
||||
throw new Error("access_token=status-secret at /srv/private/status.ts:1:2");
|
||||
},
|
||||
toString() {
|
||||
throw new Error("access_token=coercion-secret at /srv/private/coercion.ts:1:2");
|
||||
},
|
||||
};
|
||||
const upstreamReader = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.error(hostileFailure);
|
||||
},
|
||||
}).getReader();
|
||||
const errorLogs: string[] = [];
|
||||
const stream = createOpenAIArenaStream({
|
||||
reader: upstreamReader,
|
||||
model: "test-model",
|
||||
log: { error: (_scope, message) => errorLogs.push(String(message)) },
|
||||
});
|
||||
|
||||
let projectedError: unknown;
|
||||
try {
|
||||
await stream.getReader().read();
|
||||
assert.fail("reader must reject when the Arena reader fails");
|
||||
} catch (error) {
|
||||
projectedError = error;
|
||||
}
|
||||
|
||||
assert.ok(projectedError instanceof Error);
|
||||
assert.equal(projectedError.message, "Arena upstream stream error");
|
||||
assert.equal(projectedError.stack, undefined);
|
||||
assert.deepEqual(errorLogs, ["Streaming error: Arena upstream stream error"]);
|
||||
});
|
||||
|
||||
it("preserves safe disconnect classification without retaining upstream metadata", async () => {
|
||||
const upstreamError = new Error(
|
||||
"Abort failed at /srv/private/lmarena-abort.ts:17:5; " + "access_token=lmarena-abort-secret"
|
||||
) as Error & { cause?: unknown; statusCode?: number };
|
||||
upstreamError.name = "AbortError";
|
||||
upstreamError.cause = new Error("private abort cause");
|
||||
upstreamError.statusCode = 499;
|
||||
const upstreamReader = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.error(upstreamError);
|
||||
},
|
||||
}).getReader();
|
||||
const stream = createOpenAIArenaStream({ reader: upstreamReader, model: "test-model" });
|
||||
|
||||
let projectedError: unknown;
|
||||
try {
|
||||
await stream.getReader().read();
|
||||
assert.fail("reader must reject when the Arena reader aborts");
|
||||
} catch (error) {
|
||||
projectedError = error;
|
||||
}
|
||||
|
||||
assert.ok(projectedError instanceof Error);
|
||||
assert.equal(projectedError.name, "AbortError");
|
||||
assert.equal(projectedError.stack, undefined);
|
||||
assert.equal((projectedError as Error & { cause?: unknown }).cause, undefined);
|
||||
assert.equal((projectedError as Error & { statusCode?: number }).statusCode, 499);
|
||||
assert.doesNotMatch(projectedError.message, /lmarena-abort-secret|\/srv\/private/);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
/**
|
||||
* #11742 — Perplexity Web public error boundary.
|
||||
*
|
||||
* The non-streaming upstream-error path returns the provider's own wording and error
|
||||
* code to the client. Both must cross the canonical boundary first: the message
|
||||
* through `sanitizeErrorMessage` (Hard Rule #12) and the code through the bounded
|
||||
* public-identifier vocabulary (`projectPublicErrorIdentifier`).
|
||||
*/
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
const { PerplexityWebExecutor, toPublicPerplexityErrorCode } =
|
||||
await import("../../open-sse/executors/perplexity-web.ts");
|
||||
const { __setTlsFetchOverrideForTesting } =
|
||||
await import("../../open-sse/services/perplexityTlsClient.ts");
|
||||
|
||||
__setTlsFetchOverrideForTesting(async (url, opts) => {
|
||||
const res = await (globalThis.fetch as typeof fetch)(url, opts);
|
||||
return {
|
||||
status: res.status,
|
||||
headers: res.headers,
|
||||
text: res.status === 200 ? null : await res.text(),
|
||||
body: res.status === 200 ? res.body : null,
|
||||
};
|
||||
});
|
||||
|
||||
function mockPplxEvents(events: unknown[]): () => void {
|
||||
const original = globalThis.fetch;
|
||||
const encoder = new TextEncoder();
|
||||
const body =
|
||||
events.map((evt) => `event: message\r\ndata: ${JSON.stringify(evt)}\r\n\r\n`).join("") +
|
||||
"event: end_of_stream\r\n\r\n";
|
||||
globalThis.fetch = (async () =>
|
||||
new Response(
|
||||
new ReadableStream({
|
||||
start(controller) {
|
||||
controller.enqueue(encoder.encode(body));
|
||||
controller.close();
|
||||
},
|
||||
}),
|
||||
{ status: 200, headers: { "Content-Type": "text/event-stream" } }
|
||||
)) as typeof fetch;
|
||||
return () => {
|
||||
globalThis.fetch = original;
|
||||
};
|
||||
}
|
||||
|
||||
test("non-streaming upstream errors sanitize the message and bound the code", async () => {
|
||||
const restore = mockPplxEvents([
|
||||
{
|
||||
error_code: "RATE_LIMIT",
|
||||
error_message:
|
||||
"quota exhausted; access token: pplx-secret-value while reading " +
|
||||
"/srv/private/pplx/request.ts:42:7\n at send (/srv/private/pplx/request.ts:42:7)",
|
||||
},
|
||||
]);
|
||||
try {
|
||||
const result = await new PerplexityWebExecutor().execute({
|
||||
model: "pplx-auto",
|
||||
body: { messages: [{ role: "user", content: "hi" }] },
|
||||
stream: false,
|
||||
credentials: { apiKey: "test-cookie" },
|
||||
signal: AbortSignal.timeout(10_000),
|
||||
log: null,
|
||||
} as never);
|
||||
|
||||
assert.equal(result.response.status, 429, "quota wording maps to 429");
|
||||
const text = await result.response.text();
|
||||
const json = JSON.parse(text) as { error: { code: string; type: string } };
|
||||
|
||||
assert.equal(json.error.code, "quota_exhausted");
|
||||
assert.equal(json.error.type, "quota_exhausted");
|
||||
assert.doesNotMatch(text, /pplx-secret-value/);
|
||||
assert.doesNotMatch(text, /\/srv\/private/);
|
||||
assert.doesNotMatch(text, /\bat send\b/);
|
||||
} finally {
|
||||
restore();
|
||||
}
|
||||
});
|
||||
|
||||
test("an untrusted upstream error code falls back to PPLX_ERROR", () => {
|
||||
assert.equal(toPublicPerplexityErrorCode("Bearer sk-live-secret", false), "PPLX_ERROR");
|
||||
assert.equal(toPublicPerplexityErrorCode("x".repeat(65), false), "PPLX_ERROR");
|
||||
assert.equal(toPublicPerplexityErrorCode({ toString: () => "boom" }, false), "PPLX_ERROR");
|
||||
assert.equal(toPublicPerplexityErrorCode("RATE_LIMIT", true), "quota_exhausted");
|
||||
assert.equal(toPublicPerplexityErrorCode("rate_limited", false), "rate_limited");
|
||||
});
|
||||
@@ -22,7 +22,10 @@ const EXECUTOR_PATH = "../../open-sse/executors/claude-web.ts";
|
||||
|
||||
/** Calls __setTlsFetchOverrideForTesting with the given mock, resets on finish. */
|
||||
async function withTlsMock<T>(
|
||||
mock: (url: string, options: Record<string, unknown>) => Promise<{
|
||||
mock: (
|
||||
url: string,
|
||||
options: Record<string, unknown>
|
||||
) => Promise<{
|
||||
status: number;
|
||||
headers: Headers;
|
||||
text: string | null;
|
||||
@@ -68,14 +71,20 @@ test("validateClaudeWebProvider returns valid:false for 429", async () => {
|
||||
|
||||
test("validateMuseSparkWebProvider returns valid:false for 429", async () => {
|
||||
const { validateMuseSparkWebProvider } = await import(VALIDATION_PATH);
|
||||
const originalFetch = globalThis.fetch;
|
||||
globalThis.fetch = async (input, init) => {
|
||||
assert.equal(String(input), "https://www.meta.ai/api/graphql");
|
||||
assert.equal(init?.method, "POST");
|
||||
return new Response("Too Many Requests", { status: 429 });
|
||||
};
|
||||
|
||||
// validateMuseSparkWebProvider uses validationWrite() internally. We cannot
|
||||
// mock that here, but we can at least characterise the function's structure.
|
||||
// The actual 429-branch fix changes lines 64-69 from valid:true to valid:false,
|
||||
// and the integration-level exercise happens via the production proxy.
|
||||
// This test proves the validator exports and the function accepts input.
|
||||
const fn = validateMuseSparkWebProvider;
|
||||
assert.equal(typeof fn, "function");
|
||||
try {
|
||||
const result = await validateMuseSparkWebProvider({ apiKey: "ecto_1_sess=test-session" });
|
||||
assert.equal(result.valid, false, "expected valid:false for 429");
|
||||
assert.match(result.error ?? "", /429|rate limit/i);
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
// ── Test 3: errorResponseForTransport forwards Retry-After ──
|
||||
@@ -109,3 +118,55 @@ test("errorResponseForTransport forwards upstream Retry-After on 429", async ()
|
||||
const retryAfter = result.response.headers.get("Retry-After");
|
||||
assert.equal(retryAfter, "120", "expected forwarded Retry-After header");
|
||||
});
|
||||
|
||||
test("errorResponseForTransport only forwards bounded canonical Retry-After values", async () => {
|
||||
const { ClaudeWebExecutor } = await import(EXECUTOR_PATH);
|
||||
const validDate = new Date(Date.now() + 60_000).toUTCString();
|
||||
const cases = [
|
||||
{ label: "normalized delta-seconds", value: "000120", expected: "120" },
|
||||
{ label: "canonical HTTP-date", value: validDate, expected: validDate },
|
||||
{
|
||||
label: "credential and path payload",
|
||||
value: "120 access_token=sk-claude-retry /srv/private/retry.txt",
|
||||
expected: null,
|
||||
},
|
||||
{ label: "over the one-day bound", value: "86401", expected: null },
|
||||
{ label: "invalid HTTP-date", value: "Fri, 99 Foo 9999 99:99:99 GMT", expected: null },
|
||||
{ label: "missing", value: null, expected: null },
|
||||
];
|
||||
|
||||
for (const { label, value, expected } of cases) {
|
||||
const headers = new Headers({ "content-type": "application/json" });
|
||||
if (value !== null) headers.set("retry-after", value);
|
||||
const executor = new ClaudeWebExecutor({
|
||||
sendDirect: async () => ({
|
||||
status: 429,
|
||||
headers,
|
||||
body: null,
|
||||
bodyText: '{"error":"rate_limited"}',
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await executor.execute({
|
||||
model: "claude-sonnet-4-6",
|
||||
body: { messages: [{ role: "user", content: "Hello" }] },
|
||||
stream: false,
|
||||
credentials: {
|
||||
apiKey: `sessionKey=test-${label}`,
|
||||
orgId: "test-org-id",
|
||||
deviceId: "test-device-id",
|
||||
},
|
||||
log: null,
|
||||
});
|
||||
|
||||
assert.equal(result.response.status, 429, `${label}: status`);
|
||||
assert.equal(result.response.headers.get("Retry-After"), expected, `${label}: header`);
|
||||
const body = await result.response.json();
|
||||
assert.deepEqual(body.error, {
|
||||
message: "Rate limited by Claude Web API",
|
||||
type: "rate_limit_error",
|
||||
code: "rate_limit_exceeded",
|
||||
});
|
||||
assert.doesNotMatch(JSON.stringify(body), /sk-claude-retry|\/srv\/private/);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -1,13 +1,12 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
const { createStreamFailureFinalizers } = await import(
|
||||
"../../open-sse/utils/streamFailureFinalization.ts"
|
||||
);
|
||||
const { createStreamFailureFinalizers } =
|
||||
await import("../../open-sse/utils/streamFailureFinalization.ts");
|
||||
const { formatTranslatedStreamError } = await import("../../open-sse/utils/streamErrorFormat.ts");
|
||||
|
||||
test("createStreamFailureFinalizers: 499 client disconnect body is client_disconnected", () => {
|
||||
let captured: { status: number; responseBody: unknown; errorCode?: string | null } | null =
|
||||
null;
|
||||
let captured: { status: number; responseBody: unknown; errorCode?: string | null } | null = null;
|
||||
|
||||
const { onPipelineStreamError } = createStreamFailureFinalizers({
|
||||
isFailureCompletionRecorded: () => false,
|
||||
@@ -60,3 +59,45 @@ test("createStreamFailureFinalizers: caller classification survives into respons
|
||||
assert.equal(body.error.type, "stream_error");
|
||||
assert.equal(body.error.code, "stream_pipeline_error");
|
||||
});
|
||||
|
||||
test("formatTranslatedStreamError projects unsafe upstream classification fields", () => {
|
||||
const unsafeCode = "bad access_token=TOP_SECRET /home/alice/code.ts";
|
||||
const unsafeType = "bad_type\n at /home/alice/type.ts:1:2";
|
||||
const text = formatTranslatedStreamError({
|
||||
error: {
|
||||
status: 502,
|
||||
message: "Upstream failed",
|
||||
code: unsafeCode,
|
||||
type: unsafeType,
|
||||
},
|
||||
});
|
||||
const dataLine = text.split("\n").find((line) => line.startsWith("data: {"));
|
||||
assert.ok(dataLine, "OpenAI SSE must contain a JSON data line");
|
||||
const payload = JSON.parse(dataLine.slice(6)) as {
|
||||
error: { code?: string; type?: string };
|
||||
};
|
||||
|
||||
assert.equal(payload.error.code, "bad_gateway");
|
||||
assert.equal(payload.error.type, "server_error");
|
||||
assert.ok(!text.includes("TOP_SECRET"));
|
||||
assert.ok(!text.includes("/home/alice"));
|
||||
});
|
||||
|
||||
test("formatTranslatedStreamError preserves safe rate-limit classification", () => {
|
||||
const text = formatTranslatedStreamError({
|
||||
error: {
|
||||
status: 429,
|
||||
message: "Weekly quota reached",
|
||||
code: "usage_limit_reached",
|
||||
type: "rate_limit_error",
|
||||
},
|
||||
});
|
||||
const dataLine = text.split("\n").find((line) => line.startsWith("data: {"));
|
||||
assert.ok(dataLine, "OpenAI SSE must contain a JSON data line");
|
||||
const payload = JSON.parse(dataLine.slice(6)) as {
|
||||
error: { code?: string; type?: string };
|
||||
};
|
||||
|
||||
assert.equal(payload.error.code, "usage_limit_reached");
|
||||
assert.equal(payload.error.type, "rate_limit_error");
|
||||
});
|
||||
|
||||
@@ -62,6 +62,26 @@ test("model cooldown body omits retry_after / credentials_cooling when absent or
|
||||
);
|
||||
});
|
||||
|
||||
test("model cooldown body projects hostile model and retry timestamp fields", async () => {
|
||||
const { buildModelCooldownBody } = await import("../../open-sse/utils/error.ts");
|
||||
const secret = "COOLDOWN_SECRET";
|
||||
const body = buildModelCooldownBody({
|
||||
model: `model access_token=${secret} /home/alice/model.ts`,
|
||||
retryAfterSec: Number.NaN,
|
||||
retryAfterAt: `2026-07-07T12:34:56.000Z password=${secret}`,
|
||||
credentialsCoolingCount: 2,
|
||||
});
|
||||
const serialized = JSON.stringify(body);
|
||||
|
||||
assert.equal(body.error.message, "All credentials for the requested model are cooling down");
|
||||
assert.equal(body.error.model, undefined);
|
||||
assert.equal(body.error.retry_after, undefined);
|
||||
assert.equal(body.error.reset_seconds, 1);
|
||||
assert.equal(body.error.credentials_cooling, 2);
|
||||
assert.ok(!serialized.includes(secret));
|
||||
assert.ok(!serialized.includes("/home/alice"));
|
||||
});
|
||||
|
||||
test("modelCooldownResponse emits HTTP 429 with Retry-After header and retry_after ISO in body (#6460)", async () => {
|
||||
const { modelCooldownResponse } = await import("../../open-sse/utils/error.ts");
|
||||
|
||||
|
||||
Reference in New Issue
Block a user