diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index a15e82cd48..86f143b27c 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -109,6 +109,7 @@ import { formatProviderError, sanitizeErrorMessage, } from "../utils/error.ts"; +import { reportMalformed200, detectMalformedNonStream } from "../utils/diagnostics.ts"; import { checkTokenLimits, recordTokenUsage, @@ -3626,6 +3627,59 @@ export async function handleChatCore({ return createErrorResult(HTTP_STATUS.BAD_REQUEST, guardrailMessage); } + // Validate the *translated* response actually carries client-usable output. + // isEmptyContentResponse (above) runs on the raw responseBody before translation; + // this check runs after translation + sanitization + tool-call execution to catch + // cases where a provider returns a structurally valid raw body that translates into + // choices:[] or output:[] with no usable content (Responses API shape included). + const malformedTranslatedReason = detectMalformedNonStream(translatedResponse); + if (malformedTranslatedReason) { + const totalLatency = Date.now() - startTime; + const rawBytes = (() => { + try { + return JSON.stringify(responseBody || {}).length; + } catch { + return -1; + } + })(); + reportMalformed200({ + mode: "nonstream", + provider, + model, + connectionId, + reason: malformedTranslatedReason, + recvBytes: rawBytes, + recvLines: -1, + emitted: -1, + events: {}, + ttftMs: totalLatency, + elapsedMs: totalLatency, + }); + appendRequestLog({ + model, + provider, + connectionId, + status: `FAILED ${HTTP_STATUS.BAD_GATEWAY}`, + }).catch(() => {}); + const malformedMessage = `[${provider}/${model}] returned an empty response (no usable choices/output)`; + persistAttemptLogs({ + status: HTTP_STATUS.BAD_GATEWAY, + tokens: usage, + responseBody, + providerRequest: finalBody || translatedBody, + providerResponse: looksLikeSSE + ? { _streamed: true, _format: "sse-json", summary: responseBody } + : responseBody, + clientResponse: buildErrorBody(HTTP_STATUS.BAD_GATEWAY, malformedMessage), + claudeCacheMeta: claudePromptCacheLogMeta, + claudeCacheUsageMeta: cacheUsageLogMeta, + cacheSource: "upstream", + }); + persistFailureUsage(HTTP_STATUS.BAD_GATEWAY, "malformed_translated_response"); + trackPendingRequest(model, provider, pendingConnId, false); + return createErrorResult(HTTP_STATUS.BAD_GATEWAY, malformedMessage); + } + // ── Phase 9.1: Cache store (non-streaming, temp=0) ── storeSemanticCacheResponse({ enabled: semanticCacheEnabled, diff --git a/open-sse/utils/diagnostics.ts b/open-sse/utils/diagnostics.ts new file mode 100644 index 0000000000..8c58358c80 --- /dev/null +++ b/open-sse/utils/diagnostics.ts @@ -0,0 +1,222 @@ +/** + * Diagnostics for malformed HTTP-200 upstream responses. + * + * Surfaces HTTP-200-but-empty upstream responses (empty SSE stream, empty + * translated body) as structured, sanitized errors rather than silent + * `output:[]` / `choices:[]` successes. + * + * Hard Rule #12: every string that reaches an HTTP/SSE response body MUST + * route through sanitizeErrorMessage(). All helpers below enforce this. + */ + +import { sanitizeErrorMessage } from "./error.ts"; + +// ── Types ──────────────────────────────────────────────────────────────────── + +export type MalformedReason = + | "empty" + | "stall" + | "abort" + | "client_closed" + | "no_terminal" + | "parse_fail" + | "empty_choices" + | "empty_stream" + | string; + +export interface ReportMalformed200Opts { + mode: string; + provider?: string | null; + model?: string | null; + connectionId?: string | null; + reason?: MalformedReason; + recvBytes?: number; + recvLines?: number; + emitted?: number; + events?: Record; + ttftMs?: number; + elapsedMs?: number; +} + +// ── Internal helpers ───────────────────────────────────────────────────────── + +// Human-readable reason text surfaced to the client and logs. +// These strings end up in error.message — they are passed through +// sanitizeErrorMessage before being embedded in any response body. +const REASON_MESSAGES: Record = { + empty: "no content produced", + stall: "stream stalled (no data within the stall window)", + abort: "stream aborted", + client_closed: "client closed the connection", + no_terminal: "stream closed without a terminal event", + parse_fail: "failed to parse upstream stream", + empty_choices: "response had no usable choices/output", + empty_stream: "upstream stream carried no content", +}; + +function describeReason(reason?: MalformedReason): string { + if (!reason) return "empty response"; + return REASON_MESSAGES[reason] ?? reason; +} + +// ── Exports ────────────────────────────────────────────────────────────────── + +/** + * Log one structured [MALFORMED-200] line to stdout. + * Noop-safe (any field is optional). Used by streaming + non-streaming + * handlers to emit a single, grep-correlatable diagnostic entry. + */ +export function reportMalformed200(opts: ReportMalformed200Opts): void { + const { + mode, + provider, + model, + connectionId, + reason, + recvBytes, + recvLines, + emitted, + events, + ttftMs, + elapsedMs, + } = opts; + const evtStr = + events && typeof events === "object" + ? `[${Object.entries(events) + .map(([k, v]) => `${k}=${v}`) + .join(",")}]` + : "[]"; + console.log( + `[MALFORMED-200] mode=${mode || "?"} provider=${provider || "?"} model=${model || "?"} ` + + `conn=${connectionId || "-"} reason=${reason || "empty"} recvBytes=${recvBytes ?? -1} ` + + `recvLines=${recvLines ?? -1} emitted=${emitted ?? -1} events=${evtStr} ` + + `ttft=${ttftMs ?? -1}ms dur=${elapsedMs ?? -1}ms` + ); +} + +/** + * Synthesize an OpenAI chat.completion.chunk SSE line for an empty stream. + * Caller enqueues this before the terminal `data: [DONE]`. + * + * All user-visible strings are sanitized through sanitizeErrorMessage + * (Hard Rule #12) to prevent stack-trace exposure. + */ +export function synthOpenAIErrorChunk(opts: { + provider?: string | null; + model?: string | null; + reason?: MalformedReason; +}): string { + const { provider, model, reason } = opts; + const reasonText = sanitizeErrorMessage(describeReason(reason)); + const providerPart = sanitizeErrorMessage(provider ?? "?"); + const safeMessage = sanitizeErrorMessage( + `[${providerPart}] returned an empty response (${reasonText}). ` + + "Likely quota exhaustion, an overloaded upstream, or a proxy/gateway intercepting the stream." + ); + const body = { + id: `chatcmpl-empty-${Date.now()}`, + object: "chat.completion.chunk", + created: Math.floor(Date.now() / 1000), + model: sanitizeErrorMessage(model ?? "unknown"), + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + error: { + message: safeMessage, + type: "upstream_empty_response", + }, + }; + return `data: ${JSON.stringify(body)}\n\n`; +} + +/** + * Synthesize a response.failed SSE event for an empty/aborted Responses API + * passthrough stream. + * + * Message is sanitized through sanitizeErrorMessage (Hard Rule #12). + */ +export function synthResponsesFailure(reason?: MalformedReason): string { + const safeMessage = sanitizeErrorMessage( + `stream closed before response.completed (${describeReason(reason)})` + ); + const event = { + type: "response.failed", + response: { + id: null, + status: "failed", + error: { + type: "stream_error", + code: "stream_disconnected", + message: safeMessage, + }, + }, + }; + return `event: response.failed\ndata: ${JSON.stringify(event)}\n\n`; +} + +/** + * Decide whether a *translated* non-streaming body is malformed for the client. + * + * Returns a reason string ("empty_choices" | "no_terminal") when the body is + * malformed, or null when it carries usable output. + * + * This runs *after* response translation so it catches cases the raw-body + * checks above miss (e.g. a provider returning a valid non-empty raw body that + * translates into an OpenAI `choices:[]` with no content). + * + * Design notes: + * - Reasoning-only responses (content="" + reasoning_content) are intentionally + * allowed — they are valid completions, not errors. + * - Tool-call responses (content=null + tool_calls=[…]) are also valid. + * - Responses API function_call / other structural items count as output even + * when they carry no user-visible text. + */ +export function detectMalformedNonStream(resp: unknown): MalformedReason | null { + if (!resp || typeof resp !== "object") return "empty_choices"; + + const body = resp as Record; + + // ── Responses API shape ── + if (body.object === "response") { + const output = body.output; + const hasOutput = + Array.isArray(output) && + output.some((item) => { + if (!item || typeof item !== "object") return false; + const it = item as Record; + if (it.type === "message") { + return ( + Array.isArray(it.content) && + (it.content as unknown[]).some((c) => { + const part = c as Record; + return typeof part?.text === "string" && (part.text as string).length > 0; + }) + ); + } + // function_call / other structural items count + return Boolean(it.type); + }); + if (!hasOutput) return "empty_choices"; + const status = typeof body.status === "string" ? body.status : ""; + if (status && !["completed", "done"].includes(status)) return "no_terminal"; + return null; + } + + // ── Chat Completions shape ── + const choices = body.choices; + if (!Array.isArray(choices) || choices.length === 0) return "empty_choices"; + + const anyHasOutput = choices.some((choice) => { + const c = choice as Record; + const msg = c?.message as Record | undefined; + if (typeof msg?.content === "string" && (msg.content as string).length > 0) return true; + if (Array.isArray(msg?.tool_calls) && (msg.tool_calls as unknown[]).length > 0) return true; + if (typeof msg?.reasoning_content === "string" && (msg.reasoning_content as string).length > 0) + return true; + return false; + }); + + if (!anyHasOutput) return "empty_choices"; + return null; +} + +// ── Test-only export ───────────────────────────────────────────────────────── +export const __test = { describeReason }; diff --git a/tests/unit/diagnostics.test.ts b/tests/unit/diagnostics.test.ts new file mode 100644 index 0000000000..82b202e385 --- /dev/null +++ b/tests/unit/diagnostics.test.ts @@ -0,0 +1,200 @@ +/** + * Tests for open-sse/utils/diagnostics.ts + * + * Covers: + * (a) synthOpenAIErrorChunk — shape validation + * (b) synthResponsesFailure — matches a response.failed event + * (c) detectMalformedNonStream — empty/malformed input classified correctly + * (d) no stack trace leakage in error chunk message + */ + +import test from "node:test"; +import assert from "node:assert/strict"; + +import { + reportMalformed200, + synthOpenAIErrorChunk, + synthResponsesFailure, + detectMalformedNonStream, +} from "../../open-sse/utils/diagnostics.ts"; + +// ── (a) synthOpenAIErrorChunk shape ────────────────────────────────────────── + +test("synthOpenAIErrorChunk returns a valid SSE data line", () => { + const line = synthOpenAIErrorChunk({ + provider: "testprovider", + model: "gpt-test", + reason: "empty_stream", + }); + assert.match(line, /^data: /); + assert.ok(line.endsWith("\n\n"), "must end with double newline"); +}); + +test("synthOpenAIErrorChunk payload has expected OpenAI chunk shape", () => { + const line = synthOpenAIErrorChunk({ provider: "myprovider", model: "mymodel", reason: "empty" }); + const payload = JSON.parse(line.slice("data: ".length).trimEnd()); + assert.equal(payload.object, "chat.completion.chunk"); + assert.ok(Array.isArray(payload.choices), "must have choices array"); + assert.equal(payload.choices.length, 1); + assert.ok(payload.error, "must have error field"); + assert.equal(payload.error.type, "upstream_empty_response"); + assert.ok(typeof payload.error.message === "string" && payload.error.message.length > 0); +}); + +test("synthOpenAIErrorChunk references provider in message", () => { + const line = synthOpenAIErrorChunk({ + provider: "mymysteriosprovider", + model: "m", + reason: "stall", + }); + const payload = JSON.parse(line.slice("data: ".length).trimEnd()); + assert.ok( + payload.error.message.includes("mymysteriosprovider"), + `message should reference provider, got: ${payload.error.message}` + ); +}); + +// ── (b) synthResponsesFailure matches a response.failed event ──────────────── + +test("synthResponsesFailure produces a response.failed SSE event", () => { + const sseText = synthResponsesFailure("empty_stream"); + assert.match(sseText, /event: response\.failed/); + assert.match(sseText, /data: /); +}); + +test("synthResponsesFailure includes a reason in the data payload", () => { + const sseText = synthResponsesFailure("no_terminal"); + // Extract JSON after "data: " line + const dataLine = sseText.split("\n").find((l) => l.startsWith("data: ")); + assert.ok(dataLine, "must have a data line"); + const parsed = JSON.parse(dataLine.slice("data: ".length)); + assert.ok( + parsed?.response?.error?.message?.length > 0, + `response.error.message should be non-empty, got: ${JSON.stringify(parsed?.response?.error)}` + ); +}); + +// ── (c) detectMalformedNonStream ───────────────────────────────────────────── + +test("detectMalformedNonStream returns 'empty_choices' for null input", () => { + assert.equal(detectMalformedNonStream(null), "empty_choices"); +}); + +test("detectMalformedNonStream returns 'empty_choices' for empty object", () => { + assert.equal(detectMalformedNonStream({}), "empty_choices"); +}); + +test("detectMalformedNonStream returns 'empty_choices' for choices:[]", () => { + assert.equal(detectMalformedNonStream({ choices: [] }), "empty_choices"); +}); + +test("detectMalformedNonStream returns 'empty_choices' when choice message has no content", () => { + const body = { + choices: [{ message: { content: "", tool_calls: null }, finish_reason: "stop" }], + }; + assert.equal(detectMalformedNonStream(body), "empty_choices"); +}); + +test("detectMalformedNonStream returns null for valid chat completion", () => { + const body = { + choices: [{ message: { content: "Hello!", tool_calls: null }, finish_reason: "stop" }], + }; + assert.equal(detectMalformedNonStream(body), null); +}); + +test("detectMalformedNonStream returns null when tool_calls present", () => { + const body = { + choices: [ + { + message: { content: null, tool_calls: [{ id: "call_1", type: "function" }] }, + finish_reason: "tool_calls", + }, + ], + }; + assert.equal(detectMalformedNonStream(body), null); +}); + +test("detectMalformedNonStream returns null when reasoning_content present (reasoning-only)", () => { + const body = { + choices: [ + { + message: { content: "", reasoning_content: "some reasoning text", tool_calls: null }, + finish_reason: "stop", + }, + ], + }; + assert.equal(detectMalformedNonStream(body), null); +}); + +test("detectMalformedNonStream returns 'empty_choices' for Responses API with empty output", () => { + const body = { object: "response", output: [], status: "completed" }; + assert.equal(detectMalformedNonStream(body), "empty_choices"); +}); + +test("detectMalformedNonStream returns null for Responses API with text output", () => { + const body = { + object: "response", + status: "completed", + output: [ + { + type: "message", + content: [{ type: "output_text", text: "Hello there!" }], + }, + ], + }; + assert.equal(detectMalformedNonStream(body), null); +}); + +test("detectMalformedNonStream returns 'no_terminal' for Responses API with failed status", () => { + const body = { + object: "response", + status: "failed", + output: [ + { + type: "message", + content: [{ type: "output_text", text: "Some text" }], + }, + ], + }; + assert.equal(detectMalformedNonStream(body), "no_terminal"); +}); + +test("detectMalformedNonStream allows Responses API function_call items as valid output", () => { + const body = { + object: "response", + status: "completed", + output: [{ type: "function_call", name: "search", arguments: "{}" }], + }; + assert.equal(detectMalformedNonStream(body), null); +}); + +// ── (d) no stack trace leakage ─────────────────────────────────────────────── + +test("synthOpenAIErrorChunk message does NOT contain stack trace path", () => { + const line = synthOpenAIErrorChunk({ provider: "p", model: "m", reason: "empty_stream" }); + const payload = JSON.parse(line.slice("data: ".length).trimEnd()); + const msg = payload.error.message as string; + assert.ok( + !msg.includes("at /"), + `error.message must not contain stack trace patterns, got: ${msg}` + ); +}); + +test("reportMalformed200 runs without throwing", () => { + // smoke: it only logs, should not throw + assert.doesNotThrow(() => + reportMalformed200({ + mode: "nonstream", + provider: "testprov", + model: "testmodel", + connectionId: "conn-123", + reason: "empty_choices", + recvBytes: 42, + recvLines: -1, + emitted: -1, + events: { "response.completed": 1 }, + ttftMs: 100, + elapsedMs: 200, + }) + ); +});