fix(sse): surface malformed HTTP-200 upstream responses (#4942)

Integrated into release/v3.8.37 — cherry-picked defining commit onto release tip; CHANGELOG re-merged; tests green.
2026-07-31 12:22:14 +03:00 · 2026-06-26 00:20:21 -03:00
parent 20412f0e95
commit 2ddf714c2b
3 changed files with 476 additions and 0 deletions
--- a/open-sse/handlers/chatCore.ts
+++ b/open-sse/handlers/chatCore.ts
@@ -109,6 +109,7 @@ import {
  formatProviderError,
  sanitizeErrorMessage,
 } from "../utils/error.ts";
+import { reportMalformed200, detectMalformedNonStream } from "../utils/diagnostics.ts";
 import {
  checkTokenLimits,
  recordTokenUsage,
@@ -3626,6 +3627,59 @@ export async function handleChatCore({
      return createErrorResult(HTTP_STATUS.BAD_REQUEST, guardrailMessage);
    }

+    // Validate the *translated* response actually carries client-usable output.
+    // isEmptyContentResponse (above) runs on the raw responseBody before translation;
+    // this check runs after translation + sanitization + tool-call execution to catch
+    // cases where a provider returns a structurally valid raw body that translates into
+    // choices:[] or output:[] with no usable content (Responses API shape included).
+    const malformedTranslatedReason = detectMalformedNonStream(translatedResponse);
+    if (malformedTranslatedReason) {
+      const totalLatency = Date.now() - startTime;
+      const rawBytes = (() => {
+        try {
+          return JSON.stringify(responseBody || {}).length;
+        } catch {
+          return -1;
+        }
+      })();
+      reportMalformed200({
+        mode: "nonstream",
+        provider,
+        model,
+        connectionId,
+        reason: malformedTranslatedReason,
+        recvBytes: rawBytes,
+        recvLines: -1,
+        emitted: -1,
+        events: {},
+        ttftMs: totalLatency,
+        elapsedMs: totalLatency,
+      });
+      appendRequestLog({
+        model,
+        provider,
+        connectionId,
+        status: `FAILED ${HTTP_STATUS.BAD_GATEWAY}`,
+      }).catch(() => {});
+      const malformedMessage = `[${provider}/${model}] returned an empty response (no usable choices/output)`;
+      persistAttemptLogs({
+        status: HTTP_STATUS.BAD_GATEWAY,
+        tokens: usage,
+        responseBody,
+        providerRequest: finalBody || translatedBody,
+        providerResponse: looksLikeSSE
+          ? { _streamed: true, _format: "sse-json", summary: responseBody }
+          : responseBody,
+        clientResponse: buildErrorBody(HTTP_STATUS.BAD_GATEWAY, malformedMessage),
+        claudeCacheMeta: claudePromptCacheLogMeta,
+        claudeCacheUsageMeta: cacheUsageLogMeta,
+        cacheSource: "upstream",
+      });
+      persistFailureUsage(HTTP_STATUS.BAD_GATEWAY, "malformed_translated_response");
+      trackPendingRequest(model, provider, pendingConnId, false);
+      return createErrorResult(HTTP_STATUS.BAD_GATEWAY, malformedMessage);
+    }
+
    // ── Phase 9.1: Cache store (non-streaming, temp=0) ──
    storeSemanticCacheResponse({
      enabled: semanticCacheEnabled,
--- a/open-sse/utils/diagnostics.ts
+++ b/open-sse/utils/diagnostics.ts
@@ -0,0 +1,222 @@
+/**
+ * Diagnostics for malformed HTTP-200 upstream responses.
+ *
+ * Surfaces HTTP-200-but-empty upstream responses (empty SSE stream, empty
+ * translated body) as structured, sanitized errors rather than silent
+ * `output:[]` / `choices:[]` successes.
+ *
+ * Hard Rule #12: every string that reaches an HTTP/SSE response body MUST
+ * route through sanitizeErrorMessage(). All helpers below enforce this.
+ */
+
+import { sanitizeErrorMessage } from "./error.ts";
+
+// ── Types ────────────────────────────────────────────────────────────────────
+
+export type MalformedReason =
+  | "empty"
+  | "stall"
+  | "abort"
+  | "client_closed"
+  | "no_terminal"
+  | "parse_fail"
+  | "empty_choices"
+  | "empty_stream"
+  | string;
+
+export interface ReportMalformed200Opts {
+  mode: string;
+  provider?: string | null;
+  model?: string | null;
+  connectionId?: string | null;
+  reason?: MalformedReason;
+  recvBytes?: number;
+  recvLines?: number;
+  emitted?: number;
+  events?: Record<string, number>;
+  ttftMs?: number;
+  elapsedMs?: number;
+}
+
+// ── Internal helpers ─────────────────────────────────────────────────────────
+
+// Human-readable reason text surfaced to the client and logs.
+// These strings end up in error.message — they are passed through
+// sanitizeErrorMessage before being embedded in any response body.
+const REASON_MESSAGES: Record<string, string> = {
+  empty: "no content produced",
+  stall: "stream stalled (no data within the stall window)",
+  abort: "stream aborted",
+  client_closed: "client closed the connection",
+  no_terminal: "stream closed without a terminal event",
+  parse_fail: "failed to parse upstream stream",
+  empty_choices: "response had no usable choices/output",
+  empty_stream: "upstream stream carried no content",
+};
+
+function describeReason(reason?: MalformedReason): string {
+  if (!reason) return "empty response";
+  return REASON_MESSAGES[reason] ?? reason;
+}
+
+// ── Exports ──────────────────────────────────────────────────────────────────
+
+/**
+ * Log one structured [MALFORMED-200] line to stdout.
+ * Noop-safe (any field is optional). Used by streaming + non-streaming
+ * handlers to emit a single, grep-correlatable diagnostic entry.
+ */
+export function reportMalformed200(opts: ReportMalformed200Opts): void {
+  const {
+    mode,
+    provider,
+    model,
+    connectionId,
+    reason,
+    recvBytes,
+    recvLines,
+    emitted,
+    events,
+    ttftMs,
+    elapsedMs,
+  } = opts;
+  const evtStr =
+    events && typeof events === "object"
+      ? `[${Object.entries(events)
+          .map(([k, v]) => `${k}=${v}`)
+          .join(",")}]`
+      : "[]";
+  console.log(
+    `[MALFORMED-200] mode=${mode || "?"} provider=${provider || "?"} model=${model || "?"} ` +
+      `conn=${connectionId || "-"} reason=${reason || "empty"} recvBytes=${recvBytes ?? -1} ` +
+      `recvLines=${recvLines ?? -1} emitted=${emitted ?? -1} events=${evtStr} ` +
+      `ttft=${ttftMs ?? -1}ms dur=${elapsedMs ?? -1}ms`
+  );
+}
+
+/**
+ * Synthesize an OpenAI chat.completion.chunk SSE line for an empty stream.
+ * Caller enqueues this before the terminal `data: [DONE]`.
+ *
+ * All user-visible strings are sanitized through sanitizeErrorMessage
+ * (Hard Rule #12) to prevent stack-trace exposure.
+ */
+export function synthOpenAIErrorChunk(opts: {
+  provider?: string | null;
+  model?: string | null;
+  reason?: MalformedReason;
+}): string {
+  const { provider, model, reason } = opts;
+  const reasonText = sanitizeErrorMessage(describeReason(reason));
+  const providerPart = sanitizeErrorMessage(provider ?? "?");
+  const safeMessage = sanitizeErrorMessage(
+    `[${providerPart}] returned an empty response (${reasonText}). ` +
+      "Likely quota exhaustion, an overloaded upstream, or a proxy/gateway intercepting the stream."
+  );
+  const body = {
+    id: `chatcmpl-empty-${Date.now()}`,
+    object: "chat.completion.chunk",
+    created: Math.floor(Date.now() / 1000),
+    model: sanitizeErrorMessage(model ?? "unknown"),
+    choices: [{ index: 0, delta: {}, finish_reason: "stop" }],
+    error: {
+      message: safeMessage,
+      type: "upstream_empty_response",
+    },
+  };
+  return `data: ${JSON.stringify(body)}\n\n`;
+}
+
+/**
+ * Synthesize a response.failed SSE event for an empty/aborted Responses API
+ * passthrough stream.
+ *
+ * Message is sanitized through sanitizeErrorMessage (Hard Rule #12).
+ */
+export function synthResponsesFailure(reason?: MalformedReason): string {
+  const safeMessage = sanitizeErrorMessage(
+    `stream closed before response.completed (${describeReason(reason)})`
+  );
+  const event = {
+    type: "response.failed",
+    response: {
+      id: null,
+      status: "failed",
+      error: {
+        type: "stream_error",
+        code: "stream_disconnected",
+        message: safeMessage,
+      },
+    },
+  };
+  return `event: response.failed\ndata: ${JSON.stringify(event)}\n\n`;
+}
+
+/**
+ * Decide whether a *translated* non-streaming body is malformed for the client.
+ *
+ * Returns a reason string ("empty_choices" | "no_terminal") when the body is
+ * malformed, or null when it carries usable output.
+ *
+ * This runs *after* response translation so it catches cases the raw-body
+ * checks above miss (e.g. a provider returning a valid non-empty raw body that
+ * translates into an OpenAI `choices:[]` with no content).
+ *
+ * Design notes:
+ * - Reasoning-only responses (content="" + reasoning_content) are intentionally
+ *   allowed — they are valid completions, not errors.
+ * - Tool-call responses (content=null + tool_calls=[…]) are also valid.
+ * - Responses API function_call / other structural items count as output even
+ *   when they carry no user-visible text.
+ */
+export function detectMalformedNonStream(resp: unknown): MalformedReason | null {
+  if (!resp || typeof resp !== "object") return "empty_choices";
+
+  const body = resp as Record<string, unknown>;
+
+  // ── Responses API shape ──
+  if (body.object === "response") {
+    const output = body.output;
+    const hasOutput =
+      Array.isArray(output) &&
+      output.some((item) => {
+        if (!item || typeof item !== "object") return false;
+        const it = item as Record<string, unknown>;
+        if (it.type === "message") {
+          return (
+            Array.isArray(it.content) &&
+            (it.content as unknown[]).some((c) => {
+              const part = c as Record<string, unknown>;
+              return typeof part?.text === "string" && (part.text as string).length > 0;
+            })
+          );
+        }
+        // function_call / other structural items count
+        return Boolean(it.type);
+      });
+    if (!hasOutput) return "empty_choices";
+    const status = typeof body.status === "string" ? body.status : "";
+    if (status && !["completed", "done"].includes(status)) return "no_terminal";
+    return null;
+  }
+
+  // ── Chat Completions shape ──
+  const choices = body.choices;
+  if (!Array.isArray(choices) || choices.length === 0) return "empty_choices";
+
+  const anyHasOutput = choices.some((choice) => {
+    const c = choice as Record<string, unknown>;
+    const msg = c?.message as Record<string, unknown> | undefined;
+    if (typeof msg?.content === "string" && (msg.content as string).length > 0) return true;
+    if (Array.isArray(msg?.tool_calls) && (msg.tool_calls as unknown[]).length > 0) return true;
+    if (typeof msg?.reasoning_content === "string" && (msg.reasoning_content as string).length > 0)
+      return true;
+    return false;
+  });
+
+  if (!anyHasOutput) return "empty_choices";
+  return null;
+}
+
+// ── Test-only export ─────────────────────────────────────────────────────────
+export const __test = { describeReason };
--- a/tests/unit/diagnostics.test.ts
+++ b/tests/unit/diagnostics.test.ts
@@ -0,0 +1,200 @@
+/**
+ * Tests for open-sse/utils/diagnostics.ts
+ *
+ * Covers:
+ * (a) synthOpenAIErrorChunk — shape validation
+ * (b) synthResponsesFailure — matches a response.failed event
+ * (c) detectMalformedNonStream — empty/malformed input classified correctly
+ * (d) no stack trace leakage in error chunk message
+ */
+
+import test from "node:test";
+import assert from "node:assert/strict";
+
+import {
+  reportMalformed200,
+  synthOpenAIErrorChunk,
+  synthResponsesFailure,
+  detectMalformedNonStream,
+} from "../../open-sse/utils/diagnostics.ts";
+
+// ── (a) synthOpenAIErrorChunk shape ──────────────────────────────────────────
+
+test("synthOpenAIErrorChunk returns a valid SSE data line", () => {
+  const line = synthOpenAIErrorChunk({
+    provider: "testprovider",
+    model: "gpt-test",
+    reason: "empty_stream",
+  });
+  assert.match(line, /^data: /);
+  assert.ok(line.endsWith("\n\n"), "must end with double newline");
+});
+
+test("synthOpenAIErrorChunk payload has expected OpenAI chunk shape", () => {
+  const line = synthOpenAIErrorChunk({ provider: "myprovider", model: "mymodel", reason: "empty" });
+  const payload = JSON.parse(line.slice("data: ".length).trimEnd());
+  assert.equal(payload.object, "chat.completion.chunk");
+  assert.ok(Array.isArray(payload.choices), "must have choices array");
+  assert.equal(payload.choices.length, 1);
+  assert.ok(payload.error, "must have error field");
+  assert.equal(payload.error.type, "upstream_empty_response");
+  assert.ok(typeof payload.error.message === "string" && payload.error.message.length > 0);
+});
+
+test("synthOpenAIErrorChunk references provider in message", () => {
+  const line = synthOpenAIErrorChunk({
+    provider: "mymysteriosprovider",
+    model: "m",
+    reason: "stall",
+  });
+  const payload = JSON.parse(line.slice("data: ".length).trimEnd());
+  assert.ok(
+    payload.error.message.includes("mymysteriosprovider"),
+    `message should reference provider, got: ${payload.error.message}`
+  );
+});
+
+// ── (b) synthResponsesFailure matches a response.failed event ────────────────
+
+test("synthResponsesFailure produces a response.failed SSE event", () => {
+  const sseText = synthResponsesFailure("empty_stream");
+  assert.match(sseText, /event: response\.failed/);
+  assert.match(sseText, /data: /);
+});
+
+test("synthResponsesFailure includes a reason in the data payload", () => {
+  const sseText = synthResponsesFailure("no_terminal");
+  // Extract JSON after "data: " line
+  const dataLine = sseText.split("\n").find((l) => l.startsWith("data: "));
+  assert.ok(dataLine, "must have a data line");
+  const parsed = JSON.parse(dataLine.slice("data: ".length));
+  assert.ok(
+    parsed?.response?.error?.message?.length > 0,
+    `response.error.message should be non-empty, got: ${JSON.stringify(parsed?.response?.error)}`
+  );
+});
+
+// ── (c) detectMalformedNonStream ─────────────────────────────────────────────
+
+test("detectMalformedNonStream returns 'empty_choices' for null input", () => {
+  assert.equal(detectMalformedNonStream(null), "empty_choices");
+});
+
+test("detectMalformedNonStream returns 'empty_choices' for empty object", () => {
+  assert.equal(detectMalformedNonStream({}), "empty_choices");
+});
+
+test("detectMalformedNonStream returns 'empty_choices' for choices:[]", () => {
+  assert.equal(detectMalformedNonStream({ choices: [] }), "empty_choices");
+});
+
+test("detectMalformedNonStream returns 'empty_choices' when choice message has no content", () => {
+  const body = {
+    choices: [{ message: { content: "", tool_calls: null }, finish_reason: "stop" }],
+  };
+  assert.equal(detectMalformedNonStream(body), "empty_choices");
+});
+
+test("detectMalformedNonStream returns null for valid chat completion", () => {
+  const body = {
+    choices: [{ message: { content: "Hello!", tool_calls: null }, finish_reason: "stop" }],
+  };
+  assert.equal(detectMalformedNonStream(body), null);
+});
+
+test("detectMalformedNonStream returns null when tool_calls present", () => {
+  const body = {
+    choices: [
+      {
+        message: { content: null, tool_calls: [{ id: "call_1", type: "function" }] },
+        finish_reason: "tool_calls",
+      },
+    ],
+  };
+  assert.equal(detectMalformedNonStream(body), null);
+});
+
+test("detectMalformedNonStream returns null when reasoning_content present (reasoning-only)", () => {
+  const body = {
+    choices: [
+      {
+        message: { content: "", reasoning_content: "some reasoning text", tool_calls: null },
+        finish_reason: "stop",
+      },
+    ],
+  };
+  assert.equal(detectMalformedNonStream(body), null);
+});
+
+test("detectMalformedNonStream returns 'empty_choices' for Responses API with empty output", () => {
+  const body = { object: "response", output: [], status: "completed" };
+  assert.equal(detectMalformedNonStream(body), "empty_choices");
+});
+
+test("detectMalformedNonStream returns null for Responses API with text output", () => {
+  const body = {
+    object: "response",
+    status: "completed",
+    output: [
+      {
+        type: "message",
+        content: [{ type: "output_text", text: "Hello there!" }],
+      },
+    ],
+  };
+  assert.equal(detectMalformedNonStream(body), null);
+});
+
+test("detectMalformedNonStream returns 'no_terminal' for Responses API with failed status", () => {
+  const body = {
+    object: "response",
+    status: "failed",
+    output: [
+      {
+        type: "message",
+        content: [{ type: "output_text", text: "Some text" }],
+      },
+    ],
+  };
+  assert.equal(detectMalformedNonStream(body), "no_terminal");
+});
+
+test("detectMalformedNonStream allows Responses API function_call items as valid output", () => {
+  const body = {
+    object: "response",
+    status: "completed",
+    output: [{ type: "function_call", name: "search", arguments: "{}" }],
+  };
+  assert.equal(detectMalformedNonStream(body), null);
+});
+
+// ── (d) no stack trace leakage ───────────────────────────────────────────────
+
+test("synthOpenAIErrorChunk message does NOT contain stack trace path", () => {
+  const line = synthOpenAIErrorChunk({ provider: "p", model: "m", reason: "empty_stream" });
+  const payload = JSON.parse(line.slice("data: ".length).trimEnd());
+  const msg = payload.error.message as string;
+  assert.ok(
+    !msg.includes("at /"),
+    `error.message must not contain stack trace patterns, got: ${msg}`
+  );
+});
+
+test("reportMalformed200 runs without throwing", () => {
+  // smoke: it only logs, should not throw
+  assert.doesNotThrow(() =>
+    reportMalformed200({
+      mode: "nonstream",
+      provider: "testprov",
+      model: "testmodel",
+      connectionId: "conn-123",
+      reason: "empty_choices",
+      recvBytes: 42,
+      recvLines: -1,
+      emitted: -1,
+      events: { "response.completed": 1 },
+      ttftMs: 100,
+      elapsedMs: 200,
+    })
+  );
+});