OmniRoute/tests/unit/responses-transformer-cjk-split.test.ts

import test from "node:test";
import assert from "node:assert/strict";

// Regression guard for #10223 — DeepSeek /v1/responses corrupted SSE deltas.
//
// ROOT CAUSE (open-sse/transformer/responsesTransformer.ts:580): the transform()
// handler created a brand-new `new TextDecoder()` on every chunk and decoded it
// WITHOUT `{ stream: true }`. A stream:false decoder has no cross-call state, so
// whenever a multi-byte UTF-8 character (CJK: 3 bytes, emoji: 4) is split across
// two TCP chunks — the normal case in Chinese streaming text (the reporter's
// scenario), the trailing partial bytes are replaced with U+FFFD and the deltas
// accumulate garbage.
//
// This test feeds a CJK text split at a byte boundary INSIDE a multi-byte
// character and asserts a round-trip against the source text — NOT the
// `join(deltas) === done` invariant, which cannot catch this bug because done is
// rebuilt from the same corrupted buffer as the deltas.

const { createResponsesApiTransformStream } = await import(
  "../../open-sse/transformer/responsesTransformer.ts"
);

const encoder = new TextEncoder();
const decoder = new TextDecoder();

function concatBytes(parts) {
  const total = parts.reduce((sum, part) => sum + part.length, 0);
  const out = new Uint8Array(total);
  let offset = 0;
  for (const part of parts) {
    out.set(part, offset);
    offset += part.length;
  }
  return out;
}

function parseSseOutput(output) {
  return output
    .trim()
    .split("\n\n")
    .map((entry) => {
      const lines = entry.split("\n");
      const eventLine = lines.find((line) => line.startsWith("event: "));
      const dataLine = lines.find((line) => line.startsWith("data: "));
      return {
        event: eventLine ? eventLine.slice("event: ".length) : null,
        data: dataLine ? dataLine.slice("data: ".length) : null,
      };
    })
    .filter((e) => e.event !== null || e.data !== null);
}

async function runRawBytes(byteChunks, options = {}) {
  const stream = createResponsesApiTransformStream(null, 3000, options);
  const writer = stream.writable.getWriter();
  const reader = stream.readable.getReader();

  const raw = [];
  const readerTask = (async () => {
    while (true) {
      const { value, done } = await reader.read();
      if (done) break;
      if (value) raw.push(value);
    }
  })();

  for (const chunk of byteChunks) {
    await writer.write(chunk);
  }
  await writer.close();
  await readerTask;

  return decoder.decode(concatBytes(raw));
}

test("responses transform preserves multi-byte UTF-8 text split across byte chunks (#10223)", async () => {
  const source = "REASONIX_中文测试_DEEPSEEK_OK";

  const frame = (data) =>
    encoder.encode(`data: ${JSON.stringify(data)}\n\n`);

  const deltaChunk = frame({
    choices: [{ index: 0, delta: { content: source } }],
  });
  const finishChunk = frame({
    choices: [{ index: 0, delta: {}, finish_reason: "stop" }],
  });
  const full = concatBytes([deltaChunk, finishChunk]);

  // Split mid-byte inside the first 3-byte CJK character "中".
  const contentPrefix = encoder.encode(
    'data: {"choices":[{"index":0,"delta":{"content":"'
  ).length;
  const boundary = contentPrefix + encoder.encode("REASONIX_").length + 1;
  const chunkA = full.slice(0, boundary);
  const chunkB = full.slice(boundary);

  const output = await runRawBytes([chunkA, chunkB]);

  const events = parseSseOutput(output);
  const deltas = events
    .filter((e) => e.event === "response.output_text.delta")
    .map((e) => JSON.parse(e.data).delta);

  const doneEvent = events.find((e) => e.event === "response.output_text.done");
  const doneText = JSON.parse(doneEvent.data).text;

  // Round-trip against the SOURCE text — the invariant the old test missed.
  assert.equal(deltas.join(""), source, "joined deltas should round-trip to the source text");
  assert.equal(doneText, source, "done snapshot should round-trip to the source text");
});