feat(compression): add a transcript loader to the replay harness (#4246)

Integrated into release/v3.8.29

open-sse/services/compression/harness/index.ts

@@ -32,8 +32,11 @@ export {
 export {
   transcriptsToCorpus,
   replayTranscripts,
+  requestBodyToTranscript,
+  requestBodiesToTranscripts,
   type Transcript,
   type TranscriptTurn,
+  type CapturedRequestBody,
 } from "./replay.ts";
 
 export {

open-sse/services/compression/harness/replay.ts

@@ -1,4 +1,5 @@
 import { runCompressionEval, type CompressFn, type EvalCase, type EvalReport } from "./runner.ts";
+import { extractTextContent, type ChatMessageLike } from "../messageContent.ts";
 
 /**
  * Replay-bench over real transcripts (TV3). Instead of synthetic prompts, feed
@@ -36,3 +37,36 @@ export function replayTranscripts(
 ): Promise<EvalReport> {
   return runCompressionEval(transcriptsToCorpus(transcripts), compress);
 }
+
+/** Shape of a captured request body — only the messages array matters for replay. */
+export interface CapturedRequestBody {
+  messages?: Array<{ role?: unknown; content?: unknown }>;
+}
+
+/**
+ * Build a {@link Transcript} from a captured request body (a call-log / capture-store entry).
+ * Multimodal and tool-result content blocks are flattened to text via extractTextContent, so a
+ * replay corpus can be sourced from real traffic instead of synthetic prompts. A non-object body
+ * or one without a `messages` array yields a transcript with no turns (callers can filter those).
+ */
+export function requestBodyToTranscript(id: string, body: unknown): Transcript {
+  const messages =
+    body && typeof body === "object" && Array.isArray((body as CapturedRequestBody).messages)
+      ? ((body as CapturedRequestBody).messages as Array<{ role?: unknown; content?: unknown }>)
+      : [];
+  const turns: TranscriptTurn[] = messages.map((message) => ({
+    role: typeof message.role === "string" ? message.role : "user",
+    content: extractTextContent(message.content as ChatMessageLike["content"]),
+  }));
+  return { id, turns };
+}
+
+/**
+ * Map a list of captured request bodies (e.g. read from the capture store / call logs) into
+ * transcripts. Pair with {@link replayTranscripts} to benchmark compression over real traffic.
+ */
+export function requestBodiesToTranscripts(
+  entries: Array<{ id: string; body: unknown }>
+): Transcript[] {
+  return entries.map((entry) => requestBodyToTranscript(entry.id, entry.body));
+}

tests/unit/compression/harness.test.ts

@@ -10,6 +10,8 @@ import {
   checkTokensPerTaskGate,
   replayTranscripts,
   transcriptsToCorpus,
+  requestBodyToTranscript,
+  requestBodiesToTranscripts,
 } from "../../../open-sse/services/compression/harness/index.ts";
 
 const SAMPLE = "Call fetchUser() at https://api.example.com/v1 with MAX_RETRIES set to 3.0.0";
@@ -136,3 +138,45 @@ describe("compression harness — transcript replay (TV3)", () => {
     assert.equal(report.meanRetention, 1);
   });
 });
+
+describe("compression harness — transcript loader (TV3)", () => {
+  it("builds a transcript from a captured request body, flattening content blocks", () => {
+    const transcript = requestBodyToTranscript("req-1", {
+      model: "gpt-x",
+      messages: [
+        { role: "system", content: "You are helpful." },
+        {
+          role: "user",
+          content: [
+            { type: "text", text: "first block" },
+            { type: "image_url", image_url: { url: "data:..." } },
+            { type: "text", text: "second block" },
+          ],
+        },
+      ],
+    });
+    assert.equal(transcript.id, "req-1");
+    assert.equal(transcript.turns.length, 2);
+    assert.equal(transcript.turns[0].role, "system");
+    assert.equal(transcript.turns[0].content, "You are helpful.");
+    // multimodal content flattened to its text blocks (image dropped)
+    assert.equal(transcript.turns[1].content, "first block\nsecond block");
+  });
+
+  it("returns an empty transcript for a body without a messages array", () => {
+    assert.deepEqual(requestBodyToTranscript("empty", { foo: 1 }), { id: "empty", turns: [] });
+    assert.deepEqual(requestBodyToTranscript("nullish", null), { id: "nullish", turns: [] });
+  });
+
+  it("maps captured bodies into transcripts that feed the replay corpus", () => {
+    const transcripts = requestBodiesToTranscripts([
+      { id: "a", body: { messages: [{ role: "user", content: "hi" }] } },
+      { id: "b", body: { messages: [{ role: "user", content: "  " }] } }, // empty turn → skipped
+    ]);
+    assert.equal(transcripts.length, 2);
+    const corpus = transcriptsToCorpus(transcripts);
+    // transcript "a" contributes one case; "b" is all-blank so transcriptsToCorpus drops it
+    assert.equal(corpus.length, 1);
+    assert.equal(corpus[0].task, "a");
+  });
+});