feat(compression): TOON best-of-N candidate encoder + encoder A/B table (#5163)

Integrated into release/v3.8.39. TOON best-of-N candidate encoder (GCF default, fail-open). 17/17 unit tests pass on merge result; CI reds were base-stale + Quality Ratchet DRIFT.

config/quality/dependency-allowlist.json

@@ -23,6 +23,7 @@
     "@tensorflow/tfjs",
     "@testing-library/jest-dom",
     "@testing-library/react",
+    "@toon-format/toon",
     "@types/bcryptjs",
     "@types/better-sqlite3",
     "@types/bun",

open-sse/package.json

@@ -9,5 +9,8 @@
   "exports": {
     ".": "./index.js",
     "./*": "./*"
+  },
+  "dependencies": {
+    "@toon-format/toon": "^2.3.0"
   }
 }

open-sse/services/compression/engines/headroom/encoderComparison.ts

@@ -0,0 +1,72 @@
+/**
+ * encoderComparison.ts — pure A/B aggregator for the studio.
+ * Scans the same arrays SmartCrusher would compact and reports JSON vs GCF vs TOON
+ * sizes (bytes + tokens). countTokens is injected by the caller (open-sse must not
+ * import the app-side tiktoken counter). Fail-open: if TOON is unavailable on any
+ * array, toonAvailable=false and TOON is not eligible as winner.
+ */
+import { encodeTabularBlock, wrapTabular } from "./tabular.ts";
+import { encodeToonBlock, wrapToon } from "./toon.ts";
+import { collectCompactableArrays } from "./smartcrusher.ts";
+
+export interface EncoderSize {
+  bytes: number;
+  tokens: number;
+}
+export interface EncoderComparison {
+  arraysCompared: number;
+  json: EncoderSize;
+  gcf: EncoderSize;
+  toon: EncoderSize;
+  toonAvailable: boolean;
+  winner: "gcf" | "toon" | "json";
+}
+
+type MessageLike = { role?: string; content?: unknown };
+const ZERO: EncoderSize = { bytes: 0, tokens: 0 };
+
+function add(a: EncoderSize, text: string, countTokens: (t: string) => number): EncoderSize {
+  return { bytes: a.bytes + Buffer.byteLength(text, "utf8"), tokens: a.tokens + countTokens(text) };
+}
+
+function pickWinner(
+  json: EncoderSize,
+  gcf: EncoderSize,
+  toon: EncoderSize,
+  toonAvailable: boolean
+): "gcf" | "toon" | "json" {
+  const candidates: Array<["gcf" | "toon" | "json", EncoderSize]> = [
+    ["gcf", gcf],
+    ["json", json],
+  ];
+  if (toonAvailable) candidates.push(["toon", toon]);
+  candidates.sort((a, b) => a[1].tokens - b[1].tokens || a[1].bytes - b[1].bytes);
+  return candidates[0][0];
+}
+
+export function summarizeEncoderCandidates(
+  messages: MessageLike[],
+  minRows: number,
+  countTokens: (text: string) => number
+): EncoderComparison {
+  const arrays = collectCompactableArrays(messages as never, minRows);
+  let json = { ...ZERO },
+    gcf = { ...ZERO },
+    toon = { ...ZERO };
+  let toonAvailable = arrays.length > 0;
+  for (const arr of arrays) {
+    json = add(json, JSON.stringify(arr), countTokens);
+    gcf = add(gcf, wrapTabular(encodeTabularBlock(arr)), countTokens);
+    const toonInner = encodeToonBlock(arr);
+    if (toonInner === null) toonAvailable = false;
+    else toon = add(toon, wrapToon(toonInner), countTokens);
+  }
+  return {
+    arraysCompared: arrays.length,
+    json,
+    gcf,
+    toon: toonAvailable ? toon : { ...ZERO },
+    toonAvailable,
+    winner: pickWinner(json, gcf, toon, toonAvailable),
+  };
+}

open-sse/services/compression/engines/headroom/index.ts

@@ -41,6 +41,7 @@ import {
   GCF_FENCE_CLOSE,
   decodeTabular,
 } from "./tabular.ts";
+import { TOON_FENCE_OPEN, TOON_FENCE_CLOSE } from "./toon.ts";
 
 export { encodeTabular, decodeTabular } from "./tabular.ts";
 
@@ -232,35 +233,54 @@ export function reconstructHeadroom(body: Record<string, unknown>): Record<strin
  * Restore all GCF (```gcf-generic) and legacy (```omni-tabular) blocks
  * in a text string back to their original JSON.
  */
-function restoreText(text: string): string {
-  // Fast path: no fence marker present
-  if (!text.includes(TABULAR_FENCE_OPEN) && !text.includes(GCF_FENCE_OPEN)) return text;
+/** Map a fence-open marker to its matching close tag. */
+function closeTagFor(fence: string): string {
+  if (fence === GCF_FENCE_OPEN) return GCF_FENCE_CLOSE;
+  if (fence === TOON_FENCE_OPEN) return TOON_FENCE_CLOSE;
+  return TABULAR_FENCE_CLOSE;
+}
 
+/**
+ * Decode every occurrence of one fence type in `text`, replacing each block
+ * with its original JSON. Extracted from restoreText to keep that function
+ * below the cognitive-complexity gate.
+ */
+function decodeFenceOccurrences(text: string, fence: string, closeTag: string): string {
   let result = text;
+  let searchFrom = 0;
+  while (true) {
+    const fenceStart = result.indexOf(fence, searchFrom);
+    if (fenceStart === -1) break;
 
-  // Process both fence types: GCF first (new format), then legacy omni-tabular
-  for (const fence of [GCF_FENCE_OPEN, TABULAR_FENCE_OPEN]) {
-    const closeTag = fence === GCF_FENCE_OPEN ? GCF_FENCE_CLOSE : TABULAR_FENCE_CLOSE;
+    const contentStart = fenceStart + fence.length + 1; // skip "\n" after fence open
+    const fenceEnd = result.indexOf("\n" + closeTag, contentStart);
+    if (fenceEnd === -1) break;
 
-    let searchFrom = 0;
-    while (true) {
-      const fenceStart = result.indexOf(fence, searchFrom);
-      if (fenceStart === -1) break;
+    const blockContent = result.slice(contentStart, fenceEnd);
+    const decoded = decodeTabular(fence + "\n" + blockContent + "\n" + closeTag);
+    const jsonStr = JSON.stringify(decoded);
 
-      const contentStart = fenceStart + fence.length + 1; // skip "\n" after fence open
-      const fenceEnd = result.indexOf("\n" + closeTag, contentStart);
-      if (fenceEnd === -1) break;
+    const fullFence = result.slice(fenceStart, fenceEnd + closeTag.length + 1); // +1 for the "\n"
+    result = result.slice(0, fenceStart) + jsonStr + result.slice(fenceStart + fullFence.length);
 
-      const blockContent = result.slice(contentStart, fenceEnd);
-      const decoded = decodeTabular(fence + "\n" + blockContent + "\n" + closeTag);
-      const jsonStr = JSON.stringify(decoded);
+    searchFrom = fenceStart + jsonStr.length;
+  }
+  return result;
+}
 
-      const fullFence = result.slice(fenceStart, fenceEnd + closeTag.length + 1); // +1 for the "\n"
-      result = result.slice(0, fenceStart) + jsonStr + result.slice(fenceStart + fullFence.length);
+function restoreText(text: string): string {
+  // Fast path: no fence marker present
+  if (
+    !text.includes(TABULAR_FENCE_OPEN) &&
+    !text.includes(GCF_FENCE_OPEN) &&
+    !text.includes(TOON_FENCE_OPEN)
+  )
+    return text;
 
-      searchFrom = fenceStart + jsonStr.length;
-    }
+  let result = text;
+  // Process all fence types: GCF first (new format), then legacy omni-tabular, then TOON
+  for (const fence of [GCF_FENCE_OPEN, TABULAR_FENCE_OPEN, TOON_FENCE_OPEN]) {
+    result = decodeFenceOccurrences(result, fence, closeTagFor(fence));
   }
-
   return result;
 }

open-sse/services/compression/engines/headroom/smartcrusher.ts

@@ -21,6 +21,7 @@
  */
 
 import { encodeTabularBlock, wrapTabular, kindOf } from "./tabular.ts";
+import { encodeToonBlock, wrapToon } from "./toon.ts";
 
 /** Default minimum number of rows to trigger compaction. */
 export const DEFAULT_MIN_ROWS = 8;
@@ -101,8 +102,7 @@ export function tryCompactJson(jsonStr: string, minRows: number = DEFAULT_MIN_RO
   if (!allObjects(parsed)) return null;
 
   const arr = parsed as Record<string, unknown>[];
-  const blockContent = encodeTabularBlock(arr);
-  const compact = wrapTabular(blockContent);
+  const compact = pickSmallestEncoding(arr);
 
   // Only use compact form if it is strictly smaller
   if (compact.length >= jsonStr.length) return null;
@@ -110,12 +110,68 @@ export function tryCompactJson(jsonStr: string, minRows: number = DEFAULT_MIN_RO
   return compact;
 }
 
-type MessageLike = {
+/**
+ * Best-of-N encoder selection: GCF (default) vs TOON. Returns the strictly
+ * smaller fenced block; ties resolve to GCF for cache stability. Extracted so
+ * tryCompactJson stays below the complexity gate.
+ */
+export function pickSmallestEncoding(arr: Record<string, unknown>[]): string {
+  const gcf = wrapTabular(encodeTabularBlock(arr));
+  const toonInner = encodeToonBlock(arr);
+  if (toonInner !== null) {
+    const toon = wrapToon(toonInner);
+    if (toon.length < gcf.length) return toon;
+  }
+  return gcf;
+}
+
+export type MessageLike = {
   role?: string;
   content?: string | Array<Record<string, unknown>>;
   [key: string]: unknown;
 };
 
+/**
+ * Collect every JSON array (whole-string or inside a ```json fence) in non-system
+ * messages that SmartCrusher would compact (same gates as tryCompactJson). Pure;
+ * shared by summarizeEncoderCandidates so the A/B table mirrors production scope.
+ */
+export function collectCompactableArrays(
+  messages: MessageLike[],
+  minRows: number = DEFAULT_MIN_ROWS
+): Record<string, unknown>[][] {
+  const out: Record<string, unknown>[][] = [];
+  const pushIfCompactable = (jsonStr: string) => {
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(jsonStr);
+    } catch {
+      return;
+    }
+    if (!Array.isArray(parsed) || parsed.length < minRows) return;
+    if (!allObjects(parsed)) return;
+    out.push(parsed as Record<string, unknown>[]);
+  };
+  const scanText = (text: string) => {
+    const trimmed = text.trimStart();
+    if (trimmed.startsWith("[")) pushIfCompactable(text.trim());
+    const regex = new RegExp(JSON_FENCE_RE.source, "g");
+    let m: RegExpExecArray | null;
+    while ((m = regex.exec(text)) !== null) pushIfCompactable(m[1].trim());
+  };
+  for (const msg of messages) {
+    if (msg.role === "system") continue;
+    if (typeof msg.content === "string") scanText(msg.content);
+    else if (Array.isArray(msg.content)) {
+      for (const part of msg.content) {
+        if (part["type"] === "text" && typeof part["text"] === "string")
+          scanText(part["text"] as string);
+      }
+    }
+  }
+  return out;
+}
+
 /**
  * Process a single text string: try to compact it as a whole JSON array,
  * or find and compact any ```json fenced blocks inside it.

open-sse/services/compression/engines/headroom/tabular.ts

@@ -16,6 +16,7 @@
  */
 
 import { encodeGeneric, decodeGeneric } from "./gcf/index.ts";
+import { TOON_FENCE_OPEN, decodeToon } from "./toon.ts";
 
 // ─── fence markers ───────────────────────────────────────────────────────────
 
@@ -226,6 +227,8 @@ export function decodeTabularBlockLegacy(block: string): Record<string, unknown>
  * Auto-detects format from the fence marker.
  */
 export function decodeTabular(text: string): Record<string, unknown>[] {
+  if (text.startsWith(TOON_FENCE_OPEN + "\n")) return decodeToon(text);
+
   // Detect format from fence marker.
   if (text.startsWith(GCF_FENCE_OPEN + "\n") || text.startsWith("GCF ")) {
     // GCF format: strip fence if present, decode via GCF decoder.

open-sse/services/compression/engines/headroom/toon.ts

@@ -0,0 +1,43 @@
+/**
+ * toon.ts — TOON (@toon-format/toon) candidate encoder for the headroom engine.
+ *
+ * TOON is a second encoder considered alongside GCF in the SmartCrusher best-of-N
+ * gate. GCF stays the default and tiebreak winner; TOON is used only when strictly
+ * smaller. All entry points are FAIL-OPEN: any throw yields null/[] so a TOON bug
+ * can never break headroom compaction. Pure: no Date.now / Math.random.
+ */
+import { encode as toonEncode, decode as toonDecode } from "@toon-format/toon";
+
+export const TOON_FENCE_OPEN = "```toon";
+export const TOON_FENCE_CLOSE = "```";
+
+export function encodeToonBlock(arr: Record<string, unknown>[]): string | null {
+  try {
+    return toonEncode(arr);
+  } catch {
+    return null;
+  }
+}
+
+export function wrapToon(blockContent: string): string {
+  return `${TOON_FENCE_OPEN}\n${blockContent}\n${TOON_FENCE_CLOSE}`;
+}
+
+export function decodeToon(text: string): Record<string, unknown>[] {
+  let inner = text;
+  if (inner.startsWith(TOON_FENCE_OPEN + "\n")) {
+    inner = inner.slice(TOON_FENCE_OPEN.length + 1);
+    if (inner.endsWith("\n" + TOON_FENCE_CLOSE)) {
+      inner = inner.slice(0, inner.length - TOON_FENCE_CLOSE.length - 1);
+    } else if (inner.endsWith(TOON_FENCE_CLOSE)) {
+      inner = inner.slice(0, inner.length - TOON_FENCE_CLOSE.length);
+    }
+  }
+  try {
+    const decoded = toonDecode(inner);
+    if (Array.isArray(decoded)) return decoded as Record<string, unknown>[];
+    return [decoded as Record<string, unknown>];
+  } catch {
+    return [];
+  }
+}

package-lock.json

@@ -8947,6 +8947,12 @@
         }
       }
     },
+    "node_modules/@toon-format/toon": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/@toon-format/toon/-/toon-2.3.0.tgz",
+      "integrity": "sha512-/Ew9etdRQKVMnm9fDaCG0JjyAOK/O7T0M97oum1aW4W+UR8ZhVVPBanIV7oWgHBiGlnVxV9M55PWQCHofDV07w==",
+      "license": "MIT"
+    },
     "node_modules/@tufjs/canonical-json": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/@tufjs/canonical-json/-/canonical-json-2.0.0.tgz",
@@ -28551,7 +28557,10 @@
     },
     "open-sse": {
       "name": "@omniroute/open-sse",
-      "version": "3.8.39"
+      "version": "3.8.39",
+      "dependencies": {
+        "@toon-format/toon": "^2.3.0"
+      }
     }
   }
 }

src/app/(dashboard)/dashboard/compression/studio/EncoderComparisonTable.tsx

@@ -0,0 +1,57 @@
+"use client";
+import type { EncoderComparison } from "./compressionFlowModel";
+
+const fmt = (n: number) => n.toLocaleString("en-US");
+
+export function EncoderComparisonTable({ comparison }: { comparison: EncoderComparison }) {
+  if (!comparison || comparison.arraysCompared === 0) return null;
+  const rows: Array<{
+    key: "gcf" | "toon" | "json";
+    label: string;
+    size: { bytes: number; tokens: number } | null;
+  }> = [
+    { key: "gcf", label: "GCF", size: comparison.gcf },
+    { key: "toon", label: "TOON", size: comparison.toonAvailable ? comparison.toon : null },
+    { key: "json", label: "JSON", size: comparison.json },
+  ].sort((a, b) => (a.size?.tokens ?? Infinity) - (b.size?.tokens ?? Infinity));
+
+  return (
+    <section data-testid="encoder-comparison" className="rounded border p-2 text-xs">
+      <header className="mb-1 font-semibold">
+        Encoder A/B — {comparison.arraysCompared} array(s){" "}
+        <span data-testid="encoder-winner" className="font-mono">
+          vencedor: {comparison.winner}
+        </span>
+      </header>
+      <table className="w-full font-mono">
+        <thead>
+          <tr className="text-left text-muted-foreground">
+            <th>encoder</th>
+            <th>bytes</th>
+            <th>tokens (cl100k)</th>
+          </tr>
+        </thead>
+        <tbody>
+          {rows.map((r) => (
+            <tr key={r.key} className={r.key === comparison.winner ? "font-bold" : ""}>
+              <td>
+                {r.label}
+                {r.key === comparison.winner ? " ✓" : ""}
+              </td>
+              {r.size ? (
+                <>
+                  <td>{fmt(r.size.bytes)}</td>
+                  <td>{fmt(r.size.tokens)}</td>
+                </>
+              ) : (
+                <td colSpan={2} data-testid="encoder-toon-na">
+                  n/a
+                </td>
+              )}
+            </tr>
+          ))}
+        </tbody>
+      </table>
+    </section>
+  );
+}

src/app/(dashboard)/dashboard/compression/studio/PlayView.tsx

@@ -3,8 +3,13 @@ import { useState } from "react";
 import { usePreviewCompression, type Lane, type PreviewBatch } from "@/hooks/usePreviewCompression";
 import { WaterfallInspector } from "./WaterfallInspector";
 import { DiffPane } from "./DiffPane";
+import { EncoderComparisonTable } from "./EncoderComparisonTable";
 import { PlaygroundInput, LANE_ENGINES } from "./PlaygroundInput";
-export interface PlayViewProps { text: string; onText: (t: string) => void; laneEngines?: readonly string[]; }
+export interface PlayViewProps {
+  text: string;
+  onText: (t: string) => void;
+  laneEngines?: readonly string[];
+}
 
 function laneStatus(l: Lane): string {
   const rejected = l.run?.steps?.find((s) => s.rejected);
@@ -21,7 +26,12 @@ function LaneList({ lanes, onSelect }: { lanes: Lane[]; onSelect: (e: string) =>
   return (
     <>
       {lanes.map((l) => (
-        <button key={l.engine} data-testid="play-lane" onClick={() => onSelect(l.engine)} className="flex w-full items-center justify-between border-b py-1 text-left font-mono text-xs">
+        <button
+          key={l.engine}
+          data-testid="play-lane"
+          onClick={() => onSelect(l.engine)}
+          className="flex w-full items-center justify-between border-b py-1 text-left font-mono text-xs"
+        >
           <span>{l.engine}</span>
           <span>{laneStatus(l)}</span>
         </button>
@@ -37,18 +47,39 @@ export function PlayView({ text, onText, laneEngines = LANE_ENGINES }: PlayViewP
   const [fidelityGate, setFidelityGate] = useState(false);
   const { batch, loading, run } = usePreviewCompression();
   const messages = [{ role: "user", content: text }];
-  const toggle = (e: string) => setActive((a) => (a.includes(e) ? a.filter((x) => x !== e) : [...a, e]));
-  const onRun = () => run({ messages, laneEngines: [...laneEngines], activeEngines: orderByStack(active, laneEngines), fidelityGate, fuzzyDedup });
+  const toggle = (e: string) =>
+    setActive((a) => (a.includes(e) ? a.filter((x) => x !== e) : [...a, e]));
+  const onRun = () =>
+    run({
+      messages,
+      laneEngines: [...laneEngines],
+      activeEngines: orderByStack(active, laneEngines),
+      fidelityGate,
+      fuzzyDedup,
+    });
   const activeDiff = resolveActiveDiff(batch, selectedLane);
   return (
     <div className="flex h-full gap-3">
       <div className="w-[260px] shrink-0">
-        <PlaygroundInput text={text} onText={onText} active={active} onToggleActive={toggle} onRun={onRun} loading={loading} fidelityGate={fidelityGate} onToggleFidelity={() => setFidelityGate((v) => !v)} fuzzyDedup={fuzzyDedup} onToggleFuzzy={() => setFuzzyDedup((v) => !v)} />
+        <PlaygroundInput
+          text={text}
+          onText={onText}
+          active={active}
+          onToggleActive={toggle}
+          onRun={onRun}
+          loading={loading}
+          fidelityGate={fidelityGate}
+          onToggleFidelity={() => setFidelityGate((v) => !v)}
+          fuzzyDedup={fuzzyDedup}
+          onToggleFuzzy={() => setFuzzyDedup((v) => !v)}
+        />
       </div>
       <div className="flex min-w-0 flex-1 flex-col gap-3 overflow-auto">
         {batch?.combined && (
           <section data-testid="play-combined">
-            <header className="text-xs font-semibold">Fluxo combinado — {active.join(" → ")}</header>
+            <header className="text-xs font-semibold">
+              Fluxo combinado — {active.join(" → ")}
+            </header>
             <WaterfallInspector run={batch.combined} />
           </section>
         )}
@@ -56,6 +87,13 @@ export function PlayView({ text, onText, laneEngines = LANE_ENGINES }: PlayViewP
           <header className="text-xs font-semibold">Cada camada sozinha</header>
           <LaneList lanes={batch?.lanes ?? []} onSelect={setSelectedLane} />
         </section>
+        {(() => {
+          const cmp =
+            batch?.lanes.find((l) => l.engine === "headroom")?.run?.encoderComparison ??
+            batch?.combined?.encoderComparison ??
+            null;
+          return cmp ? <EncoderComparisonTable comparison={cmp} /> : null;
+        })()}
         {activeDiff && (
           <section>
             <header className="text-xs font-semibold">Diff — {selectedLane ?? "combinado"}</header>
@@ -66,4 +104,6 @@ export function PlayView({ text, onText, laneEngines = LANE_ENGINES }: PlayViewP
     </div>
   );
 }
-function orderByStack(active: string[], order: readonly string[]): string[] { return order.filter((e) => active.includes(e)); }
+function orderByStack(active: string[], order: readonly string[]): string[] {
+  return order.filter((e) => active.includes(e));
+}

src/app/(dashboard)/dashboard/compression/studio/compressionFlowModel.ts

@@ -29,6 +29,22 @@ export interface CompressionEngineStep {
 
 export type DiffSegment = { type: "same" | "removed" | "added"; text: string };
 
+// ── Encoder comparison (TOON/GCF/JSON A/B) ────────────────────────────────
+
+export interface EncoderSize {
+  bytes: number;
+  tokens: number;
+}
+
+export interface EncoderComparison {
+  arraysCompared: number;
+  json: EncoderSize;
+  gcf: EncoderSize;
+  toon: EncoderSize;
+  toonAvailable: boolean;
+  winner: "gcf" | "toon" | "json";
+}
+
 // ── Preview API response ──────────────────────────────────────────────────
 
 export interface PreviewResponse {
@@ -43,6 +59,7 @@ export interface PreviewResponse {
   diff: DiffSegment[];
   preservedBlocks: Array<{ kind: string; preview: string }>;
   ruleRemovals: string[];
+  encoderComparison?: EncoderComparison | null;
 }
 
 // ── Run Model ─────────────────────────────────────────────────────────────
@@ -57,6 +74,7 @@ export interface CompressionRunModel {
   steps: CompressionEngineStep[];
   timestamp: number;
   diff?: DiffSegment[];
+  encoderComparison?: EncoderComparison | null;
 }
 
 // ── previewToRunModel ─────────────────────────────────────────────────────
@@ -72,6 +90,7 @@ export function previewToRunModel(res: PreviewResponse, label: string): Compress
     steps: res.engineBreakdown,
     timestamp: 0,
     diff: res.diff,
+    encoderComparison: res.encoderComparison ?? null,
   };
 }
 

src/app/api/compression/preview/route.ts

@@ -14,6 +14,8 @@ import { buildCompressionPreviewDiff } from "@omniroute/open-sse/services/compre
 import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/error";
 import { countTextTokens } from "@/shared/utils/tiktokenCounter";
 import { ensureEngineBreakdown } from "@omniroute/open-sse/services/compression/engineBreakdown";
+import { summarizeEncoderCandidates } from "@omniroute/open-sse/services/compression/engines/headroom/encoderComparison";
+import { DEFAULT_MIN_ROWS } from "@omniroute/open-sse/services/compression/engines/headroom/smartcrusher";
 
 export const PreviewCompressionConfigSchema = compressionPreviewConfigSchema;
 
@@ -62,6 +64,20 @@ function buildStep(engine: string, fuzzy?: { enabled: boolean }) {
     : { engine };
 }
 
+function headroomParticipates(
+  engineId: string | undefined,
+  pipeline: string[] | undefined,
+  mode: CompressionMode
+): boolean {
+  // An explicit single-engine or pipeline override decides on its own terms:
+  // headroom only participates if it is the engine / is named in the pipeline.
+  // (effectiveMode is forced to "stacked" whenever engineId/pipeline is set, so we
+  // must not fall through to the mode check for those — e.g. engineId:"lite".)
+  if (engineId) return engineId === "headroom";
+  if (pipeline) return pipeline.includes("headroom");
+  return mode === "stacked";
+}
+
 async function dispatchCompression(
   requestBody: Record<string, unknown>,
   opts: {
@@ -117,7 +133,8 @@ export async function POST(req: Request) {
   }
 
   const { messages, mode, engineId, pipeline, config, fidelityGate, fuzzyDedup } = parsed.data;
-  const effectiveMode: CompressionMode = engineId || pipeline ? "stacked" : (mode as CompressionMode);
+  const effectiveMode: CompressionMode =
+    engineId || pipeline ? "stacked" : (mode as CompressionMode);
   const originalText = messagesToText(messages);
   const originalTokens = countTokens(originalText);
 
@@ -125,7 +142,12 @@ export async function POST(req: Request) {
     const start = Date.now();
     const requestBody = { messages };
     const result = await dispatchCompression(requestBody as Record<string, unknown>, {
-      engineId, pipeline, effectiveMode, config, fidelityGate, fuzzyDedup,
+      engineId,
+      pipeline,
+      effectiveMode,
+      config,
+      fidelityGate,
+      fuzzyDedup,
     });
     const durationMs = Date.now() - start;
 
@@ -141,7 +163,12 @@ export async function POST(req: Request) {
     const engineBreakdown = result.stats ? ensureEngineBreakdown(result.stats) : [];
     const diff = buildCompressionPreviewDiff(originalText, compressedText, result.stats);
 
+    const encoderComparison = headroomParticipates(engineId, pipeline, effectiveMode)
+      ? summarizeEncoderCandidates(messages, DEFAULT_MIN_ROWS, countTextTokens)
+      : null;
+
     return NextResponse.json({
+      encoderComparison,
       original: originalText,
       compressed: compressedText,
       originalTokens,

tests/unit/compression/encoderComparison.test.ts

@@ -0,0 +1,35 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { summarizeEncoderCandidates } from "../../../open-sse/services/compression/engines/headroom/encoderComparison.ts";
+
+const byteLen = (s: string) => Buffer.byteLength(s, "utf8");
+
+test("agrega sizes e elege winner por tokens", () => {
+  const messages = [
+    {
+      role: "user",
+      content: JSON.stringify(Array.from({ length: 20 }, (_, i) => ({ id: i, ok: true }))),
+    },
+  ];
+  const cmp = summarizeEncoderCandidates(messages, 8, byteLen);
+  assert.equal(cmp.arraysCompared, 1);
+  assert.ok(cmp.json.bytes > 0 && cmp.gcf.bytes > 0);
+  assert.ok(["gcf", "toon", "json"].includes(cmp.winner));
+  const sizes: Record<string, number> = { gcf: cmp.gcf.tokens, json: cmp.json.tokens };
+  if (cmp.toonAvailable) sizes["toon"] = cmp.toon.tokens;
+  const min = Math.min(...Object.values(sizes));
+  assert.equal(sizes[cmp.winner], min);
+});
+
+test("sem array compactável → zerado, sem winner espúrio", () => {
+  const cmp = summarizeEncoderCandidates([{ role: "user", content: "oi" }], 8, byteLen);
+  assert.equal(cmp.arraysCompared, 0);
+  assert.equal(cmp.json.bytes, 0);
+  assert.equal(cmp.gcf.bytes, 0);
+});
+
+test("system messages são ignoradas (igual ao smartcrusher)", () => {
+  const arr = JSON.stringify(Array.from({ length: 20 }, (_, i) => ({ id: i })));
+  const cmp = summarizeEncoderCandidates([{ role: "system", content: arr }], 8, byteLen);
+  assert.equal(cmp.arraysCompared, 0);
+});

tests/unit/compression/previewRouteToon.test.ts

@@ -0,0 +1,54 @@
+// tests/unit/compression/previewRouteToon.test.ts
+import test from "node:test";
+import assert from "node:assert/strict";
+import { join } from "node:path";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+const TEST_DATA_DIR = mkdtempSync(join(tmpdir(), "preview-toon-"));
+process.env.DATA_DIR = TEST_DATA_DIR;
+process.env.API_KEY_SECRET = process.env.API_KEY_SECRET ?? "test-secret-32-chars-min-aaaaaaaa";
+delete process.env.INITIAL_PASSWORD;
+const core = await import("../../../src/lib/db/core.ts");
+const route = await import("../../../src/app/api/compression/preview/route.ts");
+function makeReq(body: unknown) {
+  return new Request("http://localhost/api/compression/preview", {
+    method: "POST",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify(body),
+  });
+}
+test.beforeEach(() => core.resetDbInstance());
+test.after(() => {
+  core.resetDbInstance();
+  rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+});
+
+test("headroom engine carries encoderComparison with one array and a valid winner", async () => {
+  const big = JSON.stringify(Array.from({ length: 20 }, (_, i) => ({ id: i, ok: true })));
+  const res = await route.POST(
+    makeReq({ messages: [{ role: "user", content: big }], engineId: "headroom" })
+  );
+  assert.equal(res.status, 200);
+  const body = await res.json();
+  assert.ok(body.encoderComparison);
+  assert.equal(body.encoderComparison.arraysCompared, 1);
+  assert.ok(["gcf", "toon", "json"].includes(body.encoderComparison.winner));
+});
+
+test("non-headroom engine (lite) does not carry encoderComparison", async () => {
+  const big = JSON.stringify(Array.from({ length: 20 }, (_, i) => ({ id: i, ok: true })));
+  const res = await route.POST(
+    makeReq({ messages: [{ role: "user", content: big }], engineId: "lite" })
+  );
+  assert.equal(res.status, 200);
+  const body = await res.json();
+  assert.equal(body.encoderComparison ?? null, null);
+});
+
+test("error responses do not leak stack traces", async () => {
+  const res = await route.POST(makeReq({ messages: [] }));
+  assert.equal(res.status, 400);
+  const body = await res.json();
+  const details = JSON.stringify(body);
+  assert.ok(!String(details).includes("at /"));
+});

tests/unit/compression/smartcrusherToon.test.ts

@@ -0,0 +1,49 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import {
+  tryCompactJson,
+  pickSmallestEncoding,
+} from "../../../open-sse/services/compression/engines/headroom/smartcrusher.ts";
+import { reconstructHeadroom } from "../../../open-sse/services/compression/engines/headroom/index.ts";
+import { TOON_FENCE_OPEN } from "../../../open-sse/services/compression/engines/headroom/toon.ts";
+import { GCF_FENCE_OPEN } from "../../../open-sse/services/compression/engines/headroom/tabular.ts";
+
+const toonFavorable = Array.from({ length: 40 }, (_, i) => ({ id: i, ok: true }));
+
+test("best-of-N: usa a fence do encoder menor e nunca regride vs JSON", () => {
+  const chosen = pickSmallestEncoding(toonFavorable);
+  const json = JSON.stringify(toonFavorable);
+  assert.ok(chosen.length < json.length, "deve encolher vs JSON");
+  assert.ok(chosen.startsWith(GCF_FENCE_OPEN) || chosen.startsWith(TOON_FENCE_OPEN));
+});
+
+test("se TOON vence, tryCompactJson emite fence toon e round-trip restaura", () => {
+  const json = JSON.stringify(toonFavorable);
+  const compact = tryCompactJson(json, 8);
+  assert.notEqual(compact, null);
+  const body = { messages: [{ role: "user", content: compact as string }] };
+  const restored = reconstructHeadroom(body);
+  const text = (restored.messages as Array<{ content: string }>)[0].content;
+  assert.deepEqual(JSON.parse(text), toonFavorable);
+});
+
+test("empate/GCF-favorável: round-trip lossless via a fence escolhida", () => {
+  const gcfFavorable = Array.from({ length: 10 }, (_, i) => ({
+    id: i,
+    deep: { a: { b: [i, i + 1] }, label: `row-${i}` },
+  }));
+  const chosen = pickSmallestEncoding(gcfFavorable);
+  assert.ok(chosen.startsWith(GCF_FENCE_OPEN) || chosen.startsWith(TOON_FENCE_OPEN));
+  const compact = tryCompactJson(JSON.stringify(gcfFavorable), 8);
+  if (compact) {
+    const body = { messages: [{ role: "user", content: compact }] };
+    const restored = reconstructHeadroom(body);
+    const text = (restored.messages as Array<{ content: string }>)[0].content;
+    assert.deepEqual(JSON.parse(text), gcfFavorable);
+  }
+});
+
+test("nunca aumenta vs JSON (gate preservado): array < minRows → no-op", () => {
+  const tiny = [{ a: 1 }];
+  assert.equal(tryCompactJson(JSON.stringify(tiny), 8), null);
+});

tests/unit/compression/toon.test.ts

@@ -0,0 +1,72 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import {
+  encodeToonBlock,
+  wrapToon,
+  decodeToon,
+  TOON_FENCE_OPEN,
+} from "../../../open-sse/services/compression/engines/headroom/toon.ts";
+
+const cases: Array<{ name: string; arr: Record<string, unknown>[] }> = [
+  {
+    name: "homogêneo",
+    arr: [
+      { id: 1, name: "a" },
+      { id: 2, name: "b" },
+    ],
+  },
+  {
+    name: "heterogêneo",
+    arr: [
+      { id: 1, x: 1 },
+      { id: 2, y: 2 },
+    ],
+  },
+  {
+    name: "nested",
+    arr: [
+      { id: 1, meta: { a: [1, 2] } },
+      { id: 2, meta: { a: [3] } },
+    ],
+  },
+  {
+    name: "nullable",
+    arr: [
+      { id: 1, v: null },
+      { id: 2, v: 5 },
+    ],
+  },
+  {
+    name: "strings especiais",
+    arr: [
+      { id: 1, s: 'a,b "q"\nc' },
+      { id: 2, s: "plain" },
+    ],
+  },
+];
+
+for (const c of cases) {
+  test(`toon round-trips: ${c.name}`, () => {
+    const inner = encodeToonBlock(c.arr);
+    assert.notEqual(inner, null, "encode deve produzir string");
+    const fenced = wrapToon(inner as string);
+    assert.ok(fenced.startsWith(TOON_FENCE_OPEN));
+    const decoded = decodeToon(fenced);
+    assert.deepEqual(decoded, c.arr);
+  });
+}
+
+test("decodeToon aceita bloco sem fence", () => {
+  const arr = [{ a: 1 }, { a: 2 }];
+  const inner = encodeToonBlock(arr) as string;
+  assert.deepEqual(decodeToon(inner), arr);
+});
+
+test("encodeToonBlock é fail-open (não lança)", () => {
+  const a: Record<string, unknown> = { id: 1 };
+  a["self"] = a;
+  assert.doesNotThrow(() => {
+    const r = encodeToonBlock([a]);
+    assert.equal(r, null);
+  });
+});

tests/unit/ui/toonEncoderTable.test.tsx

@@ -0,0 +1,62 @@
+// @vitest-environment jsdom
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import { createRoot, type Root } from "react-dom/client";
+import { act } from "react";
+import { EncoderComparisonTable } from "@/app/(dashboard)/dashboard/compression/studio/EncoderComparisonTable";
+import type { EncoderComparison } from "@/app/(dashboard)/dashboard/compression/studio/compressionFlowModel";
+
+const cmp: EncoderComparison = {
+  arraysCompared: 1,
+  json: { bytes: 400, tokens: 120 },
+  gcf: { bytes: 150, tokens: 40 },
+  toon: { bytes: 170, tokens: 48 },
+  toonAvailable: true,
+  winner: "gcf",
+};
+
+let container: HTMLElement;
+let root: Root;
+beforeEach(() => {
+  (globalThis as any).IS_REACT_ACT_ENVIRONMENT = true;
+  container = document.createElement("div");
+  document.body.appendChild(container);
+  root = createRoot(container);
+});
+afterEach(() => {
+  act(() => root.unmount());
+  container.remove();
+  document.body.innerHTML = "";
+});
+
+describe("EncoderComparisonTable", () => {
+  it("renderiza GCF/TOON/JSON com tokens e marca o vencedor", () => {
+    act(() => {
+      root.render(<EncoderComparisonTable comparison={cmp} />);
+    });
+    const text = container.textContent ?? "";
+    expect(text).toMatch(/GCF/);
+    expect(text).toMatch(/TOON/);
+    expect(text).toMatch(/JSON/);
+    // tokens rendered
+    expect(text).toContain("40");
+    expect(text).toContain("48");
+    expect(text).toContain("120");
+    expect(container.querySelector('[data-testid="encoder-winner"]')?.textContent).toMatch(/gcf/i);
+  });
+
+  it("mostra TOON como n/a quando indisponível", () => {
+    act(() => {
+      root.render(
+        <EncoderComparisonTable comparison={{ ...cmp, toonAvailable: false, winner: "gcf" }} />
+      );
+    });
+    expect(container.querySelector('[data-testid="encoder-toon-na"]')).toBeTruthy();
+  });
+
+  it("não renderiza nada quando não há array comparado", () => {
+    act(() => {
+      root.render(<EncoderComparisonTable comparison={{ ...cmp, arraysCompared: 0 }} />);
+    });
+    expect(container.firstChild).toBeNull();
+  });
+});