mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-13 10:43:43 +03:00
test(compression): lock in stacked RTK+Caveman savings on redundant tool_result content (#9278)
Merge-train validated (tip 6ce4effef8). Vitest failures confirmed as base-red (#9679).
This commit is contained in:
committed by
GitHub
parent
12e5c83692
commit
41d2e4e7a1
@@ -146,6 +146,26 @@ That `78-95%` number applies when both RTK and Caveman can reduce the same input
|
||||
Caveman response output mode is separate: when enabled, use Caveman's own output savings (`65%`
|
||||
average, `~75%` headline, `22-87%` range). Total billing savings depend on your prompt/output mix.
|
||||
|
||||
### What "eligible" actually means
|
||||
|
||||
The 15-95% headline range is real, but it only applies to **redundant or verbose** content — repeated
|
||||
error lines, a build log that spams the same warning, an oversized `grep`/file-read dump. It does
|
||||
**not** mean every request saves that much.
|
||||
|
||||
Verified empirically (`tests/unit/compression/stacked-compression-tool-result-savings.test.ts`): a
|
||||
`stacked` (RTK + Caveman) run against an Anthropic-shape `tool_result` block containing 300 identical
|
||||
error lines produced **95.93% token savings / 96.26% character savings** — squarely in the advertised
|
||||
range. But the same pipeline run against normal, non-redundant tool output (a clean `grep` match list,
|
||||
a short file read, ordinary conversational text) correctly produces **near-zero savings**, because
|
||||
there is nothing repetitive to remove and `validateCompression()` (`validation.ts`) refuses to ship a
|
||||
rewrite that would drop or alter code blocks, URLs, headings, versions, or `CONST_CASE` identifiers.
|
||||
|
||||
This is expected, safe behavior, not a bug: a coding session that mostly reads/greps clean files will
|
||||
see modest total savings even with compression fully enabled, while a session that hits a failing
|
||||
loop or a chatty linter will see the full 78-95% range on that traffic. Don't use a single session's
|
||||
low aggregate savings percentage as evidence compression is misconfigured — check whether the
|
||||
underlying tool output was actually redundant first.
|
||||
|
||||
---
|
||||
|
||||
## Token Savings Visualization
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import { applyStackedCompression } from "../../../open-sse/services/compression/strategySelector.ts";
|
||||
|
||||
/**
|
||||
* Regression coverage for the documented 78-95% "stacked" savings range
|
||||
* (docs/compression/COMPRESSION_GUIDE.md § "What 'eligible' actually means").
|
||||
*
|
||||
* A near-zero-savings result was reported on a real Claude Code session and initially looked
|
||||
* like a compression-pipeline bug. Investigation showed the pipeline was working correctly —
|
||||
* that session's tool output (file reads, grep matches) was genuinely non-redundant, so there
|
||||
* was nothing safe to remove. This test locks in the other half of the story: against content
|
||||
* the pipeline is actually designed for (an Anthropic-shape `tool_result` block full of exact
|
||||
* duplicate lines, as a stuck build loop would produce), RTK + Caveman must still deliver the
|
||||
* advertised range. If this regresses to near-zero, the compression pipeline itself broke —
|
||||
* unlike a single ordinary session's low savings, which is expected and not a bug.
|
||||
*/
|
||||
test("stacked RTK+Caveman achieves >90% token savings on a redundant Anthropic tool_result block", () => {
|
||||
const spammyLog = Array.from({ length: 300 }, () => "ERROR: connection refused at line 42").join(
|
||||
"\n"
|
||||
);
|
||||
|
||||
const body = {
|
||||
model: "claude-sonnet-5",
|
||||
messages: [
|
||||
{ role: "user", content: "Run the build and show me the log." },
|
||||
{
|
||||
role: "assistant",
|
||||
content: [
|
||||
{ type: "text", text: "Running the build now." },
|
||||
{ type: "tool_use", id: "toolu_01X", name: "Bash", input: { command: "npm run build" } },
|
||||
],
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "tool_result",
|
||||
tool_use_id: "toolu_01X",
|
||||
content: [{ type: "text", text: spammyLog }],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const result = applyStackedCompression(body, [
|
||||
{ engine: "rtk", intensity: "standard" },
|
||||
{ engine: "caveman", intensity: "full" },
|
||||
]);
|
||||
|
||||
assert.equal(result.compressed, true, "expected the pipeline to report a compression happened");
|
||||
assert.ok(
|
||||
(result.stats?.savingsPercent ?? 0) > 90,
|
||||
`expected >90% token savings on redundant content, got ${result.stats?.savingsPercent}%`
|
||||
);
|
||||
assert.equal(
|
||||
result.stats?.fallbackApplied,
|
||||
undefined,
|
||||
"expected no validation fallback — the deduplicated log has nothing left to alter that " +
|
||||
"validateCompression() would flag (no code fences, URLs, versions, CONST_CASE identifiers)"
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user