mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-09-15 19:32:20 +03:00
93265eede3
* test(infra): retry recursive temp-dir removal on main (main twin of #11968)
`main` has been red since b342c1a361 on the vitest and integration gates:
✖ tests/unit/autoCombo/provider-family-combos.test.ts > auto/<family>
✖ chat pipeline applies Codex OAuth fingerprint and priority tier inside combos
Both call resetStorage() from beforeEach, which does an fs.rmSync(TEST_DATA_DIR,
{recursive: true, force: true}) with no retry, and intermittently loses the race
with a not-yet-released SQLite handle (ENOTEMPTY).
release/v3.8.51 fixed this in #11968 with a mechanical codemod adding
maxRetries/retryDelay to every recursive rm/rmSync/rmdirSync under tests/, but
that PR landed only on the release branch. Because main only receives work at
the release squash, it stayed broken for the whole cycle — and repo-wide gates
then turn every open PR into main red on checks unrelated to their diff.
This is the --base main twin: re-runs the same codemod that already shipped on
the release branch (scripts/ad-hoc/codemod-rm-maxretries.mjs), so the two
branches converge on identical test-teardown semantics. Test-only; no product
logic is touched.
The remaining three failures reported on #12133 (unit full suite exceeding its
4800s ceiling, package-artifact exceeding 1200s, and the boot-smoke that is
skipped as a consequence) are runner-contention timeouts, not code defects —
validate-release-green.mjs runs those heavy gates concurrently on one shared
hosted runner. There is no fix to port for those.
* chore(scripts): carry the rm-maxretries codemod onto main alongside its output
The codemod that generated the previous commit lives in the repo on
release/v3.8.51 (added by #11968) but was never on main. Bringing it over keeps
the tool next to the change it produced, so the transformation stays
reproducible and auditable from either branch.
455 lines
14 KiB
TypeScript
455 lines
14 KiB
TypeScript
import { describe, it, before, beforeEach, after } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { mkdtempSync, rmSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
|
|
const tmpDir = mkdtempSync(join(tmpdir(), "omniroute-test-"));
|
|
process.env.DATA_DIR = tmpDir;
|
|
|
|
const core = await import("../../../src/lib/db/core.ts");
|
|
core.resetDbInstance();
|
|
const { insertCompressionAnalyticsRow, getCompressionAnalyticsSummary } =
|
|
await import("../../../src/lib/db/compressionAnalytics.ts");
|
|
const { attachCompressionUsageReceipt } =
|
|
await import("../../../src/lib/db/compressionAnalytics.ts");
|
|
const { getDbInstance } = core;
|
|
|
|
describe("compressionAnalytics", () => {
|
|
before(() => {
|
|
const db = getDbInstance();
|
|
db.exec(`
|
|
CREATE TABLE IF NOT EXISTS compression_analytics (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
timestamp TEXT NOT NULL,
|
|
combo_id TEXT,
|
|
provider TEXT,
|
|
mode TEXT NOT NULL,
|
|
original_tokens INTEGER NOT NULL,
|
|
compressed_tokens INTEGER NOT NULL,
|
|
tokens_saved INTEGER NOT NULL,
|
|
duration_ms INTEGER,
|
|
request_id TEXT
|
|
)
|
|
`);
|
|
});
|
|
|
|
beforeEach(() => {
|
|
// Clear table before each test for full isolation
|
|
const db = getDbInstance();
|
|
db.exec("DELETE FROM compression_analytics");
|
|
});
|
|
|
|
after(() => {
|
|
core.closeDbInstance();
|
|
rmSync(tmpDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
|
});
|
|
|
|
it("empty table returns zeroed summary", () => {
|
|
const summary = getCompressionAnalyticsSummary();
|
|
assert.deepEqual(summary, {
|
|
totalRequests: 0,
|
|
totalTokensSaved: 0,
|
|
avgSavingsPct: 0,
|
|
avgDurationMs: 0,
|
|
byMode: {},
|
|
byEngine: {},
|
|
byCompressionCombo: {},
|
|
byProvider: {},
|
|
last24h: summary.last24h,
|
|
totalSkipped: 0,
|
|
bySkipReason: {},
|
|
validationFallbacks: 0,
|
|
realUsage: {
|
|
requestsWithReceipts: 0,
|
|
promptTokens: 0,
|
|
completionTokens: 0,
|
|
totalTokens: 0,
|
|
cacheReadTokens: 0,
|
|
cacheWriteTokens: 0,
|
|
estimatedUsdSaved: 0,
|
|
bySource: {},
|
|
},
|
|
mcpDescriptionCompression: {
|
|
snapshots: 0,
|
|
estimatedTokensSaved: 0,
|
|
},
|
|
});
|
|
assert.equal(summary.last24h.length, 24);
|
|
});
|
|
|
|
it("insert single row does not throw", () => {
|
|
const row = {
|
|
timestamp: new Date().toISOString(),
|
|
mode: "lite",
|
|
original_tokens: 1000,
|
|
compressed_tokens: 800,
|
|
tokens_saved: 200,
|
|
};
|
|
assert.doesNotThrow(() => insertCompressionAnalyticsRow(row));
|
|
const summary = getCompressionAnalyticsSummary();
|
|
assert.equal(summary.totalRequests, 1);
|
|
});
|
|
|
|
it("stores all RTK raw output pointers when provided", () => {
|
|
insertCompressionAnalyticsRow({
|
|
timestamp: new Date().toISOString(),
|
|
mode: "rtk",
|
|
original_tokens: 1000,
|
|
compressed_tokens: 500,
|
|
tokens_saved: 500,
|
|
rtk_raw_output_pointer: "raw_1",
|
|
rtk_raw_output_bytes: 100,
|
|
rtk_raw_output_pointers: JSON.stringify(["raw_1", "raw_2"]),
|
|
rtk_raw_output_total_bytes: 250,
|
|
});
|
|
|
|
const db = getDbInstance();
|
|
const row = db
|
|
.prepare(
|
|
"SELECT rtk_raw_output_pointer, rtk_raw_output_bytes, rtk_raw_output_pointers, rtk_raw_output_total_bytes FROM compression_analytics LIMIT 1"
|
|
)
|
|
.get() as {
|
|
rtk_raw_output_pointer: string;
|
|
rtk_raw_output_bytes: number;
|
|
rtk_raw_output_pointers: string;
|
|
rtk_raw_output_total_bytes: number;
|
|
};
|
|
|
|
assert.equal(row.rtk_raw_output_pointer, "raw_1");
|
|
assert.equal(row.rtk_raw_output_bytes, 100);
|
|
assert.equal(row.rtk_raw_output_pointers, JSON.stringify(["raw_1", "raw_2"]));
|
|
assert.equal(row.rtk_raw_output_total_bytes, 250);
|
|
});
|
|
|
|
it("summary counts correctly after multiple inserts", () => {
|
|
insertCompressionAnalyticsRow({
|
|
timestamp: new Date().toISOString(),
|
|
mode: "lite",
|
|
original_tokens: 1000,
|
|
compressed_tokens: 800,
|
|
tokens_saved: 200,
|
|
});
|
|
insertCompressionAnalyticsRow({
|
|
timestamp: new Date().toISOString(),
|
|
mode: "standard",
|
|
original_tokens: 300,
|
|
compressed_tokens: 280,
|
|
tokens_saved: 20,
|
|
});
|
|
insertCompressionAnalyticsRow({
|
|
timestamp: new Date().toISOString(),
|
|
mode: "aggressive",
|
|
original_tokens: 500,
|
|
compressed_tokens: 400,
|
|
tokens_saved: 100,
|
|
});
|
|
const summary = getCompressionAnalyticsSummary();
|
|
assert.equal(summary.totalRequests, 3);
|
|
});
|
|
|
|
it("totalTokensSaved sums correctly", () => {
|
|
insertCompressionAnalyticsRow({
|
|
timestamp: new Date().toISOString(),
|
|
mode: "lite",
|
|
original_tokens: 1000,
|
|
compressed_tokens: 800,
|
|
tokens_saved: 200,
|
|
});
|
|
insertCompressionAnalyticsRow({
|
|
timestamp: new Date().toISOString(),
|
|
mode: "standard",
|
|
original_tokens: 300,
|
|
compressed_tokens: 280,
|
|
tokens_saved: 20,
|
|
});
|
|
const summary = getCompressionAnalyticsSummary();
|
|
assert.equal(summary.totalTokensSaved, 220);
|
|
});
|
|
|
|
it("byMode groups correctly", () => {
|
|
insertCompressionAnalyticsRow({
|
|
timestamp: new Date().toISOString(),
|
|
mode: "lite",
|
|
original_tokens: 1000,
|
|
compressed_tokens: 800,
|
|
tokens_saved: 200,
|
|
});
|
|
insertCompressionAnalyticsRow({
|
|
timestamp: new Date().toISOString(),
|
|
mode: "lite",
|
|
original_tokens: 500,
|
|
compressed_tokens: 400,
|
|
tokens_saved: 100,
|
|
});
|
|
insertCompressionAnalyticsRow({
|
|
timestamp: new Date().toISOString(),
|
|
mode: "standard",
|
|
original_tokens: 300,
|
|
compressed_tokens: 270,
|
|
tokens_saved: 30,
|
|
});
|
|
const summary = getCompressionAnalyticsSummary();
|
|
assert.equal(summary.byMode["lite"].count, 2);
|
|
assert.equal(summary.byMode["lite"].tokensSaved, 300);
|
|
assert.equal(summary.byMode["standard"].count, 1);
|
|
});
|
|
|
|
it("byProvider groups correctly", () => {
|
|
insertCompressionAnalyticsRow({
|
|
timestamp: new Date().toISOString(),
|
|
mode: "lite",
|
|
original_tokens: 1000,
|
|
compressed_tokens: 800,
|
|
tokens_saved: 200,
|
|
provider: "ProviderA",
|
|
});
|
|
insertCompressionAnalyticsRow({
|
|
timestamp: new Date().toISOString(),
|
|
mode: "lite",
|
|
original_tokens: 500,
|
|
compressed_tokens: 400,
|
|
tokens_saved: 100,
|
|
provider: "ProviderB",
|
|
});
|
|
const summary = getCompressionAnalyticsSummary();
|
|
assert.deepEqual(summary.byProvider["ProviderA"], { count: 1, tokensSaved: 200 });
|
|
assert.deepEqual(summary.byProvider["ProviderB"], { count: 1, tokensSaved: 100 });
|
|
});
|
|
|
|
it("since=24h filters rows older than 24h", () => {
|
|
const oldTimestamp = new Date(Date.now() - 48 * 60 * 60 * 1000).toISOString();
|
|
const recentTimestamp = new Date().toISOString();
|
|
|
|
insertCompressionAnalyticsRow({
|
|
timestamp: oldTimestamp,
|
|
mode: "lite",
|
|
original_tokens: 2000,
|
|
compressed_tokens: 1800,
|
|
tokens_saved: 200,
|
|
});
|
|
insertCompressionAnalyticsRow({
|
|
timestamp: recentTimestamp,
|
|
mode: "lite",
|
|
original_tokens: 1000,
|
|
compressed_tokens: 900,
|
|
tokens_saved: 100,
|
|
});
|
|
|
|
const summary24h = getCompressionAnalyticsSummary("24h");
|
|
assert.equal(summary24h.totalRequests, 1);
|
|
assert.equal(summary24h.totalTokensSaved, 100);
|
|
});
|
|
|
|
it("since=undefined returns all rows including old ones", () => {
|
|
const oldTimestamp = new Date(Date.now() - 48 * 60 * 60 * 1000).toISOString();
|
|
const recentTimestamp = new Date().toISOString();
|
|
|
|
insertCompressionAnalyticsRow({
|
|
timestamp: oldTimestamp,
|
|
mode: "lite",
|
|
original_tokens: 2000,
|
|
compressed_tokens: 1800,
|
|
tokens_saved: 200,
|
|
});
|
|
insertCompressionAnalyticsRow({
|
|
timestamp: recentTimestamp,
|
|
mode: "lite",
|
|
original_tokens: 1000,
|
|
compressed_tokens: 900,
|
|
tokens_saved: 100,
|
|
});
|
|
|
|
const summaryAll = getCompressionAnalyticsSummary();
|
|
assert.equal(summaryAll.totalRequests, 2);
|
|
assert.equal(summaryAll.totalTokensSaved, 300);
|
|
});
|
|
|
|
it("avgSavingsPct calculates correctly", () => {
|
|
// 200/1000 = 20%, 100/500 = 20% → avg = 20%
|
|
insertCompressionAnalyticsRow({
|
|
timestamp: new Date().toISOString(),
|
|
mode: "lite",
|
|
original_tokens: 1000,
|
|
compressed_tokens: 800,
|
|
tokens_saved: 200,
|
|
});
|
|
insertCompressionAnalyticsRow({
|
|
timestamp: new Date().toISOString(),
|
|
mode: "standard",
|
|
original_tokens: 500,
|
|
compressed_tokens: 400,
|
|
tokens_saved: 100,
|
|
});
|
|
const summary = getCompressionAnalyticsSummary();
|
|
assert.equal(summary.avgSavingsPct, 20);
|
|
});
|
|
|
|
it("last24h hourly buckets have correct shape", () => {
|
|
insertCompressionAnalyticsRow({
|
|
timestamp: new Date().toISOString(),
|
|
mode: "lite",
|
|
original_tokens: 1000,
|
|
compressed_tokens: 800,
|
|
tokens_saved: 200,
|
|
});
|
|
const hourly = getCompressionAnalyticsSummary("24h").last24h;
|
|
assert(Array.isArray(hourly));
|
|
assert(hourly.length <= 24);
|
|
hourly.forEach((bucket) => {
|
|
assert(typeof bucket.hour === "string");
|
|
assert(typeof bucket.count === "number");
|
|
assert(typeof bucket.tokensSaved === "number");
|
|
});
|
|
});
|
|
|
|
it("attaches real usage receipts to the latest compression row", () => {
|
|
insertCompressionAnalyticsRow({
|
|
timestamp: new Date().toISOString(),
|
|
mode: "standard",
|
|
original_tokens: 1000,
|
|
compressed_tokens: 700,
|
|
tokens_saved: 300,
|
|
request_id: "req-receipt",
|
|
});
|
|
attachCompressionUsageReceipt(
|
|
"req-receipt",
|
|
{
|
|
prompt_tokens: 710,
|
|
completion_tokens: 42,
|
|
total_tokens: 752,
|
|
prompt_tokens_details: { cached_tokens: 100, cache_creation_tokens: 12 },
|
|
},
|
|
"provider"
|
|
);
|
|
const summary = getCompressionAnalyticsSummary();
|
|
assert.equal(summary.realUsage.requestsWithReceipts, 1);
|
|
assert.equal(summary.realUsage.promptTokens, 710);
|
|
assert.equal(summary.realUsage.completionTokens, 42);
|
|
assert.equal(summary.realUsage.totalTokens, 752);
|
|
assert.equal(summary.realUsage.cacheReadTokens, 100);
|
|
assert.equal(summary.realUsage.cacheWriteTokens, 12);
|
|
assert.equal(summary.realUsage.bySource.provider, 1);
|
|
});
|
|
|
|
it("aggregates estimated USD savings separately from token estimates", () => {
|
|
insertCompressionAnalyticsRow({
|
|
timestamp: new Date().toISOString(),
|
|
mode: "standard",
|
|
original_tokens: 1000,
|
|
compressed_tokens: 700,
|
|
tokens_saved: 300,
|
|
request_id: "req-usd",
|
|
estimated_usd_saved: 0.0015,
|
|
});
|
|
attachCompressionUsageReceipt("req-usd", { prompt_tokens: 700, total_tokens: 700 }, "provider");
|
|
|
|
const summary = getCompressionAnalyticsSummary();
|
|
assert.equal(summary.realUsage.requestsWithReceipts, 1);
|
|
assert.equal(summary.realUsage.estimatedUsdSaved, 0.0015);
|
|
});
|
|
|
|
it("summarizes validation fallback and output mode rows", () => {
|
|
insertCompressionAnalyticsRow({
|
|
timestamp: new Date().toISOString(),
|
|
mode: "output-caveman",
|
|
original_tokens: 900,
|
|
compressed_tokens: 900,
|
|
tokens_saved: 0,
|
|
request_id: "req-output",
|
|
validation_fallback: true,
|
|
output_mode: "full",
|
|
});
|
|
attachCompressionUsageReceipt(
|
|
"req-output",
|
|
{ prompt_tokens: 900, completion_tokens: 120, total_tokens: 1020 },
|
|
"provider"
|
|
);
|
|
|
|
const summary = getCompressionAnalyticsSummary();
|
|
assert.equal(summary.validationFallbacks, 1);
|
|
assert.equal(summary.byMode["output-caveman"].count, 1);
|
|
assert.equal(summary.realUsage.requestsWithReceipts, 1);
|
|
assert.equal(summary.realUsage.totalTokens, 1020);
|
|
});
|
|
|
|
it("summarizes MCP description estimates without counting them as provider receipts", () => {
|
|
insertCompressionAnalyticsRow({
|
|
timestamp: new Date().toISOString(),
|
|
mode: "mcp-description",
|
|
engine: "mcp-description",
|
|
original_tokens: 20,
|
|
compressed_tokens: 12,
|
|
tokens_saved: 8,
|
|
mcp_description_tokens_saved: 8,
|
|
});
|
|
|
|
const summary = getCompressionAnalyticsSummary();
|
|
assert.equal(summary.mcpDescriptionCompression.snapshots, 1);
|
|
assert.equal(summary.mcpDescriptionCompression.estimatedTokensSaved, 8);
|
|
assert.equal(summary.realUsage.requestsWithReceipts, 0);
|
|
assert.equal(summary.realUsage.bySource.mcp_metadata_estimate, undefined);
|
|
});
|
|
|
|
// #4268: attempted-but-no-op runs are recorded with skip_reason so Stacked is
|
|
// visible even when it saves nothing, while saving aggregates stay net-saving-only.
|
|
it("records skipped (no-op) runs separately without polluting saving aggregates", () => {
|
|
// One real saving run...
|
|
insertCompressionAnalyticsRow({
|
|
timestamp: new Date().toISOString(),
|
|
mode: "stacked",
|
|
original_tokens: 1000,
|
|
compressed_tokens: 600,
|
|
tokens_saved: 400,
|
|
});
|
|
// ...and two no-op stacked attempts (saved nothing).
|
|
insertCompressionAnalyticsRow({
|
|
timestamp: new Date().toISOString(),
|
|
mode: "stacked",
|
|
original_tokens: 500,
|
|
compressed_tokens: 500,
|
|
tokens_saved: 0,
|
|
skip_reason: "no_savings",
|
|
});
|
|
insertCompressionAnalyticsRow({
|
|
timestamp: new Date().toISOString(),
|
|
mode: "stacked",
|
|
original_tokens: 800,
|
|
compressed_tokens: 800,
|
|
tokens_saved: 0,
|
|
skip_reason: "no_savings",
|
|
});
|
|
|
|
const summary = getCompressionAnalyticsSummary();
|
|
|
|
// Saving aggregates count the net-saving run ONLY (skip rows excluded).
|
|
assert.equal(summary.totalRequests, 1, "totalRequests must exclude skip rows");
|
|
assert.equal(summary.totalTokensSaved, 400);
|
|
assert.equal(summary.byMode.stacked.count, 1, "byMode count excludes skip rows");
|
|
assert.equal(summary.byMode.stacked.tokensSaved, 400);
|
|
|
|
// ...but the skips are now visible instead of dropped.
|
|
assert.equal(summary.byMode.stacked.skipped, 2, "skipped attempts surfaced per mode");
|
|
assert.equal(summary.totalSkipped, 2);
|
|
assert.equal(summary.bySkipReason.no_savings, 2);
|
|
});
|
|
|
|
it("a mode with only no-op runs still appears (count 0, skipped > 0)", () => {
|
|
insertCompressionAnalyticsRow({
|
|
timestamp: new Date().toISOString(),
|
|
mode: "stacked",
|
|
original_tokens: 300,
|
|
compressed_tokens: 300,
|
|
tokens_saved: 0,
|
|
skip_reason: "no_savings",
|
|
});
|
|
|
|
const summary = getCompressionAnalyticsSummary();
|
|
assert.equal(summary.totalRequests, 0, "no net-saving runs");
|
|
assert.ok(summary.byMode.stacked, "stacked must appear even with only skip rows");
|
|
assert.equal(summary.byMode.stacked.count, 0);
|
|
assert.equal(summary.byMode.stacked.skipped, 1);
|
|
assert.equal(summary.totalSkipped, 1);
|
|
});
|
|
});
|