diff --git a/config/quality/file-size-baseline.json b/config/quality/file-size-baseline.json index 7031e19cda..2a316824e7 100644 --- a/config/quality/file-size-baseline.json +++ b/config/quality/file-size-baseline.json @@ -20,6 +20,7 @@ "_rebaseline_2026_06_16_3972_logs_autorefresh": "Issue #3972 own growth: RequestLoggerV2.tsx 1282->1287 (+5 = the auto-refresh interval now reads the live document.visibilityState each tick instead of a stale mount-time ref, plus a 3-line comment explaining the hidden-tab trap). Cohesive one-spot fix; structural shrink of this component tracked in #3501.", "_rebaseline_2026_06_16_3976_llm7_byteplus_models": "Issue #3976 own growth: models/route.ts 2489->2494 (+5 = add llm7 + byteplus to NAMED_OPENAI_STYLE_PROVIDERS with an explanatory comment so the import route does a live /models fetch instead of serving the stale hardcoded registry catalog). Structural shrink of this route tracked in #3789.", "_rebaseline_2026_06_16_3954_cooldown_epoch": "Issue #3954 own growth: accountFallback.ts 1708->1727 (+19 = a shared cooldownUntilMs() normalizer + its use in isAccountUnavailable/getEarliestRateLimitedUntil/filterAvailableAccounts so a rate_limited_until persisted as a numeric-epoch string is honored, not parsed to NaN) and auth.ts 2216->2219 (+3 = parseFutureDateMs reuses cooldownUntilMs). Cohesive cooldown read-path hardening at the existing chokepoints; one helper, not extractable.", + "_rebaseline_2026_06_16_3960_engine_breakdown": "PR #3960 capture own growth: chatCore.ts 5851->5868 (+17 = persist result.stats.engineBreakdown into the new compression_engine_breakdown table after a stacked compression run, so getPerEngineAnalytics is accurate historically and not live-only). Cohesive analytics-persistence at the existing compression chokepoint; structural shrink of chatCore.ts tracked in #3501.", "_rebaseline_2026_06_15_3938_perplexity_v218": "PR #3938 own growth: perplexity-web.ts 868->939 (+71 = rebuild buildPplxRequestBody to mirror the current www.perplexity.ai schematized request body — version 2.18, use_schematized_api + the full supported_block_use_cases list, dsl_query, shared requestId for frontend_uuid/client_search_results_cache_key, last_backend_uuid only on follow-ups — plus the x-perplexity-request-* / x-request-id headers replacing the stale X-App-ApiVersion pair that triggered HTTP 400). Cohesive upstream-schema sync in a single executor; not extractable.", "cap": 800, "frozen": { @@ -36,7 +37,7 @@ "open-sse/executors/muse-spark-web.ts": 1284, "open-sse/executors/perplexity-web.ts": 1013, "open-sse/handlers/audioSpeech.ts": 965, - "open-sse/handlers/chatCore.ts": 5851, + "open-sse/handlers/chatCore.ts": 5868, "open-sse/handlers/imageGeneration.ts": 3777, "open-sse/handlers/responseSanitizer.ts": 1103, "open-sse/handlers/search.ts": 1442, diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 531b80382e..1c896ef023 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -2705,7 +2705,7 @@ export async function handleChatCore({ compressionAnalyticsRecorded = true; compressionAnalyticsWritePromise = (async () => { try { - const { insertCompressionAnalyticsRow } = + const { insertCompressionAnalyticsRow, insertCompressionEngineBreakdown } = await import("../../src/lib/db/compressionAnalytics.ts"); const { calculateCost } = await import("../../src/lib/usage/costCalculator.ts"); const tokensSaved = Math.max( @@ -2746,6 +2746,23 @@ export async function handleChatCore({ ? rtkPointers.reduce((total, pointer) => total + pointer.bytes, 0) : null, }); + // Persist the per-engine breakdown of a stacked run so per-engine + // analytics (getPerEngineAnalytics) is accurate historically, not just + // in the live `compression.completed` event. + const engineBreakdown = result.stats.engineBreakdown ?? []; + if (engineBreakdown.length > 0) { + insertCompressionEngineBreakdown( + engineBreakdown.map((b) => ({ + timestamp: new Date().toISOString(), + request_id: skillRequestId, + engine: b.engine, + original_tokens: b.originalTokens, + compressed_tokens: b.compressedTokens, + tokens_saved: Math.max(0, b.originalTokens - b.compressedTokens), + duration_ms: b.durationMs ?? null, + })) + ); + } } catch (err) { log?.debug?.( "COMPRESSION", diff --git a/open-sse/services/compression/engines/cavemanAdapter.ts b/open-sse/services/compression/engines/cavemanAdapter.ts index 6bab174e56..41ea89dc2b 100644 --- a/open-sse/services/compression/engines/cavemanAdapter.ts +++ b/open-sse/services/compression/engines/cavemanAdapter.ts @@ -211,6 +211,29 @@ function validateUltraConfig(config: Record): EngineValidationR return { valid: errors.length === 0, errors }; } +// Lite only honors `preserveSystemPrompt` (model/vision are runtime, not user config). +// Previously this engine wrongly exposed AGGRESSIVE_SCHEMA, surfacing irrelevant +// summarizer/threshold fields in the per-engine config UI. +const LITE_SCHEMA: EngineConfigField[] = [ + { + key: "preserveSystemPrompt", + type: "boolean", + label: "Preserve system prompt", + defaultValue: true, + }, +]; + +function validateLiteConfig(config: Record): EngineValidationResult { + const errors: string[] = []; + if ( + config.preserveSystemPrompt !== undefined && + typeof config.preserveSystemPrompt !== "boolean" + ) { + errors.push("preserveSystemPrompt must be a boolean"); + } + return { valid: errors.length === 0, errors }; +} + export const liteEngine: CompressionEngine = { id: "lite", name: "Lite", @@ -240,10 +263,10 @@ export const liteEngine: CompressionEngine = { return this.apply(body, { stepConfig: config }); }, getConfigSchema() { - return AGGRESSIVE_SCHEMA; + return LITE_SCHEMA; }, validateConfig(config) { - return validateAggressiveConfig(config); + return validateLiteConfig(config); }, }; diff --git a/src/lib/db/compressionAnalytics.ts b/src/lib/db/compressionAnalytics.ts index 2a8fc448e8..0d7afd35bb 100644 --- a/src/lib/db/compressionAnalytics.ts +++ b/src/lib/db/compressionAnalytics.ts @@ -30,6 +30,21 @@ export interface CompressionAnalyticsRow { rtk_raw_output_total_bytes?: number | null; } +/** + * One row per engine that ran inside a stacked compression pipeline. A stacked + * request writes a single aggregate `compression_analytics` row (engine = mode) plus + * N of these — so per-engine savings are queryable historically, not just live. + */ +export interface CompressionEngineBreakdownRow { + timestamp: string; + request_id?: string | null; + engine: string; + original_tokens: number; + compressed_tokens: number; + tokens_saved: number; + duration_ms?: number | null; +} + export interface CompressionAnalyticsSummary { totalRequests: number; totalTokensSaved: number; @@ -139,6 +154,53 @@ export function insertCompressionAnalyticsRow(row: CompressionAnalyticsRow): voi ); } +let breakdownTableEnsuredForDb: unknown = null; + +function ensureCompressionEngineBreakdownTable(): void { + const db = getDbInstance(); + if (breakdownTableEnsuredForDb === db) return; + db.exec(` + CREATE TABLE IF NOT EXISTS compression_engine_breakdown ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + timestamp TEXT NOT NULL, + request_id TEXT, + engine TEXT NOT NULL, + original_tokens INTEGER NOT NULL DEFAULT 0, + compressed_tokens INTEGER NOT NULL DEFAULT 0, + tokens_saved INTEGER NOT NULL DEFAULT 0, + duration_ms INTEGER + ); + CREATE INDEX IF NOT EXISTS idx_ceb_engine_ts ON compression_engine_breakdown(engine, timestamp); + CREATE INDEX IF NOT EXISTS idx_ceb_request ON compression_engine_breakdown(request_id); + `); + breakdownTableEnsuredForDb = db; +} + +export function insertCompressionEngineBreakdown(rows: CompressionEngineBreakdownRow[]): void { + if (!rows.length) return; + const db = getDbInstance(); + ensureCompressionEngineBreakdownTable(); + const stmt = db.prepare( + `INSERT INTO compression_engine_breakdown + (timestamp, request_id, engine, original_tokens, compressed_tokens, tokens_saved, duration_ms) + VALUES (?, ?, ?, ?, ?, ?, ?)` + ); + const insertAll = db.transaction((items: CompressionEngineBreakdownRow[]) => { + for (const r of items) { + stmt.run( + r.timestamp, + r.request_id ?? null, + r.engine, + r.original_tokens, + r.compressed_tokens, + r.tokens_saved, + r.duration_ms ?? null + ); + } + }); + insertAll(rows); +} + export function attachCompressionUsageReceipt( requestId: string | null | undefined, usage: Record | null | undefined, @@ -205,24 +267,53 @@ function appendCondition(whereClause: string, condition: string): string { return whereClause ? `${whereClause} AND ${condition}` : `WHERE ${condition}`; } +type EngineAggRow = { runs: number; original: number; compressed: number; saved: number }; + export function getPerEngineAnalytics(engineId: string, days = 7) { const db = getDbInstance(); ensureCompressionAnalyticsColumns(); + ensureCompressionEngineBreakdownTable(); const since = new Date(Date.now() - days * 86400_000).toISOString(); - const row = db + + // (1) Per-engine contributions from stacked runs (one breakdown row per engine). + const breakdown = db + .prepare( + `SELECT COUNT(*) AS runs, + COALESCE(SUM(original_tokens), 0) AS original, + COALESCE(SUM(compressed_tokens), 0) AS compressed, + COALESCE(SUM(tokens_saved), 0) AS saved + FROM compression_engine_breakdown + WHERE engine = ? AND timestamp >= ?` + ) + .get(engineId, since) as EngineAggRow; + + // (2) Legacy single-engine rows from compression_analytics, EXCLUDING any request + // that already has a per-engine breakdown — so a stacked run's aggregate row is not + // double-counted on top of its breakdown rows. + const legacy = db .prepare( `SELECT COUNT(*) AS runs, COALESCE(SUM(original_tokens), 0) AS original, COALESCE(SUM(compressed_tokens), 0) AS compressed, COALESCE(SUM(tokens_saved), 0) AS saved FROM compression_analytics - WHERE COALESCE(engine, mode) = ? AND timestamp >= ?` + WHERE COALESCE(engine, mode) = ? AND timestamp >= ? + AND ( + request_id IS NULL + OR request_id NOT IN ( + SELECT request_id FROM compression_engine_breakdown WHERE request_id IS NOT NULL + ) + )` ) - .get(engineId, since) as { runs: number; original: number; compressed: number; saved: number }; - const tokensSaved = Math.max(0, row.saved); + .get(engineId, since) as EngineAggRow; + + const runs = breakdown.runs + legacy.runs; + const original = breakdown.original + legacy.original; + const compressed = breakdown.compressed + legacy.compressed; + const tokensSaved = Math.max(0, breakdown.saved + legacy.saved); const avgSavingsPercent = - row.original > 0 ? Math.round(((row.original - row.compressed) / row.original) * 1000) / 10 : 0; - return { engineId, runs: row.runs, tokensSaved, avgSavingsPercent, days }; + original > 0 ? Math.round(((original - compressed) / original) * 1000) / 10 : 0; + return { engineId, runs, tokensSaved, avgSavingsPercent, days }; } export function getCompressionAnalyticsSummary(since?: string): CompressionAnalyticsSummary { diff --git a/tests/unit/compression/engine-registry.test.ts b/tests/unit/compression/engine-registry.test.ts index f57422e1cb..6dffe20278 100644 --- a/tests/unit/compression/engine-registry.test.ts +++ b/tests/unit/compression/engine-registry.test.ts @@ -4,6 +4,7 @@ import assert from "node:assert/strict"; import { aggressiveEngine, cavemanEngine, + liteEngine, ultraEngine, } from "../../../open-sse/services/compression/engines/cavemanAdapter.ts"; import { rtkEngine as realRtkEngine } from "../../../open-sse/services/compression/engines/rtk/index.ts"; @@ -74,6 +75,15 @@ describe("compression engine registry contract", () => { assert.ok(rtkSchema.some((field) => field.key === "applyToCodeBlocks")); assert.ok(aggressiveSchema.some((field) => field.key === "maxTokensPerMessage")); assert.ok(ultraSchema.some((field) => field.key === "compressionRate")); + + // Lite exposes its OWN minimal schema (preserveSystemPrompt), NOT the aggressive + // summarizer/threshold fields it previously leaked. + const liteSchema = liteEngine.getConfigSchema(); + assert.ok(liteSchema.some((field) => field.key === "preserveSystemPrompt")); + assert.ok(!liteSchema.some((field) => field.key === "maxTokensPerMessage")); + assert.ok(!liteSchema.some((field) => field.key === "summarizerEnabled")); + assert.equal(liteEngine.validateConfig({ preserveSystemPrompt: true }).valid, true); + assert.equal(liteEngine.validateConfig({ preserveSystemPrompt: "yes" }).valid, false); assert.equal(cavemanEngine.validateConfig({ intensity: "full" }).valid, true); assert.equal(cavemanEngine.validateConfig({ intensity: "bad" }).valid, false); assert.equal(realRtkEngine.validateConfig({ maxLinesPerResult: 20 }).valid, true); diff --git a/tests/unit/db/per-engine-breakdown-analytics.test.ts b/tests/unit/db/per-engine-breakdown-analytics.test.ts new file mode 100644 index 0000000000..e5fec7147e --- /dev/null +++ b/tests/unit/db/per-engine-breakdown-analytics.test.ts @@ -0,0 +1,173 @@ +/** + * TDD: per-engine breakdown persistence. + * + * A real stacked run records ONE compression_analytics row (engine = stats.engine ?? + * mode, e.g. "stacked") — so per-engine savings were previously lost. The + * compression_engine_breakdown table stores one row per engine in the stacked + * pipeline, and getPerEngineAnalytics aggregates breakdown + legacy single-engine + * rows (deduped by request_id, no double counting). + * + * DB isolation mirrors tests/unit/db/per-engine-analytics.test.ts. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-ceb-analytics-")); +const originalDataDir = process.env.DATA_DIR; +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../../src/lib/db/core.ts"); +core.resetDbInstance(); + +const { insertCompressionAnalyticsRow, insertCompressionEngineBreakdown, getPerEngineAnalytics } = + await import("../../../src/lib/db/compressionAnalytics.ts"); + +function resetDb(): void { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +} + +test.beforeEach(() => { + resetDb(); +}); + +test.after(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + if (originalDataDir === undefined) delete process.env.DATA_DIR; + else process.env.DATA_DIR = originalDataDir; +}); + +test("per-engine breakdown from a stacked run is attributed to each engine", () => { + const now = new Date().toISOString(); + + // One aggregate row for the stacked request (engine column = mode, NOT per-engine). + insertCompressionAnalyticsRow({ + timestamp: now, + mode: "stacked", + engine: "stacked", + request_id: "req-stacked-1", + original_tokens: 1000, + compressed_tokens: 700, + tokens_saved: 300, + }); + + // Per-engine breakdown for that same request: rtk (1000→800) then headroom (800→700). + insertCompressionEngineBreakdown([ + { + timestamp: now, + request_id: "req-stacked-1", + engine: "rtk", + original_tokens: 1000, + compressed_tokens: 800, + tokens_saved: 200, + }, + { + timestamp: now, + request_id: "req-stacked-1", + engine: "headroom", + original_tokens: 800, + compressed_tokens: 700, + tokens_saved: 100, + }, + ]); + + const headroom = getPerEngineAnalytics("headroom"); + assert.equal(headroom.runs, 1, "headroom ran once (inside the stacked pipeline)"); + assert.equal(headroom.tokensSaved, 100, "headroom's own contribution"); + // avg = round(((800-700)/800)*1000)/10 = round(125)/10 = 12.5 + assert.equal(headroom.avgSavingsPercent, 12.5); + + const rtk = getPerEngineAnalytics("rtk"); + assert.equal(rtk.runs, 1); + assert.equal(rtk.tokensSaved, 200); +}); + +test("legacy single-engine rows still count (no breakdown present)", () => { + const now = new Date().toISOString(); + insertCompressionAnalyticsRow({ + timestamp: now, + mode: "aggressive", + engine: "aggressive", + request_id: "req-single", + original_tokens: 500, + compressed_tokens: 400, + tokens_saved: 100, + }); + + const aggressive = getPerEngineAnalytics("aggressive"); + assert.equal(aggressive.runs, 1, "single-engine run counted via the legacy engine column"); + assert.equal(aggressive.tokensSaved, 100); +}); + +test("breakdown + legacy combine for the same engine without double counting", () => { + const now = new Date().toISOString(); + + // Stacked run where headroom contributed 100 (recorded in breakdown). + insertCompressionAnalyticsRow({ + timestamp: now, + mode: "stacked", + engine: "stacked", + request_id: "req-A", + original_tokens: 1000, + compressed_tokens: 800, + tokens_saved: 200, + }); + insertCompressionEngineBreakdown([ + { + timestamp: now, + request_id: "req-A", + engine: "headroom", + original_tokens: 1000, + compressed_tokens: 900, + tokens_saved: 100, + }, + ]); + + // Separate single-engine headroom run (no breakdown) contributed 50. + insertCompressionAnalyticsRow({ + timestamp: now, + mode: "headroom", + engine: "headroom", + request_id: "req-B", + original_tokens: 200, + compressed_tokens: 150, + tokens_saved: 50, + }); + + const headroom = getPerEngineAnalytics("headroom"); + assert.equal(headroom.runs, 2, "one stacked contribution + one single-engine run"); + assert.equal(headroom.tokensSaved, 150, "100 (stacked) + 50 (single), counted once each"); +}); + +test("a stacked run does NOT double-count the aggregate row under its breakdown engines", () => { + const now = new Date().toISOString(); + insertCompressionAnalyticsRow({ + timestamp: now, + mode: "stacked", + engine: "stacked", + request_id: "req-X", + original_tokens: 1000, + compressed_tokens: 700, + tokens_saved: 300, + }); + insertCompressionEngineBreakdown([ + { + timestamp: now, + request_id: "req-X", + engine: "caveman", + original_tokens: 1000, + compressed_tokens: 700, + tokens_saved: 300, + }, + ]); + + // caveman gets exactly the breakdown contribution, not also the "stacked" aggregate. + const caveman = getPerEngineAnalytics("caveman"); + assert.equal(caveman.runs, 1); + assert.equal(caveman.tokensSaved, 300); +});