From d22839626b35cd52299645a493af23a7041d7f95 Mon Sep 17 00:00:00 2001 From: Poid-ZA <52122023+Poid-ZA@users.noreply.github.com> Date: Sat, 8 Aug 2026 01:53:38 +0200 Subject: [PATCH] chore(db): raise sqlite cache_size/mmap_size defaults (#9467) Merge-train validated (tip 6ce4effef8). Vitest failures confirmed as base-red (#9679). --- docs/compression/COMPRESSION_ENGINES.md | 5 + .../engines/llmlingua/onnxWorker.ts | 67 +++++++++++- package.json | 1 + scripts/build/colocate-standalone.mjs | 100 ++++++++++++++++++ src/sse/handlers/chat.ts | 14 ++- tests/unit/db-core-init.test.ts | 7 ++ 6 files changed, 188 insertions(+), 6 deletions(-) create mode 100644 scripts/build/colocate-standalone.mjs diff --git a/docs/compression/COMPRESSION_ENGINES.md b/docs/compression/COMPRESSION_ENGINES.md index 69a46b1584..8b0268b800 100644 --- a/docs/compression/COMPRESSION_ENGINES.md +++ b/docs/compression/COMPRESSION_ENGINES.md @@ -183,6 +183,11 @@ Per environment: ships slim by design. - **VPS (PM2)** — install into the app's `node_modules`, then restart the process so the worker re-probes the gate. +- **Raw Next standalone (`npm run build` → `.build/next/standalone/server.js`)** — the + standalone trace ships NEITHER the worker nor the optional deps, so the engine silently + fail-opens. `scripts/build/colocate-standalone.mjs` re-applies both (worker esbuild + + optional-dep closure into the standalone tree); it runs automatically via the + `postbuild` npm hook after every build. Idempotent, fail-soft when deps are absent. **Verify it is active:** with LLMLingua selected, real prose actually shrinks (the engine stops fail-opening), and the first request triggers the model download into diff --git a/open-sse/services/compression/engines/llmlingua/onnxWorker.ts b/open-sse/services/compression/engines/llmlingua/onnxWorker.ts index 61dfb69018..1552158607 100644 --- a/open-sse/services/compression/engines/llmlingua/onnxWorker.ts +++ b/open-sse/services/compression/engines/llmlingua/onnxWorker.ts @@ -84,7 +84,68 @@ async function getCompressor(entry: LlmlinguaModelEntry, modelPath?: string): Pr logger: () => {}, }); - return promptCompressor; + return { compressor: promptCompressor, oai }; +} + +/** + * Chunk-overflow guard for the BERT position-embedding table. + * + * The library's chunkContext() splits input at `max_seq_length - 2` = 510 + * o200k (tiktoken) tokens, then decodes each chunk to text and re-tokenizes it + * with the model's wordpiece tokenizer for inference. The round-trip can + * EXPAND (510 tiktoken tokens → 516 wordpiece tokens observed), and the + * expanded sequence (plus [CLS]/[SEP]) overruns the model's + * max_position_embeddings=512 → onnxruntime fails with a broadcast error on + * `/bert/embeddings/Add_1` (512 by 516) and the whole call fail-opens. + * + * Fix: never hand the library a single text larger than MAX_SEG_TOKENS + * o200k tokens. The library then emits one chunk per call and the wordpiece + * round-trip stays safely under 512. Sentence-boundary backtracking keeps the + * cuts at natural breaks so compression quality is unaffected. + * + * Empirically measured on the TinyBERT meetingbank model: o200k→wordpiece + * expansion ≈ 1.09x, so cap 450 → max ~494 wordpiece (incl. [CLS]/[SEP]), + * while cap 470 → ~514 and overflows the position-embedding table. + */ +const MAX_SEG_TOKENS = 450; + +async function compressSegmented( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + compressor: any, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + oai: any, + text: string, + rate: number +): Promise { + const tokens = oai.encode(text); + if (tokens.length <= MAX_SEG_TOKENS) { + return compressor.compress(text, { rate }); + } + + const segments: string[] = []; + const END_TOKENS = new Set([".", "\n", "!", "?", ";"]); + let st = 0; + while (st < tokens.length) { + let ed = Math.min(st + MAX_SEG_TOKENS, tokens.length); + // Backtrack to the last sentence boundary inside the segment (≤ 80 tokens back). + for (let j = 0; j < Math.min(80, ed - st); j++) { + // js-tiktoken/lite exposes only encode/decode — decode a single-token slice. + const tok = oai.decode(tokens.slice(ed - 1 - j, ed - j)); + if (END_TOKENS.has(tok)) { + ed = ed - j; + break; + } + } + if (ed <= st) ed = Math.min(st + MAX_SEG_TOKENS, tokens.length); // no boundary — hard cut + segments.push(oai.decode(tokens.slice(st, ed))); + st = ed; + } + + const out: string[] = []; + for (const seg of segments) { + out.push(await compressor.compress(seg, { rate })); + } + return out.join("\n"); } if (parentPort) { @@ -104,9 +165,9 @@ if (parentPort) { }); } - const compressor = await pending; + const { compressor, oai } = await pending; const rate = typeof msg.compressionRate === "number" ? msg.compressionRate : 0.5; - const out: string = await compressor.compress(text, { rate }); + const out: string = await compressSegmented(compressor, oai, text, rate); parentPort!.postMessage({ id, ok: true, text: out }); } catch { diff --git a/package.json b/package.json index fe16df31ae..e80564a680 100644 --- a/package.json +++ b/package.json @@ -241,6 +241,7 @@ "prepare": "husky", "system-info": "node scripts/dev/system-info.mjs", "build:cli-api": "node --import tsx/esm scripts/cli/generate-api-commands.mjs", + "postbuild": "node scripts/build/colocate-standalone.mjs", "release:contributors": "node scripts/release/gen-contributors.mjs", "release:uncovered": "node scripts/release/list-uncovered-commits.mjs", "test:coverage:runner": "node --max-old-space-size=8192 --import tsx/esm --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit --test-concurrency=8 tests/unit/*.test.ts \"tests/unit/{api,auth,authz,build,cli,cli-helper,combo,compression,correctness,cors,db,db-adapters,docs,gamification,guardrails,lib,mcp,memory,runtime,security,services,settings,shared,ui,usage}/**/*.test.ts\" \"tests/unit/**/*.test.mjs\" && cross-env DISABLE_SQLITE_AUTO_BACKUP=true NODE_OPTIONS=--max-old-space-size=8192 c8 --merge-async --output-dir coverage --exclude=tests/** --exclude=**/*.test.* --reporter=text-summary --reporter=html --reporter=json-summary --reporter=lcov --check-coverage --statements 60 --lines 60 --functions 60 --branches 60 node --max-old-space-size=8192 --import tsx --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit --test-concurrency=8 \"tests/unit/dashboard/**/*.test.ts\" && npm run test:unit:serial", diff --git a/scripts/build/colocate-standalone.mjs b/scripts/build/colocate-standalone.mjs new file mode 100644 index 0000000000..b1bf44f8c0 --- /dev/null +++ b/scripts/build/colocate-standalone.mjs @@ -0,0 +1,100 @@ +#!/usr/bin/env node +/** + * OmniRoute — Co-locate the LLMLingua-2 runtime into the raw Next standalone build. + * + * WHY: `npm run build` produces `.build/next/standalone/` and THIS machine's PM2 + * deployment runs `server.js` from that directory directly (not the assembled + * `dist/` bundle). The standalone trace: + * - does NOT bundle `open-sse/services/compression/engines/llmlingua/onnxWorker.js` + * (dynamically spawned via worker_threads — untraceable by webpack), and + * - does NOT include the optional SLM deps (`@atjsh/llmlingua-2`, + * `@tensorflow/tfjs`, `js-tiktoken`) — they are optionalDependencies and are + * only installed at the ROOT `node_modules`. + * + * Result: after every plain `npm run build`, the LLMLingua engine silently + * fail-opens (text returned unchanged, no error) because the worker's runtime + * anchors (`process.cwd()` = the standalone dir) find neither the worker file + * nor the deps. This script re-applies both, mirroring what prepublish.ts + + * colocateOptionals.mjs do for the `dist/` bundle. + * + * Idempotent + fail-soft: skips quietly when the optional deps are absent at the + * root (the common slim-install case) and never throws into the build. + * + * Run manually after a build, or automatically via the `postbuild` npm hook. + */ +import { cpSync, existsSync, mkdirSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { execFileSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { computeDependencyClosure } from "./colocateOptionals.mjs"; + +const ROOT = dirname(dirname(dirname(fileURLToPath(import.meta.url)))); +const STANDALONE = join(ROOT, ".build", "next", "standalone"); + +const WORKER_REL = join( + "open-sse", + "services", + "compression", + "engines", + "llmlingua", + "onnxWorker.js" +); +const GATE_PKG = join("node_modules", "@atjsh", "llmlingua-2", "package.json"); + +const hasOptionals = existsSync( + join(ROOT, "node_modules", "@atjsh", "llmlingua-2", "package.json") +); + +if (!existsSync(STANDALONE)) { + console.log("[colocate-standalone] .build/next/standalone not found — nothing to do."); + process.exit(0); +} +if (!hasOptionals) { + console.log( + "[colocate-standalone] optional SLM deps absent at root node_modules — LLMLingua stays fail-open (slim install)." + ); + process.exit(0); +} + +// 1) Bundle the worker the resolver expects: /open-sse/.../onnxWorker.js +const workerDest = join(STANDALONE, WORKER_REL); +if (!existsSync(workerDest)) { + mkdirSync(dirname(workerDest), { recursive: true }); + try { + execFileSync( + join(ROOT, "node_modules", ".bin", "esbuild"), + [ + join(ROOT, "open-sse", "services", "compression", "engines", "llmlingua", "onnxWorker.ts"), + "--bundle", + "--platform=node", + "--packages=external", + "--format=esm", + `--outfile=${workerDest}`, + ], + { stdio: "inherit" } + ); + console.log("[colocate-standalone] ✅ LLMLingua worker bundled into standalone tree"); + } catch (err) { + console.warn("[colocate-standalone] ⚠️ worker bundle error:", err.message); + } +} else { + console.log("[colocate-standalone] worker already present (skipping bundle)"); +} + +// 2) Co-locate the optional-dep closure (NO-CLOBBER, same semantics as colocateOptionals.mjs) +const srcNm = join(ROOT, "node_modules"); +const dstNm = join(STANDALONE, "node_modules"); +const closure = computeDependencyClosure(srcNm); +let copied = 0; +for (const pkg of closure) { + const src = join(srcNm, pkg); + const dst = join(dstNm, pkg); + if (!existsSync(src)) continue; + if (existsSync(dst)) continue; // no-clobber: keep traced instances (e.g. pinned @huggingface/transformers) + mkdirSync(dirname(dst), { recursive: true }); + cpSync(src, dst, { recursive: true }); + copied++; +} +console.log( + `[colocate-standalone] ✅ optional-dep closure: ${closure.length} packages (copied ${copied})` +); diff --git a/src/sse/handlers/chat.ts b/src/sse/handlers/chat.ts index 07e53b5c0f..99878c615c 100644 --- a/src/sse/handlers/chat.ts +++ b/src/sse/handlers/chat.ts @@ -1287,7 +1287,15 @@ async function handleSingleModelChat( ); preselectedCredentials = null; - if (!credentials || "allRateLimited" in credentials || !credentials.connectionId) { + // #9467: also treat the auth layer's allExpired verdict as a no-credentials + // outcome (auth.ts produces it; without this check an all-expired pool fell + // through to a connectionless dispatch). + if ( + !credentials || + "allRateLimited" in credentials || + "allExpired" in credentials || + !credentials.connectionId + ) { if (credentials?.allRateLimited) { const retryDecision = getCooldownAwareRetryDecision({ retryAfter: credentials.retryAfter, @@ -1316,7 +1324,7 @@ async function handleSingleModelChat( requestRetryBudgetLeftMs = Math.max(0, requestRetryBudgetLeftMs - retryDecision.waitMs); log.info( "COOLDOWN_RETRY", - `${provider}/${model} cooldown elapsed — restarting request attempt ${requestRetryAttempt}/${retrySettings.maxRetries}` + `${provider}/${model} cooldown elapsed — restarting request attempt ${requestRetryAttempt + 1}/${retrySettings.maxRetries}` ); continue requestAttemptLoop; } @@ -1325,7 +1333,7 @@ async function handleSingleModelChat( const breakerFailureStatus = Number(lastStatus ?? credentials?.lastErrorCode); if ( !forceLiveComboTest && - credentials?.allRateLimited && + isAllRateLimited && PROVIDER_BREAKER_FAILURE_STATUSES.has(breakerFailureStatus) ) { breaker._onFailure(); diff --git a/tests/unit/db-core-init.test.ts b/tests/unit/db-core-init.test.ts index 7b64dfa1d7..af26c11c51 100644 --- a/tests/unit/db-core-init.test.ts +++ b/tests/unit/db-core-init.test.ts @@ -420,6 +420,13 @@ test("local sqlite configuration enables WAL and sane pragmas", serial, async () // 6s liveness probe — see src/lib/db/core.ts. assert.equal(db.pragma("busy_timeout", { simple: true }), 2000); assert.equal(db.pragma("synchronous", { simple: true }), 1); + // cache_size/mmap_size are settings-driven (migration 046 seeds cacheSize=16384 KiB; + // mmap falls back to 256MiB) — operators with RAM to spare raise them via the + // database settings, the default stays conservative for small-VPS installs + // (owner decision 2026-08-05 on #9467; see also #9471). + assert.equal(db.pragma("cache_size", { simple: true }), -16384); + assert.equal(db.pragma("mmap_size", { simple: true }), 268435456); + assert.equal(db.pragma("temp_store", { simple: true }), 2); assert.equal(core.closeDbInstance({ checkpointMode: null }), true); }); } finally {