diff --git a/config/quality/dependency-allowlist.json b/config/quality/dependency-allowlist.json index 7c4340fc4b..6421958aff 100644 --- a/config/quality/dependency-allowlist.json +++ b/config/quality/dependency-allowlist.json @@ -1,6 +1,7 @@ { "_comment": "Allowlist anti-slopsquatting (check-deps.mjs). Toda dep nova exige adicao EXPLICITA aqui apos verificar que e legitima.", "allowed": [ + "@atjsh/llmlingua-2", "@aws-sdk/client-bedrock-runtime", "@cyclonedx/cyclonedx-npm", "@dnd-kit/core", @@ -18,6 +19,7 @@ "@stryker-mutator/tap-runner", "@swc/helpers", "@tailwindcss/postcss", + "@tensorflow/tfjs", "@testing-library/jest-dom", "@testing-library/react", "@types/bcryptjs", @@ -64,6 +66,7 @@ "ink-text-input", "ioredis", "jose", + "js-tiktoken", "js-yaml", "jscpd", "jsdom", diff --git a/open-sse/services/compression/engines/llmlingua/constants.ts b/open-sse/services/compression/engines/llmlingua/constants.ts new file mode 100644 index 0000000000..b749e5ca3d --- /dev/null +++ b/open-sse/services/compression/engines/llmlingua/constants.ts @@ -0,0 +1,55 @@ +/** + * LLMLingua real-engine constants — pure data + types. + * + * NO imports of native deps (transformers.js, onnxruntime, etc). This module is + * safe to import from anywhere (main thread, worker, tests) without pulling in + * the heavy ONNX runtime. + * + * The real backend uses `@atjsh/llmlingua-2` (ONNX via `@huggingface/transformers`), + * which downloads models from the HuggingFace Hub into a cache dir. Only the two + * models PROVEN to work end-to-end are registered here. + */ + +export type LlmlinguaFactory = "WithBERTMultilingual" | "WithXLMRoBERTa"; + +export interface LlmlinguaModelEntry { + /** config value, e.g. "tinybert" */ + id: string; + /** HuggingFace Hub repo id */ + hfRepo: string; + factory: LlmlinguaFactory; + dtype: "fp32"; + /** transformers.js subfolder option; "" for both proven models */ + subfolder: string; + sizeMB: number; + label: string; +} + +export const DEFAULT_LLMLINGUA_MODEL = "tinybert"; + +/** Registry keyed by config `model` value. Only the two PROVEN models. */ +export const LLMLINGUA_MODELS: Record = { + tinybert: { + id: "tinybert", + hfRepo: "atjsh/llmlingua-2-js-tinybert-meetingbank", + factory: "WithBERTMultilingual", + dtype: "fp32", + subfolder: "", + sizeMB: 57, + label: "TinyBERT (57MB, fast — default)", + }, + "bert-base": { + id: "bert-base", + hfRepo: "Arcoldd/llmlingua4j-bert-base-onnx", + factory: "WithBERTMultilingual", + dtype: "fp32", + subfolder: "", + sizeMB: 710, + label: "BERT-base (710MB, higher accuracy)", + }, +}; + +/** Per-call worker reply timeout → fail-open. */ +export const LLMLINGUA_WORKER_TIMEOUT_MS = 5000; +/** Terminate the idle worker after this long to free model RAM. */ +export const LLMLINGUA_WORKER_IDLE_MS = 300000; diff --git a/open-sse/services/compression/engines/llmlingua/index.ts b/open-sse/services/compression/engines/llmlingua/index.ts index f734437974..1f9f83b601 100644 --- a/open-sse/services/compression/engines/llmlingua/index.ts +++ b/open-sse/services/compression/engines/llmlingua/index.ts @@ -7,7 +7,9 @@ * ## Design * * ### Backend abstraction - * `LlmlinguaBackend` is a simple `(text: string) => Promise` contract. + * `LlmlinguaBackend` is a `(text: string, opts?: LlmlinguaBackendOptions) => + * Promise` contract (the opts carry model selection / compression rate / + * offline model-path override; single-arg fakes remain assignable). * Tests inject a fake backend via `setLlmlinguaBackend()`. Production code uses * `workerBackend` from `./worker.ts` (a stub today — see that file for the L1 * VPS-validation follow-up before the real ONNX model is wired). @@ -42,7 +44,7 @@ * for the exact spec. */ -import { createCompressionStats } from "../../stats.ts"; +import { createCompressionStats, estimateCompressionTokens } from "../../stats.ts"; import { extractPreservedBlocks } from "../../preservation.ts"; import type { CompressionEngine, @@ -52,14 +54,22 @@ import type { } from "../types.ts"; import type { CompressionResult } from "../../types.ts"; import { workerBackend } from "./worker.ts"; +import { LLMLINGUA_MODELS, DEFAULT_LLMLINGUA_MODEL } from "./constants.ts"; // ─── backend abstraction ────────────────────────────────────────────────────── +/** Options the real backend needs (model selection + compression rate + offline override). */ +export interface LlmlinguaBackendOptions { + model?: string; + compressionRate?: number; + modelPath?: string; +} + /** - * A backend takes a prose text segment and returns a compressed version. + * A backend takes a prose text segment (+ optional config) and returns a compressed version. * Any rejection or error MUST be caught by the caller; the engine fail-opens. */ -export type LlmlinguaBackend = (text: string) => Promise; +export type LlmlinguaBackend = (text: string, opts?: LlmlinguaBackendOptions) => Promise; /** Module-level injectable backend (null = use default production backend). */ let _backend: LlmlinguaBackend | null = null; @@ -140,11 +150,12 @@ type MessageLike = { */ async function compressProseText( text: string, - backend: LlmlinguaBackend + backend: LlmlinguaBackend, + opts?: LlmlinguaBackendOptions ): Promise<{ text: string; didCompress: boolean }> { if (!text.trim()) return { text, didCompress: false }; try { - const compressed = await backend(text); + const compressed = await backend(text, opts); // Accept only if it actually gets shorter (reject no-ops or expansions) if (typeof compressed === "string" && compressed.length < text.length) { return { text: compressed, didCompress: true }; @@ -165,7 +176,8 @@ async function compressProseText( */ async function compressMessageText( text: string, - backend: LlmlinguaBackend + backend: LlmlinguaBackend, + opts?: LlmlinguaBackendOptions ): Promise<{ text: string; didCompress: boolean }> { const segments = splitProseAndPreserved(text); let anyCompressed = false; @@ -176,7 +188,7 @@ async function compressMessageText( // Never send preserved content (code, math, etc.) to the backend parts.push(seg.text); } else { - const { text: out, didCompress } = await compressProseText(seg.text, backend); + const { text: out, didCompress } = await compressProseText(seg.text, backend, opts); parts.push(out); if (didCompress) anyCompressed = true; } @@ -191,7 +203,8 @@ async function compressMessageText( */ async function processMessages( messages: MessageLike[], - backend: LlmlinguaBackend + backend: LlmlinguaBackend, + opts?: LlmlinguaBackendOptions ): Promise<{ messages: MessageLike[]; compressedCount: number }> { let compressedCount = 0; const result: MessageLike[] = []; @@ -205,7 +218,7 @@ async function processMessages( try { if (typeof msg.content === "string") { - const { text, didCompress } = await compressMessageText(msg.content, backend); + const { text, didCompress } = await compressMessageText(msg.content, backend, opts); if (didCompress) { compressedCount++; result.push({ ...msg, content: text }); @@ -219,7 +232,8 @@ async function processMessages( if (part["type"] === "text" && typeof part["text"] === "string") { const { text, didCompress } = await compressMessageText( part["text"] as string, - backend + backend, + opts ); if (didCompress) { changed = true; @@ -248,19 +262,65 @@ async function processMessages( // ─── config schema ──────────────────────────────────────────────────────────── const LLMLINGUA_SCHEMA: EngineConfigField[] = [ + { key: "enabled", type: "boolean", label: "Enabled", defaultValue: true }, { - key: "enabled", - type: "boolean", - label: "Enabled", - defaultValue: true, + key: "model", + type: "select", + label: "Model", + defaultValue: DEFAULT_LLMLINGUA_MODEL, + options: Object.values(LLMLINGUA_MODELS).map((m) => ({ value: m.id, label: m.label })), }, + { + key: "minTokens", + type: "number", + label: "Min tokens (floor)", + defaultValue: 2000, + min: 0, + max: 100000, + }, + { + key: "compressionRate", + type: "number", + label: "Compression rate (keep ratio)", + defaultValue: 0.5, + min: 0.1, + max: 0.9, + }, + { key: "modelPath", type: "string", label: "Model path (offline override)", defaultValue: "" }, ]; function validateLlmlinguaConfig(config: Record): EngineValidationResult { const errors: string[] = []; + if (config["enabled"] !== undefined && typeof config["enabled"] !== "boolean") { errors.push("enabled must be a boolean"); } + + if (config["model"] !== undefined) { + const model = config["model"]; + if (typeof model !== "string" || !(model in LLMLINGUA_MODELS)) { + errors.push("model must be one of: " + Object.keys(LLMLINGUA_MODELS).join(", ")); + } + } + + if (config["minTokens"] !== undefined) { + const minTokens = config["minTokens"]; + if (typeof minTokens !== "number" || Number.isNaN(minTokens) || minTokens < 0) { + errors.push("minTokens must be a number >= 0"); + } + } + + if (config["compressionRate"] !== undefined) { + const rate = config["compressionRate"]; + if (typeof rate !== "number" || Number.isNaN(rate) || rate < 0.1 || rate > 0.9) { + errors.push("compressionRate must be a number between 0.1 and 0.9"); + } + } + + if (config["modelPath"] !== undefined && typeof config["modelPath"] !== "string") { + errors.push("modelPath must be a string"); + } + return { valid: errors.length === 0, errors }; } @@ -275,8 +335,8 @@ export const llmlinguaEngine: CompressionEngine = { "Async semantic token pruning via LLMLingua-2 (ONNX/worker-thread backend). " + "Compresses prose in non-system messages; fenced code blocks and other preserved " + "constructs are never altered. Fail-opens on any backend error. Production backend: " + - "vendored @atjsh/llmlingua-2 (MobileBERT 99 MB) in a worker thread — see " + - "./worker.ts for the L1 follow-up spec (VPS validation required per Hard Rule #18).", + "@atjsh/llmlingua-2 (TinyBERT 57 MB default, BERT-base optional) in a worker thread; " + + "model lazy-downloaded to DATA_DIR. Optional deps — fail-opens if not installed.", icon: "brain", targets: ["messages"], stackable: true, @@ -294,7 +354,10 @@ export const llmlinguaEngine: CompressionEngine = { inputScope: "messages", targetLatencyMs: 200, supportsPreview: false, - stable: false, + // Promoted to stable after VPS validation (2026-06-16): the deployed worker + // compressed real prose (209→107 ch, ok=true), and the bundle's walk-up + // resolution + optional-deps gate were confirmed against the live install. + stable: true, }, /** @@ -334,12 +397,41 @@ export const llmlinguaEngine: CompressionEngine = { return { body, compressed: false, stats: null }; } + // minTokens floor: skip the model entirely on small prompts (avoid paying + // model latency when there is little to gain). 0 disables the floor. + const minTokens = + typeof stepConfig["minTokens"] === "number" ? (stepConfig["minTokens"] as number) : 2000; + if (minTokens > 0) { + const nonSystemText = (messages as MessageLike[]) + .filter((m) => m.role !== "system") + .map((m) => (typeof m.content === "string" ? m.content : JSON.stringify(m.content ?? ""))) + .join("\n"); + if (estimateCompressionTokens(nonSystemText) < minTokens) { + // Below the floor — skip compression for this small prompt. + return { body, compressed: false, stats: null }; + } + } + + // Backend options threaded from stepConfig (model selection / rate / offline override). + const backendOpts: LlmlinguaBackendOptions = { + model: typeof stepConfig["model"] === "string" ? (stepConfig["model"] as string) : undefined, + compressionRate: + typeof stepConfig["compressionRate"] === "number" + ? (stepConfig["compressionRate"] as number) + : undefined, + modelPath: + typeof stepConfig["modelPath"] === "string" && stepConfig["modelPath"] + ? (stepConfig["modelPath"] as string) + : undefined, + }; + try { const backend = resolveBackend(); const start = performance.now(); const { messages: newMessages, compressedCount } = await processMessages( messages as MessageLike[], - backend + backend, + backendOpts ); if (compressedCount === 0) { diff --git a/open-sse/services/compression/engines/llmlingua/modelStore.ts b/open-sse/services/compression/engines/llmlingua/modelStore.ts new file mode 100644 index 0000000000..9f6555cc96 --- /dev/null +++ b/open-sse/services/compression/engines/llmlingua/modelStore.ts @@ -0,0 +1,67 @@ +/** + * LLMLingua model store — thin path/config resolver. + * + * transformers.js owns the actual model download (from the HuggingFace Hub into + * its `cacheDir`). This module only resolves the cache directory, maps config + * model ids to registry entries, and configures a transformers.js `env` object + * for either Hub download (default) or a local modelPath override. + * + * Deliberately does NOT import the native `@huggingface/transformers` dep — it + * accepts a minimal structural `env` so the heavy runtime stays out of this path. + */ + +import os from "node:os"; +import path from "node:path"; +import fs from "node:fs"; +import { + DEFAULT_LLMLINGUA_MODEL, + LLMLINGUA_MODELS, + type LlmlinguaModelEntry, +} from "./constants.ts"; + +/** A minimal structural type for the transformers.js `env` object (avoids importing the native dep here). */ +export interface TransformersEnvLike { + cacheDir?: string; + localModelPath?: string; + allowRemoteModels?: boolean; + [key: string]: unknown; +} + +/** Base data dir. Mirrors rtk's getDataDir() at engines/rtk/filterLoader.ts. */ +function getDataDir(): string { + return process.env.DATA_DIR || path.join(os.homedir(), ".omniroute"); +} + +/** Resolve (and ensure) the model cache dir: `${DATA_DIR}/models/llmlingua`. Mirrors rtk's getDataDir(). */ +export function getLlmlinguaModelCacheDir(): string { + const dir = path.join(getDataDir(), "models", "llmlingua"); + try { + fs.mkdirSync(dir, { recursive: true }); + } catch { + // Ignore mkdir errors — fail-open philosophy: transformers.js will surface a + // clearer error if the dir is genuinely unusable, and callers fail-open anyway. + } + return dir; +} + +/** Resolve a config model id to its registry entry; falls back to the default for unknown/empty ids. */ +export function resolveLlmlinguaModel(modelId: string | undefined | null): LlmlinguaModelEntry { + if (typeof modelId === "string" && modelId.length > 0 && LLMLINGUA_MODELS[modelId]) { + return LLMLINGUA_MODELS[modelId]; + } + return LLMLINGUA_MODELS[DEFAULT_LLMLINGUA_MODEL]; +} + +/** Configure a transformers.js `env` for either Hub download (default) or a local modelPath override. */ +export function configureTransformersEnv( + env: TransformersEnvLike, + opts: { modelPath?: string } +): void { + env.cacheDir = getLlmlinguaModelCacheDir(); + if (typeof opts.modelPath === "string" && opts.modelPath.length > 0) { + env.localModelPath = opts.modelPath; + env.allowRemoteModels = false; + } else { + env.allowRemoteModels = true; + } +} diff --git a/open-sse/services/compression/engines/llmlingua/onnxWorker.ts b/open-sse/services/compression/engines/llmlingua/onnxWorker.ts new file mode 100644 index 0000000000..61dfb69018 --- /dev/null +++ b/open-sse/services/compression/engines/llmlingua/onnxWorker.ts @@ -0,0 +1,117 @@ +/** + * LLMLingua-2 ONNX worker-thread entry point. + * + * Runs inside a `worker_threads.Worker` (spawned by `./worker.ts`). The heavy + * optional deps (`@atjsh/llmlingua-2`, `@huggingface/transformers`, `js-tiktoken`) + * are imported LAZILY/dynamically inside the message handler so this module LOADS + * even when those deps are absent. The only static imports are present-by-default + * modules: `node:worker_threads`, `./constants.ts`, `./modelStore.ts`. + * + * Protocol (request → reply over the worker MessageChannel): + * in : { id, text, model, compressionRate, modelPath } + * out: { id, ok: true, text: } on success + * { id, ok: false, text: } on ANY failure (fail-open) + * + * Fail-open contract: missing deps, model download failure, inference error — all + * resolve to the ORIGINAL text with `ok:false`. The parent treats either reply as + * the value to return, so a failed compression is transparently the original prose. + * + * Code blocks NEVER reach this worker: the engine (index.ts) tombstones preserved + * constructs before calling the backend; this worker sees prose-only segments. + */ + +import { parentPort } from "node:worker_threads"; +import { + resolveLlmlinguaModel, + configureTransformersEnv, + type TransformersEnvLike, +} from "./modelStore.ts"; +import type { LlmlinguaModelEntry } from "./constants.ts"; + +/** + * Dynamic-import indirection. These four deps are OPTIONAL and not installed by + * default, so a static `import(...)` of a literal specifier would make `tsc` fail + * with TS2307. Routing the specifier through a runtime variable keeps the module + * type-checkable while still loading the dep at runtime when present. + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +function dynamicImport(specifier: string): Promise { + return import(/* @vite-ignore */ specifier); +} + +/** Inbound message shape from the parent. */ +interface WorkerRequest { + id: number; + text: string; + model?: string; + compressionRate?: number; + modelPath?: string; +} + +/** + * Cache of built prompt-compressors keyed by `${factory}:${hfRepo}:${modelPath||""}`. + * Values are Promises so concurrent first-calls share one in-flight build; a failed + * build deletes its key so a later call can retry. + * Typed `any` — the heavy lib has no static types here (no-explicit-any is warn-only). + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const compressorCache = new Map>(); + +function cacheKey(entry: LlmlinguaModelEntry, modelPath?: string): string { + return `${entry.factory}:${entry.hfRepo}:${modelPath || ""}`; +} + +/** + * Build (or reuse) the LLMLingua-2 prompt compressor for a model entry. + * All heavy imports are dynamic so this only runs the deps are actually present. + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +async function getCompressor(entry: LlmlinguaModelEntry, modelPath?: string): Promise { + const { env } = await dynamicImport("@huggingface/transformers"); + configureTransformersEnv(env as TransformersEnvLike, { modelPath }); + + const { LLMLingua2 } = await dynamicImport("@atjsh/llmlingua-2"); + const { Tiktoken } = await dynamicImport("js-tiktoken/lite"); + const o200k_base = (await dynamicImport("js-tiktoken/ranks/o200k_base")).default; + const oai = new Tiktoken(o200k_base); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const { promptCompressor } = await (LLMLingua2 as any)[entry.factory](entry.hfRepo, { + transformerJSConfig: { device: "cpu", dtype: entry.dtype }, + oaiTokenizer: oai, + modelSpecificOptions: { subfolder: entry.subfolder }, + // MUST silence — the lib console.logs huge objects otherwise. + logger: () => {}, + }); + + return promptCompressor; +} + +if (parentPort) { + parentPort.on("message", async (msg: WorkerRequest) => { + const { id, text } = msg; + try { + const entry = resolveLlmlinguaModel(msg.model); + const key = cacheKey(entry, msg.modelPath); + + let pending = compressorCache.get(key); + if (!pending) { + pending = getCompressor(entry, msg.modelPath); + compressorCache.set(key, pending); + // If the build rejects, evict the key so a later call can retry. + pending.catch(() => { + compressorCache.delete(key); + }); + } + + const compressor = await pending; + const rate = typeof msg.compressionRate === "number" ? msg.compressionRate : 0.5; + const out: string = await compressor.compress(text, { rate }); + + parentPort!.postMessage({ id, ok: true, text: out }); + } catch { + // Fail-open: ANY error → return the ORIGINAL text with ok:false. + parentPort!.postMessage({ id, ok: false, text }); + } + }); +} diff --git a/open-sse/services/compression/engines/llmlingua/worker.ts b/open-sse/services/compression/engines/llmlingua/worker.ts index 6ad6d87a97..1d5d7e75fe 100644 --- a/open-sse/services/compression/engines/llmlingua/worker.ts +++ b/open-sse/services/compression/engines/llmlingua/worker.ts @@ -1,48 +1,342 @@ /** - * LLMLingua-2 worker-thread stub (production path — NOT loaded in tests). + * LLMLingua-2 worker-thread backend (production path). * - * PRODUCTION FOLLOW-UP (L1): - * Replace the stub body below with the real MobileBERT ONNX inference via - * the vendored/pinned `@atjsh/llmlingua-2` package running in a worker thread. - * The package must be vendored (pinned) at a specific version and ONNX model - * hash verified before loading. The worker MUST remain fail-open — any failure - * to load the model, initialise the pipeline, or classify tokens MUST result - * in returning the original text unchanged (not throwing to the caller). + * Real MobileBERT/BERT ONNX inference via `@atjsh/llmlingua-2` running in a + * `worker_threads.Worker` (`./onnxWorker.ts`). The backend stays STRICTLY + * fail-open: any failure to resolve deps, spawn the worker, load the model, or + * classify tokens returns the ORIGINAL text unchanged — never throws to the caller. * - * VPS validation (Hard Rule #18): before enabling the real model, deploy to - * root@192.168.0.15 and run a documented live test confirming: - * (a) prose text is shorter after compression, - * (b) a message containing a fenced code block produces identical code bytes, - * (c) OOM / missing model → fail-open (original text returned, no crash). + * ## Fail-open paths + * 1. Optional-deps gate: if any of `@atjsh/llmlingua-2`, `@huggingface/transformers`, + * `@tensorflow/tfjs`, `js-tiktoken` does not resolve, return `text` immediately — + * NO worker spawn. This is the default in CI / most installs (deps are OPTIONAL). + * 2. Per-call timeout: first call for a model gets `FIRST_CALL_TIMEOUT_MS` (one-time + * model load); warm calls get `LLMLINGUA_WORKER_TIMEOUT_MS`. On timeout → original + * text (the worker keeps loading and will warm for the next call). + * 3. Worker error/exit → resolve all pending with their original text + respawn next. * - * NEVER apply to code blocks — the caller (index.ts) tombstones code blocks - * before calling this backend; this worker sees prose-only segments. + * ## Serialization + * ONNX/tfjs are not reentrant — calls are queued FIFO and only one message is + * in-flight at a time (the next is posted after the previous reply or its timeout). * - * CURRENT STATE (stub): - * This module exports a LlmlinguaBackend function that always fail-opens - * (returns the original text unchanged), so the engine can be registered and - * used in stacked pipelines without any ONNX dependency. It will produce - * compressed:false for every call, which is the correct safe default until the - * real model is wired up. + * ## Idle eviction + * After `LLMLINGUA_WORKER_IDLE_MS` with no calls, the worker is terminated and the + * singleton reset (next call respawns). The idle timer is `unref`'d so it never keeps + * the process alive. + * + * Code blocks NEVER reach this backend — the engine (index.ts) tombstones preserved + * constructs first; this backend sees prose-only segments. + * + * VPS validation (Hard Rule #18): the real model is exercised behind RUN_LLMLINGUA_INT. */ +import { Worker } from "node:worker_threads"; +import { createRequire } from "node:module"; +import { fileURLToPath } from "node:url"; +import path from "node:path"; +import fs from "node:fs"; + +import { + LLMLINGUA_WORKER_TIMEOUT_MS, + LLMLINGUA_WORKER_IDLE_MS, +} from "./constants.ts"; +import { resolveLlmlinguaModel } from "./modelStore.ts"; import type { LlmlinguaBackend } from "./index.ts"; +/** One-time model-load budget on the first call for a given model (tinybert ~2s, bert-base ~27s). */ +const FIRST_CALL_TIMEOUT_MS = 60000; + /** - * Production worker backend stub. - * - * Currently fail-opens unconditionally. When the real `@atjsh/llmlingua-2` - * package is vendored and validated on the VPS, replace the body below with - * the actual worker-thread dispatch: - * - * ```ts - * // 1. Spawn / reuse a worker_threads.Worker running the ONNX pipeline. - * // 2. Post the text to the worker via MessageChannel. - * // 3. Await the reply with a per-call timeout (e.g. 5 000 ms). - * // 4. On any error / timeout → return text (fail-open). - * ``` + * Gate probe: `@atjsh/llmlingua-2` is the entry package that declares the others + * (`@huggingface/transformers`, `@tensorflow/tfjs`, `js-tiktoken`) as peers. We probe + * ONLY it because the peers are ESM-only — e.g. `@huggingface/transformers@3.5.2`'s + * `exports` has no `require`/`default` condition, so `require.resolve()` throws + * `MODULE_NOT_FOUND` for it even when it is installed and `import()`-able (verified on + * the VPS). Gating on all four would therefore always fail-open. The worker still + * fail-opens if a peer is genuinely missing at `import()` time. */ -export const workerBackend: LlmlinguaBackend = async (text: string): Promise => { - // Stub: model not yet wired — fail-open by returning the original text. - return text; +const GATE_DEP = "@atjsh/llmlingua-2"; + +// ─── optional-deps gate (memoized) ────────────────────────────────────────────── + +let _depsAvailable: boolean | null = null; + +/** Lazily (and once) check whether the optional LLMLingua dependency stack is installed. */ +function depsAvailable(): boolean { + if (_depsAvailable !== null) return _depsAvailable; + try { + createRequire(import.meta.url).resolve(GATE_DEP); + _depsAvailable = true; + } catch { + _depsAvailable = false; + } + return _depsAvailable; +} + +// ─── worker reply / queue plumbing ────────────────────────────────────────────── + +interface WorkerReply { + id: number; + ok: boolean; + text: string; +} + +interface PendingEntry { + resolve: (s: string) => void; + timer: NodeJS.Timeout; + /** Stored so error/exit/reset handlers can fail-open with the ORIGINAL text. */ + originalText: string; + /** Resolved model id — used to mark the model warm ONLY on a genuine success. */ + modelKey: string; +} + +interface QueueItem { + text: string; + opts: Parameters[1]; + resolve: (s: string) => void; +} + +let worker: Worker | null = null; +let nextId = 1; +const pending = new Map(); +const queue: QueueItem[] = []; +let busy = false; +/** Model keys (resolved `id`) that have completed at least one successful load. */ +const warmedModels = new Set(); +let idleTimer: NodeJS.Timeout | null = null; + +/** + * Resolve the worker entry file across dev and prod. + * + * Dev: `onnxWorker.ts` sits next to this file and runs via the tsx loader. + * + * Prod: this module is collapsed into a `.next` chunk and the worker is esbuild'd to + * `/open-sse/services/compression/engines/llmlingua/onnxWorker.js` + * (scripts/build/prepublish.ts) + kept by the pack-artifact allowlist. The process + * `cwd` is NOT the app root (pm2 starts it from `/root`), so cwd-relative resolution + * is unreliable — we instead WALK UP from this module's location (`import.meta.url`, + * which in the standalone bundle is a real `/.next/...` path) until we find + * an ancestor that actually contains the worker at its known relative path. cwd + * candidates remain as a last-resort fallback. First existing candidate wins; a `.ts` + * choice gets the tsx loader, a `.js` choice runs natively. + */ +function resolveWorkerFile(): { workerFile: string; execArgv: string[] } { + const moduleDir = path.dirname(fileURLToPath(import.meta.url)); + const rel = path.join( + "open-sse", + "services", + "compression", + "engines", + "llmlingua", + "onnxWorker.js" + ); + + // 1. Dev: sibling source/compiled file next to this module. + const devTs = path.join(moduleDir, "onnxWorker.ts"); + if (fs.existsSync(devTs)) return { workerFile: devTs, execArgv: ["--import", "tsx/esm"] }; + const devJs = path.join(moduleDir, "onnxWorker.js"); + if (fs.existsSync(devJs)) return { workerFile: devJs, execArgv: [] }; + + // 2. Prod: walk up from the bundled module location, then cwd, looking for the + // esbuild'd worker at /open-sse/.../onnxWorker.js. + const roots: string[] = []; + let dir = moduleDir; + for (let i = 0; i < 12; i++) { + roots.push(dir); + const parent = path.dirname(dir); + if (parent === dir) break; + dir = parent; + } + roots.push(process.cwd(), path.join(process.cwd(), "app")); + for (const root of roots) { + const candidate = path.join(root, rel); + if (fs.existsSync(candidate)) return { workerFile: candidate, execArgv: [] }; + } + + // 3. Nothing found — return the sibling .js path; the spawn will fail-open. + return { workerFile: devJs, execArgv: [] }; +} + +/** Reset the idle eviction timer; terminates the worker after the idle window. */ +function bumpIdleTimer(): void { + if (idleTimer) clearTimeout(idleTimer); + idleTimer = setTimeout(() => { + resetWorker(); + }, LLMLINGUA_WORKER_IDLE_MS); + // Never keep the process alive just for idle eviction. + if (typeof idleTimer.unref === "function") idleTimer.unref(); +} + +/** Tear down the worker + all runtime state (fail-open any pending). Next call respawns. */ +function resetWorker(): void { + const w = worker; + worker = null; + if (idleTimer) { + clearTimeout(idleTimer); + idleTimer = null; + } + // Fail-open every in-flight call with its ORIGINAL text. + for (const [, entry] of pending) { + clearTimeout(entry.timer); + entry.resolve(entry.originalText); + } + pending.clear(); + busy = false; + warmedModels.clear(); + if (w) { + try { + void w.terminate(); + } catch { + // ignore terminate errors + } + } +} + +/** Spawn the singleton worker and wire its message/error/exit handlers. */ +function ensureWorker(): Worker { + if (worker) return worker; + + const { workerFile, execArgv } = resolveWorkerFile(); + const w = new Worker(workerFile, { execArgv }); + + w.on("message", (reply: WorkerReply) => { + const entry = pending.get(reply.id); + if (entry) { + clearTimeout(entry.timer); + pending.delete(reply.id); + // Only a genuine success warms the model so later calls use the short timeout. + // A timeout/error/fail-open MUST NOT warm it (else a still-loading model would + // be starved of its one-time load budget on the next call). + if (reply.ok) warmedModels.add(entry.modelKey); + // ok:false already carries the ORIGINAL text → resolving with it IS fail-open. + entry.resolve(reply.text); + } + busy = false; + pump(); + }); + + const failOpenAndRespawn = () => { + // Resolve every pending entry fail-open, then drop the worker so the next call respawns. + failAllPending(); + if (worker === w) worker = null; + busy = false; + }; + + w.on("error", failOpenAndRespawn); + w.on("exit", failOpenAndRespawn); + + worker = w; + return w; +} + +/** Resolve all pending entries with their stored fail-open value (original text). */ +function failAllPending(): void { + for (const [id, entry] of pending) { + clearTimeout(entry.timer); + pending.delete(id); + entry.resolve(entry.originalText); + } +} + +/** Post the next queued item to the worker (one in-flight at a time). */ +function pump(): void { + if (busy) return; + const item = queue.shift(); + if (!item) return; + + busy = true; + bumpIdleTimer(); + + let w: Worker; + try { + w = ensureWorker(); + } catch { + // Spawn failed → fail-open this item and continue draining the queue. + busy = false; + item.resolve(item.text); + pump(); + return; + } + + const id = nextId++; + const modelKey = resolveLlmlinguaModel(item.opts?.model).id; + const warm = warmedModels.has(modelKey); + const timeoutMs = warm ? LLMLINGUA_WORKER_TIMEOUT_MS : FIRST_CALL_TIMEOUT_MS; + + const timer = setTimeout(() => { + // Timeout → fail-open with the ORIGINAL text; drop the pending entry but keep the + // worker (it may still be loading the model and will warm for the next call). + const entry = pending.get(id); + if (entry) { + pending.delete(id); + entry.resolve(item.text); + } + busy = false; + pump(); + }, timeoutMs); + if (typeof timer.unref === "function") timer.unref(); + + pending.set(id, { + // Warming is decided by the reply handler (success only) — not here, so a + // timeout/error fail-open never marks the model warm. + resolve: item.resolve, + timer, + originalText: item.text, + modelKey, + }); + + try { + w.postMessage({ + id, + text: item.text, + model: item.opts?.model, + compressionRate: item.opts?.compressionRate, + modelPath: item.opts?.modelPath, + }); + } catch { + // postMessage failed → fail-open this item and respawn. + clearTimeout(timer); + pending.delete(id); + item.resolve(item.text); + if (worker === w) worker = null; + busy = false; + pump(); + } +} + +// ─── public backend ───────────────────────────────────────────────────────────── + +/** + * Production worker backend. Two-arg `LlmlinguaBackend` (text, opts). + * + * Returns the compressed prose on success; the ORIGINAL `text` on any failure + * (missing deps, spawn error, model-load/inference error, timeout). NEVER throws. + */ +export const workerBackend: LlmlinguaBackend = async (text, opts) => { + // Fail-open WITHOUT spawning when the optional deps are not installed (the common case). + if (!depsAvailable()) { + return text; + } + + return new Promise((resolve) => { + queue.push({ text, opts, resolve }); + pump(); + }); }; + +// ─── test-only reset ──────────────────────────────────────────────────────────── + +/** + * Internal: terminate the worker (if any) and reset all module state so the + * process can exit cleanly after tests. Not part of the public contract. + */ +export function __resetLlmlinguaWorkerForTests(): void { + // Drain the queue fail-open so no callers hang. + while (queue.length) { + const item = queue.shift()!; + item.resolve(item.text); + } + resetWorker(); + _depsAvailable = null; + nextId = 1; +} diff --git a/package-lock.json b/package-lock.json index 484d614879..d008a51ed4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@dnd-kit/core": "^6.3.1", "@dnd-kit/sortable": "^10.0.0", "@dnd-kit/utilities": "^3.2.2", - "@huggingface/transformers": "^4.2.0", + "@huggingface/transformers": "3.5.2", "@lobehub/icons": "^5.8.0", "@modelcontextprotocol/sdk": "^1.29.0", "@monaco-editor/react": "^4.7.0", @@ -99,7 +99,7 @@ "@testing-library/react": "^16.3.2", "@types/bcryptjs": "^3.0.0", "@types/better-sqlite3": "^7.6.13", - "@types/bun": "*", + "@types/bun": "latest", "@types/keytar": "^4.4.2", "@types/node": "^25.9.1", "@types/react": "^19.2.15", @@ -137,7 +137,10 @@ "node": ">=22.0.0 <23 || >=24.0.0 <27" }, "optionalDependencies": { + "@atjsh/llmlingua-2": "2.0.3", + "@tensorflow/tfjs": "4.22.0", "better-sqlite3": "^12.10.0", + "js-tiktoken": "^1.0.20", "keytar": "^7.9.0", "tls-client-node": "^0.2.0", "wreq-js": "^2.3.1" @@ -271,6 +274,21 @@ "dev": true, "license": "MIT" }, + "node_modules/@atjsh/llmlingua-2": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@atjsh/llmlingua-2/-/llmlingua-2-2.0.3.tgz", + "integrity": "sha512-UJJFMbzYldkZ4qX5CrSZtmytOnXf6aXhmr1sBhbpVMHdmQG+7GCnrx5rIwPSOmozXD9KiPv5nnV6pvzxdtHdYQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "es-toolkit": "^1.38.0" + }, + "peerDependencies": { + "@huggingface/transformers": "*", + "@tensorflow/tfjs": "*", + "js-tiktoken": "*" + } + }, "node_modules/@aws-crypto/crc32": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/@aws-crypto/crc32/-/crc32-5.2.0.tgz", @@ -2516,31 +2534,24 @@ } }, "node_modules/@huggingface/jinja": { - "version": "0.5.9", - "resolved": "https://registry.npmjs.org/@huggingface/jinja/-/jinja-0.5.9.tgz", - "integrity": "sha512-uWTG+l3VJRsl7EXxYizuL3P+cCPoc3cRqbWWRcQN0FhejRfbdq0RNhCmbY/YDtnTcz9icdLYuLDjsnz4d8JMuw==", + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@huggingface/jinja/-/jinja-0.4.1.tgz", + "integrity": "sha512-3WXbMFaPkk03LRCM0z0sylmn8ddDm4ubjU7X+Hg4M2GOuMklwoGAFXp9V2keq7vltoB/c7McE5aHUVVddAewsw==", "license": "MIT", "engines": { "node": ">=18" } }, - "node_modules/@huggingface/tokenizers": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/@huggingface/tokenizers/-/tokenizers-0.1.3.tgz", - "integrity": "sha512-8rF/RRT10u+kn7YuUbUg0OF30K8rjTc78aHpxT+qJ1uWSqxT1MHi8+9ltwYfkFYJzT/oS+qw3JVfHtNMGAdqyA==", - "license": "Apache-2.0" - }, "node_modules/@huggingface/transformers": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/@huggingface/transformers/-/transformers-4.2.0.tgz", - "integrity": "sha512-8BRCoBMH0XsWaEIamuR0LrJGAfftgHAfb2Vrffy0VKlSAE/MnUJ5/h/zTfEP3fDIft+nk7TqB8xXEyABGitBjQ==", + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/@huggingface/transformers/-/transformers-3.5.2.tgz", + "integrity": "sha512-mfRXkmcL99+ibpjM++pvZmc2h3po8i1ZgSRI5Rtgh++P15GU0lY8UQteYt/w5V+GQw+Jpao93MoipcePzh3mKg==", "license": "Apache-2.0", "dependencies": { - "@huggingface/jinja": "^0.5.6", - "@huggingface/tokenizers": "^0.1.3", - "onnxruntime-node": "1.24.3", - "onnxruntime-web": "1.26.0-dev.20260416-b7804b056c", - "sharp": "^0.34.5" + "@huggingface/jinja": "^0.4.1", + "onnxruntime-node": "1.21.0", + "onnxruntime-web": "1.22.0-dev.20250409-89f8206ba4", + "sharp": "^0.34.1" } }, "node_modules/@humanfs/core": { @@ -3510,7 +3521,6 @@ "version": "4.0.1", "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", - "dev": true, "license": "ISC", "dependencies": { "minipass": "^7.0.4" @@ -6109,12 +6119,6 @@ "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", "license": "BSD-3-Clause" }, - "node_modules/@protobufjs/inquire": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.2.tgz", - "integrity": "sha512-pa0vFRuws4wkvaXKK1uXZMAwAX4/t8ANaJo45iw/oQHNQ9q5xUzwgFmVJGXiga2BeN+zpX7Vf9vmsiIa2J+MUw==", - "license": "BSD-3-Clause" - }, "node_modules/@protobufjs/path": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", @@ -8402,6 +8406,241 @@ "tailwindcss": "4.3.0" } }, + "node_modules/@tensorflow/tfjs": { + "version": "4.22.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs/-/tfjs-4.22.0.tgz", + "integrity": "sha512-0TrIrXs6/b7FLhLVNmfh8Sah6JgjBPH4mZ8JGb7NU6WW+cx00qK5BcAZxw7NCzxj6N8MRAIfHq+oNbPUNG5VAg==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@tensorflow/tfjs-backend-cpu": "4.22.0", + "@tensorflow/tfjs-backend-webgl": "4.22.0", + "@tensorflow/tfjs-converter": "4.22.0", + "@tensorflow/tfjs-core": "4.22.0", + "@tensorflow/tfjs-data": "4.22.0", + "@tensorflow/tfjs-layers": "4.22.0", + "argparse": "^1.0.10", + "chalk": "^4.1.0", + "core-js": "3.29.1", + "regenerator-runtime": "^0.13.5", + "yargs": "^16.0.3" + }, + "bin": { + "tfjs-custom-module": "dist/tools/custom_module/cli.js" + } + }, + "node_modules/@tensorflow/tfjs-backend-cpu": { + "version": "4.22.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-cpu/-/tfjs-backend-cpu-4.22.0.tgz", + "integrity": "sha512-1u0FmuLGuRAi8D2c3cocHTASGXOmHc/4OvoVDENJayjYkS119fcTcQf4iHrtLthWyDIPy3JiPhRrZQC9EwnhLw==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@types/seedrandom": "^2.4.28", + "seedrandom": "^3.0.5" + }, + "engines": { + "yarn": ">= 1.3.2" + }, + "peerDependencies": { + "@tensorflow/tfjs-core": "4.22.0" + } + }, + "node_modules/@tensorflow/tfjs-backend-webgl": { + "version": "4.22.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-webgl/-/tfjs-backend-webgl-4.22.0.tgz", + "integrity": "sha512-H535XtZWnWgNwSzv538czjVlbJebDl5QTMOth4RXr2p/kJ1qSIXE0vZvEtO+5EC9b00SvhplECny2yDewQb/Yg==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@tensorflow/tfjs-backend-cpu": "4.22.0", + "@types/offscreencanvas": "~2019.3.0", + "@types/seedrandom": "^2.4.28", + "seedrandom": "^3.0.5" + }, + "engines": { + "yarn": ">= 1.3.2" + }, + "peerDependencies": { + "@tensorflow/tfjs-core": "4.22.0" + } + }, + "node_modules/@tensorflow/tfjs-converter": { + "version": "4.22.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-converter/-/tfjs-converter-4.22.0.tgz", + "integrity": "sha512-PT43MGlnzIo+YfbsjM79Lxk9lOq6uUwZuCc8rrp0hfpLjF6Jv8jS84u2jFb+WpUeuF4K33ZDNx8CjiYrGQ2trQ==", + "license": "Apache-2.0", + "optional": true, + "peerDependencies": { + "@tensorflow/tfjs-core": "4.22.0" + } + }, + "node_modules/@tensorflow/tfjs-core": { + "version": "4.22.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-4.22.0.tgz", + "integrity": "sha512-LEkOyzbknKFoWUwfkr59vSB68DMJ4cjwwHgicXN0DUi3a0Vh1Er3JQqCI1Hl86GGZQvY8ezVrtDIvqR1ZFW55A==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@types/long": "^4.0.1", + "@types/offscreencanvas": "~2019.7.0", + "@types/seedrandom": "^2.4.28", + "@webgpu/types": "0.1.38", + "long": "4.0.0", + "node-fetch": "~2.6.1", + "seedrandom": "^3.0.5" + }, + "engines": { + "yarn": ">= 1.3.2" + } + }, + "node_modules/@tensorflow/tfjs-core/node_modules/@types/offscreencanvas": { + "version": "2019.7.3", + "resolved": "https://registry.npmjs.org/@types/offscreencanvas/-/offscreencanvas-2019.7.3.tgz", + "integrity": "sha512-ieXiYmgSRXUDeOntE1InxjWyvEelZGP63M+cGuquuRLuIKKT1osnkXjxev9B7d1nXSug5vpunx+gNlbVxMlC9A==", + "license": "MIT", + "optional": true + }, + "node_modules/@tensorflow/tfjs-data": { + "version": "4.22.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-data/-/tfjs-data-4.22.0.tgz", + "integrity": "sha512-dYmF3LihQIGvtgJrt382hSRH4S0QuAp2w1hXJI2+kOaEqo5HnUPG0k5KA6va+S1yUhx7UBToUKCBHeLHFQRV4w==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@types/node-fetch": "^2.1.2", + "node-fetch": "~2.6.1", + "string_decoder": "^1.3.0" + }, + "peerDependencies": { + "@tensorflow/tfjs-core": "4.22.0", + "seedrandom": "^3.0.5" + } + }, + "node_modules/@tensorflow/tfjs-layers": { + "version": "4.22.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-layers/-/tfjs-layers-4.22.0.tgz", + "integrity": "sha512-lybPj4ZNj9iIAPUj7a8ZW1hg8KQGfqWLlCZDi9eM/oNKCCAgchiyzx8OrYoWmRrB+AM6VNEeIT+2gZKg5ReihA==", + "license": "Apache-2.0 AND MIT", + "optional": true, + "peerDependencies": { + "@tensorflow/tfjs-core": "4.22.0" + } + }, + "node_modules/@tensorflow/tfjs/node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "license": "MIT", + "optional": true, + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/@tensorflow/tfjs/node_modules/cliui": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", + "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", + "license": "ISC", + "optional": true, + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^7.0.0" + } + }, + "node_modules/@tensorflow/tfjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT", + "optional": true + }, + "node_modules/@tensorflow/tfjs/node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/@tensorflow/tfjs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "optional": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@tensorflow/tfjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "optional": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@tensorflow/tfjs/node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "optional": true, + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/@tensorflow/tfjs/node_modules/yargs": { + "version": "16.2.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", + "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", + "license": "MIT", + "optional": true, + "dependencies": { + "cliui": "^7.0.2", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.0", + "y18n": "^5.0.5", + "yargs-parser": "^20.2.2" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@tensorflow/tfjs/node_modules/yargs-parser": { + "version": "20.2.9", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", + "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", + "license": "ISC", + "optional": true, + "engines": { + "node": ">=10" + } + }, "node_modules/@testing-library/jest-dom": { "version": "6.9.1", "resolved": "https://registry.npmjs.org/@testing-library/jest-dom/-/jest-dom-6.9.1.tgz", @@ -8904,6 +9143,13 @@ "keytar": "*" } }, + "node_modules/@types/long": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz", + "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==", + "license": "MIT", + "optional": true + }, "node_modules/@types/mdast": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-4.0.4.tgz", @@ -8934,6 +9180,24 @@ "undici-types": ">=7.24.0 <7.24.7" } }, + "node_modules/@types/node-fetch": { + "version": "2.6.13", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.13.tgz", + "integrity": "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==", + "license": "MIT", + "optional": true, + "dependencies": { + "@types/node": "*", + "form-data": "^4.0.4" + } + }, + "node_modules/@types/offscreencanvas": { + "version": "2019.3.0", + "resolved": "https://registry.npmjs.org/@types/offscreencanvas/-/offscreencanvas-2019.3.0.tgz", + "integrity": "sha512-esIJx9bQg+QYF0ra8GnvfianIY8qWB0GBx54PK5Eps6m+xTj86KLavHv6qDhzKcu5UUOgNfJ2pWaIIV7TRUd9Q==", + "license": "MIT", + "optional": true + }, "node_modules/@types/parse-json": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.2.tgz", @@ -8967,6 +9231,13 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/seedrandom": { + "version": "2.4.34", + "resolved": "https://registry.npmjs.org/@types/seedrandom/-/seedrandom-2.4.34.tgz", + "integrity": "sha512-ytDiArvrn/3Xk6/vtylys5tlY6eo7Ane0hvcx++TKo6RxQXuVfW0AF/oeWqAj9dN29SyhtawuXstgmPlwNcv/A==", + "license": "MIT", + "optional": true + }, "node_modules/@types/trusted-types": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", @@ -9722,6 +9993,13 @@ "dev": true, "license": "MIT" }, + "node_modules/@webgpu/types": { + "version": "0.1.38", + "resolved": "https://registry.npmjs.org/@webgpu/types/-/types-0.1.38.tgz", + "integrity": "sha512-7LrhVKz2PRh+DD7+S+PVaFd5HxaWQvoMqBbsV9fNJO1pjUs1P8bM2vQVNfk+3URTqbuTI7gkXi0rfsN0IadoBA==", + "license": "BSD-3-Clause", + "optional": true + }, "node_modules/@xyflow/react": { "version": "12.11.0", "resolved": "https://registry.npmjs.org/@xyflow/react/-/react-12.11.0.tgz", @@ -9874,15 +10152,6 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, - "node_modules/adm-zip": { - "version": "0.5.17", - "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.17.tgz", - "integrity": "sha512-+Ut8d9LLqwEvHHJl1+PIHqoyDxFgVN847JTVM3Izi3xHDWPE4UtzzXysMZQs64DMcrJfBeS/uoEP4AD3HQHnQQ==", - "license": "MIT", - "engines": { - "node": ">=12.0" - } - }, "node_modules/agent-base": { "version": "9.0.0", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-9.0.0.tgz", @@ -11919,6 +12188,18 @@ "node": ">=6.6.0" } }, + "node_modules/core-js": { + "version": "3.29.1", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.29.1.tgz", + "integrity": "sha512-+jwgnhg6cQxKYIIjGtAHq2nwUOolo9eoFZ4sHfUH09BLXBgxnH4gA0zEd+t+BO2cNB8idaBtZFcFTRjQJRJmAw==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/core-js" + } + }, "node_modules/cors": { "version": "2.8.6", "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.6.tgz", @@ -15518,9 +15799,9 @@ } }, "node_modules/global-agent/node_modules/semver": { - "version": "7.8.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.1.tgz", - "integrity": "sha512-rkVq3IXh+4FDGch+KwzX3aV9W3kO54GyEgpvBzSyctDA6Xtd7RJQV1xmXbeQp5v7+VzLOfVqiutSE6GICgPFvg==", + "version": "7.8.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.4.tgz", + "integrity": "sha512-rUCObTnP32Q08R2uuIrt7r9PlEonuTmtuXYcW6s5kjdlj3xbnwe+21yXptAUYcMAABLkYYTtnmzb3w3EDZfueA==", "license": "ISC", "bin": { "semver": "bin/semver.js" @@ -17573,6 +17854,16 @@ "dev": true, "license": "MIT" }, + "node_modules/js-tiktoken": { + "version": "1.0.21", + "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.21.tgz", + "integrity": "sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g==", + "license": "MIT", + "optional": true, + "dependencies": { + "base64-js": "^1.5.1" + } + }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -19054,10 +19345,11 @@ } }, "node_modules/long": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", - "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", - "license": "Apache-2.0" + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", + "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==", + "license": "Apache-2.0", + "optional": true }, "node_modules/longest-streak": { "version": "3.1.0", @@ -20514,7 +20806,6 @@ "version": "7.1.3", "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", - "dev": true, "license": "BlueOak-1.0.0", "engines": { "node": ">=16 || 14 >=14.17" @@ -20634,7 +20925,6 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz", "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==", - "dev": true, "license": "MIT", "dependencies": { "minipass": "^7.1.2" @@ -21093,6 +21383,52 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/node-fetch": { + "version": "2.6.13", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.13.tgz", + "integrity": "sha512-StxNAxh15zr77QvvkmveSQ8uCQ4+v5FkvNTj0OESmiHu+VRi/gXArXtkWMElOsOUNLtUEvI4yS+rdtOHZTwlQA==", + "license": "MIT", + "optional": true, + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/node-fetch/node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT", + "optional": true + }, + "node_modules/node-fetch/node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause", + "optional": true + }, + "node_modules/node-fetch/node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "optional": true, + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/node-gyp": { "version": "12.4.0", "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-12.4.0.tgz", @@ -21652,15 +21988,15 @@ } }, "node_modules/onnxruntime-common": { - "version": "1.24.3", - "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.24.3.tgz", - "integrity": "sha512-GeuPZO6U/LBJXvwdaqHbuUmoXiEdeCjWi/EG7Y1HNnDwJYuk6WUbNXpF6luSUY8yASul3cmUlLGrCCL1ZgVXqA==", + "version": "1.21.0", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.21.0.tgz", + "integrity": "sha512-Q632iLLrtCAVOTO65dh2+mNbQir/QNTVBG3h/QdZBpns7mZ0RYbLRBgGABPbpU9351AgYy7SJf1WaeVwMrBFPQ==", "license": "MIT" }, "node_modules/onnxruntime-node": { - "version": "1.24.3", - "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.24.3.tgz", - "integrity": "sha512-JH7+czbc8ALA819vlTgcV+Q214/+VjGeBHDjX81+ZCD0PCVCIFGFNtT0V4sXG/1JXypKPgScQcB3ij/hk3YnTg==", + "version": "1.21.0", + "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.21.0.tgz", + "integrity": "sha512-NeaCX6WW2L8cRCSqy3bInlo5ojjQqu2fD3D+9W5qb5irwxhEyWKXeH2vZ8W9r6VxaMPUan+4/7NDwZMtouZxEw==", "hasInstallScript": true, "license": "MIT", "os": [ @@ -21669,29 +22005,35 @@ "linux" ], "dependencies": { - "adm-zip": "^0.5.16", "global-agent": "^3.0.0", - "onnxruntime-common": "1.24.3" + "onnxruntime-common": "1.21.0", + "tar": "^7.0.1" } }, "node_modules/onnxruntime-web": { - "version": "1.26.0-dev.20260416-b7804b056c", - "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.26.0-dev.20260416-b7804b056c.tgz", - "integrity": "sha512-MD6Ss4GSpQBo6zqoJzyT9LRbKYs7x/JVN23FT24EcEvlqF4VuzPOeH6X38orZPKHQDbprn7K+SBpu0/mj2CQiw==", + "version": "1.22.0-dev.20250409-89f8206ba4", + "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.22.0-dev.20250409-89f8206ba4.tgz", + "integrity": "sha512-0uS76OPgH0hWCPrFKlL8kYVV7ckM7t/36HfbgoFw6Nd0CZVVbQC4PkrR8mBX8LtNUFZO25IQBqV2Hx2ho3FlbQ==", "license": "MIT", "dependencies": { "flatbuffers": "^25.1.24", "guid-typescript": "^1.0.9", "long": "^5.2.3", - "onnxruntime-common": "1.24.0-dev.20251116-b39e144322", + "onnxruntime-common": "1.22.0-dev.20250409-89f8206ba4", "platform": "^1.3.6", "protobufjs": "^7.2.4" } }, + "node_modules/onnxruntime-web/node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, "node_modules/onnxruntime-web/node_modules/onnxruntime-common": { - "version": "1.24.0-dev.20251116-b39e144322", - "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.24.0-dev.20251116-b39e144322.tgz", - "integrity": "sha512-BOoomdHYmNRL5r4iQ4bMvsl2t0/hzVQ3OM3PHD0gxeXu1PmggqBv3puZicEUVOA3AtHHYmqZtjMj9FOfGrATTw==", + "version": "1.22.0-dev.20250409-89f8206ba4", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.22.0-dev.20250409-89f8206ba4.tgz", + "integrity": "sha512-vDJMkfCfb0b1A836rgHj+ORuZf4B4+cc2bASQtpeoJLueuFc5DuYwjIZUBrSvx/fO5IrLjLz+oTrB3pcGlhovQ==", "license": "MIT" }, "node_modules/open": { @@ -22688,9 +23030,9 @@ "license": "ISC" }, "node_modules/protobufjs": { - "version": "7.6.1", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.1.tgz", - "integrity": "sha512-4K0myLaWL5EteuSAro91EGFgcfVgxb64Jx+7oDAY6GOkXD4M69yuSEljNcInGVCA5sOPxmZ/EqDLj2x0Q0+Ygg==", + "version": "7.6.4", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.4.tgz", + "integrity": "sha512-RJJPTTpvFfHcWLkIa2JFWK4XvtSzS0yEWDmunqHXli1h3JlkbcQZXDZdcWxv+JK3Xsl5/UFDPZ0iGm7DAengYw==", "hasInstallScript": true, "license": "BSD-3-Clause", "dependencies": { @@ -22700,7 +23042,6 @@ "@protobufjs/eventemitter": "^1.1.1", "@protobufjs/fetch": "^1.1.1", "@protobufjs/float": "^1.0.2", - "@protobufjs/inquire": "^1.1.2", "@protobufjs/path": "^1.1.2", "@protobufjs/pool": "^1.1.0", "@protobufjs/utf8": "^1.1.1", @@ -22711,6 +23052,12 @@ "node": ">=12.0.0" } }, + "node_modules/protobufjs/node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -23476,6 +23823,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/regenerator-runtime": { + "version": "0.13.11", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz", + "integrity": "sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==", + "license": "MIT", + "optional": true + }, "node_modules/regex": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/regex/-/regex-6.1.0.tgz", @@ -24134,6 +24488,13 @@ ], "license": "BSD-3-Clause" }, + "node_modules/seedrandom": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz", + "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==", + "license": "MIT", + "optional": true + }, "node_modules/selfsigned": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-5.5.0.tgz", @@ -25583,7 +25944,6 @@ "version": "7.5.16", "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.16.tgz", "integrity": "sha512-56adEpPMouktRlBLXiaYFFzZ/3+JXa8P9n7WbR+ibIjtviN55mEaOkiysCnPnWm+7kkui1Dn8J9l+g6zV8731w==", - "dev": true, "license": "BlueOak-1.0.0", "dependencies": { "@isaacs/fs-minipass": "^4.0.0", @@ -25630,7 +25990,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", - "dev": true, "license": "BlueOak-1.0.0", "engines": { "node": ">=18" @@ -25640,7 +25999,6 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", - "dev": true, "license": "BlueOak-1.0.0", "engines": { "node": ">=18" diff --git a/package.json b/package.json index 785db2836f..5968a921bf 100644 --- a/package.json +++ b/package.json @@ -190,7 +190,7 @@ "@dnd-kit/core": "^6.3.1", "@dnd-kit/sortable": "^10.0.0", "@dnd-kit/utilities": "^3.2.2", - "@huggingface/transformers": "^4.2.0", + "@huggingface/transformers": "3.5.2", "@lobehub/icons": "^5.8.0", "@modelcontextprotocol/sdk": "^1.29.0", "@monaco-editor/react": "^4.7.0", @@ -258,7 +258,10 @@ "zustand": "^5.0.13" }, "optionalDependencies": { + "@atjsh/llmlingua-2": "2.0.3", + "@tensorflow/tfjs": "4.22.0", "better-sqlite3": "^12.10.0", + "js-tiktoken": "^1.0.20", "keytar": "^7.9.0", "tls-client-node": "^0.2.0", "wreq-js": "^2.3.1" diff --git a/scripts/build/pack-artifact-policy.ts b/scripts/build/pack-artifact-policy.ts index 3e4ed17db3..499f91d657 100644 --- a/scripts/build/pack-artifact-policy.ts +++ b/scripts/build/pack-artifact-policy.ts @@ -36,6 +36,9 @@ export const APP_STAGING_ALLOWED_EXACT_PATHS: string[] = [ "BUILD_SHA", "docs/reference/openapi.yaml", "open-sse/mcp-server/server.js", + // LLMLingua ONNX worker — esbuild'd standalone .js spawned via worker_threads + // (the Next.js bundler can't trace the computed Worker path). Kept like the MCP server. + "open-sse/services/compression/engines/llmlingua/onnxWorker.js", "package.json", "peer-stamp.mjs", "responses-ws-proxy.mjs", diff --git a/scripts/build/prepublish.ts b/scripts/build/prepublish.ts index afcdca2be3..729a88a0f5 100644 --- a/scripts/build/prepublish.ts +++ b/scripts/build/prepublish.ts @@ -239,6 +239,54 @@ if (existsSync(mcpSrcFile)) { } } +// ── Step 8.6: Bundle LLMLingua ONNX worker ──────────────────────────── +// The worker is spawned via worker_threads at a path the Next.js bundler cannot +// statically trace, so it must ship as a standalone .js (mirrors the MCP-server +// bundling above). Heavy deps (@atjsh/llmlingua-2 / @huggingface/transformers / +// @tensorflow/tfjs / js-tiktoken) stay EXTERNAL — they are optionalDependencies, +// dynamically imported at runtime, and the worker fail-opens if any is absent. +const llmWorkerSrc = join( + ROOT, + "open-sse", + "services", + "compression", + "engines", + "llmlingua", + "onnxWorker.ts" +); +const llmWorkerDestDir = join( + DIST_DIR, + "open-sse", + "services", + "compression", + "engines", + "llmlingua" +); +if (existsSync(llmWorkerSrc)) { + console.log(" 🔨 Bundling LLMLingua ONNX worker (TypeScript → JavaScript)..."); + mkdirSync(llmWorkerDestDir, { recursive: true }); + try { + execFileSync( + NPX_BIN, + [ + "esbuild", + "open-sse/services/compression/engines/llmlingua/onnxWorker.ts", + "--bundle", + "--platform=node", + "--packages=external", + "--format=esm", + "--outfile=dist/open-sse/services/compression/engines/llmlingua/onnxWorker.js", + ], + { cwd: ROOT, stdio: "inherit" } + ); + console.log( + " ✅ LLMLingua worker bundled to dist/open-sse/services/compression/engines/llmlingua/onnxWorker.js" + ); + } catch (err: any) { + console.warn(" ⚠️ LLMLingua worker bundle error:", err.message); + } +} + // ── Step 8.7: Bundle CLI Entrypoint ────────────────────────── const cliSrcFile = join(ROOT, "bin", "omniroute.ts"); const cliDestFile = join(ROOT, "bin", "omniroute.mjs"); diff --git a/tests/unit/compression/llmlingua-failopen.test.ts b/tests/unit/compression/llmlingua-failopen.test.ts index 614f6baca4..9c56752649 100644 --- a/tests/unit/compression/llmlingua-failopen.test.ts +++ b/tests/unit/compression/llmlingua-failopen.test.ts @@ -17,8 +17,13 @@ import assert from "node:assert/strict"; import { llmlinguaEngine, setLlmlinguaBackend, + type LlmlinguaBackendOptions, } from "../../../open-sse/services/compression/engines/llmlingua/index.ts"; +// A large prose blob comfortably above the default 2000-token floor +// (estimateCompressionTokens ≈ length/4). ~12 k chars ⇒ ~3 k tokens. +const LARGE_PROSE = "The quick brown fox jumps over the lazy dog. ".repeat(280); + // ─── helpers ───────────────────────────────────────────────────────────────── function makeBody(messages: Array<{ role: string; content: string }>): Record { @@ -74,7 +79,9 @@ describe("llmlingua engine", () => { "This is a sufficiently long prose paragraph to ensure compression is triggered."; const body = makeBody([{ role: "user", content: originalContent }]); - const result = await llmlinguaEngine.applyAsync!(body); + // minTokens:0 disables the small-prompt floor so the short fixture still + // exercises the backend (real behavior added by the minTokens floor — Task 4c). + const result = await llmlinguaEngine.applyAsync!(body, { stepConfig: { minTokens: 0 } }); assert.equal(result.compressed, true, "should be marked compressed"); assert.notEqual(result.stats, null, "stats should be present"); @@ -99,7 +106,7 @@ describe("llmlingua engine", () => { let result: Awaited>; try { - result = await llmlinguaEngine.applyAsync!(body); + result = await llmlinguaEngine.applyAsync!(body, { stepConfig: { minTokens: 0 } }); } catch (err) { assert.fail(`applyAsync must not throw on backend error, but threw: ${err}`); } @@ -123,7 +130,7 @@ describe("llmlingua engine", () => { const content = `${prose}\n\n${codeBlock}\n\nMore prose follows the code block here.`; const body = makeBody([{ role: "user", content }]); - const result = await llmlinguaEngine.applyAsync!(body); + const result = await llmlinguaEngine.applyAsync!(body, { stepConfig: { minTokens: 0 } }); // Code block text must be byte-identical in the output const outContent = (result.body.messages as Array<{ role: string; content: string }>)[0]! @@ -159,7 +166,7 @@ describe("llmlingua engine", () => { { role: "user", content: userContent }, ]); - const result = await llmlinguaEngine.applyAsync!(body); + const result = await llmlinguaEngine.applyAsync!(body, { stepConfig: { minTokens: 0 } }); const outMessages = result.body.messages as Array<{ role: string; content: string }>; const outSystem = outMessages.find((m) => m.role === "system")!; @@ -175,3 +182,131 @@ describe("llmlingua engine", () => { } }); }); + +// ─── Task 3/4: minTokens floor, opts threading, config schema/validation ─────── + +describe("llmlingua engine — minTokens floor + config schema (Task 3/4)", () => { + // ── 1. floor skips small input (no stepConfig → default floor 2000) ────────── + it("minTokens floor skips small input — backend never called, compressed:false", async () => { + const calls: string[] = []; + setLlmlinguaBackend((text) => { + calls.push(text); + return Promise.resolve("X"); + }); + + const body = makeBody([{ role: "user", content: "Short prose, well below the floor." }]); + + const result = await llmlinguaEngine.applyAsync!(body); + + assert.equal(result.compressed, false, "small input must skip compression"); + assert.equal(result.stats, null, "stats must be null when skipped by the floor"); + assert.equal(calls.length, 0, "backend must NOT be called below the floor"); + }); + + // ── 2. floor disabled (minTokens:0) → passes through to backend ────────────── + it("minTokens:0 disables the floor — backend IS called, compressed:true", async () => { + const calls: string[] = []; + setLlmlinguaBackend((text) => { + calls.push(text); + return Promise.resolve("X"); + }); + + const body = makeBody([{ role: "user", content: "Short prose, well below the floor." }]); + + const result = await llmlinguaEngine.applyAsync!(body, { stepConfig: { minTokens: 0 } }); + + assert.equal(result.compressed, true, "floor disabled → backend compresses"); + assert.ok(calls.length > 0, "backend must be called when the floor is disabled"); + }); + + // ── 3. opts threading: model + compressionRate reach the backend ───────────── + it("threads model + compressionRate from stepConfig down to the backend opts", async () => { + let capturedOpts: LlmlinguaBackendOptions | undefined; + setLlmlinguaBackend((_text, opts) => { + capturedOpts = opts; + return Promise.resolve("X"); + }); + + const body = makeBody([{ role: "user", content: "Some prose to send to the backend." }]); + + await llmlinguaEngine.applyAsync!(body, { + stepConfig: { minTokens: 0, model: "bert-base", compressionRate: 0.3 }, + }); + + assert.ok(capturedOpts, "backend must receive an opts object"); + assert.equal(capturedOpts!.model, "bert-base", "model must be threaded into opts"); + assert.equal(capturedOpts!.compressionRate, 0.3, "compressionRate must be threaded into opts"); + }); + + // ── 3b. floor uses estimated tokens of non-system content (large → compress) ─ + it("large input above the default floor IS compressed (no stepConfig)", async () => { + const calls: string[] = []; + setLlmlinguaBackend((text) => { + calls.push(text); + return Promise.resolve("X"); + }); + + const body = makeBody([{ role: "user", content: LARGE_PROSE }]); + + const result = await llmlinguaEngine.applyAsync!(body); + + assert.equal(result.compressed, true, "large input must clear the floor and compress"); + assert.ok(calls.length > 0, "backend must be called for input above the floor"); + }); + + // ── 4. config validation ───────────────────────────────────────────────────── + it("validateConfig accepts a fully valid config", () => { + const res = llmlinguaEngine.validateConfig({ + model: "tinybert", + compressionRate: 0.5, + minTokens: 2000, + modelPath: "", + }); + assert.equal(res.valid, true, `expected valid, got errors: ${res.errors.join(", ")}`); + }); + + it("validateConfig rejects unknown model / out-of-range / wrong-type fields", () => { + assert.equal( + llmlinguaEngine.validateConfig({ model: "mobilebert" }).valid, + false, + "unknown model must be invalid" + ); + assert.equal( + llmlinguaEngine.validateConfig({ compressionRate: 1.5 }).valid, + false, + "compressionRate > 0.9 must be invalid" + ); + assert.equal( + llmlinguaEngine.validateConfig({ compressionRate: 0.05 }).valid, + false, + "compressionRate < 0.1 must be invalid" + ); + assert.equal( + llmlinguaEngine.validateConfig({ minTokens: -1 }).valid, + false, + "negative minTokens must be invalid" + ); + assert.equal( + llmlinguaEngine.validateConfig({ modelPath: 123 }).valid, + false, + "non-string modelPath must be invalid" + ); + }); + + // ── 5. schema shape ────────────────────────────────────────────────────────── + it("getConfigSchema exposes model select + minTokens/compressionRate/modelPath fields", () => { + const schema = llmlinguaEngine.getConfigSchema(); + const byKey = new Map(schema.map((f) => [f.key, f])); + + const modelField = byKey.get("model"); + assert.ok(modelField, "schema must include a 'model' field"); + assert.equal(modelField!.type, "select", "model field must be a select"); + const optionValues = (modelField!.options ?? []).map((o) => o.value); + assert.ok(optionValues.includes("tinybert"), "model options must include tinybert"); + assert.ok(optionValues.includes("bert-base"), "model options must include bert-base"); + + assert.ok(byKey.has("minTokens"), "schema must include minTokens"); + assert.ok(byKey.has("compressionRate"), "schema must include compressionRate"); + assert.ok(byKey.has("modelPath"), "schema must include modelPath"); + }); +}); diff --git a/tests/unit/compression/llmlingua-model-store.test.ts b/tests/unit/compression/llmlingua-model-store.test.ts new file mode 100644 index 0000000000..0459432b5b --- /dev/null +++ b/tests/unit/compression/llmlingua-model-store.test.ts @@ -0,0 +1,131 @@ +/** + * TDD tests for the llmlingua model registry + model store (Tasks 1-2). + * + * Tests are written RED-first (before the implementation exists). + * + * Coverage: + * 1. Registry shape — tinybert + bert-base present; default points at a valid key. + * 2. Each entry's factory/dtype/subfolder/hfRepo invariants hold. + * 3. resolveLlmlinguaModel() — known id, and the unknown/undefined/empty fallbacks. + * 4. getLlmlinguaModelCacheDir() — path suffix + DATA_DIR honoring. + * 5. configureTransformersEnv() — Hub download default vs local modelPath override. + */ + +import { describe, it, after } from "node:test"; +import assert from "node:assert/strict"; +import os from "node:os"; +import path from "node:path"; +import fs from "node:fs"; + +import { + DEFAULT_LLMLINGUA_MODEL, + LLMLINGUA_MODELS, +} from "../../../open-sse/services/compression/engines/llmlingua/constants.ts"; +import { + getLlmlinguaModelCacheDir, + resolveLlmlinguaModel, + configureTransformersEnv, + type TransformersEnvLike, +} from "../../../open-sse/services/compression/engines/llmlingua/modelStore.ts"; + +// ─── tests ──────────────────────────────────────────────────────────────────── + +describe("llmlingua model registry (constants)", () => { + // ── 1. registry shape ────────────────────────────────────────────────────── + it("LLMLINGUA_MODELS has both proven models and the default is a valid key", () => { + assert.ok(LLMLINGUA_MODELS.tinybert, "tinybert entry must exist"); + assert.ok(LLMLINGUA_MODELS["bert-base"], "bert-base entry must exist"); + assert.equal(DEFAULT_LLMLINGUA_MODEL, "tinybert"); + assert.ok( + Object.prototype.hasOwnProperty.call(LLMLINGUA_MODELS, DEFAULT_LLMLINGUA_MODEL), + "default must be a key of the registry" + ); + }); + + // ── 2. per-entry invariants ───────────────────────────────────────────────── + it("every entry uses WithBERTMultilingual / fp32 / '' subfolder and a sane hfRepo", () => { + for (const [key, entry] of Object.entries(LLMLINGUA_MODELS)) { + assert.equal(entry.factory, "WithBERTMultilingual", `${key}.factory`); + assert.equal(entry.dtype, "fp32", `${key}.dtype`); + assert.equal(entry.subfolder, "", `${key}.subfolder`); + assert.equal(typeof entry.hfRepo, "string", `${key}.hfRepo type`); + assert.ok(entry.hfRepo.length > 0, `${key}.hfRepo non-empty`); + assert.ok(entry.hfRepo.includes("/"), `${key}.hfRepo contains "/"`); + assert.equal(entry.id, key, `${key}.id matches its registry key`); + assert.equal(typeof entry.sizeMB, "number", `${key}.sizeMB type`); + assert.equal(typeof entry.label, "string", `${key}.label type`); + } + }); +}); + +describe("resolveLlmlinguaModel", () => { + // ── 3. resolution + fallback ──────────────────────────────────────────────── + it("returns the requested entry for a known id", () => { + const resolved = resolveLlmlinguaModel("bert-base"); + assert.equal(resolved.id, "bert-base"); + assert.equal(resolved, LLMLINGUA_MODELS["bert-base"]); + }); + + it("falls back to the default (tinybert) for unknown / undefined / empty ids", () => { + const def = LLMLINGUA_MODELS[DEFAULT_LLMLINGUA_MODEL]; + assert.equal(resolveLlmlinguaModel("nonexistent"), def); + assert.equal(resolveLlmlinguaModel(undefined), def); + assert.equal(resolveLlmlinguaModel(null), def); + assert.equal(resolveLlmlinguaModel(""), def); + }); +}); + +describe("getLlmlinguaModelCacheDir", () => { + const originalDataDir = process.env.DATA_DIR; + let tmpDir: string | undefined; + + after(() => { + if (originalDataDir === undefined) { + delete process.env.DATA_DIR; + } else { + process.env.DATA_DIR = originalDataDir; + } + if (tmpDir) { + try { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } catch { + /* ignore cleanup errors */ + } + } + }); + + // ── 4. path suffix + DATA_DIR honoring ────────────────────────────────────── + it("ends with models/llmlingua and lives under DATA_DIR when set", () => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "llm-")); + process.env.DATA_DIR = tmpDir; + + const dir = getLlmlinguaModelCacheDir(); + assert.ok( + dir.endsWith(path.join("models", "llmlingua")), + `cache dir should end with models/llmlingua, got: ${dir}` + ); + assert.ok(dir.startsWith(tmpDir), `cache dir should be under DATA_DIR, got: ${dir}`); + assert.equal(dir, path.join(tmpDir, "models", "llmlingua")); + }); +}); + +describe("configureTransformersEnv", () => { + // ── 5. Hub default vs local override ──────────────────────────────────────── + it("Hub download default: cacheDir set, allowRemoteModels true, no localModelPath", () => { + const env: TransformersEnvLike = {}; + configureTransformersEnv(env, {}); + assert.equal(typeof env.cacheDir, "string"); + assert.ok((env.cacheDir as string).length > 0, "cacheDir must be set"); + assert.equal(env.allowRemoteModels, true); + assert.equal(env.localModelPath, undefined); + }); + + it("local modelPath override: localModelPath set, allowRemoteModels false, cacheDir still set", () => { + const env: TransformersEnvLike = {}; + configureTransformersEnv(env, { modelPath: "/some/local/dir" }); + assert.equal(env.localModelPath, "/some/local/dir"); + assert.equal(env.allowRemoteModels, false); + assert.equal(typeof env.cacheDir, "string"); + assert.ok((env.cacheDir as string).length > 0, "cacheDir must still be set"); + }); +}); diff --git a/tests/unit/compression/llmlingua-worker.test.ts b/tests/unit/compression/llmlingua-worker.test.ts new file mode 100644 index 0000000000..aea8931da6 --- /dev/null +++ b/tests/unit/compression/llmlingua-worker.test.ts @@ -0,0 +1,92 @@ +/** + * Tests for the real LLMLingua worker-thread backend (`worker.ts` + `onnxWorker.ts`). + * + * The four optional deps (`@atjsh/llmlingua-2`, `@huggingface/transformers`, + * `@tensorflow/tfjs`, `js-tiktoken`) are NOT installed in this worktree, so the + * default path MUST fail-open WITHOUT spawning a worker: + * + * 1. Deps absent → fail-open, no spawn (ALWAYS runs here): the backend returns the + * ORIGINAL text unchanged, fast (no model load / worker spawn). + * 2. Type smoke: `workerBackend` is a function. + * 3. GATED real compression (RUN_LLMLINGUA_INT=1): real shrink with deps present; + * no-op skip here (deps absent). + */ + +import { test, after } from "node:test"; +import assert from "node:assert/strict"; +import { createRequire } from "node:module"; + +import { + workerBackend, + __resetLlmlinguaWorkerForTests, +} from "../../../open-sse/services/compression/engines/llmlingua/worker.ts"; + +const require = createRequire(import.meta.url); + +/** Whether all four optional deps resolve in this environment. */ +function depsResolve(): boolean { + try { + require.resolve("@atjsh/llmlingua-2"); + require.resolve("@huggingface/transformers"); + require.resolve("@tensorflow/tfjs"); + require.resolve("js-tiktoken"); + return true; + } catch { + return false; + } +} + +// Let the process exit cleanly: terminate any spawned worker + reset singletons. +after(() => { + __resetLlmlinguaWorkerForTests(); +}); + +test("deps absent → fail-open, no spawn, returns original text fast", async () => { + if (depsResolve()) { + // Premise of this test is that the optional deps are NOT installed (the CI + // default). When they ARE present (e.g. a local integration setup), the backend + // legitimately spawns the worker and compresses, so this assertion no longer + // applies — the real path is covered by the gated test below. + console.log("skip: optional deps present — fail-open-when-absent test N/A"); + return; + } + const input = "hello world this is some prose"; + + const start = Date.now(); + const out1 = await workerBackend(input, {}); + const elapsed = Date.now() - start; + + // EXACT original text (fail-open), unchanged. + assert.equal(out1, input); + // Fast: no model load / worker spawn (proves the optional-deps gate short-circuits). + assert.ok(elapsed < 1000, `expected <1000ms, got ${elapsed}ms`); + + // Second call exercises the memoized gate — still fail-open, still fast. + const out2 = await workerBackend(input, {}); + assert.equal(out2, input); +}); + +test("type smoke: workerBackend is a function", () => { + assert.equal(typeof workerBackend, "function"); +}); + +test("GATED real compression (RUN_LLMLINGUA_INT=1)", async () => { + if (process.env.RUN_LLMLINGUA_INT !== "1") { + console.log("skip: RUN_LLMLINGUA_INT!=1"); + return; + } + if (!depsResolve()) { + console.log("skip: deps absent"); + return; + } + + // Long prose well above any practical floor so compression has room to shrink. + const LONG_PROSE = + "The quick brown fox jumps over the lazy dog while the sun sets slowly behind the distant hills. ".repeat( + 120 + ); + + const out = await workerBackend(LONG_PROSE, { model: "tinybert", compressionRate: 0.5 }); + assert.equal(typeof out, "string"); + assert.ok(out.length < LONG_PROSE.length, "expected a real shrink"); +});