diff --git a/package.json b/package.json index 5ac78b8d78..426e3c5bb3 100644 --- a/package.json +++ b/package.json @@ -304,7 +304,6 @@ "smol-toml": "1.7.1", "socks": "^2.8.7", "sql.js": "^1.14.1", - "sqlite-vec": "^0.1.9", "tailwind-merge": "^3.6.0", "tsx": "^4.23.0", "undici": "^8.3.0", @@ -324,7 +323,8 @@ "js-tiktoken": "^1.0.20", "keytar": "^7.9.0", "tls-client-node": "^0.2.0", - "wreq-js": "^2.3.1" + "wreq-js": "^2.3.1", + "sqlite-vec": "^0.1.9" }, "devDependencies": { "@axe-core/playwright": "^4.11.3", diff --git a/src/shared/utils/tiktokenCounter.ts b/src/shared/utils/tiktokenCounter.ts index f36eb999a3..ec6af38913 100644 --- a/src/shared/utils/tiktokenCounter.ts +++ b/src/shared/utils/tiktokenCounter.ts @@ -1,4 +1,7 @@ -import { getEncoding, type Tiktoken } from "js-tiktoken"; +import type { Tiktoken } from "js-tiktoken"; +import { createRequire } from "module"; + +const _require = createRequire(import.meta.url); export type TokenizerEncoding = "cl100k_base" | "o200k_base"; @@ -41,9 +44,15 @@ export function resolveTokenizerEncoding(context?: TokenizerContext): TokenizerE function getEncoder(encoding: TokenizerEncoding): Tiktoken { const cached = encoders.get(encoding); if (cached) return cached; - const created = getEncoding(encoding); - encoders.set(encoding, created); - return created; + let tiktoken: { getEncoding: (name: string) => Tiktoken } | null = null; + try { + tiktoken ??= _require("js-tiktoken") as { getEncoding: (name: string) => Tiktoken }; + const created = tiktoken.getEncoding(encoding); + encoders.set(encoding, created); + return created; + } catch { + throw new Error(`js-tiktoken not available: cannot create encoder for ${encoding}`); + } } /**