feat(api): exact offline token counting for count_tokens fallback via tiktoken (#4087)

Integrated into release/v3.8.28

src/shared/utils/tiktokenCounter.ts

@@ -0,0 +1,21 @@
+import { getEncoding, type Tiktoken } from "js-tiktoken";
+
+let encoder: Tiktoken | null = null;
+
+function getEncoder(): Tiktoken {
+  if (!encoder) encoder = getEncoding("cl100k_base");
+  return encoder;
+}
+
+/**
+ * Exact token count for a string using cl100k_base (offline, no upstream call).
+ * Defensive: never throws in a counting path — falls back to a char heuristic.
+ */
+export function countTextTokens(text: string): number {
+  if (!text || typeof text !== "string") return 0;
+  try {
+    return getEncoder().encode(text).length;
+  } catch {
+    return Math.ceil(text.length / 4);
+  }
+}