mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-09-13 18:32:12 +03:00
65e81158ab
Landed with the design call resolved per the owner's pick — **option 1**: the synced store is now endpoint-agnostic (persistDiscoveredModels and managedModelImport no longer drop non-chat models at write time), and chat selectability moved to read time (auto-pool expansion in autoStrategy applies filterChatSelectableModels; the models-route projection already had its chatOnly filter). Your discovery test now passes end-to-end (3/3): /api/show capabilities persist per connection and image/embedding requests route through the advertising host. Reconciliation notes: conflicted areas merged onto the current tip (adobe discovery import, requestedModel preflight signature, resolvedProvider fast-path coexists with the synced-route override — explicit resolution wins); carried base-red drains (#10055 memoization, #11071 test variants) dropped as already-landed; the managed-model-import exclusion test was propagated to the new contract (image/video models persist; the read filter still hides them from chat pickers — pinned by a new assertion). Full battery: 205/206 focused (the one red is a confirmed periodic-timer timing flake on the loaded devbox — 20/20 isolated), autoCombo vitest 30/30, combo suites 46/46, gates + typecheck clean. Thank you @yourspraveen — the capability probe + routing design was right; it just needed the store contract opened up. Fixes #11087.
74 lines
3.2 KiB
TypeScript
74 lines
3.2 KiB
TypeScript
import { test } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import {
|
|
countTextTokens,
|
|
isCodexTokenizerContext,
|
|
resolveTokenizerEncoding,
|
|
} from "../../src/shared/utils/tiktokenCounter.ts";
|
|
|
|
test("countTextTokens returns exact tiktoken count for a known string", () => {
|
|
assert.equal(countTextTokens("hello world"), 2); // cl100k_base
|
|
});
|
|
|
|
test("Codex context selects o200k_base without changing the default", () => {
|
|
assert.equal(resolveTokenizerEncoding(), "cl100k_base");
|
|
assert.equal(resolveTokenizerEncoding({ provider: "codex" }), "o200k_base");
|
|
assert.equal(resolveTokenizerEncoding({ provider: "cx" }), "o200k_base");
|
|
assert.equal(resolveTokenizerEncoding({ model: "codex/gpt-5.6-sol" }), "o200k_base");
|
|
assert.equal(resolveTokenizerEncoding({ model: "cx/gpt-5.6-sol" }), "o200k_base");
|
|
assert.equal(resolveTokenizerEncoding({ provider: "openai", model: "gpt-5.6" }), "cl100k_base");
|
|
assert.equal(isCodexTokenizerContext({ provider: "codex" }), true);
|
|
assert.equal(isCodexTokenizerContext({ provider: "openai" }), false);
|
|
});
|
|
|
|
test("Codex token counting uses the o200k encoder", () => {
|
|
const text = "antidisestablishmentarianism 中文ภาษาไทย";
|
|
assert.notEqual(
|
|
countTextTokens(text, { provider: "codex" }),
|
|
countTextTokens(text, { provider: "openai" })
|
|
);
|
|
});
|
|
|
|
test("countTextTokens handles empty and non-string safely", () => {
|
|
assert.equal(countTextTokens(""), 0);
|
|
assert.equal(countTextTokens(undefined as unknown as string), 0);
|
|
});
|
|
|
|
test("countTextTokens is additive-ish and monotonic for longer text", () => {
|
|
const short = countTextTokens("the quick brown fox");
|
|
const long = countTextTokens("the quick brown fox jumps over the lazy dog");
|
|
assert.ok(long > short);
|
|
assert.ok(short > 0);
|
|
});
|
|
|
|
test("countTextTokens fast-paths strings over 50k chars without tokenizing (worker wedge regression)", () => {
|
|
const big = "user: please review the attached patch\ntext: ".repeat(40_000);
|
|
const start = performance.now();
|
|
const tokens = countTextTokens(big);
|
|
const elapsed = performance.now() - start;
|
|
assert.equal(tokens, Math.ceil(big.length / 4));
|
|
assert.ok(elapsed < 1000, `fast path took ${elapsed.toFixed(0)}ms`);
|
|
});
|
|
|
|
test("countTextTokens strips base64 data URIs before tokenizing (images not counted as text)", () => {
|
|
const png =
|
|
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==";
|
|
const b64 = png.repeat(60);
|
|
const withImage = countTextTokens(
|
|
`{"image_url":{"url":"data:image/png;base64,${b64}"}}`,
|
|
{ provider: "codex" }
|
|
);
|
|
const stripped = countTextTokens('{"image_url":{"url":""}}', { provider: "codex" });
|
|
assert.equal(withImage, stripped);
|
|
});
|
|
|
|
test("countTextTokens does not tokenize huge base64 image payloads (wedge repro)", () => {
|
|
const b64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==";
|
|
const body = `{"image_url":{"url":"data:image/png;base64,${b64.repeat(14_000)}"}}`;
|
|
const start = performance.now();
|
|
const tokens = countTextTokens(body);
|
|
const elapsed = performance.now() - start;
|
|
assert.ok(tokens < 1000, `base64 payload inflates token count to ${tokens}`);
|
|
assert.ok(elapsed < 1000, `took ${elapsed.toFixed(0)}ms`);
|
|
});
|