diff --git a/.env.example b/.env.example index c1a1f06e67..27c024448e 100644 --- a/.env.example +++ b/.env.example @@ -1469,28 +1469,25 @@ CURSOR_USER_AGENT="Cursor/3.4" # FIRECRAWL_BASE_URL=https://api.firecrawl.dev # FIRECRAWL_TIMEOUT_MS=30000 # Per-request timeout (default: 30000 = 30s) -# ── ChatGPT TLS sidecar (Firefox-fingerprinted client) ── -# Used by: open-sse/services/chatgptTlsClient.ts — wire-level timeout for -# the bogdanfinn/tls-client koffi binding and the JS-side grace window -# layered on top of it when the native library is wedged. +# ── ChatGPT browser transport (Firefox-fingerprinted client) ── +# Used by: open-sse/services/chatgptTlsClient.ts — native wreq-js request +# timeout and the JS-side hard-deadline grace layered on top of it. # OMNIROUTE_CHATGPT_TLS_TIMEOUT_MS=60000 # OMNIROUTE_CHATGPT_TLS_GRACE_MS=10000 -# Max wait for the FIRST streamed byte from the ChatGPT TLS sidecar before the -# request is aborted as a dead stream, in milliseconds. Default 30000 (30s). -# Raise it if upstream cold-starts routinely exceed the window. +# Max wait for the FIRST streamed byte before switching from direct streaming +# to a buffered response, in milliseconds. Default 30000 (30s). The request's +# hard deadline continues to apply while the buffered body is read. # OMNIROUTE_CHATGPT_STREAM_FIRST_BYTE_TIMEOUT_MS=30000 -# ── Claude TLS sidecar (Chromium-fingerprinted client) ── -# Used by: open-sse/services/claudeTlsClient.ts — wire-level timeout for -# the bogdanfinn/tls-client koffi binding and the JS-side grace window -# layered on top of it when the native library is wedged. +# ── Claude browser transport (Chromium-fingerprinted client) ── +# Used by: open-sse/services/claudeTlsClient.ts — native wreq-js request +# timeout and the JS-side hard-deadline grace layered on top of it. # OMNIROUTE_CLAUDE_TLS_TIMEOUT_MS=60000 # OMNIROUTE_CLAUDE_TLS_GRACE_MS=10000 -# ── Perplexity TLS sidecar (Firefox-fingerprinted client) ── -# Used by: open-sse/services/perplexityTlsClient.ts — wire-level timeout for -# the bogdanfinn/tls-client koffi binding and the JS-side grace window -# layered on top of it when the native library is wedged. +# ── Perplexity browser transport (Firefox-fingerprinted client) ── +# Used by: open-sse/services/perplexityTlsClient.ts — native wreq-js request +# timeout and the JS-side hard-deadline grace layered on top of it. # OMNIROUTE_PPLX_TLS_TIMEOUT_MS=30000 # OMNIROUTE_PPLX_TLS_GRACE_MS=10000 @@ -1502,18 +1499,16 @@ CURSOR_USER_AGENT="Cursor/3.4" # meta-commentary. Set to 1/true/yes/on to restore the old behavior. # OMNIROUTE_PPLX_SEARCH_HINT=0 -# ── Grok web TLS sidecar (Chrome-fingerprinted client) ── -# Used by: open-sse/services/grokTlsClient.ts — wire-level timeout for the -# bogdanfinn/tls-client koffi binding and the JS-side grace window layered on -# top of it when the native library is wedged. +# ── Grok web browser transport (Chrome-fingerprinted client) ── +# Used by: open-sse/services/grokTlsClient.ts — native wreq-js request timeout +# and the JS-side hard-deadline grace layered on top of it. # OMNIROUTE_GROK_TLS_TIMEOUT_MS=60000 # OMNIROUTE_GROK_TLS_GRACE_MS=10000 -# ── Notion web TLS sidecar (Chrome-fingerprinted client) ── -# Used by: open-sse/services/notionTlsClient.ts — wire-level timeout for the -# bogdanfinn/tls-client koffi binding and the JS-side grace window layered on -# top of it when the native library is wedged. The notion-web executor raises -# the wire timeout per-request to 180000 for long generations. +# ── Notion web browser transport (Chrome-fingerprinted client) ── +# Used by: open-sse/services/notionTlsClient.ts — native wreq-js request timeout +# and the JS-side hard-deadline grace layered on top of it. The notion-web +# executor raises the native timeout per-request to 180000 for long generations. # OMNIROUTE_NOTION_TLS_TIMEOUT_MS=30000 # OMNIROUTE_NOTION_TLS_GRACE_MS=10000 @@ -2525,11 +2520,6 @@ APP_LOG_TO_FILE=true # Used by: src/lib/jobs/backupScheduleJob.ts # OMNIROUTE_BACKUP_SCHEDULE_JOB_INTERVAL_MS=30000 -# ── TLS sidecar override ── -# Used by: open-sse/services/chatgptTlsClient.ts tests. Production deployments -# should leave this unset; the sidecar is auto-managed. -# OMNIROUTE_TLS_PROXY_URL= - # ── Skills sandbox (experimental) ── # Used by: src/lib/skills/builtins.ts. All values support comma lists where # noted in the source. diff --git a/.github/workflows/electron-release.yml b/.github/workflows/electron-release.yml index e899a664ea..3242a5e285 100644 --- a/.github/workflows/electron-release.yml +++ b/.github/workflows/electron-release.yml @@ -222,7 +222,7 @@ jobs: # optionals (@img/sharp-*, @img/sharp-libvips-*, @ngrok/ngrok-*, # fsevents) carry linux forks. Replace them with the forks this # leg's own `npm ci` resolved, then assert every bundled native - # (koffi triplets, better-sqlite3 prebuilds, wreq-js, onnxruntime) + # (better-sqlite3 prebuilds, wreq-js, onnxruntime) # can service this leg's platform/arch before packaging starts. run: | node scripts/build/standaloneBundle.mjs restore --archive web-bundle.tar.gz diff --git a/Dockerfile b/Dockerfile index a35f57e280..fc6b2746aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -103,25 +103,11 @@ RUN test -f package-lock.json \ # node-gyp comes from npm's own bundled copy (deterministic, already in the image) # instead of `npx --yes`, which would install an arbitrary registry version # on-demand and run its lifecycle scripts (Sonar docker:S6505). -# -# tls-client-node (chatgpt-web/claude-web/grok-web/lmarena/perplexity-web TLS -# impersonation) hits the same --ignore-scripts wall: its own postinstall.js -# fetches a platform .so/.dylib/.dll from the bogdanfinn/tls-client GitHub -# Releases API and is never invoked when npm ci skips lifecycle scripts. Unlike -# better-sqlite3 above, that script never throws on failure — it only -# `console.warn`s and exits 0 — so a rate-limited or offline build would -# otherwise succeed silently with an empty bin/ and only fail at first request -# in production (TlsClientUnavailableError, #7802). Run it explicitly here so -# a broken/rate-limited fetch fails the BUILD loudly instead of shipping a -# broken image. RUN --mount=type=cache,id=s/92ca8a61-c1ba-421f-a389-d48ac7258c2d-npm-cache,target=/root/.npm \ npm ci --include=optional --no-audit --no-fund --legacy-peer-deps --ignore-scripts \ && (cd node_modules/better-sqlite3 \ && node /usr/local/lib/node_modules/npm/node_modules/node-gyp/bin/node-gyp.js rebuild) \ - && node -e "require('better-sqlite3')(':memory:').close()" \ - && node node_modules/tls-client-node/scripts/postinstall.js \ - && (test -n "$(find node_modules/tls-client-node/bin -mindepth 1 -print -quit 2>/dev/null)" \ - || (echo "tls-client-node native binary missing after postinstall — GitHub API fetch likely rate-limited or failed (#7802)" >&2 && exit 1)) + && node -e "require('better-sqlite3')(':memory:').close()" # Build with Turbopack (stable in Next 16, the repo default). The v3.8.27-era # TurbopackInternalError panic ("entered unreachable code: there must be a path to a diff --git a/Dockerfile.bun b/Dockerfile.bun index bb547ce210..75c8d0eeb7 100644 --- a/Dockerfile.bun +++ b/Dockerfile.bun @@ -29,11 +29,6 @@ RUN if [ -d "node_modules/better-sqlite3" ]; then \ (cd node_modules/better-sqlite3 && bunx node-gyp rebuild); \ fi -# Fetch tls-client-node native binary if script exists -RUN if [ -f "node_modules/tls-client-node/scripts/postinstall.js" ]; then \ - bun node_modules/tls-client-node/scripts/postinstall.js || true; \ - fi - # Disable Turbopack for Bun builder stage (Turbopack V8 internal worker bindings require Node) ENV OMNIROUTE_USE_TURBOPACK=0 diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 45fcfed7bd..4382f6e4e9 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -24,3 +24,28 @@ NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPO NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +## wreq-js 3.0.0 + +OmniRoute distributes `wreq-js` and its seven platform-specific native addons from +[`wreq-js@3.0.0`](https://www.npmjs.com/package/wreq-js/v/3.0.0). + +MIT License + +Copyright (c) 2025 will-work-for-meal +Copyright (c) 2025 Oleksandr Herasymov + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +associated documentation files (the "Software"), to deal in the Software without restriction, +including without limitation the rights to use, copy, modify, merge, publish, distribute, +sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial +portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/changelog.d/maintenance/pending-wreq-js-transport.md b/changelog.d/maintenance/pending-wreq-js-transport.md new file mode 100644 index 0000000000..b867e48fd8 --- /dev/null +++ b/changelog.d/maintenance/pending-wreq-js-transport.md @@ -0,0 +1 @@ +- **chore(stealth):** replace the `tls-client-node` sidecar/temp-file transport used by the six web-cookie providers with the exactly pinned `wreq-js` 3.0.0 native transport, preserving streaming, proxy isolation, deadlines, EOF policies, binary responses, and cancellation while removing the obsolete downloader and native repair path. diff --git a/config/quality/.license-allowlist.json b/config/quality/.license-allowlist.json index 06dbba4aa0..f2cca50374 100644 --- a/config/quality/.license-allowlist.json +++ b/config/quality/.license-allowlist.json @@ -74,12 +74,6 @@ "justification": "CC-BY-4.0 applies to the caniuse browser-support data (a dataset, not code). The Creative Commons Attribution license requires attribution when distributing — OmniRoute does not distribute caniuse-lite data directly to end users; it is consumed by browserslist/PostCSS at build time to generate CSS compatibility info. This is a widely accepted pattern in the Node.js ecosystem (caniuse-lite is in millions of projects). Attribution is satisfied by keeping the package in node_modules with its original license file.", "risk": "low", "reviewAt": "v4.0.0" - }, - "tls-client-node": { - "license": "Custom: LICENSE (Apache-2.0 + Commons Clause)", - "justification": "TODO: revisar — tls-client-node uses Apache-2.0 with a 'Commons Clause' addendum that restricts 'Selling' the software (i.e., offering it as a hosted/commercial service whose value derives substantially from tls-client-node). OmniRoute is an open-source proxy; however if deployed as a paid SaaS/hosting service, this restriction could apply. The package is used by grokTlsClient.ts for Grok TLS fingerprinting. RISK: medium — legal review recommended before commercial deployment. Alternatives: consider replacing with a native TLS fingerprinting approach or a truly permissive library.", - "risk": "medium", - "reviewAt": "v3.9.0" } } } diff --git a/config/quality/dependency-allowlist.json b/config/quality/dependency-allowlist.json index 92e6ef7e8d..32d7aac938 100644 --- a/config/quality/dependency-allowlist.json +++ b/config/quality/dependency-allowlist.json @@ -129,7 +129,6 @@ "sqlite-vec", "tailwind-merge", "tailwindcss", - "tls-client-node", "turndown", "turndown-plugin-gfm", "tsup", diff --git a/config/release/wreq-js-native-manifest.json b/config/release/wreq-js-native-manifest.json new file mode 100644 index 0000000000..ebbf359ca5 --- /dev/null +++ b/config/release/wreq-js-native-manifest.json @@ -0,0 +1,44 @@ +{ + "package": "wreq-js", + "version": "3.0.0", + "source": "https://registry.npmjs.org/wreq-js/-/wreq-js-3.0.0.tgz", + "npmIntegrity": "sha512-RZCoRSevVPpH4A4B4MxbFGo/pVPFveWd2gbe4ENKpPWlKXEYklZSDESOjBMmrIsmnkHh+nhM4PNJvG+NL7wBPA==", + "license": "MIT", + "nativeAddons": [ + { + "path": "rust/wreq-js.darwin-arm64.node", + "size": 7690880, + "sha256": "c82eec39df691adb94f2cd09a8ff51335de8587cf132cd8b3ec797469a4b5002" + }, + { + "path": "rust/wreq-js.darwin-x64.node", + "size": 8192028, + "sha256": "073b8a8a4c26aedbce7c14eef3e5567918e62e8dbf4d28296b23f9d2beec2981" + }, + { + "path": "rust/wreq-js.linux-arm64-gnu.node", + "size": 8520824, + "sha256": "861d96a78caf7ce02c9ae8d37f1c59f5b0480e3142775c32917fcfe9b88524b0" + }, + { + "path": "rust/wreq-js.linux-arm64-musl.node", + "size": 8735472, + "sha256": "2409a3578c8c440df419b4d5abe3ac149bec48881611a6dc1571b95e6246552d" + }, + { + "path": "rust/wreq-js.linux-x64-gnu.node", + "size": 9048992, + "sha256": "55b40f4602c52111dfcdcc93db83f9d0de55d0ef7540348757709d58d05a9b64" + }, + { + "path": "rust/wreq-js.linux-x64-musl.node", + "size": 8974880, + "sha256": "bd52d15b1bb4704b11561a8aa95648a6c91150082b5af0e39dd1608b7db2d317" + }, + { + "path": "rust/wreq-js.win32-x64-msvc.node", + "size": 7967232, + "sha256": "7451a8701b82c946b03ba2be2f15257260a250b9e0ed9910611b22564fbec7a9" + } + ] +} diff --git a/docs/guides/TROUBLESHOOTING.md b/docs/guides/TROUBLESHOOTING.md index 75c8e4b601..e424b93768 100644 --- a/docs/guides/TROUBLESHOOTING.md +++ b/docs/guides/TROUBLESHOOTING.md @@ -62,7 +62,7 @@ Set these in the OmniRoute process environment (the daemon, e.g. via the LaunchA **How to verify it worked**: run your agent/cron twice in quick succession and confirm both succeed. Before the fix, the second run typically throws `429`/`401`. After the fix, failures (if any) are retried transparently and the call completes. You can also `curl /monitoring/health` and watch the `rateLimitedUntil` field on the provider connections and the `circuitBreakers.providerBreakers[].state` for the affected providers — the state is one of `CLOSED`, `DEGRADED`, `OPEN`, or `HALF_OPEN` (see `src/shared/utils/circuitBreaker.ts`), and a provider that keeps failing will flip `CLOSED → DEGRADED → OPEN` before the reset window lets a probe through (`HALF_OPEN`). -**If you still see 429**: the active account for that provider has genuinely exhausted its *quota* (not just rate). Add a second account for the same provider in the OmniRoute dashboard → Providers → Accounts, or mix in another free provider (e.g. `routeway`, `auggie`). Rotation only helps with transient rate/400/401; a hard quota exhaustion requires a second credential or a different provider. +**If you still see 429**: the active account for that provider has genuinely exhausted its _quota_ (not just rate). Add a second account for the same provider in the OmniRoute dashboard → Providers → Accounts, or mix in another free provider (e.g. `routeway`, `auggie`). Rotation only helps with transient rate/400/401; a hard quota exhaustion requires a second credential or a different provider. **If you see 403 on vision models (`auto/vision`, `bazaarlink/*`)**: the connected account lacks a paid plan that includes vision, or the API key has insufficient permissions. Verify in the provider dashboard that the key scope includes vision/multimodal, or connect a paid tier account and keep it as the vision target. @@ -75,7 +75,7 @@ When you run `npm install -g omniroute`, you may see a wall of warnings like `np The warnings come from stale peer-dependency ranges in third-party packages OmniRoute doesn't control: 1. **`marked-terminal` wants `marked >=1 <16`, found `marked@18`** — works fine in practice; the upstream peer range is just stale. -2. **`deprecated prebuild-install@7.1.3`** — the native-binary fetch helper. Only relevant later if a web-cookie provider reports a missing `tls-client-node` native binary (a separate issue, not caused by this warning). +2. **`deprecated prebuild-install@7.1.3`** — a transitive native-binary helper used by another dependency. The pinned `wreq-js@3.0.0` package bundles its seven supported platform addons directly; this warning does not diagnose the web-cookie transport. **No action needed** — the warnings cannot be fully silenced without forking upstream packages. @@ -148,9 +148,10 @@ desktop app, for example: - `resources/app/.build/next/node_modules/playwright-/lib/…/agentParser.js` and `workerProcessEntry.js` — [Playwright](https://playwright.dev), the browser-automation library used for in-app provider login and browser-backed chat. -- `resources/app/.build/next/node_modules/tls-client-node-/bin/tls-client-windows-64-.dll` - — the native binary from `tls-client-node`, used for Cloudflare-tolerant HTTP on some web - providers. +- `resources/app/.build/next/node_modules/wreq-js-/rust/wreq-js.win32-x64-msvc.node` + — the declared MIT-licensed native addon from pinned `wreq-js@3.0.0`, used for + browser-fingerprinted HTTP on some web providers. Its expected SHA-256 is recorded in + `config/release/wreq-js-native-manifest.json`. **Why it fires:** the Windows installer is **not yet code-signed**, so an unsigned NSIS installer has zero reputation and behavioral heuristics run at maximum aggression. Combined diff --git a/docs/reference/ENVIRONMENT.md b/docs/reference/ENVIRONMENT.md index b383b5696f..1f494f9ff1 100644 --- a/docs/reference/ENVIRONMENT.md +++ b/docs/reference/ENVIRONMENT.md @@ -766,18 +766,18 @@ REQUEST_TIMEOUT_MS (global override) | `OMNIROUTE_PROVIDER_PROBE_TIMEOUT_MS` | `8000` | Timeout (ms) for the `validationRead` and `modelsProbe` presets in `src/shared/network/safeOutboundFetch.ts`. Raise for slow endpoints (Cerebras, Cloudflare AI, Groq) to prevent flapping between active/error in the dashboard. Falls back to 8000ms for invalid (<1000) or non-numeric values. | | `OMNIROUTE_RELAY_FETCH_TIMEOUT_MS` | `25000` | Relay-specific fetch timeout in `open-sse/utils/proxyFetch.ts` (#9158). A hung relay must fail before the client/agent timeout (~30s) so callers see a relay-specific failure instead of a generic upstream timeout. Capped at `29000` so it always fires first. | | `OMNIROUTE_RETRY_BACKOFF_MS` | `10` | Shared retry backoff for the direct/relay/proxy retry-once paths in `open-sse/utils/proxyFetch.ts` (#9158). `0` = retry immediately. | -| `OMNIROUTE_CHATGPT_TLS_TIMEOUT_MS` | `60000` | Wire-level timeout for the bogdanfinn/tls-client koffi binding (`chatgptTlsClient.ts`). | -| `OMNIROUTE_CHATGPT_TLS_GRACE_MS` | `10000` | JS-side grace added on top of the wire timeout when the native binding is wedged. | -| `OMNIROUTE_CHATGPT_STREAM_FIRST_BYTE_TIMEOUT_MS` | `30000` (30s) | Max wait for the first streamed byte from the ChatGPT TLS sidecar (`chatgptTlsClient.ts`) before aborting a dead stream. Raise if upstream cold-starts exceed the window. | -| `OMNIROUTE_CLAUDE_TLS_TIMEOUT_MS` | `60000` | Wire-level timeout for the bogdanfinn/tls-client koffi binding (`claudeTlsClient.ts`). | -| `OMNIROUTE_CLAUDE_TLS_GRACE_MS` | `10000` | JS-side grace added on top of the wire timeout when the native binding is wedged. | -| `OMNIROUTE_PPLX_TLS_TIMEOUT_MS` | `30000` | Wire-level timeout for the bogdanfinn/tls-client koffi binding (`perplexityTlsClient.ts`). | -| `OMNIROUTE_PPLX_TLS_GRACE_MS` | `10000` | JS-side grace added on top of the wire timeout when the native binding is wedged. | +| `OMNIROUTE_CHATGPT_TLS_TIMEOUT_MS` | `60000` | Native wreq-js request timeout (`chatgptTlsClient.ts`). | +| `OMNIROUTE_CHATGPT_TLS_GRACE_MS` | `10000` | JS-side hard-deadline grace added on top of the native timeout. | +| `OMNIROUTE_CHATGPT_STREAM_FIRST_BYTE_TIMEOUT_MS` | `30000` (30s) | Max wait for the first streamed byte before ChatGPT switches to a buffered response; the hard request deadline remains active. | +| `OMNIROUTE_CLAUDE_TLS_TIMEOUT_MS` | `60000` | Native wreq-js request timeout (`claudeTlsClient.ts`). | +| `OMNIROUTE_CLAUDE_TLS_GRACE_MS` | `10000` | JS-side hard-deadline grace added on top of the native timeout. | +| `OMNIROUTE_PPLX_TLS_TIMEOUT_MS` | `30000` | Native wreq-js request timeout (`perplexityTlsClient.ts`). | +| `OMNIROUTE_PPLX_TLS_GRACE_MS` | `10000` | JS-side hard-deadline grace added on top of the native timeout. | | `OMNIROUTE_PPLX_SEARCH_HINT` | `0` (off) | Appends "You have built-in web search. Answer questions directly using search results." to the caller's system message (`perplexity-web/protocol.ts`). Off by default — Perplexity searches anyway, and the sentence leaks into replies as meta-commentary for coding clients. Set `1`/`true`/`yes`/`on` to restore. | -| `OMNIROUTE_GROK_TLS_TIMEOUT_MS` | `60000` | Wire-level timeout for the bogdanfinn/tls-client koffi binding (`grokTlsClient.ts`). | -| `OMNIROUTE_GROK_TLS_GRACE_MS` | `10000` | JS-side grace added on top of the wire timeout when the native binding is wedged. | -| `OMNIROUTE_NOTION_TLS_TIMEOUT_MS` | `30000` | Wire-level timeout for the bogdanfinn/tls-client koffi binding (`notionTlsClient.ts`); the `notion-web` executor raises it per-request to `180000` for long generations. | -| `OMNIROUTE_NOTION_TLS_GRACE_MS` | `10000` | JS-side grace added on top of the wire timeout when the native binding is wedged. | +| `OMNIROUTE_GROK_TLS_TIMEOUT_MS` | `60000` | Native wreq-js request timeout (`grokTlsClient.ts`). | +| `OMNIROUTE_GROK_TLS_GRACE_MS` | `10000` | JS-side hard-deadline grace added on top of the native timeout. | +| `OMNIROUTE_NOTION_TLS_TIMEOUT_MS` | `30000` | Native wreq-js request timeout (`notionTlsClient.ts`); `notion-web` raises it per request to `180000` for long generations. | +| `OMNIROUTE_NOTION_TLS_GRACE_MS` | `10000` | JS-side hard-deadline grace added on top of the native timeout. | | `OMNIROUTE_BROWSER_POOL` | `on` | Shared Playwright browser pool for browser-backed web-cookie chat (`browserPool.ts`); set `off` to disable. | | `WEB_COOKIE_USE_BROWSER` | `0` | Opt a web-cookie chat request into the browser-backed path (`browserBackedChat.ts`); `1` to enable. | | `KIMI_WEB_BASE_URL` | `https://www.kimi.ai` | Base URL for the Kimi Web (international kimi.ai Connect-RPC) executor (`kimi-web.ts`); override only for mirror/proxy endpoints. | @@ -1310,7 +1310,6 @@ Provider quota endpoints, network tunnels (Tailscale, Ngrok, MITM debug proxy), | `DB_BACKUP_MAX_FILES` | `20` | `src/lib/db/backup.ts`, `src/lib/db/migrationRunner.ts` | Maximum SQLite backup files retained on disk. Applies to manual/scheduled backups and to pre-migration snapshots. Overrides the value saved from Settings → Database backup retention. | | `DB_BACKUP_RETENTION_DAYS` | `0` | `src/lib/db/backup.ts`, `src/lib/db/migrationRunner.ts` | Maximum age (days) of retained backups. `0` disables age-based pruning. Applies to manual/scheduled backups and to pre-migration snapshots. Overrides the value saved from Settings → Database backup retention. | | `OMNIROUTE_BACKUP_SCHEDULE_JOB_INTERVAL_MS` | `30000` | `src/lib/jobs/backupScheduleJob.ts` | Tick interval (ms) of the server-side job that executes `backup-schedule.json`. Must stay well under the 1-minute cron granularity; values below `5000` or unparseable fall back to `30000`. | -| `OMNIROUTE_TLS_PROXY_URL` | _(unset)_ | `open-sse/services/chatgptTlsClient.ts` | Override the TLS sidecar URL for tests. Production should leave unset. | | `CONTAINER_HOST` | `docker` | `scripts/check-permissions.sh` | Container runtime hint for the entrypoint permission check. Set to `podman` for any Podman topology. Because the container cannot determine whether the engine is local or reached through Podman Machine, the warning stays topology-neutral and points to `contrib/podman/README.md`. | | `QUOTA_STORE_DRIVER` | `sqlite` | `src/lib/quota/storeFactory.ts` | Quota-share consumption store backend: `sqlite` (default) or `redis`. | | `QUOTA_STORE_REDIS_URL` | _(unset)_ | `src/lib/quota/storeFactory.ts` | Redis connection string used when `QUOTA_STORE_DRIVER=redis` (e.g. `redis://localhost:6379`). | diff --git a/docs/security/STEALTH_GUIDE.md b/docs/security/STEALTH_GUIDE.md index 07f2b7dc6a..281a812a1c 100644 --- a/docs/security/STEALTH_GUIDE.md +++ b/docs/security/STEALTH_GUIDE.md @@ -1,13 +1,13 @@ --- title: "Stealth Guide" -version: 3.8.40 -lastUpdated: 2026-06-28 +version: 3.8.50 +lastUpdated: 2026-08-26 --- # Stealth Guide -> **Source of truth:** `open-sse/utils/tlsClient.ts`, `open-sse/services/{chatgptTlsClient,claudeCodeCCH,claudeCodeFingerprint,claudeCodeObfuscation,claudeCodeCompatible}.ts`, `open-sse/config/cliFingerprints.ts`, `src/mitm/` -> **Last updated:** 2026-06-28 — v3.8.40 +> **Source of truth:** `open-sse/utils/tlsClient.ts`, `open-sse/services/{tlsClientBase,chatgptTlsClient,claudeTlsClient,perplexityTlsClient,grokTlsClient,notionTlsClient,lmarenaTlsClient,claudeCodeCCH,claudeCodeFingerprint,claudeCodeObfuscation,claudeCodeCompatible}.ts`, `open-sse/config/cliFingerprints.ts`, `src/mitm/` +> **Last updated:** 2026-08-26 — v3.8.50 > **Audience:** Engineers maintaining provider-specific stealth integrations. OmniRoute integrates with providers whose edges actively fingerprint non-official clients (TLS JA3/JA4, header ordering, JSON body shape, integrity tokens). This page documents the stealth surfaces OmniRoute exposes and where they are implemented. @@ -29,17 +29,38 @@ Lazy-loaded `wreq-js` session that impersonates **Chrome 124 on macOS**. Used as - Timeout: `TLS_CLIENT_TIMEOUT_MS` (inherits from `FETCH_TIMEOUT_MS`, default 600000) - `wreq-js` Response is fetch-compatible (`headers`, `text()`, `json()`, `clone()`, `body`). -### `open-sse/services/chatgptTlsClient.ts` — tls-client-node (Firefox 148) +### Web-cookie provider transport — wreq-js 3.0.0 -Dedicated TLS impersonator for `chatgpt.com`. ChatGPT's Cloudflare config pins `cf_clearance` to JA3/JA4 + HTTP/2 SETTINGS frame ordering — undici's handshake gets `cf-mitigated: challenge` even with valid cookies. +`open-sse/services/tlsClientBase.ts` is the shared transport for ChatGPT, Claude, Perplexity, +Grok, Notion, and LMArena web sessions. Each thin provider wrapper selects a browser/OS profile; +the base loads `wreq-js` lazily, reuses only transport-level connections keyed by +profile + OS + resolved proxy, and gives every request an ephemeral cookie scope. It never shares a +wreq session or cookie jar between accounts or requests. -- Profile: `firefox_148` (must match the Firefox 148 `User-Agent` sent) -- Mode: `runtimeMode: "native"` (koffi-loaded shared library; avoids managed sidecar HTTP) -- `withRandomTLSExtensionOrder: true` -- `tlsFetchChatGpt(url, options)` supports streaming (writes body to temp file, tailed as `ReadableStream`) -- Hang detection: `raceWithTimeout` + `TlsClientHangError` triggers `resetClientCache()` so the next call respawns the binding -- Proxy resolution (priority): per-call `proxyUrl` → `OMNIROUTE_TLS_PROXY_URL` → `HTTPS_PROXY`/`HTTP_PROXY`/`ALL_PROXY` (the native binding does **not** read these envs itself; it must be threaded through) -- Errors: `TlsClientUnavailableError` (binary missing), `TlsClientHangError` (binding deadlocked) +| Provider | Profile | Emulated OS | Stream EOF policy | +| ---------- | ------------- | ----------- | -------------------------------- | +| ChatGPT | `firefox_148` | macOS | include `[DONE]` | +| Claude | `chrome_146` | Linux | include `[DONE]` | +| Perplexity | `firefox_148` | macOS | include `event: end_of_stream` | +| Grok | `chrome_146` | Linux | exclude `[DONE]` | +| Notion | `chrome_146` | Windows | include `[DONE]` | +| LMArena | `chrome_146` | Windows | no sentinel; close on native EOF | + +- Streaming uses the native response `ReadableStream` directly; no temp file or sidecar is created. +- Up to 256 initial bytes are inspected before exposing a stream. SSE providers buffer non-SSE + errors; Grok/LMArena map Cloudflare challenges to `403` and HTML interstitials to `502`. +- The native request timeout remains wrapped by an absolute JS hard deadline. A hang invalidates + and closes only the affected profile/OS/proxy transport before the next request recreates it. +- Proxy resolution priority is per-call `proxyUrl` → request-scoped account/dashboard context → + `HTTPS_PROXY`/`HTTP_PROXY`/`ALL_PROXY` (including lowercase variants). Resolution errors fail + closed instead of leaking a direct connection. LMArena deliberately resolves against `arena.ai`. +- `byteResponse` returns a content-typed `data:` URL without UTF-8 corruption. +- Errors are `TlsClientUnavailableError` (package/addon unavailable) and `TlsClientHangError` + (deadline exceeded). + +The profiles are supported by the pinned package, but real WAF acceptance can change independently +of local contract tests. Validate fingerprint changes against an explicitly authorized live account +before claiming parity with an upstream browser. --- diff --git a/next.config.mjs b/next.config.mjs index ef67001f34..21e8e4ddca 100644 --- a/next.config.mjs +++ b/next.config.mjs @@ -288,9 +288,6 @@ const nextConfig = { "keytar", "wreq-js", "zod", - "tls-client-node", - "koffi", - "tough-cookie", "@ngrok/ngrok", "@huggingface/transformers", // copilot-m365-web.ts imports 'ws' as a client-side WebSocket. When bundled, diff --git a/open-sse/executors/chatgpt-web.ts b/open-sse/executors/chatgpt-web.ts index 438565b45c..11bb0ab5bb 100644 --- a/open-sse/executors/chatgpt-web.ts +++ b/open-sse/executors/chatgpt-web.ts @@ -2280,10 +2280,8 @@ async function imageUrlToCachedImageUrl( if (response.text == null || response.text.length === 0) return null; - // tls-client-node already returns binary bodies as a "data:;base64,..." - // string (see node_modules/tls-client-node/dist/response.js — its bytes() - // method splits on the comma to extract base64). Decode back into bytes - // so we can hand them to the cache. + // The shared browser transport returns binary bodies as a + // "data:;base64,..." string. Decode it back into bytes for the cache. let bytes: Buffer; let mime: string; if (/^data:[^;]{1,256};base64,/.test(response.text)) { diff --git a/open-sse/executors/grok-web.ts b/open-sse/executors/grok-web.ts index a99bc2590a..939a86903c 100644 --- a/open-sse/executors/grok-web.ts +++ b/open-sse/executors/grok-web.ts @@ -939,8 +939,8 @@ export class GrokWebExecutor extends BaseExecutor { // Fetch from Grok via TLS-impersonating client (#3180). // Grok sits behind Cloudflare Enterprise which rejects Node's native TLS - // fingerprint even with valid sso+sso-rw cookies. We use tls-client-node - // to send a Chrome-like handshake instead. + // fingerprint even with valid sso+sso-rw cookies. The pinned wreq-js + // transport sends a Chrome-like handshake instead. let tlsResult: TlsFetchResult; try { tlsResult = await tlsFetchGrok(GROK_CHAT_API, { diff --git a/open-sse/executors/lmarena.ts b/open-sse/executors/lmarena.ts index 42bcd9cf7e..8f2c06e670 100644 --- a/open-sse/executors/lmarena.ts +++ b/open-sse/executors/lmarena.ts @@ -2,8 +2,8 @@ * LMArenaExecutor — Arena (formerly LMArena) web-session provider. * * Routes requests through arena.ai create-evaluation with session cookies. - * Upstream sits behind Cloudflare; traffic goes through tls-client-node Chrome - * impersonation (see services/lmarenaTlsClient.ts). + * Upstream sits behind Cloudflare; traffic goes through wreq-js Chrome + * impersonation with isolated ephemeral cookies (see services/lmarenaTlsClient.ts). * * Helpers: open-sse/executors/lmarena/{cookie,models,stream,response}.ts */ @@ -174,7 +174,6 @@ export class LMArenaExecutor extends BaseExecutor { body: JSON.stringify(transformedBody), signal: ctx.signal, stream: ctx.stream, - streamEofSymbol: "__OMNIROUTE_LMARENA_EOF_NEVER__", }); const failed = mapFailedTlsResult({ diff --git a/open-sse/executors/lmarena/models.ts b/open-sse/executors/lmarena/models.ts index fbfe277809..fa8362e1c2 100644 --- a/open-sse/executors/lmarena/models.ts +++ b/open-sse/executors/lmarena/models.ts @@ -6,9 +6,9 @@ export const LMARENA_API_BASE = "https://arena.ai"; export const LMARENA_STREAM_URL = `${LMARENA_API_BASE}/nextjs-api/stream/create-evaluation`; /** * Current Chrome stable UA (header surface). - * TLS JA3 profile is separate: tls-client-node tops out at chrome_146 — see - * LMARENA_PROFILE in lmarenaTlsClient.ts. Headers track the live browser string; - * fingerprint stays at the newest native profile we can actually impersonate. + * TLS JA3/JA4 profile is separate: the provider-tested wreq-js profile is pinned + * to chrome_146 in lmarenaTlsClient.ts while headers track the live browser string. + * Treat that deliberate version skew as a WAF-sensitive compatibility surface. */ export const LMARENA_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36"; diff --git a/open-sse/executors/lmarena/response.ts b/open-sse/executors/lmarena/response.ts index acc86f915a..ea6d8313ea 100644 --- a/open-sse/executors/lmarena/response.ts +++ b/open-sse/executors/lmarena/response.ts @@ -114,7 +114,7 @@ export function mapTlsUnavailable( return { response: errorResponse( 502, - `Arena TLS impersonation unavailable: ${error.message}. Install/repair tls-client-node native binary.`, + `Arena TLS impersonation unavailable: ${error.message}. Verify the wreq-js 3.0.0 native addon.`, "upstream_error", "TLS_CLIENT_UNAVAILABLE" ), diff --git a/open-sse/executors/notion-web.ts b/open-sse/executors/notion-web.ts index b53bd69669..15e96aafd0 100644 --- a/open-sse/executors/notion-web.ts +++ b/open-sse/executors/notion-web.ts @@ -22,7 +22,7 @@ * chunk — safer than assuming unverified incremental-delta semantics. * * Auth: Cookie-based (token_v2 [+ optional space_id, notion_browser_id, user_id]) - * Method: Browser-TLS impersonation via tls-client-node (Chrome JA3). Plain + * Method: Browser-TLS impersonation via pinned wreq-js (Chrome JA3/JA4). Plain * Node/undici fetch is rejected by Notion's edge with in-band * `temporarily-unavailable` (HTTP 200, empty assistant text) — curl/Schannel * and Chrome work with the same cookie + body. See services/notionTlsClient.ts. @@ -60,10 +60,7 @@ import { messagesForNotionTranscript, type NotionAgentOptions, } from "../services/notionTranscriptBuilder.ts"; -import { - tlsFetchNotion, - TlsClientUnavailableError, -} from "../services/notionTlsClient.ts"; +import { tlsFetchNotion, TlsClientUnavailableError } from "../services/notionTlsClient.ts"; // Re-exported for unit tests that destructure `mod.` on this module. export { @@ -225,7 +222,6 @@ function extractUserIdFromCookie(cookie: string): string { return extractNotionUserIdFromCookie(cookie); } - /** * Notion's undocumented inference API does not return token usage. * Emit a cheap char-based estimate so clients don't see a constant @@ -236,9 +232,7 @@ export function estimateNotionUsage( messages: NotionMessage[] | undefined, content: string ): { prompt_tokens: number; completion_tokens: number; total_tokens: number; estimated: true } { - const promptText = (messages || []) - .map((m) => extractNotionMessageText(m?.content)) - .join("\n"); + const promptText = (messages || []).map((m) => extractNotionMessageText(m?.content)).join("\n"); // ~4 chars/token (English-ish); at least 1 when there is any text. const prompt_tokens = promptText ? Math.max(1, Math.ceil(promptText.length / 4)) : 0; const completion_tokens = content ? Math.max(1, Math.ceil(content.length / 4)) : 0; @@ -393,9 +387,8 @@ function buildNotionExecuteHeaders(opts: { const isCustom = Boolean(opts.agent?.workflowId); // Browser uses /agent/?wfv=chat for custom agents. const agentPathId = (opts.agent?.workflowId || "").replace(/-/g, ""); - const referer = isCustom && agentPathId - ? `${BASE_URL}/agent/${agentPathId}?wfv=chat` - : `${BASE_URL}/ai`; + const referer = + isCustom && agentPathId ? `${BASE_URL}/agent/${agentPathId}?wfv=chat` : `${BASE_URL}/ai`; const reqHeaders: Record = { "Content-Type": "application/json", "User-Agent": USER_AGENT, @@ -453,11 +446,8 @@ export function resolveNotionAgentOptions( "agent_id", ]) || ""; const pageFromPs = - readProviderSpecificString(ps, [ - "contextPageId", - "context_page_id", - "notionContextPageId", - ]) || ""; + readProviderSpecificString(ps, ["contextPageId", "context_page_id", "notionContextPageId"]) || + ""; const readCookie = (name: string): string => { const m = cookie.match(new RegExp(`(?:^|;\\s*)${name}=([^;]+)`, "i")); @@ -477,10 +467,7 @@ export function resolveNotionAgentOptions( readCookie("agent_id") ); const contextPageId = - pageFromPs || - readCookie("context_page_id") || - readCookie("notion_context_page_id") || - ""; + pageFromPs || readCookie("context_page_id") || readCookie("notion_context_page_id") || ""; return { workflowId: workflowId || undefined, @@ -510,8 +497,7 @@ async function sendNotionInferenceRequest(opts: { body: JSON.stringify(reqBody), signal: signal ?? undefined, // Inference can take a while (tool-autoload + LLM first token). - timeoutMs: - Number.parseInt(process.env.OMNIROUTE_NOTION_TLS_TIMEOUT_MS || "", 10) || 180_000, + timeoutMs: Number.parseInt(process.env.OMNIROUTE_NOTION_TLS_TIMEOUT_MS || "", 10) || 180_000, }); status = tlsRes.status; rawText = tlsRes.text ?? ""; @@ -634,8 +620,7 @@ export class NotionWebExecutor extends BaseExecutor { const inboundHeaders = (input.clientHeaders as Record | null | undefined) ?? ((input as { headers?: Record }).headers as - | Record - | undefined); + Record | undefined); const clientThreadId = readClientThreadId(requestBody, inboundHeaders ?? undefined); // Namespace the thread cache PER CALLER (hash of the caller's cookie) AND by custom // agent, so (a) two users of the same Notion space never share a cached thread @@ -738,7 +723,10 @@ export class NotionWebExecutor extends BaseExecutor { // One automatic retry for transient Notion faults — same threadId, never create again if (isFailedAttempt(attempt) && attempt.retryable) { - const delayMs = process.env.NODE_ENV === "test" || process.env.VITEST ? 20 : 700 + Math.floor(Math.random() * 400); + const delayMs = + process.env.NODE_ENV === "test" || process.env.VITEST + ? 20 + : 700 + Math.floor(Math.random() * 400); await new Promise((r) => setTimeout(r, delayMs)); attempt = await runOnce({ createThread: false, threadId }); } diff --git a/open-sse/executors/perplexity-web.ts b/open-sse/executors/perplexity-web.ts index fa1a0f0258..ff13b4085d 100644 --- a/open-sse/executors/perplexity-web.ts +++ b/open-sse/executors/perplexity-web.ts @@ -16,10 +16,7 @@ import { import { prepareToolMessages } from "../translator/webTools.ts"; import { buildToolModeResponse } from "./chatgptWebTools.ts"; import { sanitizeErrorMessage } from "../utils/error.ts"; -import { - buildSessionCookieHeader, - mergeRefreshedCookie, -} from "../utils/nextAuthCookie.ts"; +import { buildSessionCookieHeader, mergeRefreshedCookie } from "../utils/nextAuthCookie.ts"; import { PPLX_SSE_ENDPOINT, PPLX_USER_AGENT, @@ -362,7 +359,15 @@ export class PerplexityWebExecutor extends BaseExecutor { super("perplexity-web", { id: "perplexity-web", baseUrl: PPLX_SSE_ENDPOINT }); } - async execute({ model, body, stream, credentials, signal, log, onCredentialsRefreshed }: ExecuteInput) { + async execute({ + model, + body, + stream, + credentials, + signal, + log, + onCredentialsRefreshed, + }: ExecuteInput) { const bodyObj = (body || {}) as Record; const rawMessages = bodyObj.messages as Array> | undefined; if (!rawMessages || !Array.isArray(rawMessages) || rawMessages.length === 0) { @@ -496,7 +501,7 @@ export class PerplexityWebExecutor extends BaseExecutor { if (isCloudflareChallenge(response.text)) { errMsg = "Cloudflare blocked the request — Perplexity's edge rejected this server's TLS fingerprint " + - "(common on VPS/datacenter IPs). Ensure tls-client-node is installed with its native binary, " + + "(common on VPS/datacenter IPs). Verify the wreq-js 3.0.0 native addon, " + "or route perplexity-web through a residential proxy."; log?.error?.("PPLX-WEB", "Cloudflare challenge detected — TLS bypass failed"); } else { diff --git a/open-sse/services/__tests__/chatgptTlsClient.test.ts b/open-sse/services/__tests__/chatgptTlsClient.test.ts index 92bb78143e..3b8480c4c8 100644 --- a/open-sse/services/__tests__/chatgptTlsClient.test.ts +++ b/open-sse/services/__tests__/chatgptTlsClient.test.ts @@ -1,15 +1,15 @@ /** * Regression tests for the proxy-leak fix in chatgptTlsClient. * - * Bug context (#2022): tlsFetchChatGpt() built its native tls-client-node - * requestOptions without a `proxyUrl` field, so every chatgpt-web call + * Bug context (#2022): tlsFetchChatGpt() built its native transport options + * without a `proxyUrl` field, so every chatgpt-web call * egressed with the bare host IP regardless of the dashboard proxy config - * or HTTP_PROXY / HTTPS_PROXY env vars (the koffi-loaded Go binary does not - * consult Go's `http.ProxyFromEnvironment`). + * or HTTP_PROXY / HTTPS_PROXY env vars. Native browser transports require the + * resolved proxy to be passed explicitly. * * These tests pin the resolution-order contract: * 1. Per-call `options.proxyUrl` wins. - * 2. OMNIROUTE_TLS_PROXY_URL env var (single-flag opt-in). + * 2. Request-scoped dashboard/account proxy context. * 3. POSIX-standard HTTPS_PROXY / HTTP_PROXY / ALL_PROXY (and lowercase variants). * 4. Otherwise undefined (no proxy). * @@ -24,7 +24,6 @@ import { describe, it, beforeEach, afterEach, expect } from "vitest"; import { tlsFetchChatGpt, __setTlsFetchOverrideForTesting } from "../chatgptTlsClient.ts"; const PROXY_ENV_KEYS = [ - "OMNIROUTE_TLS_PROXY_URL", "HTTPS_PROXY", "https_proxy", "HTTP_PROXY", @@ -62,7 +61,6 @@ describe("chatgptTlsClient — proxy plumbing (#2022)", async () => { }); it("per-call proxyUrl overrides everything", async () => { - process.env.OMNIROUTE_TLS_PROXY_URL = "http://env-omni:0/"; process.env.HTTPS_PROXY = "http://env-https:0/"; let observedUrl: string | undefined; diff --git a/open-sse/services/__tests__/grokTlsClient.test.ts b/open-sse/services/__tests__/grokTlsClient.test.ts index 8b391d10d7..63a7f2af63 100644 --- a/open-sse/services/__tests__/grokTlsClient.test.ts +++ b/open-sse/services/__tests__/grokTlsClient.test.ts @@ -1,15 +1,15 @@ /** * Regression tests for the proxy-leak fix in grokTlsClient. * - * Bug context (#3180): tlsFetchGrok() built its native tls-client-node - * requestOptions without a `proxyUrl` field, so every grok-web call + * Bug context (#3180): tlsFetchGrok() built its native transport options + * without a `proxyUrl` field, so every grok-web call * egressed with the bare host IP regardless of the dashboard proxy config - * or HTTP_PROXY / HTTPS_PROXY env vars (the koffi-loaded Go binary does not - * consult Go's `http.ProxyFromEnvironment`). + * or HTTP_PROXY / HTTPS_PROXY env vars. Native browser transports require the + * resolved proxy to be passed explicitly. * * These tests pin the resolution-order contract: * 1. Per-call `options.proxyUrl` wins. - * 2. OMNIROUTE_TLS_PROXY_URL env var (single-flag opt-in). + * 2. Request-scoped dashboard/account proxy context. * 3. POSIX-standard HTTPS_PROXY / HTTP_PROXY / ALL_PROXY (and lowercase variants). * 4. Otherwise undefined (no proxy). * @@ -24,7 +24,6 @@ import { describe, it, beforeEach, afterEach, expect } from "vitest"; import { tlsFetchGrok, __setTlsFetchOverrideForTesting } from "../grokTlsClient.ts"; const PROXY_ENV_KEYS = [ - "OMNIROUTE_TLS_PROXY_URL", "HTTPS_PROXY", "https_proxy", "HTTP_PROXY", @@ -62,7 +61,6 @@ describe("grokTlsClient — proxy plumbing (#3180)", async () => { }); it("per-call proxyUrl overrides everything", async () => { - process.env.OMNIROUTE_TLS_PROXY_URL = "http://env-omni:0/"; process.env.HTTPS_PROXY = "http://env-https:0/"; let observedUrl: string | undefined; diff --git a/open-sse/services/chatgptTlsClient.ts b/open-sse/services/chatgptTlsClient.ts index fd7b3f4550..ead41d7b67 100644 --- a/open-sse/services/chatgptTlsClient.ts +++ b/open-sse/services/chatgptTlsClient.ts @@ -2,8 +2,8 @@ * Browser-TLS-impersonating HTTP client for chatgpt.com. * * Thin re-export over the shared `tlsClientBase.ts` factory - * (`createTlsClientModule`). All provider-agnostic logic (sidecar lifecycle, - * streaming tail-file, proxy resolution, error classes, SSE detection) lives + * (`createTlsClientModule`). All provider-agnostic logic (wreq-js transport + * pooling, direct streaming, proxy resolution, deadlines, SSE detection) lives * in the base module; this file supplies only ChatGPT-specific config and * preserves the original public export surface. */ @@ -24,9 +24,9 @@ const STREAM_FIRST_BYTE_TIMEOUT_MS = export const tlsClientModule = createTlsClientModule({ providerName: "ChatGPT", tlsProfile: "firefox_148", + emulationOs: "macos", domain: "https://chatgpt.com", - tempDirPrefix: "cgpt-stream-", - tailFileVariant: "A", + streamEofPolicy: "include", responseValidation: "sse", exportCloudflareCheck: false, exposeStreamingForTesting: true, diff --git a/open-sse/services/claudeTlsClient.ts b/open-sse/services/claudeTlsClient.ts index 4ab4746195..9fc19e957f 100644 --- a/open-sse/services/claudeTlsClient.ts +++ b/open-sse/services/claudeTlsClient.ts @@ -2,8 +2,8 @@ * Browser-TLS-impersonating HTTP client for claude.ai. * * Thin re-export over the shared `tlsClientBase.ts` factory - * (`createTlsClientModule`). All provider-agnostic logic (sidecar lifecycle, - * streaming tail-file, proxy resolution, error classes, SSE detection) lives + * (`createTlsClientModule`). All provider-agnostic logic (wreq-js transport + * pooling, direct streaming, proxy resolution, deadlines, SSE detection) lives * in the base module; this file supplies only Claude-specific config and * preserves the original public export surface. */ @@ -24,13 +24,13 @@ const HARD_TIMEOUT_GRACE_MS = export const tlsClientModule = createTlsClientModule({ providerName: "Claude", tlsProfile: `chrome_${CLAUDE_TLS_BROWSER_MAJOR_VERSION}`, + emulationOs: "linux", domain: "https://claude.ai", - tempDirPrefix: "cgpt-stream-", - tailFileVariant: "A", + streamEofPolicy: "include", responseValidation: "sse", exportCloudflareCheck: false, exposeStreamingForTesting: true, - // Claude waits indefinitely for the first SSE byte (original 2-arg waitForContent). + // Claude allows the native/hard request deadline to bound a slow first SSE byte. defaultTimeoutMs: DEFAULT_TIMEOUT_MS, hardTimeoutGraceMs: HARD_TIMEOUT_GRACE_MS, firstByteTimeoutMs: Number.POSITIVE_INFINITY, diff --git a/open-sse/services/claudeTurnstileSolver.ts b/open-sse/services/claudeTurnstileSolver.ts index a9164839a3..51505c20e8 100644 --- a/open-sse/services/claudeTurnstileSolver.ts +++ b/open-sse/services/claudeTurnstileSolver.ts @@ -7,7 +7,7 @@ * 3. Waits for Turnstile challenge to appear * 4. Waits for challenge to be solved (with retry) * 5. Extracts cf_clearance cookie - * 6. Returns fresh cookie for tls-client-node + * 6. Returns a fresh cookie for the isolated wreq-js request */ import type { Browser, Page } from "playwright"; diff --git a/open-sse/services/grokTlsClient.ts b/open-sse/services/grokTlsClient.ts index 00a952dd70..e37d2b170d 100644 --- a/open-sse/services/grokTlsClient.ts +++ b/open-sse/services/grokTlsClient.ts @@ -2,8 +2,8 @@ * Browser-TLS-impersonating HTTP client for grok.com. * * Thin re-export over the shared `tlsClientBase.ts` factory - * (`createTlsClientModule`). All provider-agnostic logic (sidecar lifecycle, - * streaming tail-file, proxy resolution, error classes, Cloudflare challenge + * (`createTlsClientModule`). All provider-agnostic logic (wreq-js transport + * pooling, direct streaming, proxy resolution, deadlines, Cloudflare challenge * detection) lives in the base module; this file supplies only Grok-specific * config and preserves the original public export surface. */ @@ -22,9 +22,9 @@ const HARD_TIMEOUT_GRACE_MS = export const tlsClientModule = createTlsClientModule({ providerName: "Grok", tlsProfile: "chrome_146", + emulationOs: "linux", domain: "https://grok.com", - tempDirPrefix: "grok-stream-", - tailFileVariant: "B1", + streamEofPolicy: "exclude", responseValidation: "cf", exportCloudflareCheck: true, defaultTimeoutMs: DEFAULT_TIMEOUT_MS, diff --git a/open-sse/services/lmarenaTlsClient.ts b/open-sse/services/lmarenaTlsClient.ts index 131acb550e..cc515d3218 100644 --- a/open-sse/services/lmarenaTlsClient.ts +++ b/open-sse/services/lmarenaTlsClient.ts @@ -2,8 +2,8 @@ * Browser-TLS-impersonating HTTP client for arena.ai. * * Thin re-export over the shared `tlsClientBase.ts` factory - * (`createTlsClientModule`). All provider-agnostic logic (sidecar lifecycle, - * streaming tail-file, proxy resolution, error classes, Cloudflare challenge + * (`createTlsClientModule`). All provider-agnostic logic (wreq-js transport + * pooling, direct streaming, proxy resolution, deadlines, Cloudflare challenge * detection) lives in the base module; this file supplies only LMArena-specific * config and preserves the original public export surface. */ @@ -20,11 +20,12 @@ const HARD_TIMEOUT_GRACE_MS = 10_000; export const tlsClientModule = createTlsClientModule({ providerName: "LMArena", tlsProfile: "chrome_146", + emulationOs: "windows", domain: "https://lmarena.ai", // LMArena's proxy resolution domain is hardcoded to arena.ai, not the config domain. proxyDomainOverride: "https://arena.ai", - tempDirPrefix: "LMArena-stream-", - tailFileVariant: "B2", + streamEofPolicy: "none", + streamEofSymbol: "", responseValidation: "cf", exportCloudflareCheck: true, defaultTimeoutMs: DEFAULT_TIMEOUT_MS, diff --git a/open-sse/services/notionTlsClient.ts b/open-sse/services/notionTlsClient.ts index 2dc56e5f35..6c2bd18b73 100644 --- a/open-sse/services/notionTlsClient.ts +++ b/open-sse/services/notionTlsClient.ts @@ -2,8 +2,8 @@ * Browser-TLS-impersonating HTTP client for app.notion.com. * * Thin re-export over the shared `tlsClientBase.ts` factory - * (`createTlsClientModule`). All provider-agnostic logic (sidecar lifecycle, - * streaming tail-file, proxy resolution, error classes, SSE detection, + * (`createTlsClientModule`). All provider-agnostic logic (wreq-js transport + * pooling, direct streaming, proxy resolution, deadlines, SSE detection, * Cloudflare challenge detection) lives in the base module; this file supplies * only Notion-specific config and preserves the original public export surface. */ @@ -22,9 +22,9 @@ const HARD_TIMEOUT_GRACE_MS = export const tlsClientModule = createTlsClientModule({ providerName: "Notion", tlsProfile: "chrome_146", + emulationOs: "windows", domain: "https://app.notion.com", - tempDirPrefix: "pplx-stream-", - tailFileVariant: "A", + streamEofPolicy: "include", responseValidation: "sse", exportCloudflareCheck: true, defaultTimeoutMs: DEFAULT_TIMEOUT_MS, diff --git a/open-sse/services/perplexityTlsClient.ts b/open-sse/services/perplexityTlsClient.ts index bc736476c3..c3c83c8b11 100644 --- a/open-sse/services/perplexityTlsClient.ts +++ b/open-sse/services/perplexityTlsClient.ts @@ -2,8 +2,8 @@ * Browser-TLS-impersonating HTTP client for www.perplexity.ai. * * Thin re-export over the shared `tlsClientBase.ts` factory - * (`createTlsClientModule`). All provider-agnostic logic (sidecar lifecycle, - * streaming tail-file, proxy resolution, error classes, SSE detection, + * (`createTlsClientModule`). All provider-agnostic logic (wreq-js transport + * pooling, direct streaming, proxy resolution, deadlines, SSE detection, * Cloudflare challenge detection) lives in the base module; this file supplies * only Perplexity-specific config and preserves the original public export * surface. @@ -23,9 +23,9 @@ const HARD_TIMEOUT_GRACE_MS = export const tlsClientModule = createTlsClientModule({ providerName: "Perplexity", tlsProfile: "firefox_148", + emulationOs: "macos", domain: "https://www.perplexity.ai", - tempDirPrefix: "pplx-stream-", - tailFileVariant: "A", + streamEofPolicy: "include", responseValidation: "sse", exportCloudflareCheck: true, defaultTimeoutMs: DEFAULT_TIMEOUT_MS, diff --git a/open-sse/services/tlsClientBase.ts b/open-sse/services/tlsClientBase.ts index 11249864f2..1a0ab04857 100644 --- a/open-sse/services/tlsClientBase.ts +++ b/open-sse/services/tlsClientBase.ts @@ -1,50 +1,54 @@ /** - * Shared TLS client infrastructure — a factory-style base that consolidates - * 6 nearly-identical per-provider TLS client files into one source of truth. + * Shared browser-impersonating HTTP transport for the six web-cookie providers. * - * Each provider file calls `createTlsClientModule(config)` to obtain its - * provider-specific `tlsFetch` and `__setTlsFetchOverrideForTesting` exports. + * Provider wrappers keep their existing `tlsFetch*` APIs while this module owns + * wreq-js loading, transport pooling, proxy selection, deadlines, byte responses, + * SSE/NDJSON validation, EOF handling, and cancellation. * - * TailFile variants: - * A — Uint8Array enqueue, includes EOF symbol, substring-based cleanup - * ChatGPT, Claude, Perplexity, Notion - * B1 — Buffer.from enqueue, excludes EOF symbol, inline drainRemaining loop - * Grok - * B2 — Buffer.from enqueue, excludes EOF symbol, extracted helpers - * LMArena - * - * Response validation: - * sse — checks `looksLikeSse(peek)`, falls back to buffered - * ChatGPT, Claude, Perplexity, Notion - * cf — checks `isCloudflareChallenge(peek)` → 403, HTML → 502 - * Grok, LMArena + * Every wreq request uses an ephemeral cookie scope. Transports are reused only + * within one provider module and are keyed by browser profile, emulated OS, and + * resolved proxy; no cookie jar or session identifier is shared between calls. */ -// --------------------------------------------------------------------------- -// Node imports -// --------------------------------------------------------------------------- -import { tmpdir } from "node:os"; -import { randomUUID } from "node:crypto"; -import { join, dirname } from "node:path"; -import { open, unlink, rmdir, readFile, mkdtemp, stat } from "node:fs/promises"; +import { createRequire } from "node:module"; -// --------------------------------------------------------------------------- -// Proxy resolution — every provider file imports both of these -// --------------------------------------------------------------------------- import { resolveProxyForRequest } from "../utils/proxyFetch.ts"; import { resolveTlsClientProxyUrl } from "./tlsClientProxy.ts"; -import { buildNativeTlsClientOptions } from "./tlsClientDownloadDir.ts"; -// --------------------------------------------------------------------------- -// Types -// --------------------------------------------------------------------------- +const runtimeRequire = createRequire(import.meta.url); +const transportClosers = new Set<() => void>(); +let exitHookInstalled = false; + +type EmulationOs = "windows" | "macos" | "linux" | "android" | "ios"; + +export type IterableHeaders = Iterable<[string, string]> & { + getSetCookie?: () => string[]; +}; + +export interface ReadableBodyLike { + getReader: () => ReadableStreamDefaultReader; + cancel?: (reason?: unknown) => Promise; +} export interface TlsResponseLike { status: number; - headers: Record; - body: string; + headers: Record | IterableHeaders; + body: string | ReadableBodyLike | null; + text?: () => Promise; + bytes?: () => Promise; } +interface WreqTransportLike { + close: () => Promise | void; +} + +export interface WreqRuntimeLike { + createTransport: (options: Record) => Promise; + fetch: (url: string, options: Record) => Promise; +} + +export type WreqRuntimeLoader = () => Promise; + export interface TlsFetchResult { status: number; headers: Headers; @@ -64,60 +68,41 @@ export interface TlsFetchOptions { proxyUrl?: string; } -// --------------------------------------------------------------------------- -// Factory config (one instance per provider stub) -// --------------------------------------------------------------------------- - export interface TlsClientConfig { /** Human-readable provider name for logs and error messages. */ providerName: string; - /** TLS profile identifier (e.g. "chrome_146") */ + /** Browser profile identifier, for example `chrome_146` or `firefox_148`. */ tlsProfile: string; - /** Default upstream domain for proxy resolution (e.g. "https://chatgpt.com") */ + /** Operating system paired with the browser profile. */ + emulationOs?: EmulationOs; + /** Default upstream domain used by proxy resolution. */ domain: string; - /** Temp directory prefix (e.g. "cgpt-stream-") */ - tempDirPrefix: string; - /** EOF symbol for streaming (default "[DONE]") */ + /** @deprecated wreq-js streams directly and ignores this compatibility field. */ + tempDirPrefix?: string; + /** Default EOF marker. An empty string disables marker filtering. */ streamEofSymbol?: string; - /** Default timeout in ms (default 60_000) */ + /** Native request timeout in milliseconds. */ defaultTimeoutMs?: number; - /** Hard timeout grace period in ms (default 10_000) */ + /** Additional JavaScript-side hard-timeout grace period. */ hardTimeoutGraceMs?: number; - /** First-byte timeout for waitForContent (default 5_000; ChatGPT uses 30_000) */ + /** Delay after which a late first byte is returned as a buffered response. */ firstByteTimeoutMs?: number; - /** - * TailFile variant: - * "A" — Uint8Array enqueue, includes EOF, substring cleanup - * "B1" — Buffer.from enqueue, excludes EOF, inline drainRemaining - * "B2" — Buffer.from enqueue, excludes EOF, extracted helpers - */ - tailFileVariant: "A" | "B1" | "B2"; - /** - * Response validation mode: - * "sse" — check looksLikeSse → fall back to buffered - * "cf" — check isCloudflareChallenge → 403, HTML → 502, else stream - */ + /** How a detected EOF marker is exposed; `none` disables marker filtering. */ + streamEofPolicy?: "include" | "exclude" | "none"; + /** @deprecated Compatibility alias: `A` includes EOF; `B1`/`B2` exclude it. */ + tailFileVariant?: "A" | "B1" | "B2"; + /** `sse` validates SSE prefixes; `cf` rejects Cloudflare/HTML responses. */ responseValidation: "sse" | "cf"; - /** - * Optional override for proxy resolution domain (e.g., LMArena uses - * "https://arena.ai" hardcoded instead of the config domain). - */ + /** Optional proxy-resolution domain override (LMArena uses arena.ai). */ proxyDomainOverride?: string; - /** - * Whether to export `isCloudflareChallenge` from the provider stub. - * Grok, LMArena, Perplexity, Notion all export it. - */ + /** Whether the provider module exposes the Cloudflare detection helper. */ exportCloudflareCheck: boolean; - /** - * Whether to expose `__tlsFetchStreamingForTesting` (ChatGPT only). - */ + /** Whether to expose the direct-stream dependency-injection seam. */ exposeStreamingForTesting?: boolean; + /** External-runtime seam used by focused tests; production loads wreq-js lazily. */ + wreqRuntimeLoader?: WreqRuntimeLoader; } -// --------------------------------------------------------------------------- -// Error classes -// --------------------------------------------------------------------------- - export class TlsClientUnavailableError extends Error { override name = "TlsClientUnavailableError"; } @@ -126,28 +111,93 @@ export class TlsClientHangError extends Error { override name = "TlsClientHangError"; } -// --------------------------------------------------------------------------- -// Shared helpers (identical across all 6 providers) -// --------------------------------------------------------------------------- - -export function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - export function makeAbortError(signal: AbortSignal): Error { const reason = signal.reason; if (reason instanceof Error) return reason; - const err = new Error(typeof reason === "string" ? reason : "The operation was aborted"); - err.name = "AbortError"; - return err; + const error = new Error(typeof reason === "string" ? reason : "The operation was aborted"); + error.name = "AbortError"; + return error; } -export function toHeaders(raw: Record | null | undefined): Headers { - const h = new Headers(); - for (const [k, vs] of Object.entries(raw || {})) { - for (const v of vs) h.append(k, v); +export function toHeaders( + raw: Record | IterableHeaders | null | undefined +): Headers { + const headers = new Headers(); + if (!raw) return headers; + + const iterator = (raw as Partial)[Symbol.iterator]; + if (typeof iterator === "function") { + const iterable = raw as IterableHeaders; + const setCookies = typeof iterable.getSetCookie === "function" ? iterable.getSetCookie() : []; + for (const [name, value] of iterable) { + if (name.toLowerCase() !== "set-cookie" || setCookies.length === 0) { + headers.append(name, value); + } + } + for (const value of setCookies) headers.append("set-cookie", value); + return headers; } - return h; + + for (const [name, values] of Object.entries(raw)) { + for (const value of values) headers.append(name, value); + } + return headers; +} + +function isReadableBody(body: TlsResponseLike["body"]): body is ReadableBodyLike { + return body !== null && typeof body !== "string" && typeof body.getReader === "function"; +} + +function concatChunks(chunks: Uint8Array[]): Uint8Array { + const length = chunks.reduce((total, chunk) => total + chunk.byteLength, 0); + const combined = new Uint8Array(length); + let offset = 0; + for (const chunk of chunks) { + combined.set(chunk, offset); + offset += chunk.byteLength; + } + return combined; +} + +async function readAllChunks( + reader: ReadableStreamDefaultReader, + initialChunks: Uint8Array[] = [], + readNext: () => Promise> = () => reader.read() +): Promise { + const chunks = [...initialChunks]; + while (true) { + const next = await readNext(); + if (next.done) return concatChunks(chunks); + chunks.push(next.value); + } +} + +async function readTlsResponseText( + response: TlsResponseLike, + onReader?: (reader: ReadableStreamDefaultReader) => void +): Promise { + if (typeof response.body === "string") return response.body; + if (isReadableBody(response.body)) { + const reader = response.body.getReader(); + onReader?.(reader); + return new TextDecoder().decode(await readAllChunks(reader)); + } + if (typeof response.text === "function") return response.text(); + return ""; +} + +async function readTlsResponseBytes( + response: TlsResponseLike, + onReader?: (reader: ReadableStreamDefaultReader) => void +): Promise { + if (isReadableBody(response.body)) { + const reader = response.body.getReader(); + onReader?.(reader); + return readAllChunks(reader); + } + if (typeof response.bytes === "function") return response.bytes(); + if (typeof response.body === "string") return Buffer.from(response.body, "binary"); + return new Uint8Array(); } export async function raceWithTimeout( @@ -155,111 +205,42 @@ export async function raceWithTimeout( timeoutMs: number, signal: AbortSignal | null | undefined ): Promise { - // If no signal, just race with a simple timeout. - if (!signal) { - return await Promise.race([ - promise, - new Promise((_, reject) => { - setTimeout(() => reject(new TlsClientHangError()), timeoutMs); - }), - ]); - } - - // With signal, race against both timeout and abort. return await new Promise((resolve, reject) => { let settled = false; + let timer: ReturnType | undefined; - const done = (fn: () => void) => { - if (!settled) { - settled = true; - fn(); - } + const onAbort = (): void => { + settle(() => reject(makeAbortError(signal!))); + }; + const cleanup = (): void => { + if (timer) clearTimeout(timer); + signal?.removeEventListener("abort", onAbort); + }; + const settle = (complete: () => void): void => { + if (settled) return; + settled = true; + cleanup(); + complete(); }; - const timer = setTimeout(() => { - done(() => reject(new TlsClientHangError())); - }, timeoutMs); - - const onAbort = () => { - done(() => reject(makeAbortError(signal))); - }; - - if (signal.aborted) { + timer = setTimeout( + () => settle(() => reject(new TlsClientHangError())), + Math.max(0, timeoutMs) + ); + if (signal?.aborted) { onAbort(); } else { - signal.addEventListener("abort", onAbort, { once: true }); + signal?.addEventListener("abort", onAbort, { once: true }); } promise.then( - (v) => { - done(() => { - clearTimeout(timer); - signal.removeEventListener("abort", onAbort); - resolve(v); - }); - }, - (e) => { - done(() => { - clearTimeout(timer); - signal.removeEventListener("abort", onAbort); - reject(e); - }); - } + (value) => settle(() => resolve(value)), + (error) => settle(() => reject(error)) ); }); } -/** Read up to N bytes from a file, returning the utf-8 decoded text. */ -export async function readFirstBytes(path: string, n: number): Promise { - const fd = await open(path, "r"); - try { - const buf = Buffer.alloc(n); - const { bytesRead } = await fd.read(buf, 0, n, 0); - return buf.subarray(0, bytesRead).toString("utf8"); - } finally { - await fd.close().catch(() => {}); - } -} - -/** - * Wait for the streaming output file to exist AND contain at least one byte. - * Returns false if the request settles before any bytes arrive (so the caller - * can drain `requestPromise` and surface the real upstream status). Returns - * true as soon as the file has data. - */ -export async function waitForContent( - path: string, - timeoutMs: number, - requestPromise: Promise -): Promise { - let requestSettled = false; - requestPromise.then( - () => { - requestSettled = true; - }, - () => { - requestSettled = true; - } - ); - const start = Date.now(); - while (Date.now() - start < timeoutMs) { - try { - const s = await stat(path); - if (s.size > 0) return true; - } catch { - // file doesn't exist yet - } - if (requestSettled) return false; - await sleep(25); - } - return false; -} - -/** - * Returns true if the peeked response body looks like an SSE stream — i.e., - * begins (after any leading whitespace) with one of the SSE field markers - * (`data:`, `event:`, `id:`, `retry:`) or a comment line (`:`). - */ +/** Return true when a prefix begins with an SSE field or comment marker. */ export function looksLikeSse(text: string): boolean { const trimmed = text.replace(/^[\s\r\n]+/, ""); if (!trimmed) return false; @@ -267,9 +248,7 @@ export function looksLikeSse(text: string): boolean { return /^(data|event|id|retry):/i.test(trimmed); } -/** - * Returns true if the response body is a Cloudflare challenge/interstitial page. - */ +/** Return true when a response prefix is a Cloudflare challenge/interstitial. */ export function isCloudflareChallenge(text: string | null | undefined): boolean { if (!text) return false; return /just a moment|window\._cf_chl_opt|challenges\.cloudflare\.com|attention required|cf-chl/i.test( @@ -277,430 +256,309 @@ export function isCloudflareChallenge(text: string | null | undefined): boolean ); } -// --------------------------------------------------------------------------- -// Temp-path cleanup — two variants -// --------------------------------------------------------------------------- - -/** Variant A: substring-based parent dir extraction (ChatGPT, Claude, Perplexity, Notion) */ -async function cleanupTempPathSubstring(path: string): Promise { - await unlink(path).catch(() => {}); - const dir = path.substring(0, path.lastIndexOf("/")); - await rmdir(dir).catch(() => {}); +function couldBecomeSsePrefix(text: string): boolean { + const trimmed = text.replace(/^[\s\r\n]+/, "").toLowerCase(); + return ["data:", "event:", "id:", "retry:", ":"].some((marker) => marker.startsWith(trimmed)); } -/** Variant B: dirname-based parent dir extraction (Grok, LMArena) */ -async function cleanupTempPathDirname(path: string): Promise { - await unlink(path).catch(() => {}); - await rmdir(dirname(path)).catch(() => {}); +function couldBecomeCloudflareChallenge(text: string): boolean { + const trimmed = text.trimStart().toLowerCase(); + return [ + "just a moment", + "window._cf_chl_opt", + "challenges.cloudflare.com", + "attention required", + "cf-chl", + ].some((marker) => marker.startsWith(trimmed)); } -async function readTextFileIfExists(path: string): Promise { - try { - return await readFile(path, "utf8"); - } catch { - return ""; - } +type EofControlCandidate = "possible" | "matched" | "not-control"; + +function classifyEofControlCandidate(bytes: number[], eofSymbol: string): EofControlCandidate { + const decoded = new TextDecoder().decode(Uint8Array.from(bytes), { stream: true }); + const candidate = decoded.replace(/^[\t\r ]+/, ""); + if (candidate.startsWith(eofSymbol)) return "matched"; + if (eofSymbol.startsWith(candidate)) return "possible"; + + const dataPrefix = "data:"; + const lowerCandidate = candidate.toLowerCase(); + if (dataPrefix.startsWith(lowerCandidate)) return "possible"; + if (!lowerCandidate.startsWith(dataPrefix)) return "not-control"; + + const dataValue = candidate.slice(dataPrefix.length).replace(/^[\t ]+/, ""); + if (dataValue.startsWith(eofSymbol)) return "matched"; + return eofSymbol.startsWith(dataValue) ? "possible" : "not-control"; } -// --------------------------------------------------------------------------- -// TailFile — Variant A -// Uint8Array enqueue, includes EOF symbol, substring cleanup -// Used by: ChatGPT, Claude, Perplexity, Notion -// --------------------------------------------------------------------------- - -function tailFileVariantA( - path: string, +function createEofFilteredStream( + reader: ReadableStreamDefaultReader, + initialChunks: Uint8Array[], eofSymbol: string, - done: Promise, - signal: AbortSignal | null = null, - cleanupPath: string + includeEof: boolean, + readNext: () => Promise>, + onReadError: (error: unknown) => void, + signal: AbortSignal | null, + hardDeadlineAt: number ): ReadableStream { + const queued = [...initialChunks]; + const eofBytes = new TextEncoder().encode(eofSymbol); + let controlCandidate: number[] = []; + let atLineStart = true; + let closed = false; + let deadlineTimer: ReturnType | undefined; + let removeAbortListener = (): void => {}; + + const cleanupLifecycle = (): void => { + if (deadlineTimer) clearTimeout(deadlineTimer); + deadlineTimer = undefined; + removeAbortListener(); + removeAbortListener = (): void => {}; + }; + + const errorStream = ( + controller: ReadableStreamDefaultController, + error: Error, + notifyReadError: boolean + ): void => { + if (closed) return; + closed = true; + controlCandidate = []; + cleanupLifecycle(); + if (notifyReadError) onReadError(error); + void reader.cancel(error).catch(() => {}); + try { + controller.error(error); + } catch { + // The consumer may have closed the stream concurrently. + } + }; + return new ReadableStream({ - async start(controller) { - const fd = await open(path, "r"); - const buf = Buffer.alloc(64 * 1024); - let offset = 0; - let finished = false; - let aborted = false; - let upstreamError: Error | null = null; - - done.then( - () => { - finished = true; - }, - (err) => { - upstreamError = err instanceof Error ? err : new Error(String(err)); - finished = true; - } - ); - - const onAbort = () => { - aborted = true; - }; + start(controller) { if (signal) { - if (signal.aborted) aborted = true; - else signal.addEventListener("abort", onAbort, { once: true }); + const onAbort = (): void => errorStream(controller, makeAbortError(signal), false); + if (signal.aborted) { + onAbort(); + return; + } + signal.addEventListener("abort", onAbort, { once: true }); + removeAbortListener = (): void => signal.removeEventListener("abort", onAbort); } - let errored = false; - try { - while (!aborted) { - const { bytesRead } = await fd.read(buf, 0, buf.length, offset); - if (bytesRead > 0) { - const chunk = buf.subarray(0, bytesRead); - offset += bytesRead; - const text = chunk.toString("utf8"); - if (text.includes(eofSymbol)) { - const cutAt = text.indexOf(eofSymbol) + eofSymbol.length; - controller.enqueue(new Uint8Array(chunk.subarray(0, cutAt))); - break; - } - controller.enqueue(new Uint8Array(chunk)); - } else if (finished) { - if (upstreamError) { - controller.error(upstreamError); - errored = true; - } - break; - } else { - await sleep(25); - } - } - } catch (err) { - controller.error(err); - errored = true; - } finally { - if (signal) signal.removeEventListener("abort", onAbort); - await fd.close().catch(() => {}); - await cleanupTempPathSubstring(cleanupPath); - if (!errored) controller.close(); + if (Number.isFinite(hardDeadlineAt)) { + deadlineTimer = setTimeout( + () => { + errorStream(controller, new TlsClientHangError(), true); + }, + Math.max(0, hardDeadlineAt - Date.now()) + ); + deadlineTimer.unref?.(); } }, - }); -} - -// --------------------------------------------------------------------------- -// TailFile — Variant B1 -// Buffer.from enqueue, excludes EOF symbol, inline drainRemaining loop -// Used by: Grok -// --------------------------------------------------------------------------- - -function tailFileVariantB1( - path: string, - eofSymbol: string, - done: Promise, - signal: AbortSignal | null = null, - cleanupPath: string -): ReadableStream { - return new ReadableStream({ - async start(controller) { - const fd = await open(path, "r"); - const buf = Buffer.alloc(64 * 1024); - let offset = 0; - let finished = false; - let aborted = false; - let upstreamError: Error | null = null; - - done.then( - () => { - finished = true; - }, - (err) => { - upstreamError = err instanceof Error ? err : new Error(String(err)); - finished = true; - } - ); - - const onAbort = () => { - aborted = true; - }; - if (signal) { - if (signal.aborted) aborted = true; - else signal.addEventListener("abort", onAbort, { once: true }); - } - - let errored = false; - try { - while (!aborted) { - const { bytesRead } = await fd.read(buf, 0, buf.length, offset); - if (bytesRead > 0) { - const chunk = buf.subarray(0, bytesRead); - offset += bytesRead; - const text = chunk.toString("utf8"); - - if (text.includes(eofSymbol)) { - const beforeEof = text.substring(0, text.indexOf(eofSymbol)); - if (beforeEof) { - controller.enqueue(Buffer.from(beforeEof, "utf8")); - } - controller.close(); - return; - } - - controller.enqueue(Buffer.from(chunk)); + async pull(controller) { + while (!closed) { + let chunk = queued.shift(); + if (!chunk) { + let next: ReadableStreamReadResult; + try { + next = await readNext(); + } catch (error) { + if (closed) return; + closed = true; + cleanupLifecycle(); + onReadError(error); + await reader.cancel(error).catch(() => {}); + controller.error(error); + return; } - - if (finished) { - // Request finished — drain any remaining bytes then close. - while (true) { - const { bytesRead } = await fd.read(buf, 0, buf.length, offset); - if (bytesRead === 0) break; - const chunk = buf.subarray(0, bytesRead); - offset += bytesRead; - const text = chunk.toString("utf8"); - - if (text.includes(eofSymbol)) { - const beforeEof = text.substring(0, text.indexOf(eofSymbol)); - if (beforeEof) { - controller.enqueue(Buffer.from(beforeEof, "utf8")); - } - controller.close(); - return; - } - - controller.enqueue(Buffer.from(chunk)); + if (closed) return; + if (next.done) { + if (controlCandidate.length > 0) { + controller.enqueue(Uint8Array.from(controlCandidate)); } - - if (upstreamError && !errored) { - errored = true; - controller.error(upstreamError); - return; - } - + controlCandidate = []; + closed = true; + cleanupLifecycle(); controller.close(); return; } - - await sleep(25); + chunk = next.value; } - } catch (err) { - if (!errored) { - errored = true; - controller.error(err instanceof Error ? err : new Error(String(err))); + + if (eofBytes.byteLength === 0) { + controller.enqueue(chunk); + return; } - } finally { - await fd.close().catch(() => {}); - await cleanupTempPathDirname(cleanupPath); - if (signal) signal.removeEventListener("abort", onAbort); - } - }, - }); -} -// --------------------------------------------------------------------------- -// TailFile — Variant B2 -// Buffer.from enqueue, excludes EOF symbol, extracted helpers -// Used by: LMArena -// --------------------------------------------------------------------------- - -type FileHandle = Awaited>; - -function enqueueChunkMaybeEof( - controller: ReadableStreamDefaultController, - chunk: Buffer, - eofSymbol: string -): boolean { - const text = chunk.toString("utf8"); - if (!text.includes(eofSymbol)) { - controller.enqueue(Buffer.from(chunk)); - return false; - } - const beforeEof = text.substring(0, text.indexOf(eofSymbol)); - if (beforeEof) controller.enqueue(Buffer.from(beforeEof, "utf8")); - controller.close(); - return true; -} - -async function drainRemaining( - fd: FileHandle, - buf: Buffer, - offsetRef: { offset: number }, - controller: ReadableStreamDefaultController, - eofSymbol: string -): Promise<"closed" | "drained"> { - while (true) { - const { bytesRead } = await fd.read(buf, 0, buf.length, offsetRef.offset); - if (bytesRead === 0) return "drained"; - const chunk = buf.subarray(0, bytesRead); - offsetRef.offset += bytesRead; - if (enqueueChunkMaybeEof(controller, chunk, eofSymbol)) return "closed"; - } -} - -function tailFileVariantB2( - path: string, - eofSymbol: string, - done: Promise, - signal: AbortSignal | null = null, - cleanupPath: string -): ReadableStream { - return new ReadableStream({ - async start(controller) { - const fd = await open(path, "r"); - const buf = Buffer.alloc(64 * 1024); - const offsetRef = { offset: 0 }; - let finished = false; - let aborted = false; - let upstreamError: Error | null = null; - let errored = false; - - done.then( - () => { - finished = true; - }, - (err) => { - upstreamError = err instanceof Error ? err : new Error(String(err)); - finished = true; - } - ); - - const onAbort = () => { - aborted = true; - }; - if (signal) { - if (signal.aborted) aborted = true; - else signal.addEventListener("abort", onAbort, { once: true }); - } - - try { - while (!aborted) { - const { bytesRead } = await fd.read(buf, 0, buf.length, offsetRef.offset); - if (bytesRead > 0) { - const chunk = buf.subarray(0, bytesRead); - offsetRef.offset += bytesRead; - if (enqueueChunkMaybeEof(controller, chunk, eofSymbol)) return; - } - - if (!finished) { - await sleep(25); + const output: number[] = []; + let eofReached = false; + for (const byte of chunk) { + if (!atLineStart) { + output.push(byte); + if (byte === 0x0a || byte === 0x0d) atLineStart = true; continue; } - const drained = await drainRemaining(fd, buf, offsetRef, controller, eofSymbol); - if (drained === "closed") return; - if (upstreamError && !errored) { - errored = true; - controller.error(upstreamError); - return; + if (byte === 0x0a || byte === 0x0d) { + for (const candidateByte of controlCandidate) output.push(candidateByte); + controlCandidate = []; + output.push(byte); + continue; } + + controlCandidate.push(byte); + const classification = classifyEofControlCandidate(controlCandidate, eofSymbol); + if (classification === "matched") { + if (includeEof) { + for (const candidateByte of controlCandidate) output.push(candidateByte); + } + controlCandidate = []; + eofReached = true; + break; + } + if (classification === "not-control") { + for (const candidateByte of controlCandidate) output.push(candidateByte); + controlCandidate = []; + atLineStart = false; + } + } + + if (eofReached) { + if (output.length > 0) controller.enqueue(Uint8Array.from(output)); + closed = true; + cleanupLifecycle(); + await reader.cancel("TLS stream EOF reached").catch(() => {}); controller.close(); return; } - } catch (err) { - if (!errored) { - errored = true; - controller.error(err instanceof Error ? err : new Error(String(err))); + + if (output.length > 0) { + controller.enqueue(Uint8Array.from(output)); + return; } - } finally { - await fd.close().catch(() => {}); - await cleanupTempPathDirname(cleanupPath); - if (signal) signal.removeEventListener("abort", onAbort); } }, + async cancel(reason) { + closed = true; + controlCandidate = []; + cleanupLifecycle(); + await reader.cancel(reason).catch(() => {}); + }, }); } -// --------------------------------------------------------------------------- -// Client lifecycle — TLS client singleton per provider -// --------------------------------------------------------------------------- +async function loadWreqRuntime(): Promise { + const moduleName = "wreq-js"; + const loaded = Reflect.apply(runtimeRequire, undefined, [moduleName]) as Partial; + if (typeof loaded.createTransport !== "function" || typeof loaded.fetch !== "function") { + throw new Error("wreq-js runtime is missing createTransport/fetch"); + } + return loaded as WreqRuntimeLike; +} -/** - * Create a getClient function for a provider stub. - * Uses dynamic `import("tls-client-node")` with `{ runtimeMode: "native" }` - * and `client.start()`, matching the original per-provider lifecycle. - */ +export interface TlsRequestClient { + request: (url: string, options: Record) => Promise; + invalidateTransport?: (options: Record) => void; +} + +function transportKey(browser: string, os: EmulationOs, options: Record): string { + const proxy = typeof options.proxyUrl === "string" ? options.proxyUrl : ""; + return `${browser}\0${os}\0${proxy}`; +} + +/** Create a lazy wreq-js client with transport-only pooling and no shared cookie jar. */ export function createGetClient(config: { providerName: string; tlsProfile?: string; -}): () => Promise<{ - request: (url: string, opts: Record) => Promise; -}> { - let clientPromise: Promise<{ - request: (url: string, opts: Record) => Promise; - }> | null = null; - let exitHookInstalled = false; + emulationOs?: EmulationOs; + wreqRuntimeLoader?: WreqRuntimeLoader; +}): () => Promise { + const browser = config.tlsProfile ?? "chrome_146"; + const os = config.emulationOs ?? "macos"; + const transports = new Map>(); + let runtimePromise: Promise | null = null; - const installExitHook = (client: { stop: () => Promise }): void => { - if (!exitHookInstalled) { - exitHookInstalled = true; - process.on("exit", () => { - void client.stop(); + const getRuntime = (): Promise => { + if (!runtimePromise) { + const pending = (config.wreqRuntimeLoader ?? loadWreqRuntime)().catch(() => { + if (runtimePromise === pending) runtimePromise = null; + throw new TlsClientUnavailableError( + `wreq-js 3.0.0 is not installed or unsupported on this platform — ` + + `cannot start browser transport for ${config.providerName}` + ); }); + runtimePromise = pending; } + return runtimePromise; }; - return async function getClient(): Promise<{ - request: (url: string, opts: Record) => Promise; - }> { - if (!clientPromise) { - clientPromise = (async () => { - let TLSClientCtor: { - new (config: Record): { - start: () => Promise; - request: (url: string, opts: Record) => Promise; - stop: () => Promise; - }; - }; - try { - // tls-client-node uses a native binary loaded at runtime. - // The dynamic import delays the binary load until first use — no - // point crashing startup on machines where it's not installed. - const mod = await import("tls-client-node"); - TLSClientCtor = mod.TLSClient; - } catch { - throw new TlsClientUnavailableError( - `tls-client-node is not installed — cannot start TLS client for ${config.providerName}` - ); - } - const tlsOptions: Record = { - ...buildNativeTlsClientOptions(), - }; - if (config.tlsProfile) { - tlsOptions.clientIdentifier = config.tlsProfile; - } - const client = new TLSClientCtor(tlsOptions); - // Start the native TLS client binding - await client.start(); - installExitHook(client); - - return client; - })(); - } - return clientPromise; + const closeTransport = (key: string): void => { + const pending = transports.get(key); + if (!pending) return; + transports.delete(key); + void pending.then((transport) => transport.close()).catch(() => {}); }; + + const closeAllTransports = (): void => { + for (const key of [...transports.keys()]) closeTransport(key); + }; + transportClosers.add(closeAllTransports); + if (!exitHookInstalled) { + exitHookInstalled = true; + process.once("exit", () => { + for (const close of transportClosers) close(); + transportClosers.clear(); + }); + } + + const client: TlsRequestClient = { + async request(url, options) { + const runtime = await getRuntime(); + const key = transportKey(browser, os, options); + let pending = transports.get(key); + if (!pending) { + const proxy = typeof options.proxyUrl === "string" ? options.proxyUrl : undefined; + const transportOptions: Record = { browser, os }; + if (proxy) transportOptions.proxy = proxy; + pending = runtime.createTransport(transportOptions); + transports.set(key, pending); + void pending.catch(() => { + if (transports.get(key) === pending) transports.delete(key); + }); + } + const transport = await pending; + return runtime.fetch(url, { + method: options.method, + headers: options.headers, + body: options.body, + redirect: "follow", + timeout: options.timeoutMilliseconds, + signal: options.signal, + transport, + cookieMode: "ephemeral", + }); + }, + invalidateTransport(options) { + closeTransport(transportKey(browser, os, options)); + }, + }; + + return async () => client; } -/** - * Resolve the proxy URL for a tls-client request. Per-call value wins; - * falls back to the provider-specific env var and the dashboard proxy config. - */ +/** Resolve a per-call/provider/dashboard proxy for a browser-transport request. */ export function resolveProxyUrl(domain: string, perCall: string | undefined): string | undefined { return resolveTlsClientProxyUrl(domain, perCall, resolveProxyForRequest); } -// --------------------------------------------------------------------------- -// Factory — creates provider-specific tlsFetch + helpers -// --------------------------------------------------------------------------- - -const CLEANUP_VARIANTS = { - A: cleanupTempPathSubstring, - B: cleanupTempPathDirname, -} as const; - -const TAIL_FILE_VARIANTS = { - A: tailFileVariantA, - B1: tailFileVariantB1, - B2: tailFileVariantB2, -} as const; - export interface TlsClientModule { - tlsFetch: (url: string, options: TlsFetchOptions) => Promise; + tlsFetch: (url: string, options?: TlsFetchOptions) => Promise; __setTlsFetchOverrideForTesting: ( fn: ((url: string, options: TlsFetchOptions) => Promise) | null ) => void; isCloudflareChallenge?: (text: string | null | undefined) => boolean; __tlsFetchStreamingForTesting?: ( - client: { request: (url: string, opts: Record) => Promise }, + client: TlsRequestClient, url: string, requestOptions: Record, eofSymbol?: string, @@ -710,57 +568,44 @@ export interface TlsClientModule { ) => Promise; } -/** - * Create a provider-specific TLS client module. - * - * Each provider file calls this once at module level and re-exports - * the returned `tlsFetch` (as e.g. `tlsFetchChatGpt`) and - * `__setTlsFetchOverrideForTesting`. - */ +/** Build one provider-specific facade over the shared wreq-js transport. */ export function createTlsClientModule(config: TlsClientConfig): TlsClientModule { const { providerName, tlsProfile, + emulationOs = "macos", domain, - tempDirPrefix, streamEofSymbol = "[DONE]", defaultTimeoutMs = 60_000, hardTimeoutGraceMs = 10_000, firstByteTimeoutMs = 5_000, - tailFileVariant, responseValidation, proxyDomainOverride, exportCloudflareCheck, + wreqRuntimeLoader, } = config; + const streamEofPolicy = + config.streamEofPolicy ?? (config.tailFileVariant === "A" ? "include" : "exclude"); - const getClient = createGetClient({ providerName, tlsProfile }); - - function resetClientCache(): void { - // The getClient closure holds clientPromise — by design the only - // reference is inside getClient's closure. After a hang we need - // the next call to spawn a fresh binding. We achieve this by - // clearing the local reference; the module-level tlsFetch will - // re-read via getClient which recreates it. - // Since getClient's clientPromise is a closure variable, we - // re-create getClient itself: - Object.assign(localState, { - getClient: createGetClient({ providerName, tlsProfile }), - }); - // Note: this is safe because only tlsFetch calls getClient. - // A concurrent in-flight call holds its own reference. - } - - const localState: { getClient: typeof getClient } = { getClient }; - + const getClient = createGetClient({ + providerName, + tlsProfile, + emulationOs, + wreqRuntimeLoader, + }); let testOverride: ((url: string, options: TlsFetchOptions) => Promise) | null = null; - const tailFileFn = TAIL_FILE_VARIANTS[tailFileVariant]; - - const cleanupFn = tailFileVariant === "A" ? cleanupTempPathSubstring : cleanupTempPathDirname; + const invalidateOnHang = ( + client: TlsRequestClient, + requestOptions: Record, + error: unknown + ): void => { + if (error instanceof TlsClientHangError) client.invalidateTransport?.(requestOptions); + }; async function tlsFetchStreaming( - client: { request: (url: string, opts: Record) => Promise }, + client: TlsRequestClient, url: string, requestOptions: Record, eofSymbol: string, @@ -768,157 +613,259 @@ export function createTlsClientModule(config: TlsClientConfig): TlsClientModule hardTimeoutMs: number, firstByteMs: number = firstByteTimeoutMs ): Promise { - const dir = await mkdtemp(join(tmpdir(), tempDirPrefix)); - const path = join(dir, `${randomUUID()}.sse`); + const startedAt = Date.now(); + const hardDeadlineAt = startedAt + hardTimeoutMs; + const firstByteDeadlineAt = Number.isFinite(firstByteMs) + ? startedAt + Math.max(0, firstByteMs) + : Number.POSITIVE_INFINITY; + const remainingHardTimeoutMs = (): number => Math.max(0, hardDeadlineAt - Date.now()); + let reader: ReadableStreamDefaultReader | null = null; - const streamOpts: Record = { - ...requestOptions, - streamOutputPath: path, - streamOutputBlockSize: 1024, - streamOutputEOFSymbol: eofSymbol, - }; - - let resetOnHang = true; - const requestPromise = raceWithTimeout( - client.request(url, streamOpts), - hardTimeoutMs, - signal - ).catch((err: unknown) => { - if (resetOnHang && err instanceof TlsClientHangError) { - resetClientCache(); - resetOnHang = false; - } - throw err; - }); - - // Wait for the file to exist AND have at least one byte. - const ready = await waitForContent(path, firstByteMs, requestPromise); - if (!ready) { - const r = await requestPromise.catch( - (e) => ({ status: 502, headers: {}, body: String(e) }) as TlsResponseLike + try { + const response = await raceWithTimeout( + client.request(url, requestOptions), + remainingHardTimeoutMs(), + signal ); - const fileText = await readTextFileIfExists(path); - await cleanupFn(path); - return { - status: r.status, - headers: toHeaders(r.headers), - text: r.body || fileText, - body: null, + if (!isReadableBody(response.body)) { + const text = await raceWithTimeout( + readTlsResponseText(response), + remainingHardTimeoutMs(), + signal + ); + return { status: response.status, headers: toHeaders(response.headers), text, body: null }; + } + + reader = response.body.getReader(); + const activeReader = reader; + const readBeforeHardDeadline = (): Promise> => + raceWithTimeout(activeReader.read(), remainingHardTimeoutMs(), signal); + const initialChunks: Uint8Array[] = []; + const readFirstNonEmptyChunk = async (): Promise> => { + while (true) { + const result = await readBeforeHardDeadline(); + if (result.done || result.value.byteLength > 0) return result; + } }; - } + const firstRead = readFirstNonEmptyChunk(); + let firstResult: ReadableStreamReadResult; + let firstByteTimedOut = Date.now() >= firstByteDeadlineAt; - const peek = await readFirstBytes(path, 256); + if (Number.isFinite(firstByteMs) && !firstByteTimedOut) { + let timer: ReturnType | undefined; + try { + const timed = await Promise.race([ + firstRead.then((result) => ({ kind: "read" as const, result })), + new Promise<{ kind: "timeout" }>((resolve) => { + timer = setTimeout( + () => resolve({ kind: "timeout" }), + Math.max(0, firstByteDeadlineAt - Date.now()) + ); + }), + ]); + if (timed.kind === "timeout") { + firstByteTimedOut = true; + firstResult = await firstRead; + } else { + firstResult = timed.result; + } + } finally { + if (timer) clearTimeout(timer); + } + } else { + firstResult = await firstRead; + } - if (responseValidation === "cf") { - // Cloudflare challenge check - if (isCloudflareChallenge(peek)) { - await cleanupFn(path); + if (!firstResult.done) initialChunks.push(firstResult.value); + if (firstByteTimedOut) { + const bytes = await readAllChunks(activeReader, initialChunks, readBeforeHardDeadline); + return { + status: response.status, + headers: toHeaders(response.headers), + text: new TextDecoder().decode(bytes), + body: null, + }; + } + + let previewBytes = concatChunks(initialChunks).subarray(0, 256); + let preview = new TextDecoder().decode(previewBytes, { stream: true }); + let previewReachedEof = firstResult.done; + while ( + !previewReachedEof && + previewBytes.byteLength < 256 && + ((responseValidation === "sse" && + !looksLikeSse(preview) && + couldBecomeSsePrefix(preview)) || + (responseValidation === "cf" && + !isCloudflareChallenge(preview) && + (preview.trimStart().startsWith("<") || couldBecomeCloudflareChallenge(preview)))) + ) { + const next = await readBeforeHardDeadline(); + if (next.done) { + previewReachedEof = true; + break; + } + initialChunks.push(next.value); + previewBytes = concatChunks(initialChunks).subarray(0, 256); + preview = new TextDecoder().decode(previewBytes, { stream: true }); + } + + if (previewReachedEof && previewBytes.byteLength === 0) { + return { + status: response.status, + headers: toHeaders(response.headers), + text: "", + body: null, + }; + } + + if (responseValidation === "cf" && isCloudflareChallenge(preview)) { + await activeReader.cancel("Cloudflare challenge").catch(() => {}); return { status: 403, headers: new Headers({ "Content-Type": "text/html" }), - text: peek, + text: preview, body: null, }; } - // HTML error page check - if (peek.trimStart().startsWith("<")) { - await cleanupFn(path); + + if (responseValidation === "cf" && preview.trimStart().startsWith("<")) { + await activeReader.cancel("HTML response").catch(() => {}); return { status: 502, headers: new Headers({ "Content-Type": "text/html" }), - text: peek, + text: preview, body: null, }; } - } else { - // SSE validation — if it doesn't look like SSE, return buffered - if (!looksLikeSse(peek)) { - const r = await requestPromise.catch( - (e) => ({ status: 502, headers: {}, body: String(e) }) as TlsResponseLike - ); - const fileText = await readTextFileIfExists(path); - await cleanupFn(path); + + if (response.status < 200 || response.status >= 300) { + const bytes = await readAllChunks(activeReader, initialChunks, readBeforeHardDeadline); return { - status: r.status, - headers: toHeaders(r.headers), - text: r.body || fileText, + status: response.status, + headers: toHeaders(response.headers), + text: new TextDecoder().decode(bytes), body: null, }; } + + if (responseValidation === "sse" && !looksLikeSse(preview)) { + const bytes = await readAllChunks(activeReader, initialChunks, readBeforeHardDeadline); + return { + status: response.status, + headers: toHeaders(response.headers), + text: new TextDecoder().decode(bytes), + body: null, + }; + } + + const headers = toHeaders(response.headers); + headers.set( + "Content-Type", + responseValidation === "cf" ? "application/x-ndjson" : "text/event-stream" + ); + headers.set("Cache-Control", "no-cache"); + const stream = createEofFilteredStream( + activeReader, + initialChunks, + streamEofPolicy === "none" ? "" : eofSymbol, + streamEofPolicy === "include", + readBeforeHardDeadline, + (error) => invalidateOnHang(client, requestOptions, error), + signal, + hardDeadlineAt + ); + reader = null; + return { status: 200, headers, text: null, body: stream }; + } catch (error) { + invalidateOnHang(client, requestOptions, error); + if (reader) await reader.cancel(error).catch(() => {}); + throw error; } - - // Looks valid — create streaming response. - const stream = tailFileFn(path, eofSymbol, requestPromise, signal, path); - - const contentType = responseValidation === "cf" ? "application/x-ndjson" : "text/event-stream"; - - const headers = new Headers({ - "Content-Type": contentType, - "Cache-Control": "no-cache", - }); - return { status: 200, headers, text: null, body: stream }; } async function tlsFetch(url: string, options: TlsFetchOptions = {}): Promise { - // Resolve proxyUrl early so test overrides and the real path both see it. const resolvedProxyUrl = resolveProxyUrl(proxyDomainOverride ?? domain, options.proxyUrl); if (testOverride) return testOverride(url, { ...options, proxyUrl: resolvedProxyUrl }); + if (options.signal?.aborted) throw makeAbortError(options.signal); - if (options.signal?.aborted) { - throw makeAbortError(options.signal); - } - const client = await localState.getClient(); - if (options.signal?.aborted) { - throw makeAbortError(options.signal); - } + const client = await getClient(); + if (options.signal?.aborted) throw makeAbortError(options.signal); const requestOptions: Record = { method: options.method || "GET", headers: options.headers || {}, body: options.body, - tlsClientIdentifier: tlsProfile, timeoutMilliseconds: options.timeoutMs ?? defaultTimeoutMs, - followRedirects: true, - withRandomTLSExtensionOrder: true, proxyUrl: resolvedProxyUrl, + signal: options.signal, }; - - requestOptions.isByteResponse = options.byteResponse === true; + const hardTimeoutMs = (options.timeoutMs ?? defaultTimeoutMs) + hardTimeoutGraceMs; if (options.stream) { - return await tlsFetchStreaming( + return tlsFetchStreaming( client, url, requestOptions, - options.streamEofSymbol || streamEofSymbol, + options.streamEofSymbol ?? streamEofSymbol, options.signal ?? null, - (options.timeoutMs ?? defaultTimeoutMs) + hardTimeoutGraceMs, + hardTimeoutMs, firstByteTimeoutMs ); } - let tlsResponse: TlsResponseLike; + const hardDeadlineAt = Date.now() + hardTimeoutMs; + const remainingHardTimeoutMs = (): number => Math.max(0, hardDeadlineAt - Date.now()); + let response: TlsResponseLike | null = null; + let bodyReader: ReadableStreamDefaultReader | null = null; try { - tlsResponse = await raceWithTimeout( + response = await raceWithTimeout( client.request(url, requestOptions), - (options.timeoutMs ?? defaultTimeoutMs) + hardTimeoutGraceMs, + remainingHardTimeoutMs(), options.signal ?? null ); - } catch (err) { - if (err instanceof TlsClientHangError) { - resetClientCache(); + if (options.signal?.aborted) throw makeAbortError(options.signal); + const headers = toHeaders(response.headers); + if (options.byteResponse) { + const bytes = await raceWithTimeout( + readTlsResponseBytes(response, (reader) => { + bodyReader = reader; + }), + remainingHardTimeoutMs(), + options.signal ?? null + ); + bodyReader = null; + const mime = + headers.get("content-type")?.split(";", 1)[0]?.trim() || "application/octet-stream"; + return { + status: response.status, + headers, + text: `data:${mime};base64,${Buffer.from(bytes).toString("base64")}`, + body: null, + }; } - throw err; + const text = await raceWithTimeout( + readTlsResponseText(response, (reader) => { + bodyReader = reader; + }), + remainingHardTimeoutMs(), + options.signal ?? null + ); + bodyReader = null; + return { status: response.status, headers, text, body: null }; + } catch (error) { + invalidateOnHang(client, requestOptions, error); + if (bodyReader) { + await bodyReader.cancel(error).catch(() => {}); + } else if ( + response && + isReadableBody(response.body) && + typeof response.body.cancel === "function" + ) { + await response.body.cancel(error).catch(() => {}); + } + throw error; } - if (options.signal?.aborted) { - throw makeAbortError(options.signal); - } - return { - status: tlsResponse.status, - headers: toHeaders(tlsResponse.headers), - text: tlsResponse.body, - body: null, - }; } const module: TlsClientModule = { @@ -927,11 +874,7 @@ export function createTlsClientModule(config: TlsClientConfig): TlsClientModule testOverride = fn; }, }; - - if (exportCloudflareCheck) { - module.isCloudflareChallenge = isCloudflareChallenge; - } - + if (exportCloudflareCheck) module.isCloudflareChallenge = isCloudflareChallenge; if (config.exposeStreamingForTesting) { module.__tlsFetchStreamingForTesting = ( client, @@ -941,18 +884,8 @@ export function createTlsClientModule(config: TlsClientConfig): TlsClientModule signal = null, hardTimeoutMs = defaultTimeoutMs + hardTimeoutGraceMs, firstByteMs = firstByteTimeoutMs - ): Promise => { - return tlsFetchStreaming( - client, - url, - requestOptions, - eofSymbol, - signal, - hardTimeoutMs, - firstByteMs - ); - }; + ) => + tlsFetchStreaming(client, url, requestOptions, eofSymbol, signal, hardTimeoutMs, firstByteMs); } - return module; } diff --git a/open-sse/services/tlsClientDownloadDir.ts b/open-sse/services/tlsClientDownloadDir.ts deleted file mode 100644 index 4ded7fbf01..0000000000 --- a/open-sse/services/tlsClientDownloadDir.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { join } from "node:path"; -import { resolveDataDir } from "@/lib/dataPaths"; - -/** - * Writable cache directory for tls-client-node's native binary. - * - * Without an explicit `downloadDir`, the library defaults to its own package - * `node_modules/tls-client-node/bin`, which is root-owned on global installs - * and fails with EACCES for normal users (#8579). - */ -export function resolveTlsClientDownloadDir(): string { - return join(resolveDataDir(), "tls-client", "bin"); -} - -export function buildNativeTlsClientOptions(): { - runtimeMode: "native"; - downloadDir: string; -} { - return { - runtimeMode: "native", - downloadDir: resolveTlsClientDownloadDir(), - }; -} diff --git a/package-lock.json b/package-lock.json index 10f668c47e..0780afccc1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -162,8 +162,7 @@ "keytar": "^7.9.0", "onnxruntime-node": "1.24.3", "sqlite-vec": "^0.1.9", - "tls-client-node": "^0.2.0", - "wreq-js": "^3.0.0" + "wreq-js": "3.0.0" } }, "node_modules/@adobe/css-tools": { @@ -25388,17 +25387,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/koffi": { - "version": "2.16.1", - "resolved": "https://registry.npmjs.org/koffi/-/koffi-2.16.1.tgz", - "integrity": "sha512-0Ie6CfD026dNfWSosDw9dPxPzO9Rlyo0N8m5r05S8YjytIpuilzMFDMY4IDy/8xQsTwpuVinhncD+S8n3bcYZQ==", - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "funding": { - "url": "https://liberapay.com/Koromix" - } - }, "node_modules/kuler": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", @@ -25588,17 +25576,6 @@ "node": ">= 14" } }, - "node_modules/libxmljs2/node_modules/brace-expansion": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.4.tgz", - "integrity": "sha512-hGfVzPxthbf3+2yjg/RBs60cB0FhqBS/zvdV/4wn4/BmN0bNMMHPc4V/BbFieqf1TKAGGAHnY4eSjajCl0f2Xg==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "balanced-match": "^1.0.0" - } - }, "node_modules/libxmljs2/node_modules/cacache": { "version": "19.0.1", "resolved": "https://registry.npmjs.org/cacache/-/cacache-19.0.1.tgz", @@ -35641,7 +35618,7 @@ "version": "7.0.27", "resolved": "https://registry.npmjs.org/tldts/-/tldts-7.0.27.tgz", "integrity": "sha512-I4FZcVFcqCRuT0ph6dCDpPuO4Xgzvh+spkcTr1gK7peIvxWauoloVO0vuy1FQnijT63ss6AsHB6+OIM4aXHbPg==", - "devOptional": true, + "dev": true, "license": "MIT", "dependencies": { "tldts-core": "^7.0.27" @@ -35654,28 +35631,9 @@ "version": "7.0.27", "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-7.0.27.tgz", "integrity": "sha512-YQ7uPjgWUibIK6DW5lrKujGwUKhLevU4hcGbP5O6TcIUb+oTjJYJVWPS4nZsIHrEEEG6myk/oqAJUEQmpZrHsg==", - "devOptional": true, + "dev": true, "license": "MIT" }, - "node_modules/tls-client-node": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/tls-client-node/-/tls-client-node-0.2.0.tgz", - "integrity": "sha512-0PHJgaGPvMK9ly7xohviOoe8Oxos43IOIdsEhibgku4ce/3/YLhxJTPPKNQZII0PdcOjlfPweB9eRs13mWaWIg==", - "hasInstallScript": true, - "license": "SEE LICENSE IN LICENSE", - "optional": true, - "dependencies": { - "koffi": "^2.8.9", - "tough-cookie": "^6.0.1" - }, - "engines": { - "node": ">=18.17" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/fatihkabakk" - } - }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -35728,7 +35686,7 @@ "version": "6.0.2", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-6.0.2.tgz", "integrity": "sha512-exgYmnmL/sJpR3upZfXG5PoatXQii55xAiXGXzY+sROLZ/Y+SLcp9PgJNI9Vz37HpQ74WvDcLT8eqm+kV3FzrA==", - "devOptional": true, + "dev": true, "license": "BSD-3-Clause", "dependencies": { "tldts": "^7.0.5" diff --git a/package.json b/package.json index 54a0bc149e..ec4181f671 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,6 @@ "src/types/", ".env.example", "scripts/build/postinstall.mjs", - "scripts/build/fixTlsClientNodeBinary.mjs", "scripts/build/fixPlaywrightAndroid.mjs", "bin/cli/runtime/", "scripts/postinstall.mjs", @@ -38,6 +37,8 @@ "scripts/build/backendOnlyPages.mjs", "scripts/build/build-tproxy-native.mjs", "scripts/build/native-binary-compat.mjs", + "scripts/build/wreqJsNative.mjs", + "config/release/wreq-js-native-manifest.json", "scripts/build/build-next-isolated.mjs", "scripts/build/runtime-env.mjs", "scripts/packs/optionalPackManifest.mjs", @@ -347,8 +348,7 @@ "keytar": "^7.9.0", "onnxruntime-node": "1.24.3", "sqlite-vec": "^0.1.9", - "tls-client-node": "^0.2.0", - "wreq-js": "^3.0.0" + "wreq-js": "3.0.0" }, "devDependencies": { "@axe-core/playwright": "^4.13.0", diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index c37a4e47a7..66883433a5 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -12,12 +12,10 @@ allowBuilds: core-js: true esbuild: true keytar: true - koffi: true libxmljs2: true onnxruntime-node: true protobufjs: true sharp: true - tls-client-node: true unrs-resolver: true onlyBuiltDependencies: - "@parcel/watcher" @@ -26,11 +24,9 @@ onlyBuiltDependencies: - "core-js" - "esbuild" - "keytar" - - "koffi" - "libxmljs2" - "onnxruntime-node" - "omniroute" - "protobufjs" - "sharp" - - "tls-client-node" - "unrs-resolver" diff --git a/pnpm.json b/pnpm.json index b07b72ab41..9d2741a0be 100644 --- a/pnpm.json +++ b/pnpm.json @@ -6,13 +6,11 @@ "core-js", "esbuild", "keytar", - "koffi", "libxmljs2", "omniroute", "onnxruntime-node", "protobufjs", "sharp", - "tls-client-node", "unrs-resolver" ] } diff --git a/scripts/build/fixTlsClientNodeBinary.mjs b/scripts/build/fixTlsClientNodeBinary.mjs deleted file mode 100644 index 9db5f4bbb4..0000000000 --- a/scripts/build/fixTlsClientNodeBinary.mjs +++ /dev/null @@ -1,148 +0,0 @@ -#!/usr/bin/env node - -/** - * tls-client-node postinstall repair (#7802). - * - * tls-client-node's own postinstall.js fetches a platform-specific native - * binary (.so/.dylib/.dll) from the bogdanfinn/tls-client GitHub Releases - * API. That script is blocked by `npm ci --ignore-scripts` (the Dockerfile - * builder stage runs with scripts disabled for supply-chain hygiene) and, - * even when it does run, silently no-ops on a rate-limited/failed GitHub API - * call instead of raising — so `node_modules/tls-client-node/bin/` can end - * up empty with no visible signal until the first live request throws - * TlsClientUnavailableError (chatgpt-web/claude-web/grok-web/lmarena/ - * perplexity-web all share this transport). - * - * This module: - * 1. Copies an already-fetched root `bin/` into the standalone - * `dist/node_modules/tls-client-node/bin/` bundle (same pattern as - * fixWreqJsBinary), so the published npm package works even though its - * own `files` allowlist never ships the binary. - * 2. When the root `bin/` is empty (--ignore-scripts blocked it, or a - * transient GitHub rate-limit ate the first attempt), retries the - * module's own postinstall.js with exponential backoff instead of - * giving up on the first failure. - * - * Best-effort throughout: a failure here never throws out of postinstall.mjs - * — it only warns, matching the other fix*Binary() steps. The runtime layer - * (perplexityTlsClient.ts and its 4 siblings) already surfaces a clear - * TlsClientUnavailableError pointing at the missing binary, so an operator - * who hits a still-empty bin/ after this repair gets an actionable message - * rather than an opaque crash. - */ - -import { copyFileSync, existsSync, mkdirSync, readdirSync } from "node:fs"; -import { join } from "node:path"; - -const DEFAULT_RETRY_DELAYS_MS = [1_000, 3_000, 8_000]; - -function hasAnyFile(dir) { - if (!existsSync(dir)) return false; - try { - return readdirSync(dir).length > 0; - } catch { - return false; - } -} - -function copyBinDir(sourceDir, destDir) { - mkdirSync(destDir, { recursive: true }); - for (const file of readdirSync(sourceDir)) { - copyFileSync(join(sourceDir, file), join(destDir, file)); - } -} - -async function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -/** - * Re-run tls-client-node's own postinstall.js in-process, retrying with - * backoff when the attempt leaves `bin/` empty (covers transient GitHub API - * rate-limiting — the upstream script itself never throws on failure, it - * only warns, so "still empty after running it" is the only failure signal - * available). - */ -async function downloadWithRetry(rootTlsClientDir, retryDelaysMs, log) { - const postinstallScript = join(rootTlsClientDir, "scripts", "postinstall.js"); - const binDir = join(rootTlsClientDir, "bin"); - if (!existsSync(postinstallScript)) return false; - - for (let attempt = 0; attempt <= retryDelaysMs.length; attempt++) { - if (attempt > 0) { - log( - ` ⏳ tls-client-node native binary still missing — retrying download ` + - `(attempt ${attempt + 1}/${retryDelaysMs.length + 1}) after rate-limit/backoff...` - ); - await sleep(retryDelaysMs[attempt - 1]); - } - - try { - const { execFileSync } = await import("node:child_process"); - execFileSync(process.execPath, [postinstallScript], { - cwd: rootTlsClientDir, - stdio: "pipe", - timeout: 30_000, - }); - } catch (err) { - log(` ⚠️ tls-client-node postinstall attempt failed: ${err.message.split("\n")[0]}`); - } - - if (hasAnyFile(binDir)) return true; - } - - return false; -} - -/** - * @param {object} opts - * @param {string} opts.rootDir - repo root - * @param {(msg: string) => void} [opts.log] - * @param {number[]} [opts.retryDelaysMs] - override for tests (avoid real sleeps) - */ -export async function fixTlsClientNodeBinary({ - rootDir, - log = (m) => console.log(m), - retryDelaysMs = DEFAULT_RETRY_DELAYS_MS, -} = {}) { - const rootTlsClientDir = join(rootDir, "node_modules", "tls-client-node"); - const rootBinDir = join(rootTlsClientDir, "bin"); - const distTlsClientDir = join(rootDir, "dist", "node_modules", "tls-client-node"); - - if (!existsSync(rootTlsClientDir)) return; - - if (!hasAnyFile(rootBinDir)) { - log( - "\n 🔧 tls-client-node native binary missing (blocked by --ignore-scripts or a " + - "failed fetch) — attempting repair...\n" - ); - const recovered = await downloadWithRetry(rootTlsClientDir, retryDelaysMs, log); - if (!recovered) { - console.warn( - "\n ⚠️ Could not fetch tls-client-node's native binary " + - "(GitHub API rate-limited or unreachable after retries)." - ); - console.warn( - " chatgpt-web/claude-web/grok-web/lmarena/perplexity-web will raise a clear " + - "TlsClientUnavailableError on first use until this is resolved." - ); - console.warn( - ` Manual fix: node ${join(rootTlsClientDir, "scripts", "postinstall.js")}\n` - ); - return; - } - log(" ✅ tls-client-node native binary fetched successfully!\n"); - } - - if (!existsSync(distTlsClientDir) || !hasAnyFile(rootBinDir)) return; - - const distBinDir = join(distTlsClientDir, "bin"); - if (hasAnyFile(distBinDir)) return; - - try { - copyBinDir(rootBinDir, distBinDir); - log(" ✅ tls-client-node native binary copied to standalone dist/node_modules.\n"); - } catch (err) { - console.warn(` ⚠️ Could not copy tls-client-node binary into dist/: ${err.message}`); - } -} diff --git a/scripts/build/hydrateNativeDeps.mjs b/scripts/build/hydrateNativeDeps.mjs index 4b7d4a2f9a..8fe6b1bb9c 100644 --- a/scripts/build/hydrateNativeDeps.mjs +++ b/scripts/build/hydrateNativeDeps.mjs @@ -7,9 +7,9 @@ * matrix leg. Everything except install-machine-forked optional packages is * platform-independent: * - * - Bundled-for-all (verify only): koffi ships every triplet under - * `build/koffi/_`, better-sqlite3 v13 ships Node-API prebuilds for - * 8 platforms, wreq-js ships `rust/wreq-js.-[-libc].node`, and + * - Bundled-for-all (verify only): better-sqlite3 v13 ships Node-API prebuilds + * for 8 platforms, wreq-js ships + * `rust/wreq-js.-[-libc].node`, and * onnxruntime-node ships `bin/napi-v6//`. * - Install-machine-forked (hydrate): `@img/sharp-*`, `@img/sharp-libvips-*`, * `@ngrok/ngrok-*` and macOS-only `fsevents` resolve to whichever platform @@ -33,8 +33,7 @@ export const HYDRATED_ROOT_PACKAGES = ["fsevents"]; export const BUNDLED_EXEMPTIONS = new Set(["onnxruntime-node:darwin-x64"]); function platformTriple(platform, arch) { - // koffi uses underscore triplets; better-sqlite3/wreq-js/onnx use dashes. - return { koffi: `${platform}_${arch}`, dash: `${platform}-${arch}` }; + return { dash: `${platform}-${arch}` }; } function rmrf(target) { @@ -106,9 +105,6 @@ export function verifyBundledNatives({ nodeModulesDir, platform, arch }) { const errors = []; const triple = platformTriple(platform, arch); - const koffiDir = path.join(nodeModulesDir, "koffi", "build", "koffi", triple.koffi); - if (!fs.existsSync(koffiDir)) errors.push(`koffi: missing bundled triplet ${triple.koffi}`); - const sqlitePrebuild = path.join( nodeModulesDir, "better-sqlite3", diff --git a/scripts/build/pack-artifact-policy.ts b/scripts/build/pack-artifact-policy.ts index 0e6d37908b..f23d5975e1 100644 --- a/scripts/build/pack-artifact-policy.ts +++ b/scripts/build/pack-artifact-policy.ts @@ -94,6 +94,7 @@ export const PACK_ARTIFACT_ROOT_ALLOWED_EXACT_PATHS: string[] = [ "LICENSE", "README.md", "THIRD_PARTY_NOTICES.md", + "config/release/wreq-js-native-manifest.json", "bin/aliasResolver.mjs", "bin/chatgpt-web-codex-mcp.mjs", // #7808: ESM loader hook split out of bin/aliasResolver.mjs to silence CodeQL @@ -136,12 +137,10 @@ export const PACK_ARTIFACT_ROOT_ALLOWED_EXACT_PATHS: string[] = [ "scripts/build/build-next-isolated.mjs", "scripts/check/check-supported-node-runtime.ts", "scripts/build/native-binary-compat.mjs", + "scripts/build/wreqJsNative.mjs", "scripts/build/postinstall.mjs", "scripts/build/postinstallSupport.mjs", "scripts/build/colocateOptionals.mjs", - // #7802: imported by scripts/build/postinstall.mjs to repair tls-client-node's - // native binary (chatgpt-web/claude-web/grok-web/lmarena/perplexity-web transport). - "scripts/build/fixTlsClientNodeBinary.mjs", // #8859: imported by scripts/build/postinstall.mjs to repair playwright-core's // browser resolution on Termux/Android (no glibc, no bundled browsers). "scripts/build/fixPlaywrightAndroid.mjs", @@ -220,13 +219,14 @@ export const PACK_ARTIFACT_REQUIRED_PATHS: string[] = [ // or the CLI fails to boot — list them REQUIRED so a regression is loud. "bin/aliasResolver.mjs", "bin/aliasResolverHook.mjs", + "config/release/wreq-js-native-manifest.json", "package.json", "scripts/build/native-binary-compat.mjs", "scripts/build/postinstall.mjs", "scripts/build/postinstallSupport.mjs", "scripts/build/colocateOptionals.mjs", - "scripts/build/fixTlsClientNodeBinary.mjs", "scripts/build/runtime-env.mjs", + "scripts/build/wreqJsNative.mjs", // #10382: runtime imports of bin/cli/commands/packs.mjs (optional packs CLI) — // listed REQUIRED so their absence from the tarball fails loudly. "scripts/packs/optionalPackInstaller.mjs", diff --git a/scripts/build/postinstall.mjs b/scripts/build/postinstall.mjs index 1628aca7cf..e190f88343 100644 --- a/scripts/build/postinstall.mjs +++ b/scripts/build/postinstall.mjs @@ -14,8 +14,7 @@ * * Modules repaired: * - better-sqlite3 (SQLite bindings) - * - wreq-js (TLS client for OAuth providers) - * - tls-client-node (TLS client for chatgpt-web/claude-web/grok-web/lmarena/perplexity-web) + * - wreq-js (TLS client for OAuth and web-cookie providers) * - sql.js (WASM SQLite fallback runtime) * - node-machine-id (local CLI machine-token server runtime) * @@ -26,15 +25,7 @@ * Fixes: https://github.com/diegosouzapw/OmniRoute/issues/7802 */ -import { - copyFileSync, - cpSync, - existsSync, - mkdirSync, - readFileSync, - readdirSync, - writeFileSync, -} from "node:fs"; +import { copyFileSync, cpSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { createRequire } from "node:module"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -42,8 +33,8 @@ import { fileURLToPath } from "node:url"; import { PUBLISHED_BUILD_ARCH, PUBLISHED_BUILD_PLATFORM } from "./native-binary-compat.mjs"; import { hasStandaloneAppBundle, isTermux } from "./postinstallSupport.mjs"; import { colocateLlmlinguaOptionals } from "./colocateOptionals.mjs"; -import { fixTlsClientNodeBinary } from "./fixTlsClientNodeBinary.mjs"; import { fixPlaywrightAndroid } from "./fixPlaywrightAndroid.mjs"; +import { resolveWreqJsNativeBinaryName } from "./wreqJsNative.mjs"; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); @@ -271,7 +262,7 @@ async function fixWreqJsBinary() { if (process.platform === "android" || isTermux()) { console.log( " [postinstall] wreq-js: skipped on Termux/Android " + - "(libgcc not available — OAuth TLS fingerprinting will use the fallback path)" + "(wreq-js 3.0.0 does not publish an Android native addon)" ); return; } @@ -283,7 +274,16 @@ async function fixWreqJsBinary() { return; } - const binaryName = `wreq-js.${process.platform}-${process.arch}.node`; + const binaryName = resolveWreqJsNativeBinaryName({ + platform: process.platform, + arch: process.arch, + }); + if (!binaryName) { + console.warn( + ` ⚠️ wreq-js 3.0.0 has no native addon for ${process.platform}-${process.arch}.` + ); + return; + } const appBinaryPath = join(appWreqDir, binaryName); const rootBinaryPath = join(rootWreqDir, binaryName); @@ -312,27 +312,7 @@ async function fixWreqJsBinary() { } } - // Strategy 2: Copy entire rust/ directory from root (gets all platform binaries) - if (existsSync(rootWreqDir)) { - try { - mkdirSync(appWreqDir, { recursive: true }); - const files = readdirSync(rootWreqDir); - for (const file of files) { - if (file.endsWith(".node")) { - copyFileSync(join(rootWreqDir, file), join(appWreqDir, file)); - } - } - if (existsSync(appBinaryPath)) { - process.dlopen({ exports: {} }, appBinaryPath); - console.log(" ✅ wreq-js native module fixed (full copy) successfully!\n"); - return; - } - } catch (err) { - console.warn(` ⚠️ wreq-js full copy failed: ${err.message}`); - } - } - - // Strategy 3: Rebuild wreq-js inside dist/ + // Strategy 2: Rebuild wreq-js inside dist/ console.log(" 📥 Attempting npm rebuild wreq-js..."); try { const { execSync } = await import("node:child_process"); @@ -353,8 +333,10 @@ async function fixWreqJsBinary() { console.warn( `\n ⚠️ Could not fix wreq-js native module for ${process.platform}-${process.arch}.` ); - console.warn(" OAuth-based providers (Codex, Cursor, etc.) may not work."); - console.warn(` Manual fix: cd ${join(ROOT, "dist")} && npm install wreq-js --no-save\n`); + console.warn(" Browser-TLS OAuth and web-cookie providers may not work."); + console.warn( + ` Manual fix: cd ${join(ROOT, "dist")} && npm install wreq-js@3.0.0 --save-exact\n` + ); } async function ensureSwcHelpers() { @@ -464,7 +446,6 @@ async function ensureStandaloneRuntimePackages() { await verifyDevNativeModules(); await fixBetterSqliteBinary(); await fixWreqJsBinary(); -await fixTlsClientNodeBinary({ rootDir: ROOT }); await fixPlaywrightAndroid({ rootDir: ROOT }); await ensureSwcHelpers(); await ensureStandaloneRuntimePackages(); diff --git a/scripts/build/wreqJsNative.mjs b/scripts/build/wreqJsNative.mjs new file mode 100644 index 0000000000..26065a9eb0 --- /dev/null +++ b/scripts/build/wreqJsNative.mjs @@ -0,0 +1,37 @@ +/** Exact native-addon layout published by wreq-js 3.0.0. */ +export const WREQ_JS_NATIVE_BINARY_NAMES = Object.freeze([ + "wreq-js.darwin-arm64.node", + "wreq-js.darwin-x64.node", + "wreq-js.linux-arm64-gnu.node", + "wreq-js.linux-arm64-musl.node", + "wreq-js.linux-x64-gnu.node", + "wreq-js.linux-x64-musl.node", + "wreq-js.win32-x64-msvc.node", +]); + +/** Detect the C library used by the current Linux runtime. */ +export function detectRuntimeLibc() { + if (process.platform !== "linux") return undefined; + try { + const report = process.report?.getReport(); + return report?.header?.glibcVersionRuntime ? "gnu" : "musl"; + } catch { + return "musl"; + } +} + +/** Resolve the exact addon filename that wreq-js 3.0.0 will load. */ +export function resolveWreqJsNativeBinaryName({ platform, arch, libc }) { + if (platform === "darwin" && (arch === "arm64" || arch === "x64")) { + return `wreq-js.darwin-${arch}.node`; + } + if (platform === "linux" && (arch === "arm64" || arch === "x64")) { + const linuxLibc = libc ?? detectRuntimeLibc(); + if (linuxLibc !== "gnu" && linuxLibc !== "musl") return null; + return `wreq-js.linux-${arch}-${linuxLibc}.node`; + } + if (platform === "win32" && arch === "x64") { + return "wreq-js.win32-x64-msvc.node"; + } + return null; +} diff --git a/src/lib/providers/validation/webProvidersA.ts b/src/lib/providers/validation/webProvidersA.ts index 52e9aa4c0e..82a671d089 100644 --- a/src/lib/providers/validation/webProvidersA.ts +++ b/src/lib/providers/validation/webProvidersA.ts @@ -404,7 +404,7 @@ export async function validateGrokWebProvider({ apiKey, providerSpecificData = { errorDetail = (response.text || "").slice(0, 240); } catch {} - // Detect Cloudflare challenge pages even with a 200 status from tls-client-node + // Detect Cloudflare challenge pages even when the browser transport reports status 200. if (isCloudflareChallenge(errorDetail)) { return { valid: false, @@ -680,7 +680,7 @@ export async function validatePerplexityWebProvider({ apiKey, providerSpecificDa valid: false, error: "Cloudflare is blocking connections from this server's IP (TLS fingerprint rejected). " + - "The session cookie may still be valid — install tls-client-node's native binary or route " + + "The session cookie may still be valid — verify the wreq-js 3.0.0 native addon or route " + "perplexity-web through a residential proxy.", }; } diff --git a/tests/unit/build/check-licenses.test.ts b/tests/unit/build/check-licenses.test.ts index 6ee28694db..746f9cc9dc 100644 --- a/tests/unit/build/check-licenses.test.ts +++ b/tests/unit/build/check-licenses.test.ts @@ -205,17 +205,17 @@ test("classifyLicense: exception does not apply to different package", () => { assert.equal(result.status, "denied", "exception must be per-package, not per-license"); }); -test("classifyLicense: exception with risk=medium still returns 'exception' (not denied)", () => { +test("classifyLicense: a medium-risk custom exception still returns 'exception'", () => { const allowlist = makeAllowlist({ exceptions: { - "tls-client-node": { - license: "Custom: LICENSE", - justification: "Commons Clause + Apache-2.0. TODO: revisar.", + "custom-runtime": { + license: "Custom: reviewed terms", + justification: "Reviewed custom runtime terms.", risk: "medium", }, }, }); - const result = classifyLicense("tls-client-node@0.2.0", "Custom: LICENSE", allowlist); + const result = classifyLicense("custom-runtime@1.0.0", "Custom: reviewed terms", allowlist); assert.equal(result.status, "exception"); }); @@ -285,13 +285,6 @@ test("loadAllowlist: exceptions entries have required fields", () => { } }); -test("loadAllowlist: tls-client-node exception has risk=medium (Commons Clause)", () => { - const allowlist = loadAllowlist(); - const exc = allowlist.exceptions["tls-client-node"] as any; - assert.ok(exc, "tls-client-node exception must be registered"); - assert.equal(exc.risk, "medium", "tls-client-node is a medium-risk exception (Commons Clause)"); -}); - test("loadAllowlist: LGPL packages have registered exceptions", () => { const allowlist = loadAllowlist(); const lgplPkgs = ["@img/sharp-libvips-linux-x64", "@img/sharp-libvips-linuxmusl-x64"]; @@ -326,12 +319,6 @@ test("integration: classifyLicense passes MIT packages against real allowlist", assert.equal(result.status, "allowed"); }); -test("integration: classifyLicense passes tls-client-node as exception against real allowlist", () => { - const allowlist = loadAllowlist(); - const result = classifyLicense("tls-client-node@0.2.0", "Custom: LICENSE", allowlist); - assert.equal(result.status, "exception"); -}); - test("integration: classifyLicense denies GPL-3.0 against real allowlist", () => { const allowlist = loadAllowlist(); const result = classifyLicense("hypothetical-gpl@1.0.0", "GPL-3.0", allowlist); diff --git a/tests/unit/build/standalone-bundle.test.ts b/tests/unit/build/standalone-bundle.test.ts index 2444469202..dc95e829c7 100644 --- a/tests/unit/build/standalone-bundle.test.ts +++ b/tests/unit/build/standalone-bundle.test.ts @@ -266,7 +266,6 @@ test("verifyBundledNatives asserts serviceability and honors the onnx darwin-x64 const root = tmpDir("s8-natives-"); try { const nm = path.join(root, "node_modules"); - writeNative(nm, "koffi/build/koffi/linux_x64/koffi.node", "elf"); writeNative(nm, "better-sqlite3/prebuilds/linux-x64.node", "napi"); writeNative(nm, "wreq-js/rust/wreq-js.linux-x64-gnu.node", "rust"); writeNative(nm, "onnxruntime-node/bin/napi-v6/linux/x64/libonnxruntime.so", "ort"); @@ -278,18 +277,18 @@ test("verifyBundledNatives asserts serviceability and honors the onnx darwin-x64 `expected serviceable: ${(good as { errors?: string[] }).errors?.join("; ")}` ); - const missingKoffi = verifyBundledNatives({ + const missingPlatformNatives = verifyBundledNatives({ nodeModulesDir: nm, platform: "darwin", arch: "arm64", }); - assert.equal(missingKoffi.ok, false); - assert.ok((missingKoffi as { errors: string[] }).errors.some((e) => e.startsWith("koffi:"))); - + assert.equal(missingPlatformNatives.ok, false); + assert.ok( + (missingPlatformNatives as { errors: string[] }).errors.some((e) => e.startsWith("wreq-js:")) + ); // darwin-x64 has no onnxruntime-node prebuild at all — the exemption must keep it green // as long as the other bundled natives service that triple. const nm2 = path.join(root, "node_modules2"); - writeNative(nm2, "koffi/build/koffi/darwin_x64/koffi.node", "macho"); writeNative(nm2, "better-sqlite3/prebuilds/darwin-x64.node", "napi"); writeNative(nm2, "wreq-js/rust/wreq-js.darwin-x64.node", "rust"); const exempted = verifyBundledNatives({ nodeModulesDir: nm2, platform: "darwin", arch: "x64" }); diff --git a/tests/unit/chatgpt-web.test.ts b/tests/unit/chatgpt-web.test.ts index 267c0eb6dc..9f93b6ecc1 100644 --- a/tests/unit/chatgpt-web.test.ts +++ b/tests/unit/chatgpt-web.test.ts @@ -1,6 +1,5 @@ import test from "node:test"; import assert from "node:assert/strict"; -import { writeFile } from "node:fs/promises"; import type { TlsFetchOptions } from "../../open-sse/services/chatgptTlsClient.ts"; const { ChatGptWebExecutor, __derivePublicBaseUrlForTesting, __resetChatGptWebCachesForTesting } = @@ -255,8 +254,8 @@ function installMockFetch({ return { status: cfg.status, headers: makeHeaders({ "Content-Type": "image/png" }), - // tls-client-node packages binary bodies as a data:;base64,... - // string when isByteResponse is set; the mock mirrors that contract. + // The shared browser transport packages byte responses as a + // data:;base64,... string; the mock mirrors that contract. text: `data:image/png;base64,${tinyPng.toString("base64")}`, body: null, }; @@ -1901,30 +1900,30 @@ test("looksLikeSse: rejects non-SSE bodies that previously passed as 200", () => assert.equal(looksLikeSse("error: rate limit"), false, "non-SSE field name"); }); -test("tls streaming: late first byte is read from streamOutputPath instead of empty body", async () => { +test("tls streaming: a late first byte falls back to the complete buffered body", async () => { const fakeClient = { - async request(_url, opts) { - await new Promise((resolve) => setTimeout(resolve, 25)); - await writeFile( - String(opts.streamOutputPath), - mockChatGptStreamText([ - { - conversation_id: "conv-late", - message: { - id: "msg-late", - author: { role: "assistant" }, - content: { content_type: "text", parts: ["Late title answer"] }, - status: "finished_successfully", - }, + async request() { + const payload = mockChatGptStreamText([ + { + conversation_id: "conv-late", + message: { + id: "msg-late", + author: { role: "assistant" }, + content: { content_type: "text", parts: ["Late title answer"] }, + status: "finished_successfully", }, - ]), - "utf8" + }, + ]); + return new Response( + new ReadableStream({ + async pull(controller) { + await new Promise((resolve) => setTimeout(resolve, 25)); + controller.enqueue(new TextEncoder().encode(payload)); + controller.close(); + }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } } ); - return { - status: 200, - headers: { "content-type": ["text/event-stream"] }, - body: "", - }; }, }; @@ -2345,9 +2344,8 @@ test("Image gen: signed URL bytes are cached and exposed via /v1/chatgpt-web/ima } if (u.startsWith(downloadUrl)) { calls.signed++; - // tls-client-node returns binary bodies as a "data:;base64,..." - // string (see its response.js bytes() impl); the executor decodes it - // back into bytes before putting the image in OmniRoute's cache. + // The shared browser transport returns binary bodies as a + // "data:;base64,..." string; the executor decodes it for the cache. return { status: 200, headers: makeHeaders({ "Content-Type": "image/png" }), diff --git a/tests/unit/claude-web-slow-first-byte.test.ts b/tests/unit/claude-web-slow-first-byte.test.ts index 7a67e74151..2d1386db99 100644 --- a/tests/unit/claude-web-slow-first-byte.test.ts +++ b/tests/unit/claude-web-slow-first-byte.test.ts @@ -1,5 +1,4 @@ import assert from "node:assert/strict"; -import { writeFile } from "node:fs/promises"; import test from "node:test"; import { tlsFetchStreaming } from "../../open-sse/services/claudeTlsClient.ts"; @@ -15,19 +14,17 @@ const SSE_BODY = [ test("Claude Web keeps waiting when the first Opus SSE event takes longer than five seconds", async () => { const client = { - request: async (_url: string, options: Record) => { - await new Promise((resolve) => setTimeout(resolve, SLOW_FIRST_BYTE_MS)); - await writeFile(String(options.streamOutputPath), SSE_BODY); - return { - status: 200, - headers: {}, - body: "", - cookies: {}, - text: async () => "", - json: async () => ({}), - bytes: async () => new Uint8Array(), - }; - }, + request: async () => + new Response( + new ReadableStream({ + async pull(controller) { + await new Promise((resolve) => setTimeout(resolve, SLOW_FIRST_BYTE_MS)); + controller.enqueue(new TextEncoder().encode(SSE_BODY)); + controller.close(); + }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } } + ), }; const result = await tlsFetchStreaming( diff --git a/tests/unit/fix-tls-client-node-binary-7802.test.ts b/tests/unit/fix-tls-client-node-binary-7802.test.ts deleted file mode 100644 index c4c1209949..0000000000 --- a/tests/unit/fix-tls-client-node-binary-7802.test.ts +++ /dev/null @@ -1,115 +0,0 @@ -import { test } from "node:test"; -import assert from "node:assert/strict"; -import { mkdtempSync, mkdirSync, writeFileSync, existsSync, readdirSync, rmSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; - -import { fixTlsClientNodeBinary } from "../../scripts/build/fixTlsClientNodeBinary.mjs"; - -function makeRoot() { - return mkdtempSync(join(tmpdir(), "fix-tls-client-node-binary-7802-")); -} - -function collectLogs() { - const logs: string[] = []; - return { logs, log: (m: string) => logs.push(m) }; -} - -test("no-ops when node_modules/tls-client-node is absent (module not installed)", async () => { - const rootDir = makeRoot(); - try { - const { logs, log } = collectLogs(); - await fixTlsClientNodeBinary({ rootDir, log }); - assert.deepEqual(logs, []); - } finally { - rmSync(rootDir, { recursive: true, force: true }); - } -}); - -test("copies an already-populated root bin/ into the standalone dist bundle (#7802 item 2)", async () => { - const rootDir = makeRoot(); - try { - const rootBin = join(rootDir, "node_modules", "tls-client-node", "bin"); - mkdirSync(rootBin, { recursive: true }); - writeFileSync(join(rootBin, "tls-client-linux-ubuntu-amd64-1.0.0.so"), "fake-binary"); - - const distTlsClientDir = join(rootDir, "dist", "node_modules", "tls-client-node"); - mkdirSync(distTlsClientDir, { recursive: true }); - - const { log } = collectLogs(); - await fixTlsClientNodeBinary({ rootDir, log }); - - const distBin = join(distTlsClientDir, "bin"); - assert.ok(existsSync(distBin), "dist bin/ should have been created"); - assert.deepEqual(readdirSync(distBin), ["tls-client-linux-ubuntu-amd64-1.0.0.so"]); - } finally { - rmSync(rootDir, { recursive: true, force: true }); - } -}); - -test("retries the download when root bin/ is empty, and stops once a file appears (#7802 item 3)", async () => { - const rootDir = makeRoot(); - try { - const tlsClientDir = join(rootDir, "node_modules", "tls-client-node"); - const rootBin = join(tlsClientDir, "bin"); - mkdirSync(rootBin, { recursive: true }); - - const scriptsDir = join(tlsClientDir, "scripts"); - mkdirSync(scriptsDir, { recursive: true }); - // A postinstall.js stand-in that drops a file into bin/ on its 2nd invocation — - // simulating a first attempt eaten by a GitHub rate-limit and a 2nd that recovers. - writeFileSync( - join(scriptsDir, "postinstall.js"), - `const fs = require("fs"); - const path = require("path"); - const marker = path.join(__dirname, "..", ".attempts"); - const attempts = fs.existsSync(marker) ? Number(fs.readFileSync(marker, "utf8")) : 0; - fs.writeFileSync(marker, String(attempts + 1)); - if (attempts + 1 >= 2) { - fs.writeFileSync(path.join(__dirname, "..", "bin", "tls-client-linux-ubuntu-amd64-1.0.0.so"), "ok"); - }` - ); - - const { logs, log } = collectLogs(); - await fixTlsClientNodeBinary({ rootDir, log, retryDelaysMs: [1, 1, 1] }); - - assert.ok(existsSync(join(rootBin, "tls-client-linux-ubuntu-amd64-1.0.0.so"))); - assert.ok( - logs.some((m) => m.includes("fetched successfully")), - "expected a success log once the retry recovered" - ); - } finally { - rmSync(rootDir, { recursive: true, force: true }); - } -}); - -test("warns without throwing when every retry leaves bin/ empty (still rate-limited)", async () => { - const rootDir = makeRoot(); - try { - const tlsClientDir = join(rootDir, "node_modules", "tls-client-node"); - mkdirSync(join(tlsClientDir, "bin"), { recursive: true }); - const scriptsDir = join(tlsClientDir, "scripts"); - mkdirSync(scriptsDir, { recursive: true }); - // A postinstall.js stand-in that always fails to produce a binary (persistent rate-limit). - writeFileSync(join(scriptsDir, "postinstall.js"), `process.exitCode = 0;`); - - const originalWarn = console.warn; - const warnings: string[] = []; - console.warn = (m: string) => warnings.push(m); - try { - const { log } = collectLogs(); - await assert.doesNotReject( - fixTlsClientNodeBinary({ rootDir, log, retryDelaysMs: [1, 1] }) - ); - } finally { - console.warn = originalWarn; - } - - assert.ok( - warnings.some((m) => m.includes("Could not fetch tls-client-node")), - "expected a clear warning pointing at the manual fix, not a silent no-op" - ); - } finally { - rmSync(rootDir, { recursive: true, force: true }); - } -}); diff --git a/tests/unit/pack-artifact-policy.test.ts b/tests/unit/pack-artifact-policy.test.ts index 065ba83251..066b566222 100644 --- a/tests/unit/pack-artifact-policy.test.ts +++ b/tests/unit/pack-artifact-policy.test.ts @@ -252,6 +252,7 @@ test("findMissingArtifactPaths flags missing root runtime files in the tarball", "bin/mcp-server.mjs", "bin/mcpStdioConsoleGuard.mjs", "bin/nodeRuntimeSupport.mjs", + "config/release/wreq-js-native-manifest.json", "dist/head-response-guard.cjs", "dist/http-method-guard.cjs", "dist/main-server-timeouts.mjs", @@ -266,9 +267,9 @@ test("findMissingArtifactPaths flags missing root runtime files in the tarball", "dist/tls-options.mjs", "dist/webdav-handler.mjs", "scripts/build/colocateOptionals.mjs", - "scripts/build/fixTlsClientNodeBinary.mjs", "scripts/build/native-binary-compat.mjs", "scripts/build/runtime-env.mjs", + "scripts/build/wreqJsNative.mjs", "scripts/packs/optionalPackInstaller.mjs", "scripts/packs/optionalPackManifest.mjs", "src/shared/utils/nodeRuntimeSupport.ts", diff --git a/tests/unit/probe-7134-claude-web-empty-error-body.test.ts b/tests/unit/probe-7134-claude-web-empty-error-body.test.ts index b62659616a..17ccbff391 100644 --- a/tests/unit/probe-7134-claude-web-empty-error-body.test.ts +++ b/tests/unit/probe-7134-claude-web-empty-error-body.test.ts @@ -1,25 +1,16 @@ import test from "node:test"; import assert from "node:assert/strict"; -import { writeFile } from "node:fs/promises"; // Issue #7134 — claude-web reported "Claude Web API error (400) with no // response body" even when Claude's upstream DID send a real JSON error body. // -// Root cause: tlsFetchStreaming() streams the upstream response to a temp -// file via tls-client-node's `streamOutputPath` mode. For a non-SSE, -// non-2xx response, the native binding resolves with an EMPTY in-memory -// `body` field (it only populates `body` for its non-streaming mode) even -// though the real error bytes were already written to the temp file and -// even peeked (`looksLikeSse`) to decide the response wasn't SSE. The old -// code read the empty `r.body` instead of the file it just peeked, throwing -// away the real upstream error detail. +// The browser transport must peek a requested stream to distinguish SSE from +// an upstream JSON error. Once it decides the response is not SSE, it must +// buffer the same native body stream without discarding the bytes it peeked. // // This test injects a fake `client` (matching the `{ request }` shape -// tlsFetchStreaming already accepts for DI) that reproduces the exact -// tls-client-node contract under `streamOutputPath`: write bytes to the file, -// resolve with an empty `body`. No `--experimental-test-module-mocks` flag -// needed — this exercises the real, unmodified `tlsFetchStreaming` via -// dependency injection instead of module-mocking `tls-client-node`. +// tlsFetchStreaming accepts for DI). No experimental module mocks are needed: +// the test exercises the production wreq response-stream path directly. const { tlsFetchStreaming } = await import("../../open-sse/services/claudeTlsClient.ts"); @@ -33,21 +24,16 @@ const REAL_CLAUDE_ERROR_BODY = JSON.stringify({ function makeFakeClient(status: number, bodyOnFile: string) { return { - request: async (_url: string, opts: Record) => { - const streamOutputPath = opts.streamOutputPath as string; - await writeFile(streamOutputPath, bodyOnFile); - return { - status, - headers: {}, - // tls-client-node does not populate `body` for streamed requests — - // this is the exact defect condition. - body: "", - cookies: {}, - text: async () => "", - json: async () => ({}), - bytes: async () => new Uint8Array(), - }; - }, + request: async () => + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(bodyOnFile)); + controller.close(); + }, + }), + { status } + ), }; } @@ -73,19 +59,11 @@ test("issue #7134: tlsFetchStreaming surfaces the real error body for a non-SSE test("issue #7134: tlsFetchStreaming still uses r.body when the native client DOES populate it", async () => { const client = { - request: async (_url: string, opts: Record) => { - const streamOutputPath = opts.streamOutputPath as string; - await writeFile(streamOutputPath, "{}"); - return { - status: 403, - headers: {}, - body: "populated body from native client", - cookies: {}, - text: async () => "", - json: async () => ({}), - bytes: async () => new Uint8Array(), - }; - }, + request: async () => ({ + status: 403, + headers: {}, + body: "populated body from native client", + }), }; const result = await tlsFetchStreaming( diff --git a/tests/unit/provider-validation-specialty.test.ts b/tests/unit/provider-validation-specialty.test.ts index 9e0fb92366..ba68aa2642 100644 --- a/tests/unit/provider-validation-specialty.test.ts +++ b/tests/unit/provider-validation-specialty.test.ts @@ -16,7 +16,6 @@ const { __setTlsFetchOverrideForTesting: __setPplxTlsFetchOverride } = const { __setTlsFetchOverrideForTesting: __setGrokTlsFetchOverride } = await import("../../open-sse/services/grokTlsClient.ts"); - const originalFetch = globalThis.fetch; test.afterEach(() => { @@ -2458,7 +2457,7 @@ test("claude-web validator: 500 → Claude.ai unavailable", async () => { test("claude-web validator: TLS client unavailable → clear error", async () => { const { TlsClientUnavailableError } = await import("../../open-sse/services/claudeTlsClient.ts"); __setClaudeTlsFetchOverride(async () => { - throw new TlsClientUnavailableError("tls-client-node not installed"); + throw new TlsClientUnavailableError("wreq-js 3.0.0 native addon unavailable"); }); const result = await validateProviderApiKey({ @@ -2467,7 +2466,7 @@ test("claude-web validator: TLS client unavailable → clear error", async () => }); assert.equal(result.valid, false); - assert.match(result.error || "", /tls-client-node not installed/i); + assert.match(result.error || "", /wreq-js 3\.0\.0 native addon unavailable/i); __setClaudeTlsFetchOverride(null); }); diff --git a/tests/unit/tls-client-download-dir-8579.test.ts b/tests/unit/tls-client-download-dir-8579.test.ts deleted file mode 100644 index 272ee73245..0000000000 --- a/tests/unit/tls-client-download-dir-8579.test.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { test, afterEach } from "node:test"; -import assert from "node:assert/strict"; -import { mkdtempSync } from "node:fs"; -import { join } from "node:path"; -import { tmpdir } from "node:os"; -import { readFileSync } from "node:fs"; -import { fileURLToPath } from "node:url"; -import { dirname } from "node:path"; - -const __dirname = dirname(fileURLToPath(import.meta.url)); -const ROOT = join(__dirname, "..", ".."); - -const TLS_CLIENT_WRAPPERS = [ - "open-sse/services/chatgptTlsClient.ts", - "open-sse/services/claudeTlsClient.ts", - "open-sse/services/grokTlsClient.ts", - "open-sse/services/perplexityTlsClient.ts", - "open-sse/services/lmarenaTlsClient.ts", - "open-sse/services/notionTlsClient.ts", -] as const; - -const originalDataDir = process.env.DATA_DIR; - -afterEach(() => { - if (originalDataDir === undefined) { - delete process.env.DATA_DIR; - } else { - process.env.DATA_DIR = originalDataDir; - } -}); - -test("resolveTlsClientDownloadDir caches native binary under DATA_DIR/tls-client/bin (#8579)", async () => { - const dataDir = mkdtempSync(join(tmpdir(), "omniroute-tls-client-8579-")); - process.env.DATA_DIR = dataDir; - - const { resolveTlsClientDownloadDir } = - await import("../../open-sse/services/tlsClientDownloadDir.ts"); - - assert.equal(resolveTlsClientDownloadDir(), join(dataDir, "tls-client", "bin")); -}); - -test("buildNativeTlsClientOptions passes downloadDir to tls-client-node (#8579)", async () => { - const dataDir = mkdtempSync(join(tmpdir(), "omniroute-tls-client-opts-8579-")); - process.env.DATA_DIR = dataDir; - - const { buildNativeTlsClientOptions } = - await import("../../open-sse/services/tlsClientDownloadDir.ts"); - - const options = buildNativeTlsClientOptions(); - - assert.equal(options.runtimeMode, "native"); - assert.equal(options.downloadDir, join(dataDir, "tls-client", "bin")); -}); - -test("all web-provider tls clients wire downloadDir through buildNativeTlsClientOptions (#8579)", () => { - const base = readFileSync(join(ROOT, "open-sse/services/tlsClientBase.ts"), "utf8"); - assert.match( - base, - /buildNativeTlsClientOptions\(\)/, - "tlsClientBase.ts must pass buildNativeTlsClientOptions() to TLSClient" - ); - assert.doesNotMatch( - base, - /new TLSClient\(\{\s*runtimeMode:\s*"native"\s*\}\)/, - "tlsClientBase.ts must not construct TLSClient without downloadDir" - ); - - for (const relPath of TLS_CLIENT_WRAPPERS) { - const source = readFileSync(join(ROOT, relPath), "utf8"); - assert.match( - source, - /createTlsClientModule\(/, - `${relPath} must go through createTlsClientModule so downloadDir is inherited` - ); - assert.doesNotMatch( - source, - /new TLSClient\(\{\s*runtimeMode:\s*"native"\s*\}\)/, - `${relPath} must not construct TLSClient without downloadDir` - ); - } -}); diff --git a/tests/unit/tls-client-node-docker-binary-7802.test.ts b/tests/unit/tls-client-node-docker-binary-7802.test.ts deleted file mode 100644 index ff4aab0a60..0000000000 --- a/tests/unit/tls-client-node-docker-binary-7802.test.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { test } from "node:test"; -import assert from "node:assert/strict"; -import { readFileSync } from "node:fs"; -import { fileURLToPath } from "node:url"; -import { dirname, join } from "node:path"; - -const __dirname = dirname(fileURLToPath(import.meta.url)); -const ROOT = join(__dirname, "..", ".."); - -test("Dockerfile's --ignore-scripts npm ci is compensated for tls-client-node's native binary, same as it is for wreq-js and better-sqlite3 (#7802)", () => { - const dockerfile = readFileSync(join(ROOT, "Dockerfile"), "utf8"); - const postinstall = readFileSync(join(ROOT, "scripts/build/postinstall.mjs"), "utf8"); - - assert.match( - dockerfile, - // Flag-order tolerant on purpose: the assertion is about the --ignore-scripts - // PRECONDITION, not the exact flag list. #9185 inserted --include=optional - // (LLMLingua optional deps) and broke the literal pin without touching intent. - /npm ci(?: --[\w-]+(?:=[\w-]+)?)* --ignore-scripts/, - "expected the builder stage to install with --ignore-scripts (precondition of #7802)" - ); - - assert.match( - dockerfile, - /better-sqlite3[\s\S]*node-gyp\.js rebuild/, - "expected an explicit better-sqlite3 rebuild step after --ignore-scripts" - ); - - assert.match( - postinstall, - /fixWreqJsBinary/, - "expected postinstall.mjs to repair wreq-js's native binary" - ); - - const dockerfileHandlesIt = /tls-client-node[\s\S]{0,200}(postinstall|rebuild|download)/i.test( - dockerfile - ); - const postinstallHandlesIt = /tls-client-node/i.test(postinstall); - - assert.ok( - dockerfileHandlesIt || postinstallHandlesIt, - "tls-client-node has no --ignore-scripts compensation in Dockerfile or " + - "scripts/build/postinstall.mjs (unlike better-sqlite3 and wreq-js) — " + - "node_modules/tls-client-node/bin/ is never populated in the official " + - "Docker image, so chatgpt-web/claude-web/grok-web/lmarena/perplexity-web " + - "all fail with TlsClientUnavailableError at first request (#7802)" - ); -}); diff --git a/tests/unit/tls-client-wreq-migration.test.ts b/tests/unit/tls-client-wreq-migration.test.ts new file mode 100644 index 0000000000..b953931eef --- /dev/null +++ b/tests/unit/tls-client-wreq-migration.test.ts @@ -0,0 +1,1078 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { createTlsClientModule } from "../../open-sse/services/tlsClientBase.ts"; + +const encoder = new TextEncoder(); + +test("stream EOF filtering recognizes a sentinel fragmented across native chunks", async () => { + const module = createTlsClientModule({ + providerName: "Test", + tlsProfile: "chrome_146", + domain: "https://example.test", + streamEofPolicy: "exclude", + responseValidation: "cf", + exportCloudflareCheck: false, + exposeStreamingForTesting: true, + }); + const client = { + async request() { + const chunks = ['{"answer":"ok"}\n[DO', "NE]ignored"]; + return new Response( + new ReadableStream({ + async pull(controller) { + const next = chunks.shift(); + if (next === undefined) { + controller.close(); + return; + } + if (chunks.length === 0) await new Promise((resolve) => setTimeout(resolve, 25)); + controller.enqueue(encoder.encode(next)); + }, + }), + { status: 200 } + ); + }, + }; + + const result = await module.__tlsFetchStreamingForTesting!( + client, + "https://example.test/stream", + { method: "POST" }, + "[DONE]", + null, + 1_000, + 1_000 + ); + + assert.ok(result.body); + assert.equal(await new Response(result.body).text(), '{"answer":"ok"}\n'); +}); + +test("stream EOF filtering recognizes a fragmented sentinel after an isolated CR", async () => { + const module = createTlsClientModule({ + providerName: "Test", + tlsProfile: "chrome_146", + domain: "https://example.test", + streamEofPolicy: "exclude", + responseValidation: "cf", + exportCloudflareCheck: false, + exposeStreamingForTesting: true, + }); + const client = { + async request() { + const chunks = ['{"answer":"ok"}\r[DO', "NE]ignored"]; + return new Response( + new ReadableStream({ + pull(controller) { + const next = chunks.shift(); + if (next === undefined) controller.close(); + else controller.enqueue(encoder.encode(next)); + }, + }), + { status: 200 } + ); + }, + }; + + const result = await module.__tlsFetchStreamingForTesting!( + client, + "https://example.test/stream", + { method: "POST" }, + "[DONE]", + null, + 1_000, + 1_000 + ); + + assert.ok(result.body); + assert.equal(await new Response(result.body).text(), '{"answer":"ok"}\r'); +}); + +test("wreq transports are isolated by browser, OS, and resolved proxy without a cookie jar", async () => { + const transportOptions: Array> = []; + const transports: Array<{ id: number; close(): Promise }> = []; + const fetchCalls: Array<{ url: string; options: Record }> = []; + const runtime = { + async createTransport(options: Record) { + transportOptions.push(options); + const transport = { id: transports.length + 1, async close() {} }; + transports.push(transport); + return transport; + }, + async fetch(url: string, options: Record) { + fetchCalls.push({ url, options }); + return new Response("ok", { + status: 200, + headers: [["set-cookie", "upstream=one; Path=/"]], + }); + }, + }; + const module = createTlsClientModule({ + providerName: "Test", + tlsProfile: "chrome_146", + emulationOs: "linux", + domain: "https://example.test", + streamEofPolicy: "include", + responseValidation: "sse", + exportCloudflareCheck: false, + wreqRuntimeLoader: async () => runtime, + }); + + const first = await module.tlsFetch("https://example.test/one", { + proxyUrl: "http://proxy-a.test:8080", + }); + const second = await module.tlsFetch("https://example.test/two", { + proxyUrl: "http://proxy-a.test:8080", + }); + await module.tlsFetch("https://example.test/three", { + proxyUrl: "http://proxy-b.test:8080", + }); + + assert.equal(first.text, "ok"); + assert.equal(second.text, "ok"); + assert.deepEqual(transportOptions, [ + { browser: "chrome_146", os: "linux", proxy: "http://proxy-a.test:8080" }, + { browser: "chrome_146", os: "linux", proxy: "http://proxy-b.test:8080" }, + ]); + assert.equal(fetchCalls[0]?.options.transport, transports[0]); + assert.equal(fetchCalls[1]?.options.transport, transports[0]); + assert.equal(fetchCalls[2]?.options.transport, transports[1]); + assert.equal(fetchCalls[0]?.options.cookieMode, "ephemeral"); + assert.equal("session" in (fetchCalls[0]?.options ?? {}), false); + assert.equal("sessionId" in (fetchCalls[0]?.options ?? {}), false); +}); + +test("wreq response streaming validates a fragmented SSE prefix and filters a fragmented EOF", async () => { + const runtime = { + async createTransport() { + return { async close() {} }; + }, + async fetch() { + const chunks = ["da", 'ta: {"answer":"ok"}\n[D', "ONE]ignored"]; + return new Response( + new ReadableStream({ + pull(controller) { + const next = chunks.shift(); + if (next === undefined) { + controller.close(); + } else { + controller.enqueue(encoder.encode(next)); + } + }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } } + ); + }, + }; + const module = createTlsClientModule({ + providerName: "Test", + tlsProfile: "firefox_148", + emulationOs: "macos", + domain: "https://example.test", + streamEofPolicy: "exclude", + responseValidation: "sse", + exportCloudflareCheck: false, + wreqRuntimeLoader: async () => runtime, + }); + + const result = await module.tlsFetch("https://example.test/stream", { + method: "POST", + stream: true, + streamEofSymbol: "[DONE]", + }); + + assert.ok(result.body, "a valid SSE response must stay streaming"); + assert.equal(result.text, null); + assert.equal(await new Response(result.body).text(), 'data: {"answer":"ok"}\n'); +}); + +test("byteResponse preserves arbitrary bytes as a content-typed data URL", async () => { + const runtime = { + async createTransport() { + return { async close() {} }; + }, + async fetch() { + return { + status: 200, + headers: new Headers({ "content-type": "image/png; charset=binary" }), + body: null, + async bytes() { + return new Uint8Array([0, 255, 1, 254]); + }, + async text(): Promise { + throw new Error("binary response must not be decoded as UTF-8"); + }, + }; + }, + }; + const module = createTlsClientModule({ + providerName: "Test", + tlsProfile: "firefox_148", + domain: "https://example.test", + streamEofPolicy: "include", + responseValidation: "sse", + exportCloudflareCheck: false, + wreqRuntimeLoader: async () => runtime, + }); + + const result = await module.tlsFetch("https://example.test/image", { + byteResponse: true, + }); + + assert.equal(result.text, "data:image/png;base64,AP8B/g=="); +}); + +test("a response that misses the first-byte deadline falls back to a buffered body", async () => { + const runtime = { + async createTransport() { + return { async close() {} }; + }, + async fetch() { + let sent = false; + return new Response( + new ReadableStream({ + async pull(controller) { + if (sent) { + controller.close(); + return; + } + sent = true; + await new Promise((resolve) => setTimeout(resolve, 60)); + controller.enqueue(encoder.encode('data: {"late":true}\n\n[DONE]')); + }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } } + ); + }, + }; + const module = createTlsClientModule({ + providerName: "Test", + tlsProfile: "chrome_146", + domain: "https://example.test", + streamEofPolicy: "include", + responseValidation: "sse", + exportCloudflareCheck: false, + firstByteTimeoutMs: 10, + wreqRuntimeLoader: async () => runtime, + }); + + const result = await module.tlsFetch("https://example.test/stream", { + stream: true, + timeoutMs: 500, + }); + + assert.equal(result.body, null); + assert.equal(result.text, 'data: {"late":true}\n\n[DONE]'); +}); + +test("an empty native chunk does not satisfy the first-byte deadline", async () => { + const runtime = { + async createTransport() { + return { async close() {} }; + }, + async fetch() { + let pullCount = 0; + return new Response( + new ReadableStream({ + async pull(controller) { + pullCount += 1; + if (pullCount === 1) { + controller.enqueue(new Uint8Array(0)); + return; + } + if (pullCount === 2) { + await new Promise((resolve) => setTimeout(resolve, 60)); + controller.enqueue(encoder.encode('data: {"late":true}\n\n[DONE]')); + return; + } + controller.close(); + }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } } + ); + }, + }; + const module = createTlsClientModule({ + providerName: "Test", + tlsProfile: "chrome_146", + domain: "https://example.test", + streamEofPolicy: "include", + responseValidation: "sse", + exportCloudflareCheck: false, + firstByteTimeoutMs: 10, + wreqRuntimeLoader: async () => runtime, + }); + + const result = await module.tlsFetch("https://example.test/stream", { + stream: true, + timeoutMs: 500, + }); + + assert.equal(result.body, null); + assert.equal(result.text, 'data: {"late":true}\n\n[DONE]'); +}); + +test("the first-byte deadline includes request and response-header latency", async () => { + const module = createTlsClientModule({ + providerName: "Test", + tlsProfile: "chrome_146", + domain: "https://example.test", + streamEofPolicy: "include", + responseValidation: "sse", + exportCloudflareCheck: false, + exposeStreamingForTesting: true, + }); + const client = { + async request() { + await new Promise((resolve) => setTimeout(resolve, 40)); + return new Response('data: {"lateHeaders":true}\n\n[DONE]', { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); + }, + }; + + const result = await module.__tlsFetchStreamingForTesting!( + client, + "https://example.test/stream", + { method: "POST" }, + "[DONE]", + null, + 1_000, + 10 + ); + + assert.equal(result.body, null); + assert.equal(result.text, 'data: {"lateHeaders":true}\n\n[DONE]'); +}); + +test("the hard timeout also bounds a wreq body that never produces its first byte", async () => { + const runtime = { + async createTransport() { + return { async close() {} }; + }, + async fetch() { + return new Response( + new ReadableStream({ + pull() { + return new Promise(() => {}); + }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } } + ); + }, + }; + const module = createTlsClientModule({ + providerName: "Test", + tlsProfile: "chrome_146", + domain: "https://example.test", + streamEofPolicy: "include", + responseValidation: "sse", + exportCloudflareCheck: false, + defaultTimeoutMs: 20, + hardTimeoutGraceMs: 10, + firstByteTimeoutMs: Number.POSITIVE_INFINITY, + wreqRuntimeLoader: async () => runtime, + }); + + const outcome = await Promise.race([ + module.tlsFetch("https://example.test/stream", { stream: true }).then( + () => ({ kind: "resolved" as const }), + (error: unknown) => ({ kind: "rejected" as const, error }) + ), + new Promise<{ kind: "hung" }>((resolve) => setTimeout(() => resolve({ kind: "hung" }), 250)), + ]); + + assert.notEqual(outcome.kind, "hung", "the body read must remain bounded"); + assert.equal(outcome.kind, "rejected"); + if (outcome.kind === "rejected") { + assert.equal((outcome.error as Error).name, "TlsClientHangError"); + } +}); + +test("the hard timeout remains active after streaming headers and the first chunk", async () => { + const runtime = { + async createTransport() { + return { async close() {} }; + }, + async fetch() { + let first = true; + return new Response( + new ReadableStream({ + pull(controller) { + if (first) { + first = false; + controller.enqueue(encoder.encode('data: {"partial":true}\n\n')); + return; + } + return new Promise(() => {}); + }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } } + ); + }, + }; + const module = createTlsClientModule({ + providerName: "Test", + tlsProfile: "chrome_146", + domain: "https://example.test", + streamEofPolicy: "include", + responseValidation: "sse", + exportCloudflareCheck: false, + defaultTimeoutMs: 300, + hardTimeoutGraceMs: 200, + firstByteTimeoutMs: Number.POSITIVE_INFINITY, + wreqRuntimeLoader: async () => runtime, + }); + + const result = await module.tlsFetch("https://example.test/stream", { stream: true }); + assert.ok(result.body); + const reader = result.body.getReader(); + const first = await reader.read(); + assert.equal(first.done, false); + const outcome = await Promise.race([ + reader.read().then( + () => ({ kind: "resolved" as const }), + (error: unknown) => ({ kind: "rejected" as const, error }) + ), + new Promise<{ kind: "hung" }>((resolve) => setTimeout(() => resolve({ kind: "hung" }), 2_000)), + ]); + + assert.equal(outcome.kind, "rejected"); + if (outcome.kind === "rejected") { + assert.equal((outcome.error as Error).name, "TlsClientHangError"); + } +}); + +test("an empty native stream preserves the upstream error status instead of becoming 200", async () => { + const module = createTlsClientModule({ + providerName: "Test", + tlsProfile: "chrome_146", + domain: "https://example.test", + streamEofPolicy: "none", + responseValidation: "cf", + exportCloudflareCheck: false, + exposeStreamingForTesting: true, + }); + const client = { + async request() { + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(0)); + controller.close(); + }, + }), + { status: 403, headers: { "x-upstream": "preserved" } } + ); + }, + }; + + const result = await module.__tlsFetchStreamingForTesting!( + client, + "https://example.test/stream", + { method: "POST" }, + "", + null, + 1_000, + 1_000 + ); + + assert.equal(result.status, 403); + assert.equal(result.headers.get("x-upstream"), "preserved"); + assert.equal(result.text, ""); + assert.equal(result.body, null); +}); + +test("Cloudflare detection peeks across fragmented wreq chunks before exposing a stream", async () => { + const runtime = { + async createTransport() { + return { async close() {} }; + }, + async fetch() { + const chunks = [ + "Ju", + "st a moment...", + ]; + return new Response( + new ReadableStream({ + pull(controller) { + const next = chunks.shift(); + if (next === undefined) controller.close(); + else controller.enqueue(encoder.encode(next)); + }, + }), + { status: 200, headers: { "content-type": "text/html" } } + ); + }, + }; + const module = createTlsClientModule({ + providerName: "Test", + tlsProfile: "chrome_146", + domain: "https://example.test", + streamEofPolicy: "exclude", + responseValidation: "cf", + exportCloudflareCheck: true, + wreqRuntimeLoader: async () => runtime, + }); + + const result = await module.tlsFetch("https://example.test/stream", { stream: true }); + + assert.equal(result.status, 403); + assert.equal(result.body, null); + assert.match(result.text ?? "", /just a moment/i); +}); + +test("a non-success native response is buffered without rewriting its status to 200", async () => { + const module = createTlsClientModule({ + providerName: "Test", + tlsProfile: "chrome_146", + domain: "https://example.test", + streamEofPolicy: "none", + responseValidation: "cf", + exportCloudflareCheck: false, + exposeStreamingForTesting: true, + }); + const client = { + async request() { + const chunks = ['{"error":"rate ', 'limited"}']; + return new Response( + new ReadableStream({ + pull(controller) { + const next = chunks.shift(); + if (next === undefined) controller.close(); + else controller.enqueue(encoder.encode(next)); + }, + }), + { status: 429, headers: { "retry-after": "30" } } + ); + }, + }; + + const result = await module.__tlsFetchStreamingForTesting!( + client, + "https://example.test/stream", + { method: "POST" }, + "", + null, + 1_000, + 1_000 + ); + + assert.equal(result.status, 429); + assert.equal(result.headers.get("retry-after"), "30"); + assert.equal(result.text, '{"error":"rate limited"}'); + assert.equal(result.body, null); +}); + +test("a non-success HTML response keeps the generic HTML classification", async () => { + const module = createTlsClientModule({ + providerName: "Test", + tlsProfile: "chrome_146", + domain: "https://example.test", + streamEofPolicy: "none", + responseValidation: "cf", + exportCloudflareCheck: false, + exposeStreamingForTesting: true, + }); + const client = { + async request() { + return new Response("Service unavailable", { + status: 503, + headers: { "content-type": "text/html" }, + }); + }, + }; + + const result = await module.__tlsFetchStreamingForTesting!( + client, + "https://example.test/stream", + { method: "POST" }, + "", + null, + 1_000, + 1_000 + ); + + assert.equal(result.status, 502); + assert.match(result.text ?? "", /service unavailable/i); + assert.equal(result.body, null); +}); + +test("SSE validation tolerates a UTF-8 BOM fragmented across native chunks", async () => { + const module = createTlsClientModule({ + providerName: "Test", + tlsProfile: "chrome_146", + domain: "https://example.test", + streamEofPolicy: "exclude", + responseValidation: "sse", + exportCloudflareCheck: false, + exposeStreamingForTesting: true, + }); + const payload = encoder.encode('data: {"answer":"ok"}\n\n[DONE]'); + const chunks = [ + new Uint8Array([0xef]), + new Uint8Array([0xbb]), + new Uint8Array([0xbf, ...payload]), + ]; + const client = { + async request() { + return new Response( + new ReadableStream({ + pull(controller) { + const next = chunks.shift(); + if (next === undefined) controller.close(); + else controller.enqueue(next); + }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } } + ); + }, + }; + + const result = await module.__tlsFetchStreamingForTesting!( + client, + "https://example.test/stream", + { method: "POST" }, + "[DONE]", + null, + 1_000, + 1_000 + ); + + assert.ok(result.body); + assert.equal(await new Response(result.body).text(), 'data: {"answer":"ok"}\n\n'); +}); + +test("EOF filtering ignores a sentinel literal inside an SSE data frame", async () => { + const module = createTlsClientModule({ + providerName: "Test", + tlsProfile: "chrome_146", + domain: "https://example.test", + streamEofPolicy: "exclude", + responseValidation: "sse", + exportCloudflareCheck: false, + exposeStreamingForTesting: true, + }); + const expected = 'data: {"content":"literal [DONE] survives"}\n\n'; + const client = { + async request() { + return new Response(`${expected}data: [DONE]ignored`, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); + }, + }; + + const result = await module.__tlsFetchStreamingForTesting!( + client, + "https://example.test/stream", + { method: "POST" }, + "[DONE]", + null, + 1_000, + 1_000 + ); + + assert.ok(result.body); + assert.equal(await new Response(result.body).text(), expected); +}); + +test("the include policy preserves a fragmented Perplexity end_of_stream marker", async () => { + const runtime = { + async createTransport() { + return { async close() {} }; + }, + async fetch() { + const chunks = ['data: {"answer":"ok"}\n\nev', "ent: end_of_", "stream\nignored"]; + return new Response( + new ReadableStream({ + pull(controller) { + const next = chunks.shift(); + if (next === undefined) controller.close(); + else controller.enqueue(encoder.encode(next)); + }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } } + ); + }, + }; + const module = createTlsClientModule({ + providerName: "Perplexity", + tlsProfile: "firefox_148", + emulationOs: "macos", + domain: "https://www.perplexity.ai", + streamEofPolicy: "include", + responseValidation: "sse", + exportCloudflareCheck: true, + wreqRuntimeLoader: async () => runtime, + }); + + const result = await module.tlsFetch("https://www.perplexity.ai/rest/sse/perplexity_ask", { + stream: true, + streamEofSymbol: "event: end_of_stream", + }); + + assert.ok(result.body); + assert.equal( + await new Response(result.body).text(), + 'data: {"answer":"ok"}\n\nevent: end_of_stream' + ); +}); + +test("the no-sentinel policy leaves an LMArena stream untouched through native EOF", async () => { + const payload = '{"text":"[DONE] is data"}\n[DONE]still-data'; + const runtime = { + async createTransport() { + return { async close() {} }; + }, + async fetch() { + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(payload)); + controller.close(); + }, + }), + { status: 200, headers: { "content-type": "application/x-ndjson" } } + ); + }, + }; + const module = createTlsClientModule({ + providerName: "LMArena", + tlsProfile: "chrome_146", + emulationOs: "windows", + domain: "https://lmarena.ai", + streamEofPolicy: "none", + responseValidation: "cf", + exportCloudflareCheck: true, + wreqRuntimeLoader: async () => runtime, + }); + + const result = await module.tlsFetch("https://arena.ai/api/stream", { + stream: true, + streamEofSymbol: "[DONE]", + }); + + assert.ok(result.body); + assert.equal(await new Response(result.body).text(), payload); +}); + +test("duplicate response headers and Set-Cookie values survive the adapter", async () => { + const rawHeaders = { + *[Symbol.iterator](): IterableIterator<[string, string]> { + yield ["x-trace", "one"]; + yield ["x-trace", "two"]; + yield ["set-cookie", "collapsed-value-must-not-win"]; + }, + getSetCookie() { + return ["session=one; Path=/; HttpOnly", "affinity=two; Path=/"]; + }, + }; + const runtime = { + async createTransport() { + return { async close() {} }; + }, + async fetch() { + return { status: 202, headers: rawHeaders, body: "accepted" }; + }, + }; + const module = createTlsClientModule({ + providerName: "Headers", + tlsProfile: "chrome_146", + domain: "https://example.test", + streamEofPolicy: "include", + responseValidation: "sse", + exportCloudflareCheck: false, + wreqRuntimeLoader: async () => runtime, + }); + + const result = await module.tlsFetch("https://example.test/headers"); + + assert.equal(result.status, 202); + assert.equal(result.headers.get("x-trace"), "one, two"); + assert.deepEqual(result.headers.getSetCookie(), [ + "session=one; Path=/; HttpOnly", + "affinity=two; Path=/", + ]); +}); + +test("caller abort errors the exposed stream and cancels the native reader", async () => { + let cancelReason: unknown; + let first = true; + const runtime = { + async createTransport() { + return { async close() {} }; + }, + async fetch() { + return new Response( + new ReadableStream({ + pull(controller) { + if (first) { + first = false; + controller.enqueue(encoder.encode('data: {"partial":true}\n\n')); + return; + } + return new Promise(() => {}); + }, + cancel(reason) { + cancelReason = reason; + }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } } + ); + }, + }; + const module = createTlsClientModule({ + providerName: "Abort", + tlsProfile: "chrome_146", + domain: "https://example.test", + streamEofPolicy: "include", + responseValidation: "sse", + exportCloudflareCheck: false, + defaultTimeoutMs: 1_000, + hardTimeoutGraceMs: 1_000, + wreqRuntimeLoader: async () => runtime, + }); + const abort = new AbortController(); + const result = await module.tlsFetch("https://example.test/stream", { + stream: true, + signal: abort.signal, + }); + assert.ok(result.body); + const reader = result.body.getReader(); + assert.equal((await reader.read()).done, false); + + abort.abort(); + await assert.rejects(reader.read(), (error: unknown) => (error as Error).name === "AbortError"); + assert.equal((cancelReason as Error).name, "AbortError"); +}); + +test("caller abort cancels a native reader while a non-stream response is buffering", async () => { + let cancelReason: unknown; + let pullCount = 0; + let notifySecondPull!: () => void; + const secondPullStarted = new Promise((resolve) => { + notifySecondPull = resolve; + }); + const runtime = { + async createTransport() { + return { async close() {} }; + }, + async fetch() { + return new Response( + new ReadableStream({ + pull(controller) { + pullCount += 1; + if (pullCount === 1) { + controller.enqueue(encoder.encode("partial")); + return; + } + notifySecondPull(); + return new Promise(() => {}); + }, + cancel(reason) { + cancelReason = reason; + }, + }), + { status: 200 } + ); + }, + }; + const module = createTlsClientModule({ + providerName: "Abort", + tlsProfile: "chrome_146", + domain: "https://example.test", + streamEofPolicy: "include", + responseValidation: "sse", + exportCloudflareCheck: false, + defaultTimeoutMs: 1_000, + hardTimeoutGraceMs: 1_000, + wreqRuntimeLoader: async () => runtime, + }); + const abort = new AbortController(); + const pending = module.tlsFetch("https://example.test/buffer", { signal: abort.signal }); + + await secondPullStarted; + abort.abort(); + + await assert.rejects(pending, (error: unknown) => (error as Error).name === "AbortError"); + assert.equal((cancelReason as Error | undefined)?.name, "AbortError"); +}); + +test("caller abort cancels an exposed stream even without another consumer read", async () => { + let cancelReason: unknown; + const runtime = { + async createTransport() { + return { async close() {} }; + }, + async fetch() { + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode('{"partial":true}\n')); + }, + cancel(reason) { + cancelReason = reason; + }, + }), + { status: 200, headers: { "content-type": "application/x-ndjson" } } + ); + }, + }; + const module = createTlsClientModule({ + providerName: "Abort", + tlsProfile: "chrome_146", + domain: "https://example.test", + streamEofPolicy: "none", + responseValidation: "cf", + exportCloudflareCheck: false, + defaultTimeoutMs: 1_000, + hardTimeoutGraceMs: 1_000, + wreqRuntimeLoader: async () => runtime, + }); + const abort = new AbortController(); + const result = await module.tlsFetch("https://example.test/stream", { + stream: true, + signal: abort.signal, + }); + assert.ok(result.body); + + abort.abort(); + await new Promise((resolve) => setTimeout(resolve, 20)); + + assert.equal((cancelReason as Error | undefined)?.name, "AbortError"); +}); + +test("the absolute hard deadline cancels an exposed stream under consumer backpressure", async () => { + let cancelReason: unknown; + let invalidated = false; + const module = createTlsClientModule({ + providerName: "Deadline", + tlsProfile: "chrome_146", + domain: "https://example.test", + streamEofPolicy: "none", + responseValidation: "cf", + exportCloudflareCheck: false, + exposeStreamingForTesting: true, + }); + const client = { + async request() { + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode('{"partial":true}\n')); + }, + cancel(reason) { + cancelReason = reason; + }, + }), + { status: 200 } + ); + }, + invalidateTransport() { + invalidated = true; + }, + }; + + const result = await module.__tlsFetchStreamingForTesting!( + client, + "https://example.test/stream", + { method: "POST" }, + "", + null, + 40, + 1_000 + ); + assert.ok(result.body); + + await new Promise((resolve) => setTimeout(resolve, 80)); + + assert.equal((cancelReason as Error | undefined)?.name, "TlsClientHangError"); + assert.equal(invalidated, true); +}); + +test("consumer cancellation propagates to the native wreq response reader", async () => { + let cancelReason: unknown; + const runtime = { + async createTransport() { + return { async close() {} }; + }, + async fetch() { + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode('data: {"partial":true}\n\n')); + }, + cancel(reason) { + cancelReason = reason; + }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } } + ); + }, + }; + const module = createTlsClientModule({ + providerName: "Cancel", + tlsProfile: "chrome_146", + domain: "https://example.test", + streamEofPolicy: "include", + responseValidation: "sse", + exportCloudflareCheck: false, + wreqRuntimeLoader: async () => runtime, + }); + const result = await module.tlsFetch("https://example.test/stream", { stream: true }); + assert.ok(result.body); + + await result.body.cancel("consumer stopped"); + + assert.equal(cancelReason, "consumer stopped"); +}); + +test("a hard timeout evicts and closes only the affected pooled transport", async () => { + const created: Array<{ id: number; closed: boolean }> = []; + let requestCount = 0; + const runtime = { + async createTransport() { + const state = { id: created.length + 1, closed: false }; + created.push(state); + return { + async close() { + state.closed = true; + }, + }; + }, + async fetch() { + requestCount += 1; + if (requestCount === 1) return new Promise(() => {}); + return new Response("recovered", { status: 200 }); + }, + }; + const module = createTlsClientModule({ + providerName: "Reset", + tlsProfile: "chrome_146", + emulationOs: "linux", + domain: "https://example.test", + streamEofPolicy: "include", + responseValidation: "sse", + exportCloudflareCheck: false, + defaultTimeoutMs: 10, + hardTimeoutGraceMs: 10, + wreqRuntimeLoader: async () => runtime, + }); + + await assert.rejects( + module.tlsFetch("https://example.test/hang", { proxyUrl: "http://proxy.test:8080" }), + (error: unknown) => (error as Error).name === "TlsClientHangError" + ); + const recovered = await module.tlsFetch("https://example.test/recovered", { + proxyUrl: "http://proxy.test:8080", + timeoutMs: 100, + }); + + assert.equal(recovered.text, "recovered"); + assert.equal(created.length, 2); + assert.equal(created[0]?.closed, true); + assert.equal(created[1]?.closed, false); +}); diff --git a/tests/unit/tls-client-wreq-residue.test.ts b/tests/unit/tls-client-wreq-residue.test.ts new file mode 100644 index 0000000000..c25cc37b3a --- /dev/null +++ b/tests/unit/tls-client-wreq-residue.test.ts @@ -0,0 +1,54 @@ +import assert from "node:assert/strict"; +import { existsSync, readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +const ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); + +test("the distributable is pinned to wreq-js 3.0.0 with no tls-client-node or orphaned koffi residue", () => { + const packageJson = JSON.parse(readFileSync(join(ROOT, "package.json"), "utf8")) as { + files: string[]; + optionalDependencies: Record; + }; + assert.equal(packageJson.optionalDependencies["wreq-js"], "3.0.0"); + assert.equal(packageJson.optionalDependencies["tls-client-node"], undefined); + assert.equal(packageJson.files.includes("scripts/build/fixTlsClientNodeBinary.mjs"), false); + + for (const relativePath of [ + "package-lock.json", + "next.config.mjs", + "Dockerfile", + "Dockerfile.bun", + "pnpm.json", + "pnpm-workspace.yaml", + "config/quality/dependency-allowlist.json", + "config/quality/.license-allowlist.json", + "scripts/build/postinstall.mjs", + "scripts/build/pack-artifact-policy.ts", + ]) { + const source = readFileSync(join(ROOT, relativePath), "utf8"); + assert.doesNotMatch( + source, + /tls-client-node/i, + `${relativePath} still references tls-client-node` + ); + assert.doesNotMatch(source, /\bkoffi\b/i, `${relativePath} still references orphaned koffi`); + } + + assert.equal(existsSync(join(ROOT, "open-sse/services/tlsClientDownloadDir.ts")), false); + assert.equal(existsSync(join(ROOT, "scripts/build/fixTlsClientNodeBinary.mjs")), false); + + for (const relativePath of [ + ".env.example", + "docs/reference/ENVIRONMENT.md", + "docs/security/STEALTH_GUIDE.md", + ]) { + const source = readFileSync(join(ROOT, relativePath), "utf8"); + assert.doesNotMatch( + source, + /OMNIROUTE_TLS_PROXY_URL/, + `${relativePath} still documents the removed sidecar override` + ); + } +}); diff --git a/tests/unit/tls-profiles-valid-5591.test.mjs b/tests/unit/tls-profiles-valid-5591.test.mjs index 87f2a41e11..46dccd7639 100644 --- a/tests/unit/tls-profiles-valid-5591.test.mjs +++ b/tests/unit/tls-profiles-valid-5591.test.mjs @@ -4,7 +4,7 @@ import fs from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; -// #5591 regression guard: every chrome_* TLS impersonation profile referenced in +// #5591 regression guard: every TLS impersonation profile referenced in // the source must be a real wreq-js BrowserProfile. PR #5237 set them to // "chrome_149", which does not exist in wreq-js 2.3.1 (the union tops out at // chrome_147) — the native layer then produced a degenerate fingerprint and the @@ -19,15 +19,24 @@ function supportedProfiles() { path.join(ROOT, "node_modules", "wreq-js", "dist", "wreq-js.d.ts"), "utf8" ); - return new Set([...dts.matchAll(/chrome_(\d+)/g)].map((m) => `chrome_${m[1]}`)); + const union = dts.match(/type BrowserProfile = ([^;]+);/)?.[1] ?? ""; + return new Set([...union.matchAll(/'([^']+)'/g)].map((match) => match[1])); } -// Source files that hand a `browser`/PROFILE value to wreq-js. +const PROVIDER_PROFILES = { + "open-sse/services/chatgptTlsClient.ts": ["firefox_148", "macos"], + "open-sse/services/claudeTlsClient.ts": ["chrome_146", "linux"], + "open-sse/services/perplexityTlsClient.ts": ["firefox_148", "macos"], + "open-sse/services/grokTlsClient.ts": ["chrome_146", "linux"], + "open-sse/services/notionTlsClient.ts": ["chrome_146", "windows"], + "open-sse/services/lmarenaTlsClient.ts": ["chrome_146", "windows"], +}; + +// Other source files that hand a browser profile directly to wreq-js. const SOURCES = [ "src/app/api/internal/codex-responses-ws/route.ts", "scripts/dev/responses-ws-proxy.mjs", - "open-sse/services/grokTlsClient.ts", - "open-sse/services/claudeTlsClient.ts", + ...Object.keys(PROVIDER_PROFILES), ]; // Strip comments before scanning — explanatory comments may name the bad @@ -36,16 +45,16 @@ function stripComments(line) { return line.replace(/\/\*.*?\*\//g, "").replace(/\/\/.*$/, ""); } -test("#5591 all configured chrome_* TLS profiles exist in wreq-js", () => { +test("#5591 all configured browser TLS profiles exist in pinned wreq-js", () => { const supported = supportedProfiles(); - assert.ok(supported.size > 0, "expected to parse chrome_* profiles from wreq-js d.ts"); + assert.ok(supported.size > 0, "expected to parse BrowserProfile from wreq-js d.ts"); for (const rel of SOURCES) { const lines = fs.readFileSync(path.join(ROOT, rel), "utf8").split("\n"); lines.forEach((line, i) => { const code = stripComments(line); - for (const m of code.matchAll(/chrome_(\d+)/g)) { - const profile = `chrome_${m[1]}`; + for (const m of code.matchAll(/\b(?:chrome|firefox|edge|opera|safari|okhttp)_[\w.]+/g)) { + const profile = m[0]; assert.ok( supported.has(profile), `${rel}:${i + 1} uses ${profile} which is NOT a wreq-js BrowserProfile ` + @@ -54,4 +63,16 @@ test("#5591 all configured chrome_* TLS profiles exist in wreq-js", () => { } }); } + + for (const [rel, [profile, os]] of Object.entries(PROVIDER_PROFILES)) { + const source = fs.readFileSync(path.join(ROOT, rel), "utf8"); + assert.ok(supported.has(profile), `${rel} expected unsupported ${profile}`); + if (rel.endsWith("claudeTlsClient.ts")) { + assert.match(source, /CLAUDE_TLS_BROWSER_MAJOR_VERSION = "146"/); + assert.match(source, /tlsProfile: `chrome_\$\{CLAUDE_TLS_BROWSER_MAJOR_VERSION\}`/); + } else { + assert.match(source, new RegExp(`tlsProfile: ["']${profile}["']`)); + } + assert.match(source, new RegExp(`emulationOs: ["']${os}["']`)); + } }); diff --git a/tests/unit/ui/add-api-key-modal-validation-error-5088.test.tsx b/tests/unit/ui/add-api-key-modal-validation-error-5088.test.tsx index 2495674223..6119bf0c04 100644 --- a/tests/unit/ui/add-api-key-modal-validation-error-5088.test.tsx +++ b/tests/unit/ui/add-api-key-modal-validation-error-5088.test.tsx @@ -19,9 +19,9 @@ const { default: AddApiKeyModal } = await import("../../../src/app/(dashboard)/dashboard/providers/[id]/components/modals/AddApiKeyModal"); const TLS_EACCES_ERROR = - "TLS impersonation client failed to start: EACCES: permission denied, mkdir " + - "'/usr/lib/node_modules/omniroute/dist/node_modules/tls-client-node/bin'. " + - "Verify tls-client-node is installed and its native binary downloaded. " + + "TLS impersonation client failed to start: EACCES: permission denied, open " + + "'/usr/lib/node_modules/omniroute/dist/node_modules/wreq-js/rust'. " + + "Verify wreq-js 3.0.0 and its platform addon are installed. " + "(claude-web requires this — without it, Cloudflare blocks every request)"; const containers: Array<{ root: ReturnType; el: HTMLDivElement }> = []; @@ -32,7 +32,12 @@ function render(props: Record) { const root = createRoot(el); act(() => { root.render( - undefined} onClose={() => {}} {...(props as any)} /> + undefined} + onClose={() => {}} + {...(props as any)} + /> ); }); containers.push({ root, el }); @@ -68,7 +73,10 @@ beforeEach(() => { json: () => Promise.resolve({ valid: false, error: TLS_EACCES_ERROR }), } as Response); } - return Promise.resolve({ ok: true, json: () => Promise.resolve({ valid: true }) } as Response); + return Promise.resolve({ + ok: true, + json: () => Promise.resolve({ valid: true }), + } as Response); }) ); }); @@ -92,8 +100,7 @@ describe("AddApiKeyModal — surfaces the detailed validation error (#5088)", () // The validate ("check") button is the first button that follows the // credential input in DOM order (it sits right next to it). const checkBtn = Array.from(el.querySelectorAll("button")).find( - (b) => - (apiKeyInput.compareDocumentPosition(b) & Node.DOCUMENT_POSITION_FOLLOWING) !== 0 + (b) => (apiKeyInput.compareDocumentPosition(b) & Node.DOCUMENT_POSITION_FOLLOWING) !== 0 )!; expect(checkBtn).toBeTruthy(); act(() => { diff --git a/tests/unit/wreq-native-manifest.test.ts b/tests/unit/wreq-native-manifest.test.ts new file mode 100644 index 0000000000..ab2c497d2f --- /dev/null +++ b/tests/unit/wreq-native-manifest.test.ts @@ -0,0 +1,64 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import test from "node:test"; + +const ROOT = process.cwd(); +const MANIFEST_PATH = join(ROOT, "config/release/wreq-js-native-manifest.json"); + +interface NativeEntry { + path: string; + size: number; + sha256: string; +} + +interface NativeManifest { + package: string; + version: string; + npmIntegrity: string; + license: string; + nativeAddons: NativeEntry[]; +} + +test("wreq-js 3.0.0 has an exact seven-platform native hash manifest and notice", () => { + assert.equal(existsSync(MANIFEST_PATH), true, "native manifest must be committed"); + const manifest = JSON.parse(readFileSync(MANIFEST_PATH, "utf8")) as NativeManifest; + assert.equal(manifest.package, "wreq-js"); + assert.equal(manifest.version, "3.0.0"); + assert.equal( + manifest.npmIntegrity, + "sha512-RZCoRSevVPpH4A4B4MxbFGo/pVPFveWd2gbe4ENKpPWlKXEYklZSDESOjBMmrIsmnkHh+nhM4PNJvG+NL7wBPA==" + ); + assert.equal(manifest.license, "MIT"); + assert.equal(manifest.nativeAddons.length, 7); + assert.deepEqual(manifest.nativeAddons.map((entry) => entry.path).sort(), [ + "rust/wreq-js.darwin-arm64.node", + "rust/wreq-js.darwin-x64.node", + "rust/wreq-js.linux-arm64-gnu.node", + "rust/wreq-js.linux-arm64-musl.node", + "rust/wreq-js.linux-x64-gnu.node", + "rust/wreq-js.linux-x64-musl.node", + "rust/wreq-js.win32-x64-msvc.node", + ]); + + const packageRoot = join(ROOT, "node_modules/wreq-js"); + const installedPackage = JSON.parse(readFileSync(join(packageRoot, "package.json"), "utf8")) as { + version: string; + }; + assert.equal(installedPackage.version, manifest.version); + for (const entry of manifest.nativeAddons) { + const bytes = readFileSync(join(packageRoot, entry.path)); + assert.equal(bytes.byteLength, entry.size, `${entry.path}: byte size`); + assert.equal( + createHash("sha256").update(bytes).digest("hex"), + entry.sha256, + `${entry.path}: sha256` + ); + } + + const notices = readFileSync(join(ROOT, "THIRD_PARTY_NOTICES.md"), "utf8"); + assert.match(notices, /^## wreq-js 3\.0\.0$/m); + assert.match(notices, /Copyright \(c\) 2025 will-work-for-meal/); + assert.match(notices, /Copyright \(c\) 2025 Oleksandr Herasymov/); +}); diff --git a/tests/unit/wreq-postinstall-native.test.ts b/tests/unit/wreq-postinstall-native.test.ts new file mode 100644 index 0000000000..38aa95f44e --- /dev/null +++ b/tests/unit/wreq-postinstall-native.test.ts @@ -0,0 +1,49 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + WREQ_JS_NATIVE_BINARY_NAMES, + resolveWreqJsNativeBinaryName, +} from "../../scripts/build/wreqJsNative.mjs"; + +test("wreq-js 3.0.0 resolver covers every shipped native addon name", () => { + assert.deepEqual([...WREQ_JS_NATIVE_BINARY_NAMES].sort(), [ + "wreq-js.darwin-arm64.node", + "wreq-js.darwin-x64.node", + "wreq-js.linux-arm64-gnu.node", + "wreq-js.linux-arm64-musl.node", + "wreq-js.linux-x64-gnu.node", + "wreq-js.linux-x64-musl.node", + "wreq-js.win32-x64-msvc.node", + ]); + assert.equal( + resolveWreqJsNativeBinaryName({ platform: "darwin", arch: "arm64" }), + "wreq-js.darwin-arm64.node" + ); + assert.equal( + resolveWreqJsNativeBinaryName({ platform: "darwin", arch: "x64" }), + "wreq-js.darwin-x64.node" + ); + assert.equal( + resolveWreqJsNativeBinaryName({ platform: "linux", arch: "arm64", libc: "gnu" }), + "wreq-js.linux-arm64-gnu.node" + ); + assert.equal( + resolveWreqJsNativeBinaryName({ platform: "linux", arch: "arm64", libc: "musl" }), + "wreq-js.linux-arm64-musl.node" + ); + assert.equal( + resolveWreqJsNativeBinaryName({ platform: "linux", arch: "x64", libc: "gnu" }), + "wreq-js.linux-x64-gnu.node" + ); + assert.equal( + resolveWreqJsNativeBinaryName({ platform: "linux", arch: "x64", libc: "musl" }), + "wreq-js.linux-x64-musl.node" + ); + assert.equal( + resolveWreqJsNativeBinaryName({ platform: "win32", arch: "x64" }), + "wreq-js.win32-x64-msvc.node" + ); + assert.equal(resolveWreqJsNativeBinaryName({ platform: "win32", arch: "arm64" }), null); + assert.equal(resolveWreqJsNativeBinaryName({ platform: "android", arch: "arm64" }), null); +});